From 0c84998bf4234e2df40a16132e8da9607bb0e7ae Mon Sep 17 00:00:00 2001 From: JasonAtClockwork Date: Tue, 1 Sep 2026 14:15:58 -0700 Subject: [PATCH 1/8] Initial split of sdk-test into separate command --- .github/workflows/ci.yml | 242 ++++++++++++-- Cargo.lock | 11 + Cargo.toml | 1 + crates/testing/src/sdk.rs | 406 +++++++++++++++++++++--- sdks/rust/tests/test.rs | 71 +++-- tools/ci/commands/sdk-tests/Cargo.toml | 14 + tools/ci/commands/sdk-tests/src/main.rs | 235 ++++++++++++++ tools/ci/commands/test/src/main.rs | 49 +-- tools/ci/src/main.rs | 4 + 9 files changed, 885 insertions(+), 148 deletions(-) create mode 100644 tools/ci/commands/sdk-tests/Cargo.toml create mode 100644 tools/ci/commands/sdk-tests/src/main.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d349ae85819..54503f26471 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -589,8 +589,219 @@ jobs: - name: Skip duplicate merge queue smoketest run: echo "Merge queue commit has the same tree as the PR head; smoketest already ran for the PR." + sdk_test_build: + needs: [upload-build-artifacts-linux] + name: Build SDK tests + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 30 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux + SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli + steps: + - *find-git-ref + - *checkout-sources + + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain + - *set-native-cache-keys + + - name: Restore rusty_v8 (debug) + uses: actions/cache/restore@v4 + with: + path: ${{ env.CARGO_TARGET_DIR }}/debug/gn_out/obj + key: ${{ steps.native-cache-keys.outputs.v8-debug-key }} + - name: Restore jemalloc + id: cache-jemalloc + uses: actions/cache/restore@v4 + with: *jemalloc-cache + - *configure-cached-jemalloc + - name: Restore Zstd + id: cache-zstd + uses: actions/cache/restore@v4 + with: *zstd-cache + - *configure-cached-zstd + - *restore-openssl + - *configure-cached-openssl + + - *download-build-artifacts + - *extract-build-artifacts + + - uses: actions/setup-dotnet@v4 + with: + global-json-file: global.json + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - uses: ./.github/actions/setup-pnpm + with: + run_install: true + + - name: Install cmake and emscripten + run: | + sudo apt-get update + sudo apt-get install -y cmake + git clone https://github.com/emscripten-core/emsdk.git ~/emsdk + cd ~/emsdk + ./emsdk install 4.0.21 + ./emsdk activate 4.0.21 + + - name: Override NuGet packages + shell: bash + run: | + dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime + dotnet pack -c Release crates/bindings-csharp/Runtime + cd sdks/csharp + ./tools~/write-nuget-config.sh ../.. + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Install wasm-bindgen CLI + run: | + REQUIRED_WASM_BINDGEN_VERSION="$( + awk ' + $1 == "name" && $3 == "\"wasm-bindgen\"" { in_pkg = 1; next } + in_pkg && $1 == "version" { + gsub(/"/, "", $3); + print $3; + exit; + } + ' Cargo.lock + )" + if [ -z "${REQUIRED_WASM_BINDGEN_VERSION}" ]; then + echo "Failed to determine wasm-bindgen version from Cargo.lock" + exit 1 + fi + cargo install --locked --force wasm-bindgen-cli --version "${REQUIRED_WASM_BINDGEN_VERSION}" + + - name: Build SDK test modules and archives + shell: bash + run: | + source ~/emsdk/emsdk_env.sh + cargo run --timings -p ci-sdk-tests -- prepare-modules --output-dir sdk-test-modules + cargo run --timings -p ci-sdk-tests -- prepare-clients --mode native --module-dir sdk-test-modules --output-dir sdk-test-clients/native + cargo run --timings -p ci-sdk-tests -- archive --mode native --archive-file sdk-tests-native-nextest.tar.zst + cargo run --timings -p ci-sdk-tests -- prepare-clients --mode browser --module-dir sdk-test-modules --output-dir sdk-test-clients/browser + cargo run --timings -p ci-sdk-tests -- archive --mode browser --archive-file sdk-tests-browser-nextest.tar.zst + tar -czf sdk-test-support.tar.gz \ + target/debug/ci-sdk-tests \ + sdk-test-modules \ + sdk-test-clients + + - name: Upload SDK test build + uses: actions/upload-artifact@v4 + with: + name: sdk-test-build-linux + path: | + sdk-tests-native-nextest.tar.zst + sdk-tests-browser-nextest.tar.zst + sdk-test-support.tar.gz + if-no-files-found: error + overwrite: true + retention-days: 14 + + - name: Upload Cargo timing reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: sdk-test-build-cargo-timings + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ + if-no-files-found: warn + overwrite: true + retention-days: 14 + + sdk_tests_native: + needs: [sdk_test_build] + name: SDK Tests (native ${{ matrix.partition }}/4) + strategy: + fail-fast: false + matrix: + partition: [1, 2, 3, 4] + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 10 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux + PARTITION_COUNT: 4 + SDK_TEST_ARCHIVE: sdk-tests-native-nextest.tar.zst + SDK_TEST_CLIENT_DIR: sdk-test-clients/native + SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli + steps: &sdk-test-partition-steps + - *find-git-ref + - *checkout-sources + + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - *download-build-artifacts + - *extract-build-artifacts + + - name: Download SDK test build + uses: actions/download-artifact@v4 + with: + name: sdk-test-build-linux + + - name: Extract SDK test support files + run: tar -xzf sdk-test-support.tar.gz + + - name: Run SDK test partition + run: | + ./target/debug/ci-sdk-tests run-archive \ + --archive-file "${SDK_TEST_ARCHIVE}" \ + --module-dir sdk-test-modules \ + --client-dir "${SDK_TEST_CLIENT_DIR}" \ + -- \ + --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} \ + -E 'not test(/unreal/)' + + sdk_tests_browser: + needs: [sdk_test_build] + name: SDK Tests (browser ${{ matrix.partition }}/4) + strategy: + fail-fast: false + matrix: + partition: [1, 2, 3, 4] + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 10 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux + PARTITION_COUNT: 4 + SDK_TEST_ARCHIVE: sdk-tests-browser-nextest.tar.zst + SDK_TEST_CLIENT_DIR: sdk-test-clients/browser + SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli + steps: *sdk-test-partition-steps + + sdk_tests_noop: + needs: [merge_queue_noop] + if: ${{ needs.merge_queue_noop.outputs.skip == 'true' }} + name: SDK Tests (${{ matrix.mode }} ${{ matrix.partition }}/4) + strategy: + matrix: + mode: [native, browser] + partition: [1, 2, 3, 4] + runs-on: ubuntu-latest + steps: + - name: Skip duplicate merge queue SDK test + run: echo "Merge queue commit has the same tree as the PR head; SDK tests already ran for the PR." + test: - needs: [merge_queue_noop, lints, upload-build-artifacts-linux] + needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Test Suite runs-on: spacetimedb-new-runner-2 @@ -598,8 +809,6 @@ jobs: env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full - ARTIFACT_SUFFIX: linux - SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: - name: Find Git ref env: @@ -649,9 +858,6 @@ jobs: - *restore-openssl - *configure-cached-openssl - - *download-build-artifacts - - *extract-build-artifacts - - uses: actions/setup-dotnet@v3 with: global-json-file: global.json @@ -675,30 +881,6 @@ jobs: ./emsdk install 4.0.21 ./emsdk activate 4.0.21 - - name: Install wasm-bindgen CLI - run: | - REQUIRED_WASM_BINDGEN_VERSION="$( - awk ' - $1 == "name" && $3 == "\"wasm-bindgen\"" { in_pkg = 1; next } - in_pkg && $1 == "version" { - gsub(/"/, "", $3); - print $3; - exit; - } - ' Cargo.lock - )" - if [ -z "${REQUIRED_WASM_BINDGEN_VERSION}" ]; then - echo "Failed to determine wasm-bindgen version from Cargo.lock" - exit 1 - fi - - INSTALLED_WASM_BINDGEN_VERSION="$(wasm-bindgen --version 2>/dev/null | awk '{print $2}' || true)" - if [ "${INSTALLED_WASM_BINDGEN_VERSION}" != "${REQUIRED_WASM_BINDGEN_VERSION}" ]; then - cargo install --locked --force wasm-bindgen-cli --version "${REQUIRED_WASM_BINDGEN_VERSION}" - fi - - wasm-bindgen --version - # Source emsdk environment to make emcc (Emscripten compiler) available in PATH. - name: Run tests run: | diff --git a/Cargo.lock b/Cargo.lock index 0543315066c..565eb54dae5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -977,6 +977,17 @@ dependencies = [ "duct", ] +[[package]] +name = "ci-sdk-tests" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-common", + "clap 4.5.50", + "duct", + "spacetimedb-testing", +] + [[package]] name = "ci-smoketest-checks" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index d98d2a8ecfb..b5f4c32a230 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ members = [ "tools/ci/commands/lint", "tools/ci/commands/module-latest-deps", "tools/ci/commands/smoketests", + "tools/ci/commands/sdk-tests", "tools/ci/commands/smoketest-checks", "tools/ci/commands/keynote-bench", "tools/ci/commands/update-flow", diff --git a/crates/testing/src/sdk.rs b/crates/testing/src/sdk.rs index eb45353aa2b..d9803d8e3ce 100644 --- a/crates/testing/src/sdk.rs +++ b/crates/testing/src/sdk.rs @@ -4,7 +4,8 @@ use spacetimedb::messages::control_db::HostType; use spacetimedb_data_structures::map::HashMap; use spacetimedb_guard::SpacetimeDbGuard; use spacetimedb_paths::{RootDir, SpacetimePaths}; -use std::fs::create_dir_all; +use std::fs::{copy, create_dir_all, read_dir}; +use std::path::{Path, PathBuf}; use std::sync::Mutex; use crate::invoke_cli; @@ -69,12 +70,29 @@ pub struct Test { /// - `SPACETIME_SDK_TEST_DB_NAME` bound to the database identity or name. /// - `SPACETIME_SDK_TEST_SERVER_URL` bound to the server URL for this test. run_command: String, + + prepared_client: Option, + prepared_client_key: Option, +} + +#[derive(Clone)] +// These are artifact execution strategies, not SDK test modes. The test suite +// still has only native and browser modes; `Node` packages existing TypeScript +// clients that run in both modes. +enum PreparedClient { + Native { binary_name: String, args: Vec }, + Browser { artifact_name: String, selector: String }, + Node { entrypoint: PathBuf, args: Vec }, } pub const TEST_MODULE_PROJECT_ENV_VAR: &str = "SPACETIME_SDK_TEST_MODULE_PROJECT"; pub const TEST_DB_NAME_ENV_VAR: &str = "SPACETIME_SDK_TEST_DB_NAME"; pub const TEST_SERVER_URL_ENV_VAR: &str = "SPACETIME_SDK_TEST_SERVER_URL"; pub const TEST_CLIENT_PROJECT_ENV_VAR: &str = "SPACETIME_SDK_TEST_CLIENT_PROJECT"; +pub const PRECOMPILED_MODULE_DIR_ENV_VAR: &str = "SPACETIME_SDK_TEST_MODULE_DIR"; +pub const PREPARED_CLIENT_DIR_ENV_VAR: &str = "SPACETIME_SDK_TEST_CLIENT_DIR"; +pub const PREPARE_CLIENT_DIR_ENV_VAR: &str = "SPACETIME_SDK_TEST_PREPARE_CLIENT_DIR"; +pub const TEST_WORKSPACE_ROOT_ENV_VAR: &str = "SPACETIME_SDK_TEST_WORKSPACE_ROOT"; fn language_is_unreal(language: &str) -> bool { language.eq_ignore_ascii_case("unrealcpp") @@ -90,26 +108,199 @@ impl Test { let (file, host_type) = compile_module(&self.module_name); + let prepared_client_dir = std::env::var_os(PREPARED_CLIENT_DIR_ENV_VAR).map(PathBuf::from); + if prepared_client_dir.is_none() { + self.generate_bindings(paths, &file, host_type); + compile_client(&self.compile_command, &self.client_project, &self.module_name); + } + + let guard = SpacetimeDbGuard::spawn_in_temp_data_dir(); + let server_url = guard.host_url.as_str(); + let db_name = publish_module(paths, server_url, &file, host_type); + + if let Some(prepared_client_dir) = prepared_client_dir { + self.run_prepared_client(&prepared_client_dir, server_url, &db_name); + } else { + run_client(&self.run_command, &self.client_project, server_url, &db_name); + } + } + + pub fn prepare(self) { + let output_dir = PathBuf::from( + std::env::var_os(PREPARE_CLIENT_DIR_ENV_VAR) + .unwrap_or_else(|| panic!("{PREPARE_CLIENT_DIR_ENV_VAR} is not set")), + ); + let sdk_paths = SdkTestPaths::new(); + let (file, host_type) = compile_module(&self.module_name); + self.generate_bindings(&sdk_paths.paths, &file, host_type); + compile_client(&self.compile_command, &self.client_project, &self.module_name); + self.export_prepared_client(&output_dir); + } + + fn generate_bindings(&self, paths: &SpacetimePaths, file: &str, host_type: HostType) { generate_bindings( paths, &self.generate_language, - &file, + file, host_type, &self.client_project, &self.generate_subdir, self.generate_include_private, ); + } - compile_client(&self.compile_command, &self.client_project); + fn client_artifact_dir(&self, root: &Path) -> PathBuf { + let key = self.prepared_client_key.as_deref().unwrap_or_else(|| { + Path::new(&self.client_project) + .file_name() + .and_then(|name| name.to_str()) + .expect("SDK client project should end in a UTF-8 directory name") + }); + root.join(key) + } - let guard = SpacetimeDbGuard::spawn_in_temp_data_dir(); - let server_url = guard.host_url.as_str(); - let db_name = publish_module(paths, server_url, &file, host_type); + fn export_prepared_client(&self, output_dir: &Path) { + let prepared = self + .prepared_client + .as_ref() + .expect("SDK test does not describe how to prepare its client"); + let artifact_dir = self.client_artifact_dir(output_dir); + match prepared { + PreparedClient::Native { binary_name, .. } => { + let source = cargo_target_dir().join("debug").join(binary_name); + copy_file(&source, &artifact_dir.join("bin").join(binary_name)); + } + PreparedClient::Browser { artifact_name, .. } => { + let package_name = Path::new(&self.client_project).file_name().unwrap(); + let source = Path::new(&self.client_project) + .join("target/sdk-test-web-bindgen") + .join(package_name); + copy_dir(&source, &artifact_dir.join("web")); + assert!( + artifact_dir.join("web").join(format!("{artifact_name}.cjs")).is_file(), + "Prepared browser client is missing its CommonJS entrypoint" + ); + } + PreparedClient::Node { entrypoint, .. } => { + let source = Path::new(&self.client_project).join(entrypoint); + copy_file(&source, &artifact_dir.join(entrypoint)); + copy_file( + &Path::new(&self.client_project).join("package.json"), + &artifact_dir.join("package.json"), + ); + } + } + } + + fn run_prepared_client(&self, root: &Path, server_url: &str, db_name: &str) { + let prepared = self + .prepared_client + .as_ref() + .expect("SDK test does not describe how to run its prepared client"); + let artifact_dir = self.client_artifact_dir(root); + let (exe, args) = match prepared { + PreparedClient::Native { binary_name, args } => ( + artifact_dir.join("bin").join(binary_name).into_os_string(), + args.iter().map(Into::into).collect(), + ), + PreparedClient::Browser { + artifact_name, + selector, + } => { + let js_module = artifact_dir.join("web").join(format!("{artifact_name}.cjs")); + let script = browser_node_script(&js_module, selector); + ( + "node".into(), + vec!["--experimental-websocket".into(), "-e".into(), script.into()], + ) + } + PreparedClient::Node { entrypoint, args } => { + let mut node_args = vec![artifact_dir.join(entrypoint).into_os_string()]; + node_args.extend(args.iter().map(Into::into)); + ("node".into(), node_args) + } + }; - run_client(&self.run_command, &self.client_project, server_url, &db_name); + run_client_command(exe, args, &self.client_project, server_url, db_name, &self.run_command); + } +} + +pub fn workspace_path(path: impl AsRef) -> PathBuf { + let path = path.as_ref(); + if path.exists() { + return path.to_path_buf(); + } + + let build_workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("spacetimedb-testing should be two directories below the workspace root"); + match path.strip_prefix(build_workspace_root) { + Ok(relative) => runtime_workspace_root().join(relative), + Err(_) => path.to_path_buf(), + } +} + +fn runtime_workspace_root() -> PathBuf { + std::env::var_os(TEST_WORKSPACE_ROOT_ENV_VAR) + .map(PathBuf::from) + .unwrap_or_else(|| { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("spacetimedb-testing should be two directories below the workspace root") + .to_path_buf() + }) +} + +fn cargo_target_dir() -> PathBuf { + std::env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| runtime_workspace_root().join("target")) +} + +fn copy_file(source: &Path, destination: &Path) { + create_dir_all(destination.parent().unwrap()).unwrap(); + copy(source, destination).unwrap_or_else(|error| { + panic!( + "Failed to copy {} to {}: {error}", + source.display(), + destination.display() + ) + }); +} + +fn copy_dir(source: &Path, destination: &Path) { + create_dir_all(destination).unwrap(); + for entry in read_dir(source).unwrap_or_else(|error| panic!("Failed to read {}: {error}", source.display())) { + let entry = entry.unwrap(); + let source_path = entry.path(); + let destination_path = destination.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_dir(&source_path, &destination_path); + } else { + copy_file(&source_path, &destination_path); + } } } +fn browser_node_script(js_module: &Path, selector: &str) -> String { + format!( + "(async () => {{ \ + const m = require({js_module:?}); \ + if (m.default) {{ await m.default(); }} \ + const run = m.run || m.main || m.start; \ + if (!run) throw new Error(\"No exported run/main/start function from wasm module\"); \ + const dbName = process.env.SPACETIME_SDK_TEST_DB_NAME; \ + if (!dbName) throw new Error(\"Missing SPACETIME_SDK_TEST_DB_NAME\"); \ + const serverUrl = process.env.SPACETIME_SDK_TEST_SERVER_URL; \ + if (!serverUrl) throw new Error(\"Missing SPACETIME_SDK_TEST_SERVER_URL\"); \ + await run({selector:?}, dbName, serverUrl); \ + process.exit(0); \ + }})().catch((e) => {{ console.error(e); process.exit(1); }});" + ) +} + fn status_ok_or_panic(output: std::process::Output, command: &str, test_name: &str) { if !output.status.success() { panic!( @@ -170,6 +361,20 @@ macro_rules! memoized { // with toolchains like .NET which don't expect parallel invocations // of their build tools on the same project folder. fn compile_module(module: &str) -> (String, HostType) { + if let Some(module_dir) = std::env::var_os(PRECOMPILED_MODULE_DIR_ENV_VAR) { + let module_dir = PathBuf::from(module_dir); + for (extension, host_type) in [("wasm", HostType::Wasm), ("js", HostType::Js)] { + let path = module_dir.join(module).with_extension(extension); + if path.is_file() { + return (path.to_string_lossy().into_owned(), host_type); + } + } + panic!( + "No precompiled SDK test module found for {module:?} in {}", + module_dir.display() + ); + } + let module = module.to_owned(); memoized!(|module: String| -> (String, HostType) { @@ -178,6 +383,33 @@ fn compile_module(module: &str) -> (String, HostType) { }) } +/// Compile all SDK test modules into a stable directory for archived CI tests. +pub fn build_precompiled_modules(output_dir: &Path) -> anyhow::Result { + let workspace_root = std::env::current_dir()?; + let modules_dir = workspace_root.join("modules"); + let mut module_names = std::fs::read_dir(&modules_dir)? + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir())) + .filter_map(|entry| entry.file_name().into_string().ok()) + .filter(|name| name.starts_with("sdk-test")) + .collect::>(); + module_names.sort(); + + create_dir_all(output_dir)?; + for module_name in &module_names { + eprintln!("Building precompiled SDK test module {module_name}..."); + let module = CompiledModule::compile(module_name, CompilationMode::Debug); + let extension = match module.host_type { + HostType::Wasm => "wasm", + HostType::Js => "js", + }; + let destination = output_dir.join(module_name).with_extension(extension); + std::fs::copy(module.path(), &destination)?; + } + + Ok(module_names.len()) +} + // Note: this function does not memoize because we want each test to publish the same // module as a separate clean database instance for isolation purposes. fn publish_module(paths: &SpacetimePaths, server_url: &str, wasm_file: &str, host_type: HostType) -> String { @@ -218,7 +450,7 @@ fn publish_module(paths: &SpacetimePaths, server_url: &str, wasm_file: &str, hos /// and the `generate_subdir` an arbitrary relative path within it. /// These will be combined as `"{client_project}/{generate_subdir}"` to produce the `--out-dir`. /// -/// Note: this function is memoized to ensure we only run `spacetime generate` once for each target directory. +/// This function memoizes the complete generation input, not just the output directory. /// /// Without this lock, if multiple `Test`s ran concurrently in the same process /// with the same `client_project` and `generate_subdir`, @@ -226,13 +458,9 @@ fn publish_module(paths: &SpacetimePaths, server_url: &str, wasm_file: &str, hos /// each of which would remove and re-populate the bindings directory, /// potentially sweeping them out from under a compile or run process. /// -/// This lock ensures that only one `spacetime generate` process runs at a time, -/// and the `HashSet` ensures that we run `spacetime generate` only once for each output directory. +/// This lock ensures that only one `spacetime generate` process runs at a time. /// /// Circumstances where this will still break: -/// - If multiple tests want to use the same client_project/generate_subdir pair, -/// but for different modules' bindings, only one module's bindings will ever be generated. -/// If you need bindings for multiple different modules, put them in different subdirs. /// - If multiple distinct test harness processes run concurrently, /// they will encounter the race condition described above, /// because the binding-generation lock is not shared between harness processes. @@ -253,47 +481,57 @@ fn generate_bindings( generate_subdir: &str, generate_include_private: bool, ) { - // We need these to be owned `String`s so we can memoize on them. + // We need these to be owned values so we can memoize on them. let client_project = client_project.to_owned(); let generate_subdir = generate_subdir.to_owned(); + let language = language.to_owned(); + let wasm_file = wasm_file.to_owned(); + let host_is_js = matches!(host_type, HostType::Js); // Codegen is side-effecting and doesn't meaningfully return a Rust value, // so our memoization has unit as the value. // This makes it run at most once for each key. - memoized!(|(client_project, generate_subdir): (String, String)| -> () { - let mut args: Vec<&str> = vec![ - "generate", - "--yes", - "--lang", - language, - match host_type { - HostType::Wasm => "--bin-path", - HostType::Js => "--js-path", - }, - wasm_file, - ]; - - if generate_include_private { - args.push("--include-private"); - } + memoized!( + |(client_project, generate_subdir, language, wasm_file, host_is_js, generate_include_private): ( + String, + String, + String, + String, + bool, + bool, + )| + -> () { + let mut args: Vec<&str> = vec![ + "generate", + "--yes", + "--lang", + &language, + if *host_is_js { "--js-path" } else { "--bin-path" }, + &wasm_file, + ]; + + if *generate_include_private { + args.push("--include-private"); + } - let generate_dir: String; + let generate_dir: String; + + // `generate --lang unrealcpp` takes different arguments from non-Unreal languages + // to account for some quirks of Unreal project structure. + if language_is_unreal(language) { + // For unreal, we use `client_project` as the uproject directory, + // and `generate_subdir` as the module name. + args.extend_from_slice(&["--uproject-dir", client_project]); + args.extend_from_slice(&["--module-name", generate_subdir]); + } else { + generate_dir = format!("{client_project}/{generate_subdir}"); + create_dir_all(&generate_dir).unwrap(); + args.extend_from_slice(&["--out-dir", &generate_dir]); + } - // `generate --lang unrealcpp` takes different arguments from non-Unreal languages - // to account for some quirks of Unreal project structure. - if language_is_unreal(language) { - // For unreal, we use `client_project` as the uproject directory, - // and `generate_subdir` as the module name. - args.extend_from_slice(&["--uproject-dir", client_project]); - args.extend_from_slice(&["--module-name", generate_subdir]); - } else { - generate_dir = format!("{client_project}/{generate_subdir}"); - create_dir_all(&generate_dir).unwrap(); - args.extend_from_slice(&["--out-dir", &generate_dir]); + invoke_cli(paths, &args); } - - invoke_cli(paths, &args); - }) + ) } fn split_command_string(command: &str) -> (String, Vec) { @@ -329,10 +567,12 @@ fn split_command_string(command: &str) -> (String, Vec) { } // Note: this function is memoized to ensure we only compile each client once. -fn compile_client(compile_command: &str, client_project: &str) { +fn compile_client(compile_command: &str, client_project: &str, module_name: &str) { let client_project = client_project.to_owned(); + let module_name = module_name.to_owned(); - memoized!(|client_project: String| -> () { + memoized!(|(client_project, module_name): (String, String)| -> () { + let _ = module_name; let (exe, args) = split_command_string(compile_command); let output = cmd(exe, args) @@ -351,6 +591,24 @@ fn compile_client(compile_command: &str, client_project: &str) { fn run_client(run_command: &str, client_project: &str, server_url: &str, db_name: &str) { let (exe, args) = split_command_string(run_command); + run_client_command( + exe.into(), + args.into_iter().map(Into::into).collect(), + client_project, + server_url, + db_name, + run_command, + ); +} + +fn run_client_command( + exe: std::ffi::OsString, + args: Vec, + client_project: &str, + server_url: &str, + db_name: &str, + command_description: &str, +) { let output = cmd(exe, args) .dir(client_project) .env(TEST_CLIENT_PROJECT_ENV_VAR, client_project) @@ -366,7 +624,7 @@ fn run_client(run_command: &str, client_project: &str, server_url: &str, db_name .run() .expect("Error running run command"); - status_ok_or_panic(output, run_command, "(running)"); + status_ok_or_panic(output, command_description, "(running)"); } #[derive(Clone, Default)] @@ -379,6 +637,8 @@ pub struct TestBuilder { generate_subdir: Option, compile_command: Option, run_command: Option, + prepared_client: Option, + prepared_client_key: Option, } impl TestBuilder { @@ -397,8 +657,9 @@ impl TestBuilder { } pub fn with_client(self, client_project: impl Into) -> Self { + let client_project = client_project.into(); TestBuilder { - client_project: Some(client_project.into()), + client_project: Some(workspace_path(client_project).to_string_lossy().into_owned()), ..self } } @@ -439,6 +700,51 @@ impl TestBuilder { } } + pub fn with_prepared_native_client( + self, + binary_name: impl Into, + args: impl IntoIterator>, + ) -> Self { + TestBuilder { + prepared_client: Some(PreparedClient::Native { + binary_name: binary_name.into(), + args: args.into_iter().map(Into::into).collect(), + }), + ..self + } + } + + pub fn with_prepared_client_key(self, key: impl Into) -> Self { + TestBuilder { + prepared_client_key: Some(key.into()), + ..self + } + } + + pub fn with_prepared_browser_client(self, artifact_name: impl Into, selector: impl Into) -> Self { + TestBuilder { + prepared_client: Some(PreparedClient::Browser { + artifact_name: artifact_name.into(), + selector: selector.into(), + }), + ..self + } + } + + pub fn with_prepared_node_client( + self, + entrypoint: impl Into, + args: impl IntoIterator>, + ) -> Self { + TestBuilder { + prepared_client: Some(PreparedClient::Node { + entrypoint: entrypoint.into(), + args: args.into_iter().map(Into::into).collect(), + }), + ..self + } + } + pub fn with_generate_private_items(self, include_private: bool) -> Self { TestBuilder { generate_include_private: include_private, @@ -477,6 +783,8 @@ impl TestBuilder { run_command: self .run_command .expect("Supply a run command using TestBuilder::with_run_command"), + prepared_client: self.prepared_client, + prepared_client_key: self.prepared_client_key, } } } diff --git a/sdks/rust/tests/test.rs b/sdks/rust/tests/test.rs index af7c535824b..18e443fa5b5 100644 --- a/sdks/rust/tests/test.rs +++ b/sdks/rust/tests/test.rs @@ -1,22 +1,20 @@ -#[cfg(feature = "browser")] use std::path::Path; use spacetimedb_testing::sdk::{Test, TestBuilder}; fn platform_test_builder(client_project: &str, run_selector: Option<&str>) -> TestBuilder { - let builder = Test::builder(); - let builder = builder.with_client(client_project); + let package_name = Path::new(client_project) + .file_name() + .and_then(|name| name.to_str()) + .expect("client project path should end in a UTF-8 directory name") + .to_owned(); + let builder = Test::builder().with_client(client_project); // Note: `run_selector` is intentionally interpreted differently by mode: // - Native mode uses it as a CLI subcommand (`cargo run -- `), with `None` => `cargo run`. // - Web mode assembles the Node/wasm-bindgen commands directly in this test harness. #[cfg(feature = "browser")] { - let package_name = Path::new(client_project) - .file_name() - .and_then(|name| name.to_str()) - .expect("client project path should end in a UTF-8 directory name") - .to_owned(); let artifact_name = package_name.replace('-', "_"); let target_dir = std::env::var("CARGO_TARGET_DIR").unwrap_or_else(|_| { // Cargo workspace members emit into the workspace target directory, not each crate's @@ -77,6 +75,7 @@ fn platform_test_builder(client_project: &str, run_selector: Option<&str>) -> Te builder .with_compile_command(compile_command) .with_run_command(run_command) + .with_prepared_browser_client(artifact_name, run_selector) } #[cfg(not(feature = "browser"))] @@ -89,6 +88,7 @@ fn platform_test_builder(client_project: &str, run_selector: Option<&str>) -> Te builder .with_compile_command("cargo build") .with_run_command(run_command) + .with_prepared_native_client(package_name, run_selector) } } @@ -100,7 +100,7 @@ macro_rules! declare_tests_with_suffix { const MODULE: &str = concat!("sdk-test", $suffix); const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/test-client"); - fn make_test(subcommand: &str) -> Test { + pub(super) fn make_test(subcommand: &str) -> Test { super::platform_test_builder(CLIENT, Some(subcommand)) .with_name(subcommand) .with_module(MODULE) @@ -285,8 +285,7 @@ macro_rules! declare_tests_with_suffix { make_test("reconnect-different-connection-id").run(); } - #[test] - fn connect_disconnect_callbacks() { + pub(super) fn make_connect_disconnect_test() -> Test { const CONNECT_DISCONNECT_CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/connect_disconnect_client"); @@ -302,7 +301,11 @@ macro_rules! declare_tests_with_suffix { .with_generate_private_items(true) .with_bindings_dir("src/module_bindings") .build() - .run(); + } + + #[test] + fn connect_disconnect_callbacks() { + make_connect_disconnect_test().run(); } #[test] @@ -418,7 +421,7 @@ mod event_table_tests { const MODULE: &str = "sdk-test-event-table"; const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/event-table-client"); - fn make_test(subcommand: &str) -> Test { + pub(super) fn make_test(subcommand: &str) -> Test { super::platform_test_builder(CLIENT, Some(subcommand)) .with_name(subcommand) .with_module(MODULE) @@ -456,7 +459,7 @@ macro_rules! procedure_tests { const MODULE: &str = concat!("sdk-test-procedure", $suffix); const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/procedure-client"); - fn make_test(subcommand: &str) -> Test { + pub(super) fn make_test(subcommand: &str) -> Test { super::platform_test_builder(CLIENT, Some(subcommand)) .with_name(subcommand) .with_module(MODULE) @@ -520,7 +523,7 @@ mod rust_procedure_concurrency { const MODULE: &str = "sdk-test-procedure-concurrency"; const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/procedure-concurrency-client"); - fn make_test(subcommand: &str) -> Test { + pub(super) fn make_test(subcommand: &str) -> Test { super::platform_test_builder(CLIENT, Some(subcommand)) .with_name(subcommand) .with_module(MODULE) @@ -564,7 +567,7 @@ macro_rules! view_tests { const MODULE: &str = concat!("sdk-test-view", $suffix); const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/view-client"); - fn make_test(subcommand: &str) -> Test { + pub(super) fn make_test(subcommand: &str) -> Test { super::platform_test_builder(CLIENT, Some(subcommand)) .with_name(subcommand) .with_module(MODULE) @@ -626,7 +629,7 @@ mod case_conversion_ts { const MODULE: &str = "sdk-test-case-conversion-ts"; const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/case-conversion-client"); - fn make_test(subcommand: &str) -> Test { + pub(super) fn make_test(subcommand: &str) -> Test { Test::builder() .with_name(subcommand) .with_module(MODULE) @@ -635,6 +638,8 @@ mod case_conversion_ts { .with_bindings_dir("src/module_bindings") .with_compile_command("cargo build") .with_run_command(format!("cargo run -- {}", subcommand)) + .with_prepared_native_client("case-conversion-client", [subcommand]) + .with_prepared_client_key("case-conversion-client-ts-module") .build() } @@ -675,7 +680,7 @@ mod case_conversion_rust { const MODULE: &str = "sdk-test-case-conversion"; const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/case-conversion-client"); - fn make_test(subcommand: &str) -> Test { + pub(super) fn make_test(subcommand: &str) -> Test { Test::builder() .with_name(subcommand) .with_module(MODULE) @@ -684,6 +689,7 @@ mod case_conversion_rust { .with_bindings_dir("src/module_bindings") .with_compile_command("cargo build") .with_run_command(format!("cargo run -- {}", subcommand)) + .with_prepared_native_client("case-conversion-client", [subcommand]) .build() } @@ -732,7 +738,7 @@ mod case_conversion_rust_ts_client { "/../../crates/bindings-typescript/case-conversion-test-client" ); - fn make_test(subcommand: &str) -> Test { + pub(super) fn make_test(subcommand: &str) -> Test { Test::builder() .with_name(subcommand) .with_module(MODULE) @@ -743,6 +749,7 @@ mod case_conversion_rust_ts_client { "sh -c 'pnpm install && pnpm --dir .. run build && pnpm exec prettier --write src/module_bindings && pnpm run build'", ) .with_run_command(format!("node dist/index.js {}", subcommand)) + .with_prepared_node_client("dist/index.js", [subcommand]) .build() } @@ -782,7 +789,7 @@ macro_rules! view_pk_tests { const MODULE: &str = concat!("sdk-test-view-pk", $suffix); const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/view-pk-client"); - fn make_test(subcommand: &str) -> Test { + pub(super) fn make_test(subcommand: &str) -> Test { super::platform_test_builder(CLIENT, Some(subcommand)) .with_name(subcommand) .with_module(MODULE) @@ -822,7 +829,7 @@ macro_rules! procedural_view_pk_tests { const MODULE: &str = $module; const CLIENT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/procedural-view-pk-client"); - fn make_test(subcommand: &str) -> Test { + pub(super) fn make_test(subcommand: &str) -> Test { super::platform_test_builder(CLIENT, Some(subcommand)) .with_name(subcommand) .with_module(MODULE) @@ -853,3 +860,25 @@ procedural_view_pk_tests!(rust_procedural_view_pk, "sdk-test-procedural-view-pk" procedural_view_pk_tests!(csharp_procedural_view_pk, "sdk-test-procedural-view-pk-cs"); procedural_view_pk_tests!(typescript_procedural_view_pk, "sdk-test-procedural-view-pk-ts"); procedural_view_pk_tests!(cpp_procedural_view_pk, "sdk-test-procedural-view-pk-cpp"); + +#[test] +#[ignore = "CI prepares shared SDK client artifacts before partitioning"] +fn prepare_clients() { + let prepared_clients = [ + rust::make_test("insert-primitive"), + rust::make_connect_disconnect_test(), + event_table_tests::make_test("event-table"), + rust_procedures::make_test("procedure-return-values"), + rust_procedure_concurrency::make_test("procedure-reducer-interleaving"), + rust_view::make_test("view-anonymous-subscribe"), + case_conversion_ts::make_test("insert-player"), + case_conversion_rust::make_test("insert-player"), + case_conversion_rust_ts_client::make_test("insert-player"), + rust_view_pk::make_test("view-pk-on-update"), + rust_procedural_view_pk::make_test("sender-scoped-pk-view"), + ]; + + for client in prepared_clients { + client.prepare(); + } +} diff --git a/tools/ci/commands/sdk-tests/Cargo.toml b/tools/ci/commands/sdk-tests/Cargo.toml new file mode 100644 index 00000000000..9729944171e --- /dev/null +++ b/tools/ci/commands/sdk-tests/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "ci-sdk-tests" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +duct.workspace = true +ci-common = { path = "../../common" } +spacetimedb-testing = { path = "../../../../crates/testing" } + +[lints] +workspace = true diff --git a/tools/ci/commands/sdk-tests/src/main.rs b/tools/ci/commands/sdk-tests/src/main.rs new file mode 100644 index 00000000000..e4a6dd79f18 --- /dev/null +++ b/tools/ci/commands/sdk-tests/src/main.rs @@ -0,0 +1,235 @@ +#![allow(clippy::disallowed_macros)] + +use anyhow::{ensure, Context, Result}; +use ci_common::pnpm; +use clap::{Parser, Subcommand, ValueEnum}; +use duct::cmd; +use spacetimedb_testing::sdk::{ + build_precompiled_modules, PRECOMPILED_MODULE_DIR_ENV_VAR, PREPARED_CLIENT_DIR_ENV_VAR, PREPARE_CLIENT_DIR_ENV_VAR, + TEST_WORKSPACE_ROOT_ENV_VAR, +}; +use std::env; +use std::path::{Path, PathBuf}; +use std::process::Command; + +#[derive(Parser)] +#[command(about = "Builds and runs the Rust SDK test suite")] +struct Args { + #[command(subcommand)] + command: Option, +} + +#[derive(Clone, Copy, ValueEnum)] +enum Mode { + Native, + Browser, +} + +impl Mode { + fn features(self) -> &'static str { + match self { + Self::Native => "allow_loopback_http_for_tests", + Self::Browser => "allow_loopback_http_for_tests,browser", + } + } +} + +#[derive(Subcommand)] +enum SdkTestCommand { + /// Compile all sdk-test modules without running tests. + PrepareModules { + #[arg(long)] + output_dir: PathBuf, + }, + /// Generate, compile, and export one artifact for each shared SDK test client. + PrepareClients { + #[arg(long)] + mode: Mode, + #[arg(long)] + module_dir: PathBuf, + #[arg(long)] + output_dir: PathBuf, + }, + /// Compile the SDK test binary into a nextest archive. + Archive { + #[arg(long)] + mode: Mode, + #[arg(long)] + archive_file: PathBuf, + }, + /// Run a partition from an existing nextest archive. + RunArchive { + #[arg(long)] + archive_file: PathBuf, + #[arg(long)] + module_dir: PathBuf, + #[arg(long)] + client_dir: PathBuf, + #[arg(trailing_var_arg = true)] + args: Vec, + }, +} + +fn main() -> Result<()> { + ci_common::ensure_repo_root()?; + match Args::parse().command { + Some(SdkTestCommand::PrepareModules { output_dir }) => { + let count = build_precompiled_modules(&output_dir)?; + ensure!(count > 0, "No SDK test modules were found"); + eprintln!("Built {count} precompiled SDK test modules."); + Ok(()) + } + Some(SdkTestCommand::Archive { mode, archive_file }) => archive(mode, &archive_file), + Some(SdkTestCommand::PrepareClients { + mode, + module_dir, + output_dir, + }) => prepare_clients(mode, &module_dir, &output_dir), + Some(SdkTestCommand::RunArchive { + archive_file, + module_dir, + client_dir, + args, + }) => run_archive(&archive_file, &module_dir, &client_dir, args), + None => run_local(), + } +} + +fn ensure_runtime() -> Result<()> { + if env::var_os("SPACETIME_BIN").is_some() { + return ci_common::require_runtime(); + } + + let status = Command::new("cargo") + .args([ + "build", + "--release", + "-p", + "spacetimedb-cli", + "-p", + "spacetimedb-standalone", + "--features", + "spacetimedb-standalone/allow_loopback_http_for_tests", + ]) + .status()?; + ensure!(status.success(), "Failed to build the SDK test runtime"); + Ok(()) +} + +fn run_local() -> Result<()> { + ensure_runtime()?; + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + for mode in [Mode::Native, Mode::Browser] { + let status = Command::new("cargo") + .args([ + "test", + "-p", + "spacetimedb-sdk", + "--features", + mode.features(), + "--", + "--test-threads=2", + "--skip", + "unreal", + ]) + .status()?; + ensure!(status.success(), "SDK tests failed"); + } + Ok(()) +} + +fn archive(mode: Mode, archive_file: &Path) -> Result<()> { + let status = Command::new("cargo") + .args([ + "nextest", + "archive", + "--timings", + "-p", + "spacetimedb-sdk", + "--features", + mode.features(), + "--archive-file", + ]) + .arg(archive_file) + .status()?; + ensure!(status.success(), "Failed to archive SDK tests"); + Ok(()) +} + +fn prepare_clients(mode: Mode, module_dir: &Path, output_dir: &Path) -> Result<()> { + let workspace_root = env::current_dir()?; + let module_dir = absolute_from_workspace(module_dir)?; + let output_dir = absolute_from_workspace(output_dir)?; + ensure!( + module_dir.is_dir(), + "SDK module directory does not exist: {}", + module_dir.display() + ); + std::fs::create_dir_all(&output_dir)?; + + let status = Command::new("cargo") + .args([ + "test", + "--timings", + "-p", + "spacetimedb-sdk", + "--features", + mode.features(), + "--test", + "test", + "prepare_clients", + "--", + "--ignored", + "--exact", + "--test-threads=1", + ]) + .env(PRECOMPILED_MODULE_DIR_ENV_VAR, module_dir) + .env(PREPARE_CLIENT_DIR_ENV_VAR, output_dir) + .env(TEST_WORKSPACE_ROOT_ENV_VAR, workspace_root) + .status()?; + ensure!(status.success(), "Failed to prepare SDK test clients"); + Ok(()) +} + +fn absolute_from_workspace(path: &Path) -> Result { + if path.is_absolute() { + Ok(path.to_path_buf()) + } else { + Ok(env::current_dir()?.join(path)) + } +} + +fn run_archive(archive_file: &Path, module_dir: &Path, client_dir: &Path, args: Vec) -> Result<()> { + ci_common::require_runtime()?; + let workspace_root = env::current_dir()?; + let archive_file = absolute_from_workspace(archive_file)?; + let module_dir = absolute_from_workspace(module_dir)?; + let client_dir = absolute_from_workspace(client_dir)?; + ensure!( + module_dir.is_dir(), + "SDK module directory does not exist: {}", + module_dir.display() + ); + ensure!( + client_dir.is_dir(), + "Prepared SDK client directory does not exist: {}", + client_dir.display() + ); + + let status = Command::new("cargo") + .args(["nextest", "run", "--archive-file"]) + .arg(archive_file) + .arg("--workspace-remap") + .arg(&workspace_root) + .args(["--no-fail-fast", "--no-tests", "pass", "-j", "1"]) + .args(args) + .env(PRECOMPILED_MODULE_DIR_ENV_VAR, module_dir) + .env(PREPARED_CLIENT_DIR_ENV_VAR, client_dir) + .env(TEST_WORKSPACE_ROOT_ENV_VAR, workspace_root) + .status() + .context("Failed to start cargo nextest")?; + ensure!(status.success(), "SDK tests failed"); + + cmd!("bash", "tools/check-diff.sh").run()?; + Ok(()) +} diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs index 1c57d2b8ab7..26242e4f480 100644 --- a/tools/ci/commands/test/src/main.rs +++ b/tools/ci/commands/test/src/main.rs @@ -14,10 +14,6 @@ struct Cli {} fn main() -> Result<()> { Cli::parse(); - let use_prebuilt_runtime = std::env::var_os("SPACETIME_BIN").is_some(); - if use_prebuilt_runtime { - ci_common::require_runtime()?; - } pnpm(["build"]).dir("crates/bindings-typescript").run()?; @@ -55,50 +51,7 @@ fn main() -> Result<()> { "--test-threads=2", ) .run()?; - // The SDK test harness uses the same child-process server guard as smoketests, - // which expects release CLI/standalone binaries to already exist. - if !use_prebuilt_runtime { - cmd!( - "cargo", - "build", - "--release", - "-p", - "spacetimedb-cli", - "-p", - "spacetimedb-standalone", - "--features", - "spacetimedb-standalone/allow_loopback_http_for_tests", - ) - .run()?; - } - // SDK procedure tests intentionally make localhost HTTP requests. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-sdk", - "--features", - "allow_loopback_http_for_tests", - "--", - "--test-threads=2", - "--skip", - "unreal" - ) - .run()?; - // Run the same SDK suite against wasm/browser test clients. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-sdk", - "--features", - "allow_loopback_http_for_tests,browser", - "--", - "--test-threads=2", - "--skip", - "unreal" - ) - .run()?; + // SDK tests have their own dedicated, sharded command: `cargo ci sdk-tests`. // TODO: This should check for a diff at the start. If there is one, we should alert the user // that we're disabling diff checks because they have a dirty git repo, and to re-run in a clean one // if they want those checks. diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 5562a8b8b12..3eab988484e 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -28,6 +28,10 @@ const COMMANDS: &[Command] = &[ path: &["smoketests"], package: "ci-smoketests", }, + Command { + path: &["sdk-tests"], + package: "ci-sdk-tests", + }, Command { path: &["smoketests", "check-mod-list"], package: "ci-smoketest-checks", From 87a837d0cf25301578581c98b1f894c4931db181 Mon Sep 17 00:00:00 2001 From: JasonAtClockwork Date: Tue, 1 Sep 2026 14:36:24 -0700 Subject: [PATCH 2/8] Linting fix --- crates/testing/src/sdk.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/testing/src/sdk.rs b/crates/testing/src/sdk.rs index d9803d8e3ce..878d65304e3 100644 --- a/crates/testing/src/sdk.rs +++ b/crates/testing/src/sdk.rs @@ -397,7 +397,7 @@ pub fn build_precompiled_modules(output_dir: &Path) -> anyhow::Result { create_dir_all(output_dir)?; for module_name in &module_names { - eprintln!("Building precompiled SDK test module {module_name}..."); + log::info!("Building precompiled SDK test module {module_name}..."); let module = CompiledModule::compile(module_name, CompilationMode::Debug); let extension = match module.host_type { HostType::Wasm => "wasm", From 6f5a910b7b73054645c5fe808ea5d482a450283f Mon Sep 17 00:00:00 2001 From: JasonAtClockwork Date: Tue, 1 Sep 2026 14:56:26 -0700 Subject: [PATCH 3/8] Build typescript early --- tools/ci/commands/sdk-tests/src/main.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/ci/commands/sdk-tests/src/main.rs b/tools/ci/commands/sdk-tests/src/main.rs index e4a6dd79f18..038eb31bfcc 100644 --- a/tools/ci/commands/sdk-tests/src/main.rs +++ b/tools/ci/commands/sdk-tests/src/main.rs @@ -74,6 +74,7 @@ fn main() -> Result<()> { ci_common::ensure_repo_root()?; match Args::parse().command { Some(SdkTestCommand::PrepareModules { output_dir }) => { + build_typescript_sdk()?; let count = build_precompiled_modules(&output_dir)?; ensure!(count > 0, "No SDK test modules were found"); eprintln!("Built {count} precompiled SDK test modules."); @@ -118,7 +119,7 @@ fn ensure_runtime() -> Result<()> { fn run_local() -> Result<()> { ensure_runtime()?; - pnpm(["build"]).dir("crates/bindings-typescript").run()?; + build_typescript_sdk()?; for mode in [Mode::Native, Mode::Browser] { let status = Command::new("cargo") .args([ @@ -138,6 +139,11 @@ fn run_local() -> Result<()> { Ok(()) } +fn build_typescript_sdk() -> Result<()> { + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + Ok(()) +} + fn archive(mode: Mode, archive_file: &Path) -> Result<()> { let status = Command::new("cargo") .args([ From ea5a65f24f6d16672a4e510e63045a85b0612ddd Mon Sep 17 00:00:00 2001 From: JasonAtClockwork Date: Tue, 1 Sep 2026 15:58:37 -0700 Subject: [PATCH 4/8] Refactor to speed up building sdk-tests --- .github/workflows/ci.yml | 66 ++++++++++++++++++------ tools/ci/commands/sdk-tests/src/main.rs | 40 ++++++++------- tools/ci/run-sdk-test-archive.sh | 67 +++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 33 deletions(-) create mode 100644 tools/ci/run-sdk-test-archive.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54503f26471..37948e7910d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -684,23 +684,44 @@ jobs: run: | source ~/emsdk/emsdk_env.sh cargo run --timings -p ci-sdk-tests -- prepare-modules --output-dir sdk-test-modules - cargo run --timings -p ci-sdk-tests -- prepare-clients --mode native --module-dir sdk-test-modules --output-dir sdk-test-clients/native - cargo run --timings -p ci-sdk-tests -- archive --mode native --archive-file sdk-tests-native-nextest.tar.zst - cargo run --timings -p ci-sdk-tests -- prepare-clients --mode browser --module-dir sdk-test-modules --output-dir sdk-test-clients/browser - cargo run --timings -p ci-sdk-tests -- archive --mode browser --archive-file sdk-tests-browser-nextest.tar.zst - tar -czf sdk-test-support.tar.gz \ - target/debug/ci-sdk-tests \ - sdk-test-modules \ - sdk-test-clients - - - name: Upload SDK test build + ./target/debug/ci-sdk-tests archive --mode native --archive-file sdk-tests-native-nextest.tar.zst + ./target/debug/ci-sdk-tests archive --mode browser --archive-file sdk-tests-browser-nextest.tar.zst + ./target/debug/ci-sdk-tests prepare-clients --archive-file sdk-tests-native-nextest.tar.zst --module-dir sdk-test-modules --output-dir sdk-test-clients/native + ./target/debug/ci-sdk-tests prepare-clients --archive-file sdk-tests-browser-nextest.tar.zst --module-dir sdk-test-modules --output-dir sdk-test-clients/browser + tar -I 'zstd -T0 -3' -cf sdk-test-modules.tar.zst sdk-test-modules + tar -I 'zstd -T0 -3' -cf sdk-test-clients-native.tar.zst sdk-test-clients/native + tar -I 'zstd -T0 -3' -cf sdk-test-clients-browser.tar.zst sdk-test-clients/browser + + - name: Upload common SDK test build uses: actions/upload-artifact@v4 with: - name: sdk-test-build-linux + name: sdk-test-common-linux + path: sdk-test-modules.tar.zst + compression-level: 0 + if-no-files-found: error + overwrite: true + retention-days: 14 + + - name: Upload native SDK test build + uses: actions/upload-artifact@v4 + with: + name: sdk-test-native-linux path: | sdk-tests-native-nextest.tar.zst + sdk-test-clients-native.tar.zst + compression-level: 0 + if-no-files-found: error + overwrite: true + retention-days: 14 + + - name: Upload browser SDK test build + uses: actions/upload-artifact@v4 + with: + name: sdk-test-browser-linux + path: | sdk-tests-browser-nextest.tar.zst - sdk-test-support.tar.gz + sdk-test-clients-browser.tar.zst + compression-level: 0 if-no-files-found: error overwrite: true retention-days: 14 @@ -729,7 +750,9 @@ jobs: RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux PARTITION_COUNT: 4 + SDK_TEST_BUILD_ARTIFACT: sdk-test-native-linux SDK_TEST_ARCHIVE: sdk-tests-native-nextest.tar.zst + SDK_TEST_CLIENT_ARCHIVE: sdk-test-clients-native.tar.zst SDK_TEST_CLIENT_DIR: sdk-test-clients/native SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: &sdk-test-partition-steps @@ -750,17 +773,26 @@ jobs: - *download-build-artifacts - *extract-build-artifacts - - name: Download SDK test build + - name: Download common SDK test build uses: actions/download-artifact@v4 with: - name: sdk-test-build-linux + name: sdk-test-common-linux + + - name: Download mode-specific SDK test build + uses: actions/download-artifact@v4 + with: + name: ${{ env.SDK_TEST_BUILD_ARTIFACT }} - name: Extract SDK test support files - run: tar -xzf sdk-test-support.tar.gz + run: | + tar --zstd -xf sdk-test-modules.tar.zst + tar --zstd -xf "${SDK_TEST_CLIENT_ARCHIVE}" - name: Run SDK test partition run: | - ./target/debug/ci-sdk-tests run-archive \ + # Use the checked-in wrapper instead of shipping the large ci-sdk-tests + # helper binary from the build job in every SDK test artifact. + bash tools/ci/run-sdk-test-archive.sh \ --archive-file "${SDK_TEST_ARCHIVE}" \ --module-dir sdk-test-modules \ --client-dir "${SDK_TEST_CLIENT_DIR}" \ @@ -782,7 +814,9 @@ jobs: RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux PARTITION_COUNT: 4 + SDK_TEST_BUILD_ARTIFACT: sdk-test-browser-linux SDK_TEST_ARCHIVE: sdk-tests-browser-nextest.tar.zst + SDK_TEST_CLIENT_ARCHIVE: sdk-test-clients-browser.tar.zst SDK_TEST_CLIENT_DIR: sdk-test-clients/browser SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: *sdk-test-partition-steps diff --git a/tools/ci/commands/sdk-tests/src/main.rs b/tools/ci/commands/sdk-tests/src/main.rs index 038eb31bfcc..8b481529232 100644 --- a/tools/ci/commands/sdk-tests/src/main.rs +++ b/tools/ci/commands/sdk-tests/src/main.rs @@ -44,7 +44,7 @@ enum SdkTestCommand { /// Generate, compile, and export one artifact for each shared SDK test client. PrepareClients { #[arg(long)] - mode: Mode, + archive_file: PathBuf, #[arg(long)] module_dir: PathBuf, #[arg(long)] @@ -82,10 +82,10 @@ fn main() -> Result<()> { } Some(SdkTestCommand::Archive { mode, archive_file }) => archive(mode, &archive_file), Some(SdkTestCommand::PrepareClients { - mode, + archive_file, module_dir, output_dir, - }) => prepare_clients(mode, &module_dir, &output_dir), + }) => prepare_clients(&archive_file, &module_dir, &output_dir), Some(SdkTestCommand::RunArchive { archive_file, module_dir, @@ -162,10 +162,16 @@ fn archive(mode: Mode, archive_file: &Path) -> Result<()> { Ok(()) } -fn prepare_clients(mode: Mode, module_dir: &Path, output_dir: &Path) -> Result<()> { +fn prepare_clients(archive_file: &Path, module_dir: &Path, output_dir: &Path) -> Result<()> { let workspace_root = env::current_dir()?; + let archive_file = absolute_from_workspace(archive_file)?; let module_dir = absolute_from_workspace(module_dir)?; let output_dir = absolute_from_workspace(output_dir)?; + ensure!( + archive_file.is_file(), + "SDK test archive does not exist: {}", + archive_file.display() + ); ensure!( module_dir.is_dir(), "SDK module directory does not exist: {}", @@ -174,20 +180,20 @@ fn prepare_clients(mode: Mode, module_dir: &Path, output_dir: &Path) -> Result<( std::fs::create_dir_all(&output_dir)?; let status = Command::new("cargo") + .args(["nextest", "run", "--archive-file"]) + .arg(archive_file) + .arg("--workspace-remap") + .arg(&workspace_root) .args([ - "test", - "--timings", - "-p", - "spacetimedb-sdk", - "--features", - mode.features(), - "--test", - "test", - "prepare_clients", - "--", - "--ignored", - "--exact", - "--test-threads=1", + "--run-ignored", + "only", + "-E", + "test(prepare_clients)", + "--no-fail-fast", + "--no-tests", + "fail", + "-j", + "1", ]) .env(PRECOMPILED_MODULE_DIR_ENV_VAR, module_dir) .env(PREPARE_CLIENT_DIR_ENV_VAR, output_dir) diff --git a/tools/ci/run-sdk-test-archive.sh b/tools/ci/run-sdk-test-archive.sh new file mode 100644 index 00000000000..5153115fb31 --- /dev/null +++ b/tools/ci/run-sdk-test-archive.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Keep partition jobs independent of the large ci-sdk-tests helper binary so it +# does not have to be packaged, uploaded, and downloaded with every test build. +usage() { + echo "Usage: $0 --archive-file PATH --module-dir PATH --client-dir PATH -- [nextest arguments]" >&2 + exit 2 +} + +archive_file="" +module_dir="" +client_dir="" + +while (( $# > 0 )); do + case "$1" in + --archive-file) + (( $# >= 2 )) || usage + archive_file="$2" + shift 2 + ;; + --module-dir) + (( $# >= 2 )) || usage + module_dir="$2" + shift 2 + ;; + --client-dir) + (( $# >= 2 )) || usage + client_dir="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + usage + ;; + esac +done + +[[ -n "${archive_file}" && -n "${module_dir}" && -n "${client_dir}" ]] || usage +[[ -f "${archive_file}" ]] || { echo "SDK test archive does not exist: ${archive_file}" >&2; exit 1; } +[[ -d "${module_dir}" ]] || { echo "SDK module directory does not exist: ${module_dir}" >&2; exit 1; } +[[ -d "${client_dir}" ]] || { echo "Prepared SDK client directory does not exist: ${client_dir}" >&2; exit 1; } +: "${SPACETIME_BIN:?SPACETIME_BIN must point to the spacetimedb CLI}" +[[ -f "${SPACETIME_BIN}" ]] || { echo "SpacetimeDB CLI does not exist: ${SPACETIME_BIN}" >&2; exit 1; } +[[ -f "$(dirname "${SPACETIME_BIN}")/spacetimedb-standalone" ]] || { + echo "SpacetimeDB standalone runtime was not found alongside ${SPACETIME_BIN}" >&2 + exit 1 +} + +workspace_root="$(pwd -P)" +export SPACETIME_SDK_TEST_MODULE_DIR="$(realpath "${module_dir}")" +export SPACETIME_SDK_TEST_CLIENT_DIR="$(realpath "${client_dir}")" +export SPACETIME_SDK_TEST_WORKSPACE_ROOT="${workspace_root}" + +cargo nextest run \ + --archive-file "$(realpath "${archive_file}")" \ + --workspace-remap "${workspace_root}" \ + --no-fail-fast \ + --no-tests pass \ + -j 1 \ + "$@" + +bash tools/check-diff.sh From 6ce8b94a69392c005bae71593e4c935d1df9cc27 Mon Sep 17 00:00:00 2001 From: JasonAtClockwork Date: Wed, 2 Sep 2026 07:45:18 -0700 Subject: [PATCH 5/8] Reduce partitions --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37948e7910d..2df5a0bed49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -738,18 +738,18 @@ jobs: sdk_tests_native: needs: [sdk_test_build] - name: SDK Tests (native ${{ matrix.partition }}/4) + name: SDK Tests (native ${{ matrix.partition }}/2) strategy: fail-fast: false matrix: - partition: [1, 2, 3, 4] + partition: [1, 2] runs-on: spacetimedb-new-runner-2 timeout-minutes: 10 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux - PARTITION_COUNT: 4 + PARTITION_COUNT: 2 SDK_TEST_BUILD_ARTIFACT: sdk-test-native-linux SDK_TEST_ARCHIVE: sdk-tests-native-nextest.tar.zst SDK_TEST_CLIENT_ARCHIVE: sdk-test-clients-native.tar.zst @@ -802,18 +802,18 @@ jobs: sdk_tests_browser: needs: [sdk_test_build] - name: SDK Tests (browser ${{ matrix.partition }}/4) + name: SDK Tests (browser ${{ matrix.partition }}/2) strategy: fail-fast: false matrix: - partition: [1, 2, 3, 4] + partition: [1, 2] runs-on: spacetimedb-new-runner-2 timeout-minutes: 10 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux - PARTITION_COUNT: 4 + PARTITION_COUNT: 2 SDK_TEST_BUILD_ARTIFACT: sdk-test-browser-linux SDK_TEST_ARCHIVE: sdk-tests-browser-nextest.tar.zst SDK_TEST_CLIENT_ARCHIVE: sdk-test-clients-browser.tar.zst @@ -824,11 +824,11 @@ jobs: sdk_tests_noop: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip == 'true' }} - name: SDK Tests (${{ matrix.mode }} ${{ matrix.partition }}/4) + name: SDK Tests (${{ matrix.mode }} ${{ matrix.partition }}/2) strategy: matrix: mode: [native, browser] - partition: [1, 2, 3, 4] + partition: [1, 2] runs-on: ubuntu-latest steps: - name: Skip duplicate merge queue SDK test From 341567c5e33a33b5300bee97bc0ce0cdc51581c0 Mon Sep 17 00:00:00 2001 From: JasonAtClockwork Date: Wed, 2 Sep 2026 11:56:26 -0700 Subject: [PATCH 6/8] Missing pieces and cleanup --- .github/workflows/ci.yml | 1 + crates/testing/src/sdk.rs | 164 ++++++++++++++---------- tools/ci/commands/sdk-tests/src/main.rs | 1 + 3 files changed, 99 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 838a0cc02a4..e32436714bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -688,6 +688,7 @@ jobs: ./target/debug/ci-sdk-tests archive --mode browser --archive-file sdk-tests-browser-nextest.tar.zst ./target/debug/ci-sdk-tests prepare-clients --archive-file sdk-tests-native-nextest.tar.zst --module-dir sdk-test-modules --output-dir sdk-test-clients/native ./target/debug/ci-sdk-tests prepare-clients --archive-file sdk-tests-browser-nextest.tar.zst --module-dir sdk-test-modules --output-dir sdk-test-clients/browser + bash tools/check-diff.sh tar -I 'zstd -T0 -3' -cf sdk-test-modules.tar.zst sdk-test-modules tar -I 'zstd -T0 -3' -cf sdk-test-clients-native.tar.zst sdk-test-clients/native tar -I 'zstd -T0 -3' -cf sdk-test-clients-browser.tar.zst sdk-test-clients/browser diff --git a/crates/testing/src/sdk.rs b/crates/testing/src/sdk.rs index 878d65304e3..981add2598c 100644 --- a/crates/testing/src/sdk.rs +++ b/crates/testing/src/sdk.rs @@ -111,7 +111,7 @@ impl Test { let prepared_client_dir = std::env::var_os(PREPARED_CLIENT_DIR_ENV_VAR).map(PathBuf::from); if prepared_client_dir.is_none() { self.generate_bindings(paths, &file, host_type); - compile_client(&self.compile_command, &self.client_project, &self.module_name); + compile_client(&self.compile_command, &self.client_project); } let guard = SpacetimeDbGuard::spawn_in_temp_data_dir(); @@ -132,8 +132,10 @@ impl Test { ); let sdk_paths = SdkTestPaths::new(); let (file, host_type) = compile_module(&self.module_name); - self.generate_bindings(&sdk_paths.paths, &file, host_type); - compile_client(&self.compile_command, &self.client_project, &self.module_name); + // Preparation is serial and must rebuild each explicitly requested artifact, + // including artifacts for distinct modules that share a client project. + self.generate_bindings_uncached(&sdk_paths.paths, &file, host_type); + compile_client_uncached(&self.compile_command, &self.client_project); self.export_prepared_client(&output_dir); } @@ -149,6 +151,18 @@ impl Test { ); } + fn generate_bindings_uncached(&self, paths: &SpacetimePaths, file: &str, host_type: HostType) { + generate_bindings_uncached( + paths, + &self.generate_language, + file, + host_type, + &self.client_project, + &self.generate_subdir, + self.generate_include_private, + ); + } + fn client_artifact_dir(&self, root: &Path) -> PathBuf { let key = self.prepared_client_key.as_deref().unwrap_or_else(|| { Path::new(&self.client_project) @@ -450,7 +464,50 @@ fn publish_module(paths: &SpacetimePaths, server_url: &str, wasm_file: &str, hos /// and the `generate_subdir` an arbitrary relative path within it. /// These will be combined as `"{client_project}/{generate_subdir}"` to produce the `--out-dir`. /// -/// This function memoizes the complete generation input, not just the output directory. +fn generate_bindings_uncached( + paths: &SpacetimePaths, + language: &str, + wasm_file: &str, + host_type: HostType, + client_project: &str, + generate_subdir: &str, + generate_include_private: bool, +) { + let mut args: Vec<&str> = vec![ + "generate", + "--yes", + "--lang", + language, + match host_type { + HostType::Wasm => "--bin-path", + HostType::Js => "--js-path", + }, + wasm_file, + ]; + + if generate_include_private { + args.push("--include-private"); + } + + let generate_dir: String; + + // `generate --lang unrealcpp` takes different arguments from non-Unreal languages + // to account for some quirks of Unreal project structure. + if language_is_unreal(language) { + // For unreal, we use `client_project` as the uproject directory, + // and `generate_subdir` as the module name. + args.extend_from_slice(&["--uproject-dir", client_project]); + args.extend_from_slice(&["--module-name", generate_subdir]); + } else { + generate_dir = format!("{client_project}/{generate_subdir}"); + create_dir_all(&generate_dir).unwrap(); + args.extend_from_slice(&["--out-dir", &generate_dir]); + } + + invoke_cli(paths, &args); +} + +/// Note: this function is memoized to ensure we only run `spacetime generate` once for each target directory. /// /// Without this lock, if multiple `Test`s ran concurrently in the same process /// with the same `client_project` and `generate_subdir`, @@ -458,9 +515,13 @@ fn publish_module(paths: &SpacetimePaths, server_url: &str, wasm_file: &str, hos /// each of which would remove and re-populate the bindings directory, /// potentially sweeping them out from under a compile or run process. /// -/// This lock ensures that only one `spacetime generate` process runs at a time. +/// This lock ensures that only one `spacetime generate` process runs at a time, +/// and the `HashSet` ensures that we run `spacetime generate` only once for each output directory. /// /// Circumstances where this will still break: +/// - If multiple tests want to use the same client_project/generate_subdir pair, +/// but for different modules' bindings, only one module's bindings will ever be generated. +/// If you need bindings for multiple different modules, put them in different subdirs. /// - If multiple distinct test harness processes run concurrently, /// they will encounter the race condition described above, /// because the binding-generation lock is not shared between harness processes. @@ -481,57 +542,24 @@ fn generate_bindings( generate_subdir: &str, generate_include_private: bool, ) { - // We need these to be owned values so we can memoize on them. + // We need these to be owned `String`s so we can memoize on them. let client_project = client_project.to_owned(); let generate_subdir = generate_subdir.to_owned(); - let language = language.to_owned(); - let wasm_file = wasm_file.to_owned(); - let host_is_js = matches!(host_type, HostType::Js); // Codegen is side-effecting and doesn't meaningfully return a Rust value, // so our memoization has unit as the value. // This makes it run at most once for each key. - memoized!( - |(client_project, generate_subdir, language, wasm_file, host_is_js, generate_include_private): ( - String, - String, - String, - String, - bool, - bool, - )| - -> () { - let mut args: Vec<&str> = vec![ - "generate", - "--yes", - "--lang", - &language, - if *host_is_js { "--js-path" } else { "--bin-path" }, - &wasm_file, - ]; - - if *generate_include_private { - args.push("--include-private"); - } - - let generate_dir: String; - - // `generate --lang unrealcpp` takes different arguments from non-Unreal languages - // to account for some quirks of Unreal project structure. - if language_is_unreal(language) { - // For unreal, we use `client_project` as the uproject directory, - // and `generate_subdir` as the module name. - args.extend_from_slice(&["--uproject-dir", client_project]); - args.extend_from_slice(&["--module-name", generate_subdir]); - } else { - generate_dir = format!("{client_project}/{generate_subdir}"); - create_dir_all(&generate_dir).unwrap(); - args.extend_from_slice(&["--out-dir", &generate_dir]); - } - - invoke_cli(paths, &args); - } - ) + memoized!(|(client_project, generate_subdir): (String, String)| -> () { + generate_bindings_uncached( + paths, + language, + wasm_file, + host_type, + client_project, + generate_subdir, + generate_include_private, + ); + }) } fn split_command_string(command: &str) -> (String, Vec) { @@ -566,25 +594,27 @@ fn split_command_string(command: &str) -> (String, Vec) { (exe, iter.collect()) } +fn compile_client_uncached(compile_command: &str, client_project: &str) { + let (exe, args) = split_command_string(compile_command); + + let output = cmd(exe, args) + .dir(client_project) + .env(TEST_CLIENT_PROJECT_ENV_VAR, client_project) + .stderr_to_stdout() + .stdout_capture() + .unchecked() + .run() + .expect("Error running compile command"); + + status_ok_or_panic(output, compile_command, "(compiling)"); +} + // Note: this function is memoized to ensure we only compile each client once. -fn compile_client(compile_command: &str, client_project: &str, module_name: &str) { +fn compile_client(compile_command: &str, client_project: &str) { let client_project = client_project.to_owned(); - let module_name = module_name.to_owned(); - - memoized!(|(client_project, module_name): (String, String)| -> () { - let _ = module_name; - let (exe, args) = split_command_string(compile_command); - - let output = cmd(exe, args) - .dir(client_project) - .env(TEST_CLIENT_PROJECT_ENV_VAR, client_project) - .stderr_to_stdout() - .stdout_capture() - .unchecked() - .run() - .expect("Error running compile command"); - - status_ok_or_panic(output, compile_command, "(compiling)"); + + memoized!(|client_project: String| -> () { + compile_client_uncached(compile_command, client_project); }) } diff --git a/tools/ci/commands/sdk-tests/src/main.rs b/tools/ci/commands/sdk-tests/src/main.rs index 8b481529232..8e6b39889de 100644 --- a/tools/ci/commands/sdk-tests/src/main.rs +++ b/tools/ci/commands/sdk-tests/src/main.rs @@ -135,6 +135,7 @@ fn run_local() -> Result<()> { ]) .status()?; ensure!(status.success(), "SDK tests failed"); + cmd!("bash", "tools/check-diff.sh").run()?; } Ok(()) } From 07c90266905864666e4a2a629c2f885d463bc1ff Mon Sep 17 00:00:00 2001 From: JasonAtClockwork Date: Wed, 2 Sep 2026 13:16:19 -0700 Subject: [PATCH 7/8] C++ to be first client for diff check --- sdks/rust/tests/test.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/rust/tests/test.rs b/sdks/rust/tests/test.rs index 18e443fa5b5..33495a31d2f 100644 --- a/sdks/rust/tests/test.rs +++ b/sdks/rust/tests/test.rs @@ -865,7 +865,10 @@ procedural_view_pk_tests!(cpp_procedural_view_pk, "sdk-test-procedural-view-pk-c #[ignore = "CI prepares shared SDK client artifacts before partitioning"] fn prepare_clients() { let prepared_clients = [ - rust::make_test("insert-primitive"), + // The Rust module exposes additional private reducers when generating with + // `--include-private`; use the C++ module whose schema matches the committed + // bindings shared by these otherwise equivalent module variants. + cpp::make_test("insert-primitive"), rust::make_connect_disconnect_test(), event_table_tests::make_test("event-table"), rust_procedures::make_test("procedure-return-values"), From 33d16ca5ae17edc8a5e2f349f06e96a0aa9cff1a Mon Sep 17 00:00:00 2001 From: JasonAtClockwork Date: Thu, 3 Sep 2026 11:03:47 -0700 Subject: [PATCH 8/8] Split build job more before heavy refactor --- .github/workflows/ci.yml | 249 ++++++++++++++++++++++++++++++--------- 1 file changed, 193 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4900d307139..3a80205ff8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -589,16 +589,15 @@ jobs: - name: Skip duplicate merge queue smoketest run: echo "Merge queue commit has the same tree as the PR head; smoketest already ran for the PR." - sdk_test_build: - needs: [upload-build-artifacts-linux] - name: Build SDK tests + sdk_test_modules: + needs: [merge_queue_noop, lints] + if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} + name: Build SDK test modules runs-on: spacetimedb-new-runner-2 - timeout-minutes: 30 + timeout-minutes: 15 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full - ARTIFACT_SUFFIX: linux - SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: - *find-git-ref - *checkout-sources @@ -625,9 +624,6 @@ jobs: - *restore-openssl - *configure-cached-openssl - - *download-build-artifacts - - *extract-build-artifacts - - uses: actions/setup-dotnet@v4 with: global-json-file: global.json @@ -658,42 +654,14 @@ jobs: cd sdks/csharp ./tools~/write-nuget-config.sh ../.. - - name: Install cargo-nextest - uses: taiki-e/install-action@nextest - - - name: Install wasm-bindgen CLI - run: | - REQUIRED_WASM_BINDGEN_VERSION="$( - awk ' - $1 == "name" && $3 == "\"wasm-bindgen\"" { in_pkg = 1; next } - in_pkg && $1 == "version" { - gsub(/"/, "", $3); - print $3; - exit; - } - ' Cargo.lock - )" - if [ -z "${REQUIRED_WASM_BINDGEN_VERSION}" ]; then - echo "Failed to determine wasm-bindgen version from Cargo.lock" - exit 1 - fi - cargo install --locked --force wasm-bindgen-cli --version "${REQUIRED_WASM_BINDGEN_VERSION}" - - - name: Build SDK test modules and archives + - name: Build SDK test modules shell: bash run: | source ~/emsdk/emsdk_env.sh cargo run --timings -p ci-sdk-tests -- prepare-modules --output-dir sdk-test-modules - ./target/debug/ci-sdk-tests archive --mode native --archive-file sdk-tests-native-nextest.tar.zst - ./target/debug/ci-sdk-tests archive --mode browser --archive-file sdk-tests-browser-nextest.tar.zst - ./target/debug/ci-sdk-tests prepare-clients --archive-file sdk-tests-native-nextest.tar.zst --module-dir sdk-test-modules --output-dir sdk-test-clients/native - ./target/debug/ci-sdk-tests prepare-clients --archive-file sdk-tests-browser-nextest.tar.zst --module-dir sdk-test-modules --output-dir sdk-test-clients/browser - bash tools/check-diff.sh tar -I 'zstd -T0 -3' -cf sdk-test-modules.tar.zst sdk-test-modules - tar -I 'zstd -T0 -3' -cf sdk-test-clients-native.tar.zst sdk-test-clients/native - tar -I 'zstd -T0 -3' -cf sdk-test-clients-browser.tar.zst sdk-test-clients/browser - - name: Upload common SDK test build + - name: Upload SDK test modules uses: actions/upload-artifact@v4 with: name: sdk-test-common-linux @@ -703,25 +671,77 @@ jobs: overwrite: true retention-days: 14 - - name: Upload native SDK test build + - name: Upload Cargo timing reports + if: always() uses: actions/upload-artifact@v4 with: - name: sdk-test-native-linux - path: | - sdk-tests-native-nextest.tar.zst - sdk-test-clients-native.tar.zst + name: sdk-test-modules-cargo-timings + path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ + if-no-files-found: warn + overwrite: true + retention-days: 14 + + sdk_test_archives: + needs: [merge_queue_noop, lints] + if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} + name: Build SDK test archives + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 15 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + steps: + - *find-git-ref + - *checkout-sources + + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain + - *set-native-cache-keys + + - name: Restore rusty_v8 (debug) + uses: actions/cache/restore@v4 + with: + path: ${{ env.CARGO_TARGET_DIR }}/debug/gn_out/obj + key: ${{ steps.native-cache-keys.outputs.v8-debug-key }} + - name: Restore jemalloc + id: cache-jemalloc + uses: actions/cache/restore@v4 + with: *jemalloc-cache + - *configure-cached-jemalloc + - name: Restore Zstd + id: cache-zstd + uses: actions/cache/restore@v4 + with: *zstd-cache + - *configure-cached-zstd + - *restore-openssl + - *configure-cached-openssl + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + # Build the archives independently of the modules so both expensive phases + # can run in parallel before client preparation joins their artifacts. + - name: Build SDK test archives + shell: bash + run: | + cargo nextest archive --timings -p spacetimedb-sdk --features allow_loopback_http_for_tests --archive-file sdk-tests-native-nextest.tar.zst + cargo nextest archive --timings -p spacetimedb-sdk --features allow_loopback_http_for_tests,browser --archive-file sdk-tests-browser-nextest.tar.zst + + - name: Upload native SDK test archive + uses: actions/upload-artifact@v4 + with: + name: sdk-test-native-archive-linux + path: sdk-tests-native-nextest.tar.zst compression-level: 0 if-no-files-found: error overwrite: true retention-days: 14 - - name: Upload browser SDK test build + - name: Upload browser SDK test archive uses: actions/upload-artifact@v4 with: - name: sdk-test-browser-linux - path: | - sdk-tests-browser-nextest.tar.zst - sdk-test-clients-browser.tar.zst + name: sdk-test-browser-archive-linux + path: sdk-tests-browser-nextest.tar.zst compression-level: 0 if-no-files-found: error overwrite: true @@ -731,14 +751,124 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: sdk-test-build-cargo-timings + name: sdk-test-archives-cargo-timings path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ if-no-files-found: warn overwrite: true retention-days: 14 + sdk_test_prepare_native: + needs: [sdk_test_modules, sdk_test_archives] + name: Prepare SDK test clients (native) + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 10 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + SDK_TEST_MODE: native + SDK_TEST_ARCHIVE_ARTIFACT: sdk-test-native-archive-linux + SDK_TEST_ARCHIVE: sdk-tests-native-nextest.tar.zst + SDK_TEST_CLIENT_ARTIFACT: sdk-test-native-clients-linux + SDK_TEST_CLIENT_ARCHIVE: sdk-test-clients-native.tar.zst + SDK_TEST_CLIENT_DIR: sdk-test-clients/native + steps: &sdk-test-prepare-steps + - *find-git-ref + - *checkout-sources + + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - uses: ./.github/actions/setup-pnpm + with: + run_install: true + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Install wasm-bindgen CLI + if: env.SDK_TEST_MODE == 'browser' + run: | + REQUIRED_WASM_BINDGEN_VERSION="$( + awk ' + $1 == "name" && $3 == "\"wasm-bindgen\"" { in_pkg = 1; next } + in_pkg && $1 == "version" { + gsub(/"/, "", $3); + print $3; + exit; + } + ' Cargo.lock + )" + if [ -z "${REQUIRED_WASM_BINDGEN_VERSION}" ]; then + echo "Failed to determine wasm-bindgen version from Cargo.lock" + exit 1 + fi + cargo install --locked --force wasm-bindgen-cli --version "${REQUIRED_WASM_BINDGEN_VERSION}" + + - name: Download SDK test modules + uses: actions/download-artifact@v4 + with: + name: sdk-test-common-linux + + - name: Download SDK test archive + uses: actions/download-artifact@v4 + with: + name: ${{ env.SDK_TEST_ARCHIVE_ARTIFACT }} + + # Preparation needs both independently-built inputs. Keeping it in a + # separate job also allows a failed client build to reuse those artifacts. + - name: Prepare SDK test clients + shell: bash + run: | + tar --zstd -xf sdk-test-modules.tar.zst + mkdir -p "${SDK_TEST_CLIENT_DIR}" + cargo nextest run \ + --archive-file "${SDK_TEST_ARCHIVE}" \ + --workspace-remap "${GITHUB_WORKSPACE}" \ + --run-ignored only \ + -E 'test(prepare_clients)' \ + --no-fail-fast \ + --no-tests fail \ + -j 1 + bash tools/check-diff.sh + tar -I 'zstd -T0 -3' -cf "${SDK_TEST_CLIENT_ARCHIVE}" "${SDK_TEST_CLIENT_DIR}" + env: + SPACETIME_SDK_TEST_MODULE_DIR: ${{ github.workspace }}/sdk-test-modules + SPACETIME_SDK_TEST_PREPARE_CLIENT_DIR: ${{ github.workspace }}/${{ env.SDK_TEST_CLIENT_DIR }} + SPACETIME_SDK_TEST_WORKSPACE_ROOT: ${{ github.workspace }} + + - name: Upload prepared SDK test clients + uses: actions/upload-artifact@v4 + with: + name: ${{ env.SDK_TEST_CLIENT_ARTIFACT }} + path: ${{ env.SDK_TEST_CLIENT_ARCHIVE }} + compression-level: 0 + if-no-files-found: error + overwrite: true + retention-days: 14 + + sdk_test_prepare_browser: + needs: [sdk_test_modules, sdk_test_archives] + name: Prepare SDK test clients (browser) + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 10 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + SDK_TEST_MODE: browser + SDK_TEST_ARCHIVE_ARTIFACT: sdk-test-browser-archive-linux + SDK_TEST_ARCHIVE: sdk-tests-browser-nextest.tar.zst + SDK_TEST_CLIENT_ARTIFACT: sdk-test-browser-clients-linux + SDK_TEST_CLIENT_ARCHIVE: sdk-test-clients-browser.tar.zst + SDK_TEST_CLIENT_DIR: sdk-test-clients/browser + steps: *sdk-test-prepare-steps + sdk_tests_native: - needs: [sdk_test_build] + needs: [upload-build-artifacts-linux, sdk_test_prepare_native] name: SDK Tests (native ${{ matrix.partition }}/2) strategy: fail-fast: false @@ -751,7 +881,8 @@ jobs: RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux PARTITION_COUNT: 2 - SDK_TEST_BUILD_ARTIFACT: sdk-test-native-linux + SDK_TEST_ARCHIVE_ARTIFACT: sdk-test-native-archive-linux + SDK_TEST_CLIENT_ARTIFACT: sdk-test-native-clients-linux SDK_TEST_ARCHIVE: sdk-tests-native-nextest.tar.zst SDK_TEST_CLIENT_ARCHIVE: sdk-test-clients-native.tar.zst SDK_TEST_CLIENT_DIR: sdk-test-clients/native @@ -779,10 +910,15 @@ jobs: with: name: sdk-test-common-linux - - name: Download mode-specific SDK test build + - name: Download SDK test archive + uses: actions/download-artifact@v4 + with: + name: ${{ env.SDK_TEST_ARCHIVE_ARTIFACT }} + + - name: Download prepared SDK test clients uses: actions/download-artifact@v4 with: - name: ${{ env.SDK_TEST_BUILD_ARTIFACT }} + name: ${{ env.SDK_TEST_CLIENT_ARTIFACT }} - name: Extract SDK test support files run: | @@ -802,7 +938,7 @@ jobs: -E 'not test(/unreal/)' sdk_tests_browser: - needs: [sdk_test_build] + needs: [upload-build-artifacts-linux, sdk_test_prepare_browser] name: SDK Tests (browser ${{ matrix.partition }}/2) strategy: fail-fast: false @@ -815,7 +951,8 @@ jobs: RUST_BACKTRACE: full ARTIFACT_SUFFIX: linux PARTITION_COUNT: 2 - SDK_TEST_BUILD_ARTIFACT: sdk-test-browser-linux + SDK_TEST_ARCHIVE_ARTIFACT: sdk-test-browser-archive-linux + SDK_TEST_CLIENT_ARTIFACT: sdk-test-browser-clients-linux SDK_TEST_ARCHIVE: sdk-tests-browser-nextest.tar.zst SDK_TEST_CLIENT_ARCHIVE: sdk-test-clients-browser.tar.zst SDK_TEST_CLIENT_DIR: sdk-test-clients/browser