diff --git a/.github/workflows/community.yml b/.github/workflows/community.yml new file mode 100644 index 0000000..8125b37 --- /dev/null +++ b/.github/workflows/community.yml @@ -0,0 +1,42 @@ +name: Community extension + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: community-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + distribution: + name: DuckDB 2.0 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@8f59772b4ef9a13837b3e353db032b3aca300d04 + with: + extension_name: sqlnb + duckdb_version: v2.0.0 + duckdb_commit: d54cbbe3df17e8e5ce68312f483b4155836112aa + ci_tools_version: 8f59772b4ef9a13837b3e353db032b3aca300d04 + extra_toolchains: rust + # The server requires native sockets/filesystems. MinGW is not supported + # by the Windows filesystem implementation; MSVC is tested instead. + exclude_archs: wasm_mvp;wasm_eh;wasm_threads;windows_amd64_mingw;windows_amd64_rtools + opt_in_archs: osx_amd64 + reduced_ci_mode: disabled + save_cache: false + + rust: + name: Rust contracts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.99.0 + - name: Install build cache + run: bash scripts/configure-ci.sh + - name: Test HTTP, MCP, history, and filesystem contracts + run: .ci/tools/bin/mbx test --manifest-path http/Cargo.toml --locked -- --test-threads=1 diff --git a/.gitignore b/.gitignore index dfe7e66..38206c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,17 @@ .artifacts/ .deps/ .agents/ +.ci/ +.cache/ +configure/ +vcpkg/ +local_vcpkg_installation/ +duckdb_downloads/ +duckdb_extensions/ +duckdb_binaries/ +testext/ +duckdb_test_dir/ +duckdb_unittest_tempdir/ build/ node_modules/ test-results/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0aec35e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "extension-ci-tools"] + path = extension-ci-tools + url = https://github.com/duckdb/extension-ci-tools.git +[submodule "duckdb"] + path = duckdb + url = https://github.com/duckdb/duckdb.git diff --git a/CMakeLists.txt b/CMakeLists.txt index c0109e8..0855dba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,19 +21,26 @@ set(SIDEQUERY_HTTP_LOCK ${CMAKE_CURRENT_SOURCE_DIR}/http/Cargo.lock) file(GLOB_RECURSE SIDEQUERY_HTTP_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/http/src/*.rs) set(SIDEQUERY_HTTP_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR}/http-cargo) +set(SIDEQUERY_HTTP_PROFILE_DIR ${SIDEQUERY_HTTP_TARGET_DIR}/release) +set(SIDEQUERY_CARGO_TARGET_ARGS) +if(Rust_CARGO_TARGET) + # Community CI cross-compiles Intel macOS on Apple Silicon runners. + list(APPEND SIDEQUERY_CARGO_TARGET_ARGS --target ${Rust_CARGO_TARGET}) + set(SIDEQUERY_HTTP_PROFILE_DIR ${SIDEQUERY_HTTP_TARGET_DIR}/${Rust_CARGO_TARGET}/release) +endif() if(MSVC) - set(SIDEQUERY_HTTP_LIBRARY ${SIDEQUERY_HTTP_TARGET_DIR}/release/sidequery_http.lib) + set(SIDEQUERY_HTTP_LIBRARY ${SIDEQUERY_HTTP_PROFILE_DIR}/sidequery_http.lib) # DuckDB uses the static MSVC runtime; Rust and its C dependencies must agree. set(SIDEQUERY_RUST_ENV "RUSTFLAGS=$ENV{RUSTFLAGS} -C target-feature=+crt-static") else() - set(SIDEQUERY_HTTP_LIBRARY ${SIDEQUERY_HTTP_TARGET_DIR}/release/libsidequery_http.a) + set(SIDEQUERY_HTTP_LIBRARY ${SIDEQUERY_HTTP_PROFILE_DIR}/libsidequery_http.a) endif() set(SIDEQUERY_REPO_MBX ${CMAKE_CURRENT_SOURCE_DIR}/scripts/mbx) if(EXISTS ${SIDEQUERY_REPO_MBX}) set(SIDEQUERY_CARGO_BUILD ${SIDEQUERY_REPO_MBX} build) else() - find_program(SIDEQUERY_MBX_EXECUTABLE mbx) + find_program(SIDEQUERY_MBX_EXECUTABLE mbx HINTS ${CMAKE_CURRENT_SOURCE_DIR}/.ci/tools/bin) if(SIDEQUERY_MBX_EXECUTABLE AND "$ENV{RUSTC_WRAPPER}" STREQUAL "" AND "$ENV{RUSTC_WORKSPACE_WRAPPER}" STREQUAL "") set(SIDEQUERY_CARGO_BUILD ${SIDEQUERY_MBX_EXECUTABLE} build) @@ -50,7 +57,7 @@ endif() add_custom_command( OUTPUT ${SIDEQUERY_HTTP_LIBRARY} COMMAND ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${SIDEQUERY_HTTP_TARGET_DIR} ${SIDEQUERY_RUST_ENV} - ${SIDEQUERY_CARGO_BUILD} --manifest-path ${SIDEQUERY_HTTP_MANIFEST} --locked --release + ${SIDEQUERY_CARGO_BUILD} --manifest-path ${SIDEQUERY_HTTP_MANIFEST} --locked --release ${SIDEQUERY_CARGO_TARGET_ARGS} DEPENDS ${SIDEQUERY_HTTP_MANIFEST} ${SIDEQUERY_HTTP_LOCK} ${SIDEQUERY_HTTP_SOURCES} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/http COMMENT "Building Sidequery's in-process Axum HTTP server" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f328555 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 SQLNB contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3e36e78 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +EXT_NAME := sqlnb +EXT_CONFIG := $(PROJ_DIR)extension_config.cmake + +include extension-ci-tools/makefiles/duckdb_extension.Makefile +include extension-ci-tools/makefiles/vcpkg.Makefile + +.PHONY: sqlnb_ci_setup +configure_ci: sqlnb_ci_setup +sqlnb_ci_setup: + bash scripts/configure-ci.sh diff --git a/README.md b/README.md index a4a9f95..1ef006e 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,10 @@ load sqlnb from sidequery; select sqlnb_start(); ``` -Keep DuckDB running while you use the browser. SQLNB loads its Quack, `httpfs`, -and `icu` dependencies automatically. The first launch needs an internet -connection to download the UI; subsequent launches can use a verified cached copy. +Keep DuckDB running while you use the browser. SQLNB loads the official Quack, +`httpfs`, and `icu` dependencies from DuckDB core automatically. The first launch +needs an internet connection to download the UI; subsequent launches can use a +verified cached copy. Use `select sqlnb_status();` to see the local URL and `select sqlnb_stop();` to stop the server. Restart the server to pick up UI updates. @@ -56,3 +57,8 @@ To connect an MCP client, start with `mcp := true` and follow the - [MCP reference](docs/mcp.md) — authentication, tools, and execution behavior. - [UI assets](docs/ui-assets.md) — custom frontends, updates, and the deployment contract. - [Release procedures](docs/releases.md) — maintainer builds, signing, and publication. + +## License + +The native extension is [MIT licensed](LICENSE). The separately hosted Sidequery +UI is not included in this repository or covered by its license. diff --git a/docs/development.md b/docs/development.md index a611765..a8045e0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -8,6 +8,29 @@ checkout or frontend build is required. Run the commands below from the repository root. +## Community build + +The DuckDB and extension-ci-tools submodules pin DuckDB 2.0.0 and its community +build tooling. Initialize them before using the standard extension targets: + +```sh +git submodule update --init --recursive +make setup-vcpkg +VCPKG_TOOLCHAIN_PATH="$PWD/vcpkg/scripts/buildsystems/vcpkg.cmake" GEN=ninja make release +make test +``` + +The SQL tests start the server with a small local UI fixture, exercise MCP owner +isolation and shutdown, and verify invalid startup options and disabled external +access. They install runtime dependencies from DuckDB's core repository; they +do not need a Sidequery repository registration or the hosted UI. + +The [community workflow](../.github/workflows/community.yml) uses DuckDB's reusable +distribution workflow for Linux x86-64/ARM64, macOS Intel/Apple Silicon, and Windows +MSVC. Wasm cannot provide the native server's sockets and filesystem; Windows +MinGW is not supported. The CI setup hook installs Boxington into the disposable +workspace when another compiler wrapper is not already configured. + ## Rust tests ```sh diff --git a/docs/releases.md b/docs/releases.md index c964098..010b487 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -8,6 +8,29 @@ Building this repository does not require that access. Run the commands below from the repository root. +## Community repository + +The community build uses the standard `Makefile`, `extension_config.cmake`, +and `test/sql` suite. Its CI pins DuckDB 2.0.0 and the matching community tooling; +it builds and tests artifacts without publishing them. + +Submit `extensions/sqlnb/description.yml` to +[duckdb/community-extensions](https://github.com/duckdb/community-extensions), +pinning `repo.ref` to a validated commit of this repository. The community +repository builds, signs, and distributes accepted submissions. A passing build +here does not mean the extension has been accepted or published there. + +After acceptance and publication, users can install with: + +```sql +install sqlnb from community; +load sqlnb; +select sqlnb_start(); +``` + +Quack, `httpfs`, and `icu` load from DuckDB core, including when SQLNB itself was +installed from the Sidequery repository. + ## DuckDB 2.0 For official DuckDB `v2.0.0` (source `d54cbbe3df`), use the matching official CLI: @@ -27,10 +50,10 @@ distributable standard library so Homebrew's host deployment target cannot leak into the extension. The OpenSSL build also requires Curl, Perl, and Make. -Sign both artifacts using the monorepo's `tools/extension-repository` operator +Sign the SQLNB artifact using the monorepo's `tools/extension-repository` operator tool. Run its signed native smoke and browser qualification before publishing -`quack`, then `sqlnb`, to `https://extensions.sidequery.dev`. Core `httpfs` and -`icu` remain official DuckDB dependencies and are not republished by Sidequery. +`sqlnb` to `https://extensions.sidequery.dev`. Quack is built for local qualification; +installed SQLNB uses the official core Quack, `httpfs`, and `icu` dependencies. After publication, verify a fresh signed installation: ```sh @@ -79,5 +102,5 @@ bun scripts/runtime-smoke.mjs --duckdb /absolute/path/to/alpha41489/duckdb \ A local repository directory is also accepted. Each client uses fresh extension and trust directories, allows only signed extensions, and proves that starting -`sqlnb` bootstraps Quack from Sidequery. The suite checks queries, notebook files, caller +`sqlnb` bootstraps Quack from core. The suite checks queries, notebook files, caller isolation, authentication, server cleanup, and remote UI generation updates. diff --git a/duckdb b/duckdb new file mode 160000 index 0000000..d54cbbe --- /dev/null +++ b/duckdb @@ -0,0 +1 @@ +Subproject commit d54cbbe3df17e8e5ce68312f483b4155836112aa diff --git a/extension-ci-tools b/extension-ci-tools new file mode 160000 index 0000000..8f59772 --- /dev/null +++ b/extension-ci-tools @@ -0,0 +1 @@ +Subproject commit 8f59772b4ef9a13837b3e353db032b3aca300d04 diff --git a/extension_config.cmake b/extension_config.cmake index 96688d1..ea49d9c 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -1,15 +1,5 @@ -duckdb_extension_load(json) -duckdb_extension_load(autocomplete) -duckdb_extension_load(icu) - -duckdb_extension_load(httpfs - GIT_URL https://github.com/duckdb/duckdb-httpfs - GIT_TAG 13e18b3c9f3810334f5972b76a3acc247b28e537 +# Community builds use DuckDB's core repository for runtime dependencies. +duckdb_extension_load(sqlnb + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} + LOAD_TESTS ) - -duckdb_extension_load(quack - GIT_URL https://github.com/duckdb/duckdb-quack - GIT_TAG 40de7badae4193c29d9c0834473fb76acc6c51e6 -) - -duckdb_extension_load(sqlnb SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}) diff --git a/extension_config_legacy.cmake b/extension_config_legacy.cmake new file mode 100644 index 0000000..96688d1 --- /dev/null +++ b/extension_config_legacy.cmake @@ -0,0 +1,15 @@ +duckdb_extension_load(json) +duckdb_extension_load(autocomplete) +duckdb_extension_load(icu) + +duckdb_extension_load(httpfs + GIT_URL https://github.com/duckdb/duckdb-httpfs + GIT_TAG 13e18b3c9f3810334f5972b76a3acc247b28e537 +) + +duckdb_extension_load(quack + GIT_URL https://github.com/duckdb/duckdb-quack + GIT_TAG 40de7badae4193c29d9c0834473fb76acc6c51e6 +) + +duckdb_extension_load(sqlnb SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}) diff --git a/http/Cargo.toml b/http/Cargo.toml index 5d57b9f..cd08e7b 100644 --- a/http/Cargo.toml +++ b/http/Cargo.toml @@ -2,6 +2,7 @@ name = "sidequery-ui-http" version = "0.1.0" edition = "2021" +license = "MIT" publish = false [lib] diff --git a/http/src/history.rs b/http/src/history.rs index ad4da39..5f10570 100644 --- a/http/src/history.rs +++ b/http/src/history.rs @@ -521,6 +521,11 @@ impl Transaction<'_> { sync_history_path(&entry.map_err(err)?.path()).map_err(err)?; } sync_history_path(&packs).map_err(err)?; + let pack_name = builder.name().map(|s| format!("pack-{s}")); + drop(builder); + // libgit2 retains pack handles without delete sharing on Windows. Reopen + // its lazy object database before removing old packs; keep our lock held. + self.repo = Repository::open_bare(self.repo.path()).map_err(err)?; // All live objects are durably packed. Remove loose objects and superseded packs only. for entry in fs::read_dir(&objects).map_err(err)? { let entry = entry.map_err(err)?; @@ -530,7 +535,6 @@ impl Transaction<'_> { fs::remove_dir_all(entry.path()).map_err(err)?; } } - let pack_name = builder.name().map(|s| format!("pack-{s}")); for path in previous { if pack_name .as_ref() diff --git a/http/src/platform/windows.rs b/http/src/platform/windows.rs index a047c16..3f08ebb 100644 --- a/http/src/platform/windows.rs +++ b/http/src/platform/windows.rs @@ -10,8 +10,9 @@ use windows_sys::{ Wdk::{ Foundation::OBJECT_ATTRIBUTES, Storage::FileSystem::{ - NtCreateFile, FILE_CREATE, FILE_DIRECTORY_FILE, FILE_NON_DIRECTORY_FILE, FILE_OPEN, - FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT, + FileRenameInformation, NtCreateFile, NtSetInformationFile, FILE_CREATE, + FILE_DIRECTORY_FILE, FILE_NON_DIRECTORY_FILE, FILE_OPEN, FILE_OPEN_REPARSE_POINT, + FILE_RENAME_INFORMATION, FILE_SYNCHRONOUS_IO_NONALERT, }, }, Win32::{ @@ -121,6 +122,16 @@ fn wide_component(name: &CStr) -> Result> { // the point itself, and checking that handle rejects junctions and every other tag. fn open_at(parent: HANDLE, name: &CStr, access: u32, disposition: u32, kind: u32) -> Result { let mut wide = wide_component(name)?; + open_relative(parent, &mut wide, access, disposition, kind) +} + +fn open_relative( + parent: HANDLE, + wide: &mut [u16], + access: u32, + disposition: u32, + kind: u32, +) -> Result { let mut unicode = UNICODE_STRING { Length: (wide.len() * 2) as u16, MaximumLength: (wide.len() * 2) as u16, @@ -175,6 +186,19 @@ pub(crate) fn open_directory_at(parent: HANDLE, name: &CStr) -> Result { FILE_DIRECTORY_FILE, ) } + +fn reopen_directory(directory: HANDLE) -> Result { + // NT paths do not interpret "." as Win32 paths do. An empty relative name + // reopens this exact directory with an independent enumeration cursor. + // Keep empty names private to this operation, not caller-supplied paths. + open_relative( + directory, + &mut [], + FILE_LIST_DIRECTORY | FILE_READ_ATTRIBUTES, + FILE_OPEN, + FILE_DIRECTORY_FILE, + ) +} pub(crate) fn open_file_at(parent: HANDLE, name: &CStr) -> Result { open_at( parent, @@ -222,24 +246,29 @@ pub(crate) fn metadata_at(parent: HANDLE, name: &CStr) -> Result { } fn rename_handle(file: &File, destination: HANDLE, name: &CStr, replace: bool) -> Result<()> { let wide = wide_component(name)?; - let offset = std::mem::offset_of!(FILE_RENAME_INFO, FileName); - let size = offset + wide.len() * 2; + let size = std::mem::size_of::() + wide.len() * 2; // A u64 allocation supplies the structure's required alignment. let mut storage = vec![0u64; size.div_ceil(8)]; - let info = storage.as_mut_ptr().cast::(); + let info = storage.as_mut_ptr().cast::(); + let mut status = IO_STATUS_BLOCK::default(); unsafe { (*info).Anonymous.ReplaceIfExists = replace; (*info).RootDirectory = destination; (*info).FileNameLength = (wide.len() * 2) as u32; ptr::copy_nonoverlapping(wide.as_ptr(), (*info).FileName.as_mut_ptr(), wide.len()); - if SetFileInformationByHandle( + // Use the native directory-relative rename contract, matching NtCreateFile. + // Resolving a pathname here would lose the pinned-parent boundary. + let result = NtSetInformationFile( file.platform_handle(), - FileRenameInfo, + &mut status, info.cast(), size as u32, - ) == 0 - { - return Err(Error::last_os_error()); + FileRenameInformation, + ); + if result < 0 { + return Err(Error::from_raw_os_error( + RtlNtStatusToDosError(result) as i32 + )); } } Ok(()) @@ -330,7 +359,7 @@ pub(crate) fn list_directory_fd( files: &mut Vec, directories: &mut Vec, ) -> std::result::Result<(), ApiError> { - let directory = open_directory_at(directory, c".").map_err(file_io)?; + let directory = reopen_directory(directory).map_err(file_io)?; let mut storage = vec![0u64; 8192]; let mut first = true; loop { @@ -537,6 +566,59 @@ mod tests { b"select 1" ); assert!(!outside.join("test.sql").exists()); + + let revision = revision(b"select 1"); + atomic_write_at( + parent.platform_handle(), + c"test.sql", + b"select 2", + &revision, + ) + .unwrap(); + assert_eq!( + fs::read(root.join("original/test.sql")).unwrap(), + b"select 2" + ); + let mut files = Vec::new(); + let mut directories = Vec::new(); + list_directory_fd( + parent.platform_handle(), + Path::new(""), + &mut files, + &mut directories, + ) + .unwrap(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].path, "test.sql"); + assert!(!outside.join("test.sql").exists()); + } + + #[test] + fn native_write_flush_and_relative_rename_preserve_collision_semantics() { + let temp = tempfile::tempdir().unwrap(); + let root = open_directory(temp.path()).unwrap(); + let mut file = open_at( + root.platform_handle(), + c"temporary", + FILE_WRITE_DATA | FILE_READ_ATTRIBUTES | DELETE, + FILE_CREATE, + FILE_NON_DIRECTORY_FILE, + ) + .expect("create a file relative to the root handle"); + file.write_all(b"select 42;").expect("write file contents"); + file.sync_all().expect("flush file contents"); + rename_handle(&file, root.platform_handle(), c"π.sql", false) + .expect("rename relative to the root handle"); + assert_eq!(fs::read(temp.path().join("π.sql")).unwrap(), b"select 42;"); + drop(file); + + assert_eq!( + atomic_write_at(root.platform_handle(), c"π.sql", b"overwrite", "") + .unwrap_err() + .kind(), + ErrorKind::AlreadyExists + ); + assert_eq!(fs::read(temp.path().join("π.sql")).unwrap(), b"select 42;"); } #[test] diff --git a/scripts/build-and-test.sh b/scripts/build-and-test.sh index 482a9a5..431c46a 100755 --- a/scripts/build-and-test.sh +++ b/scripts/build-and-test.sh @@ -103,7 +103,7 @@ export VCPKG_ROOT="$vcpkg_root" export VCPKG_DOWNLOADS="$task_root/vcpkg-downloads" mkdir -p "$VCPKG_DOWNLOADS" "$generated_manifest_dir" -extension_config="$extension_dir/extension_config.cmake" +extension_config="$extension_dir/extension_config_legacy.cmake" toolchain_file="$vcpkg_root/scripts/buildsystems/vcpkg.cmake" manifest_cmake_args=( diff --git a/scripts/configure-ci-windows.ps1 b/scripts/configure-ci-windows.ps1 new file mode 100644 index 0000000..ae43c6a --- /dev/null +++ b/scripts/configure-ci-windows.ps1 @@ -0,0 +1,11 @@ +$ErrorActionPreference = 'Stop' +$repoRoot = Split-Path $PSScriptRoot -Parent +$vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio/Installer/vswhere.exe' +$installation = & $vswhere -latest -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath +if ($LASTEXITCODE -or -not $installation) { + throw 'MSVC installation not found' +} +$vcvars = Join-Path $installation 'VC/Auxiliary/Build/vcvars64.bat' +# Keep activation and Cargo in the same process so C dependencies see MSVC. +& cmd.exe /d /s /c "call `"$vcvars`" && cargo install mbx --version 1.27.0 --locked --root `"$repoRoot\.ci\tools`"" +if ($LASTEXITCODE) { exit $LASTEXITCODE } diff --git a/scripts/configure-ci.sh b/scripts/configure-ci.sh new file mode 100644 index 0000000..3cebcd0 --- /dev/null +++ b/scripts/configure-ci.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Linux setup runs once on the host and again inside the build container. +# Install the cache in the environment that actually compiles Rust. +if [[ "${LINUX_CI_IN_DOCKER:-}" == 0 ]]; then + exit 0 +fi +if [[ -n "${RUSTC_WRAPPER:-}" || -n "${RUSTC_WORKSPACE_WRAPPER:-}" ]]; then + echo "Using the configured Rust compiler wrapper" + exit 0 +fi +if command -v mbx >/dev/null 2>&1; then + exit 0 +fi + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ "${OS:-}" == Windows_NT ]]; then + # Community setup precedes the build phase that normally activates MSVC. + powershell -NoProfile -File "$repo_root/scripts/configure-ci-windows.ps1" + exit 0 +fi +cargo install mbx --version 1.27.0 --locked --root "$repo_root/.ci/tools" diff --git a/scripts/qualify-windows.ps1 b/scripts/qualify-windows.ps1 index 6e120e6..2e6c5c1 100644 --- a/scripts/qualify-windows.ps1 +++ b/scripts/qualify-windows.ps1 @@ -16,7 +16,7 @@ foreach ($Checkout in @(@('https://github.com/duckdb/duckdb.git', $DuckDBRevisio if ((git -C $Checkout[2] rev-parse HEAD) -ne $Checkout[1]) { throw 'Dependency revision mismatch' } } & (Join-Path $Vcpkg 'bootstrap-vcpkg.bat') -disableMetrics -& (Join-Path $Vcpkg 'vcpkg.exe') install openssl:x64-windows-static zlib:x64-windows-static +& (Join-Path $Vcpkg 'vcpkg.exe') install --classic openssl:x64-windows-static zlib:x64-windows-static cmake -S $DuckDB -B $Build -G Ninja -DCMAKE_BUILD_TYPE=Release ` "-DCMAKE_TOOLCHAIN_FILE=$Vcpkg/scripts/buildsystems/vcpkg.cmake" ` -DVCPKG_MANIFEST_MODE=OFF -DVCPKG_TARGET_TRIPLET=x64-windows-static ` diff --git a/scripts/runtime-smoke.mjs b/scripts/runtime-smoke.mjs index 840dadd..1a2dc0a 100644 --- a/scripts/runtime-smoke.mjs +++ b/scripts/runtime-smoke.mjs @@ -27,11 +27,13 @@ const httpfsExtension = values.get('--httpfs-extension'); const repository = values.get('--repository'); assert( duckdb && (repository || (extension && quackExtension && httpfsExtension)), - 'usage: runtime-smoke.mjs --duckdb PATH [--repository PREFIX | --extension PATH --quack-extension PATH --httpfs-extension PATH_OR_core]', + 'usage: runtime-smoke.mjs --duckdb PATH [--repository PREFIX | --extension PATH --quack-extension PATH_OR_core --httpfs-extension PATH_OR_core]', ); if (!repository) { assert(statSync(extension).size > 0, `empty SQLNB extension: ${extension}`); - assert(statSync(quackExtension).size > 0, `empty Quack extension: ${quackExtension}`); + if (quackExtension !== 'core') { + assert(statSync(quackExtension).size > 0, `empty Quack extension: ${quackExtension}`); + } if (httpfsExtension !== 'core') { assert(statSync(httpfsExtension).size > 0, `empty httpfs extension: ${httpfsExtension}`); } @@ -70,13 +72,17 @@ function loadExtensions(includeUI = true) { } if (repository) { if (includeUI) { - // SQLNB installs and loads its private Quack dependency from this repository. + // SQLNB comes from this repository; its runtime dependencies come from core. statements.push('install sqlnb from sidequery', 'load sqlnb from sidequery'); } else { - statements.push('install quack from sidequery', 'load quack from sidequery'); + statements.push('install quack from core', 'load quack from core'); } } else { - statements.push(`load ${sqlLiteral(quackExtension)}`); + if (quackExtension !== 'core') { + statements.push(`load ${sqlLiteral(quackExtension)}`); + } else if (!includeUI) { + statements.push('install quack from core', 'load quack from core'); + } if (includeUI) statements.push(`load ${sqlLiteral(extension)}`); } return statements.join('; '); @@ -189,7 +195,11 @@ await expectSQLError( /port must be between 0 and 65535/, ); assert.equal(firstValue(await runSQL('select 42 as recovered')), 42); -assert.equal(firstValue(await runSQL('select count(*) as count from quack_server_list()')), 0); +if (repository || quackExtension === 'core') { + assert.equal(firstValue(await runSQL("select loaded from duckdb_extensions() where extension_name = 'quack'")), false); +} else { + assert.equal(firstValue(await runSQL('select count(*) as count from quack_server_list()')), 0); +} assert.equal( firstValue(await runSQL('create table shared_table as select 7::integer as value; select true')), true, @@ -225,6 +235,13 @@ const startRows = await runSQL( const start = JSON.parse(firstValue(startRows)); assert.equal(start.engine, 'native-duckdb'); assert.equal(start.protocol, 'quack'); +if (repository || quackExtension === 'core') { + assert.equal( + firstValue(await runSQL("select installed_from from duckdb_extensions() where extension_name = 'quack'")), + 'core', + 'SQLNB must use the official core Quack', + ); +} assert.equal( firstValue(await runSQL("select loaded from duckdb_extensions() where extension_name = 'icu'")), true, @@ -656,8 +673,10 @@ assert.match(await remoteAsset.text(), /sidequeryRemoteFixture = 'one'/); assert.equal((await fetch(`${remote.baseUrl}/../outside`)).status, 404); assert(upstreamPaths.includes('/notebook/manifest.json')); assert(!upstreamPaths.includes('/notebook/outside')); -assert.equal(firstValue(await runSQL('select sqlnb_stop()')), true); release = 'two'; +// Publishing a new UI must not replace assets beneath an active notebook. +assert.match(await (await fetch(`${remote.baseUrl}/asset.js`)).text(), /sidequeryRemoteFixture = 'one'/); +assert.equal(firstValue(await runSQL('select sqlnb_stop()')), true); const updatedUi = JSON.parse(firstValue(await runSQL( `select sqlnb_start(ui := 'http://127.0.0.1:${upstreamPort}/notebook/', open_browser := false, notebook_root := ${sqlLiteral(notebookRoot)}) as status`, ))); diff --git a/src/sidequery_ui_extension.cpp b/src/sidequery_ui_extension.cpp index be0c75f..2e4059e 100644 --- a/src/sidequery_ui_extension.cpp +++ b/src/sidequery_ui_extension.cpp @@ -62,9 +62,6 @@ namespace duckdb { namespace { static constexpr const char *DEFAULT_SQLNB_UI = "https://app.sidequery.dev/sqlnb/"; -#ifdef SQLNB_HAS_USER_EXTENSION_REPOSITORIES -static constexpr const char *SIDEQUERY_REPOSITORY = "sidequery"; -#endif static string JSONEscape(const string &value) { std::ostringstream out; @@ -106,29 +103,6 @@ static string RandomHex(idx_t bytes) { return out.str(); } -static void LoadQuack(DatabaseInstance &database) { - if (database.ExtensionIsLoaded("quack")) { - return; - } -#ifdef SQLNB_HAS_USER_EXTENSION_REPOSITORIES - auto &file_system = FileSystem::GetFileSystem(database); - ExtensionRepository repository; - if (ExtensionRepositoryManager::TryGetRepository(database, file_system, SIDEQUERY_REPOSITORY, - repository)) { - ExtensionInstallOptions install_options; - install_options.repository = &repository; - install_options.throw_on_origin_mismatch = true; - ExtensionHelper::InstallExtension(database, file_system, "quack", install_options); - - ExtensionLoadOptions load_options("quack"); - load_options.repository = repository.name; - ExtensionHelper::LoadExternalExtension(database, file_system, load_options); - return; - } -#endif - ExtensionHelper::AutoLoadExtension(database, "quack"); -} - static void EnsureDependenciesLoaded(DatabaseInstance &database) { // Source-column date profiles use timezone conversion. Load ICU before // accepting notebook queries so their first date query never installs it. @@ -148,11 +122,12 @@ static void EnsureDependenciesLoaded(DatabaseInstance &database) { throw MissingExtensionException("SQLNB requires the DuckDB httpfs extension for Quack sessions"); } if (!database.ExtensionIsLoaded("quack")) { - LoadQuack(database); + // Use DuckDB's core-signed Quack regardless of where SQLNB was installed. + ExtensionHelper::AutoLoadExtension(database, "quack"); } if (!database.ExtensionIsLoaded("quack")) { throw MissingExtensionException( - "SQLNB requires DuckDB Quack; install it from the same Sidequery extension repository"); + "SQLNB requires DuckDB Quack; install it from core"); } } diff --git a/test/fixtures/ui/index.html b/test/fixtures/ui/index.html new file mode 100644 index 0000000..23304f9 --- /dev/null +++ b/test/fixtures/ui/index.html @@ -0,0 +1,6 @@ + + + +SQLNB native test fixture +Local assets for the SQLNB lifecycle tests. + diff --git a/test/sql/lifecycle.test b/test/sql/lifecycle.test new file mode 100644 index 0000000..277499d --- /dev/null +++ b/test/sql/lifecycle.test @@ -0,0 +1,109 @@ +# No remote UI or browser is needed. Runtime dependencies come from DuckDB core, +# exercising the same startup path as an installation from community. +require sqlnb + +require json + +# Dependency download failures must fail qualification, never skip this test. +set ignore_error_messages + +query T +select json_extract(sqlnb_status(), '$.running'); +---- +false + +query I +select sqlnb_stop(); +---- +false + +statement error +select sqlnb_start(port := -1, open_browser := false); +---- +port must be between 0 and 65535 + +statement error +select sqlnb_start(port := 65536, open_browser := false); +---- +port must be between 0 and 65535 + +query I +select sqlnb_start(assets := NULL, open_browser := false) is null; +---- +true + +statement ok +set autoinstall_known_extensions = true; + +statement ok +set autoload_known_extensions = true; + +# The SQL test runner normally points autoinstall at its local build repository. +statement ok +set autoinstall_extension_repository = 'http://extensions.duckdb.org'; + +# A registered Sidequery repository must not override DuckDB's official Quack. +# This is the public fixture key from DuckDB's extension_repositories_lock test; +# the unreachable repository intentionally cannot serve any extensions. +statement ok +set extension_repository_directory = '{TEMP_DIR_ABSOLUTE}/sqlnb-repositories'; + +statement ok +set allow_extension_repositories = 'allowed'; + +statement ok +create extension repository sidequery with prefix 'http://127.0.0.1:1' using public key 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt9+KeDkWJa7SiBb7AMdXAPV2qpRunztoC2c7DtRmN9+Lq2YQm4rMQmmAQcwyxTxcK8vwPKOiE4mLMwmQMvTty9tL+buREfvg5KJ+FY3Bv05cZYsf7VuYgUDcw41zSEE5W1thh5PiafcrMn7CPF0jN0oFfVvAHB/fAM0yaZGPKs3WYqnHAHa6v/cVMeiWZ/lQORCBqqL2hii1fEiuIqw6AOEoZVsN0/lqm7GfnD899zrzLOPRuP4YFg46drL1UyNdg+4TquJ2OV9kR4CAfdGRexqHFWLmoxkTfnUxJ4yL+gkV57K1H+smEdKrDhDX68pEFM/34E2eVKZoRU/0paxDZQIDAQAB'; + +query T +select json_extract(sqlnb_start(assets := '{WORKING_DIRECTORY}/test/fixtures/ui', notebook_root := '{TEMP_DIR_ABSOLUTE}/sqlnb-lifecycle', open_browser := false, mcp := true), '$.running'); +---- +true + +query T +select json_extract(sqlnb_status(), '$.mcp.enabled'); +---- +true + +query IT +select count(*), bool_and(loaded) from duckdb_extensions() where extension_name in ('quack', 'httpfs', 'icu'); +---- +3 true + +query T +select installed_from from duckdb_extensions() where extension_name = 'quack'; +---- +core + +# The capability belongs to the connection that started the server. +statement error other +select sqlnb_status(); +---- +may only be called from the connection that started SQLNB + +statement error other +select sqlnb_stop(); +---- +may only be called from the connection that started SQLNB + +query I +select sqlnb_stop(); +---- +true + +query T +select json_extract(sqlnb_status(), '$.running'); +---- +false + +query I +select sqlnb_stop(); +---- +false + +statement ok +set enable_external_access = false; + +statement error +select sqlnb_start(assets := '{WORKING_DIRECTORY}/test/fixtures/ui', open_browser := false); +---- +SQLNB requires enable_external_access=true