diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml new file mode 100644 index 0000000..cbcca05 --- /dev/null +++ b/.github/workflows/windows.yml @@ -0,0 +1,33 @@ +name: Windows qualification +on: + pull_request: + workflow_dispatch: +permissions: + contents: read +jobs: + native: + runs-on: windows-2025 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.99.0 + - uses: oven-sh/setup-bun@v2 + - uses: ilammy/msvc-dev-cmd@v1 + - name: Allow symlink security fixtures + shell: pwsh + run: | + New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock' -Name AllowDevelopmentWithoutDevLicense -Value 1 -PropertyType DWord -Force + - name: Install build cache in the disposable runner + shell: pwsh + run: | + cargo install mbx --version 1.27.0 --locked --root "$env:RUNNER_TEMP/mbx" + if ($LASTEXITCODE) { exit $LASTEXITCODE } + "$env:RUNNER_TEMP/mbx/bin" | Out-File -FilePath $env:GITHUB_PATH -Append + - name: Rust contracts and Windows filesystem security + shell: pwsh + run: | + mbx test --manifest-path http/Cargo.toml --locked -- --test-threads=1 + if ($LASTEXITCODE) { exit $LASTEXITCODE } + - name: Build native extension and exercise HTTP bridge + shell: pwsh + run: ./scripts/qualify-windows.ps1 diff --git a/CMakeLists.txt b/CMakeLists.txt index b8d8dd4..c0109e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,10 +21,13 @@ set(SIDEQUERY_HTTP_LOCK ${CMAKE_CURRENT_SOURCE_DIR}/http/Cargo.lock) file(GLOB_RECURSE SIDEQUERY_HTTP_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/http/src/*.rs) set(SIDEQUERY_HTTP_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR}/http-cargo) -if(NOT UNIX) - message(FATAL_ERROR "SQLNB's local notebook server currently supports macOS and Linux") +if(MSVC) + set(SIDEQUERY_HTTP_LIBRARY ${SIDEQUERY_HTTP_TARGET_DIR}/release/sidequery_http.lib) + # DuckDB uses the static MSVC runtime; Rust and its C dependencies must agree. + set(SIDEQUERY_RUST_ENV "RUSTFLAGS=$ENV{RUSTFLAGS} -C target-feature=+crt-static") +else() + set(SIDEQUERY_HTTP_LIBRARY ${SIDEQUERY_HTTP_TARGET_DIR}/release/libsidequery_http.a) endif() -set(SIDEQUERY_HTTP_LIBRARY ${SIDEQUERY_HTTP_TARGET_DIR}/release/libsidequery_http.a) set(SIDEQUERY_REPO_MBX ${CMAKE_CURRENT_SOURCE_DIR}/scripts/mbx) if(EXISTS ${SIDEQUERY_REPO_MBX}) @@ -46,7 +49,7 @@ endif() add_custom_command( OUTPUT ${SIDEQUERY_HTTP_LIBRARY} - COMMAND ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${SIDEQUERY_HTTP_TARGET_DIR} + COMMAND ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${SIDEQUERY_HTTP_TARGET_DIR} ${SIDEQUERY_RUST_ENV} ${SIDEQUERY_CARGO_BUILD} --manifest-path ${SIDEQUERY_HTTP_MANIFEST} --locked --release DEPENDS ${SIDEQUERY_HTTP_MANIFEST} ${SIDEQUERY_HTTP_LOCK} ${SIDEQUERY_HTTP_SOURCES} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/http @@ -67,6 +70,8 @@ foreach(target ${EXTENSION_NAME} ${LOADABLE_EXTENSION_NAME}) target_link_libraries(${target} iconv) elseif(UNIX) target_link_libraries(${target} rt util m) + elseif(WIN32) + target_link_libraries(${target} ws2_32 userenv bcrypt ntdll crypt32 secur32 shell32 advapi32 winhttp rpcrt4 ole32) endif() endforeach() diff --git a/README.md b/README.md index 4d01b2c..d488cd8 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,50 @@ For local UI development, use `assets := '/absolute/path/to/assets'`; local asse The authenticated `GET /api/v1/recent-notebook` endpoint returns `{ "path": "relative.sqlnb" }`, or `{ "path": null }` when no readable recent notebook is available. The UI records a successful open or save using `PUT` to the same endpoint with `{ "path": "relative.sqlnb" }`. The path must identify an existing notebook within the configured root. This preference survives server restarts and port changes and is isolated by canonical notebook root under the OS cache's `sidequery/sqlnb-extension/notebooks` directory. It contains only the relative path; unavailable or corrupt preferences do not affect normal notebook file operations. +## DuckDB UI notebooks + +The local DuckDB UI launched by `duckdb -ui` stores its notebooks in a DuckDB +database at `~/.duckdb/extension_data/ui/ui.db`. SQLNB exposes existing notebooks +as a second live source alongside local SQLNB files. Opening a notebook does not +execute it; saving edits writes a new version to that same database. + +The browser uses three native SQL functions, each returning a JSON string: + +```sql +select sqlnb_duckdb_ui_notebooks(); +select sqlnb_duckdb_ui_notebook('notebook-uuid'); +select sqlnb_duckdb_ui_save('notebook-uuid', expected_entity_version, 'Title', 'edited notebook JSON'); +``` + +Reads return the official ID, entity version, title and stored notebook JSON. +Listing omits the JSON bodies. Saves require serialization format 3 and both the +expected entity version and unchanged content version. An accepted save advances +both versions and expires the preceding current revision in seven days, matching +the official UI. Conflicts and malformed edits roll back. Notebook identity, +creation time, internal name, current-notebook preference, unknown JSON fields +and historical revisions remain intact. Browsing never creates or migrates a +database, and `enable_external_access` must be enabled. + +An official UI catalog already attached to the host instance takes precedence; +otherwise each operation opens the existing default database for its duration. +Normal DuckDB locks apply. A separate process holding a write lock prevents +access; stop that process or run both UIs in the same DuckDB instance. No lock +bypass or database copy is used. The current adapter supports editing existing +notebooks, not creating or deleting notebook records. + +The frontend owns conversion through the canonical Rust SQLNB parser and +serializer. Cell IDs, database choices, grouping, comments and SQLNB directives +survive saves. Features the official UI does not implement remain SQLNB source +comments, with a compatibility notice in SQLNB. The official UI must reopen or +refresh a notebook to see saved edits. SQLNB refreshes clean documents and retains +dirty drafts when another editor changes the source. + +The native source smoke uses an isolated home directory and database: + +```sh +bun scripts/duckdb-ui-smoke.mjs /absolute/path/to/duckdb /absolute/path/to/sqlnb.duckdb_extension +``` + ## Deployment contract The stable base URL serves `manifest.json` with `version` (7–64 lowercase hexadecimal characters), `bridgeVersion: 1`, an optional relative `assetBase` directory, and a sorted `files` array of `{path, sha256, size}`. File paths are relative to the asset base and include `index.html`. Publish the manifest together with its complete immutable generation. Incompatible bridge versions are rejected; capabilities beyond the native bridge require an extension update. diff --git a/http/Cargo.lock b/http/Cargo.lock index 7718099..ef1c262 100644 --- a/http/Cargo.lock +++ b/http/Cargo.lock @@ -1255,6 +1255,7 @@ dependencies = [ "tower", "url", "uuid", + "windows-sys 0.61.2", ] [[package]] diff --git a/http/Cargo.toml b/http/Cargo.toml index 6bf2294..5d57b9f 100644 --- a/http/Cargo.toml +++ b/http/Cargo.toml @@ -27,5 +27,8 @@ uuid = { version = "1", features = ["v4"] } rmcp = { version = "=3.4.0", default-features = false, features = ["server", "transport-streamable-http-server"] } tokio-util = "0.7" +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61.2", features = ["Wdk_Foundation", "Wdk_Storage_FileSystem", "Win32_Foundation", "Win32_Security", "Win32_Storage_FileSystem", "Win32_System_IO", "Win32_System_JobObjects", "Win32_System_Threading", "Win32_System_Diagnostics_ToolHelp"] } + [dev-dependencies] tempfile = "3" diff --git a/http/src/cache_location.rs b/http/src/cache_location.rs index 6c9c31d..d1219f6 100644 --- a/http/src/cache_location.rs +++ b/http/src/cache_location.rs @@ -1,4 +1,4 @@ -#[cfg(any(target_os = "linux", test))] +#[cfg(any(target_os = "linux", all(test, unix)))] use std::ffi::OsStr; use std::path::PathBuf; use url::Url; @@ -10,10 +10,9 @@ pub(super) fn asset_cache_scope(remote: &Url) -> Result { } pub(super) fn notebook_state_path(root: &std::path::Path) -> Result { - use std::os::unix::ffi::OsStrExt; Ok(cache_root()? .join("sidequery/sqlnb-extension/notebooks") - .join(crate::revision(root.as_os_str().as_bytes())) + .join(crate::revision(root.as_os_str().as_encoded_bytes())) .join("recent-notebook.json")) } @@ -27,10 +26,12 @@ fn cache_root() -> Result { let root = linux_cache_root(std::env::var_os("XDG_CACHE_HOME").as_deref(), || { std::env::var_os("HOME") })?; + #[cfg(windows)] + let root = windows_local_data()?.join("Cache"); Ok(root) } -#[cfg(any(target_os = "linux", test))] +#[cfg(any(target_os = "linux", all(test, unix)))] fn linux_cache_root( xdg_cache_home: Option<&OsStr>, home: impl FnOnce() -> Option, @@ -49,7 +50,7 @@ fn linux_cache_root( Ok(PathBuf::from(home).join(".cache")) } -#[cfg(test)] +#[cfg(all(test, unix))] mod tests { use super::*; @@ -86,3 +87,11 @@ mod tests { } } } + +#[cfg(windows)] +pub(super) fn windows_local_data() -> Result { + std::env::var_os("LOCALAPPDATA") + .map(PathBuf::from) + .filter(|path| path.is_absolute()) + .ok_or_else(|| "LOCALAPPDATA is unavailable or not absolute".into()) +} diff --git a/http/src/duckdb_ui.rs b/http/src/duckdb_ui.rs new file mode 100644 index 0000000..9d675c0 --- /dev/null +++ b/http/src/duckdb_ui.rs @@ -0,0 +1,180 @@ +//! The local DuckDB UI's stored notebook format. Keep unknown metadata intact. +use serde_json::Value; +use std::collections::HashSet; + +const MAX_BYTES: usize = 16 * 1024 * 1024; + +fn notebook(source: &str) -> Result { + if source.len() > MAX_BYTES { + return Err("DuckDB UI notebook exceeds the 16 MiB limit".into()); + } + let value: Value = serde_json::from_str(source) + .map_err(|error| format!("Invalid DuckDB UI notebook JSON: {error}"))?; + if value + .get("notebookSerializationFormat") + .and_then(Value::as_u64) + != Some(3) + { + return Err("Unsupported DuckDB UI notebook format; expected format 3".into()); + } + if !value + .get("version") + .and_then(Value::as_i64) + .is_some_and(|version| (1..=i32::MAX as i64).contains(&version)) + { + return Err("Invalid DuckDB UI notebook content version".into()); + } + if !matches!( + value.get("currentDatabase"), + Some(Value::Null | Value::String(_)) + ) { + return Err("Invalid DuckDB UI current database".into()); + } + let view = value + .get("viewMode") + .and_then(Value::as_object) + .ok_or("Invalid DuckDB UI notebook view mode")?; + match view.get("mode").and_then(Value::as_str) { + Some("default") => {} + Some("fullScreen") + if view.get("cellId").is_some_and(Value::is_number) + && matches!( + view.get("focus").and_then(Value::as_str), + Some("editor" | "results" | "split") + ) => {} + _ => return Err("Invalid DuckDB UI notebook view mode".into()), + } + let cells = value + .get("cells") + .and_then(Value::as_array) + .ok_or("Invalid DuckDB UI notebook cells")?; + let mut ids = HashSet::new(); + for cell in cells { + let id = cell + .get("cellId") + .filter(|id| id.is_number()) + .ok_or("Invalid DuckDB UI cell ID")?; + if !ids.insert(id.to_string()) { + return Err("Duplicate DuckDB UI cell ID".into()); + } + if !matches!(cell.get("query"), Some(Value::Null | Value::String(_))) + || !matches!( + cell.get("useDatabase"), + None | Some(Value::Null | Value::String(_)) + ) + || !matches!(cell.get("isActive"), None | Some(Value::Bool(_))) + || !matches!( + cell.get("runMode").and_then(Value::as_str), + None | Some("default" | "instant") + ) + || cell.get("runMode").is_some_and(|mode| !mode.is_string()) + { + return Err("Invalid DuckDB UI notebook cell".into()); + } + } + Ok(value) +} + +pub(crate) fn initial_version(source: &str) -> Result { + let mut value = notebook(source)?; + value["version"] = Value::from(1); + serde_json::to_string(&value).map_err(|error| error.to_string()) +} + +pub(crate) fn next_version(previous: &str, edited: &str) -> Result { + let previous = notebook(previous)?; + let mut edited = notebook(edited)?; + if edited["version"] != previous["version"] { + return Err("DuckDB UI notebook content version changed; reload before saving".into()); + } + let version = previous["version"].as_i64().unwrap(); + if version == i32::MAX as i64 { + return Err("DuckDB UI notebook content version is exhausted".into()); + } + edited["version"] = Value::from(version + 1); + let result = serde_json::to_string(&edited).map_err(|error| error.to_string())?; + if result.len() > MAX_BYTES { + return Err("DuckDB UI notebook exceeds the 16 MiB limit".into()); + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn fixture() -> Value { + json!({"notebookSerializationFormat":3,"version":2,"currentDatabase":"analytics", + "viewMode":{"mode":"default"},"futureNotebookOption":{"keep":true}, + "cells":[{"cellId":2,"query":"-- A database-aware cell\nselect answer, 'π' as label from sample;", + "useDatabase":"analytics","isActive":true,"runMode":"instant","futureCellOption":42}, + {"cellId":3,"query":null,"useDatabase":null}]}) + } + + #[test] + fn advances_official_content_version_without_losing_unknown_metadata() { + let previous = fixture(); + let mut edited = previous.clone(); + edited["cells"][0]["query"] = json!("select 'edited π' as value;"); + let result: Value = serde_json::from_str( + &next_version(&previous.to_string(), &edited.to_string()).unwrap(), + ) + .unwrap(); + edited["version"] = json!(3); + assert_eq!(result, edited); + } + + #[test] + fn rejects_incompatible_or_malformed_edits_before_a_write() { + let previous = fixture().to_string(); + for (field, invalid) in [ + ("notebookSerializationFormat", json!(4)), + ("version", json!(3)), + ("currentDatabase", json!(42)), + ("viewMode", json!({"mode":"unknown"})), + ("cells", json!({})), + ] { + let mut edited = fixture(); + edited[field] = invalid; + assert!( + next_version(&previous, &edited.to_string()).is_err(), + "{field}" + ); + } + for (field, invalid) in [ + ("query", json!(7)), + ("cellId", json!("2")), + ("useDatabase", json!(true)), + ("isActive", json!("yes")), + ("runMode", json!("unknown")), + ] { + let mut edited = fixture(); + edited["cells"][0][field] = invalid; + assert!( + next_version(&previous, &edited.to_string()).is_err(), + "{field}" + ); + } + let mut duplicate = fixture(); + duplicate["cells"][1]["cellId"] = json!(2); + assert!(next_version(&previous, &duplicate.to_string()).is_err()); + assert!(next_version(&previous, "{").is_err()); + } +} + +#[cfg(test)] +mod creation_tests { + use super::*; + #[test] + fn new_identity_resets_version_preserving_opaque_metadata() { + let source = r#"{"notebookSerializationFormat":3,"version":9,"currentDatabase":null,"viewMode":{"mode":"default","future":42},"cells":[{"cellId":7,"query":"select 1","opaque":[1,2]}],"future":{"keep":true}}"#; + let mut expected: Value = serde_json::from_str(source).unwrap(); + expected["version"] = Value::from(1); + assert_eq!( + serde_json::from_str::(&initial_version(source).unwrap()).unwrap(), + expected + ); + assert!(initial_version("{}").is_err()); + } +} diff --git a/http/src/history.rs b/http/src/history.rs index 569af90..ad4da39 100644 --- a/http/src/history.rs +++ b/http/src/history.rs @@ -2,7 +2,7 @@ use super::*; use git2::{Oid, Repository, Signature}; use std::collections::BTreeMap; -use std::os::unix::ffi::OsStrExt; +#[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::time::{SystemTime, UNIX_EPOCH}; @@ -57,7 +57,7 @@ struct Journal { pub(super) struct History { directory: PathBuf, root: PathBuf, - root_fd: OwnedFd, + root_fd: File, } pub(super) struct Transaction<'a> { history: &'a History, @@ -80,17 +80,12 @@ fn now() -> u64 { } fn persist(path: &Path, bytes: &[u8]) -> Result<(), ApiError> { let temporary = path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4())); - let mut file = OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(&temporary) - .map_err(err)?; + let mut file = private_new_file(&temporary).map_err(err)?; file.write_all(bytes).map_err(err)?; file.sync_all().map_err(err)?; fs::rename(&temporary, path).map_err(err)?; - File::open(path.parent().unwrap()) - .and_then(|f| f.sync_all()) + open_directory(path.parent().unwrap()) + .and_then(|f| sync_directory(f.platform_handle())) .map_err(err) } impl History { @@ -98,27 +93,26 @@ impl History { #[cfg(target_os = "macos")] let base = PathBuf::from(std::env::var_os("HOME").ok_or("HOME unavailable for history")?) .join("Library/Application Support"); - #[cfg(not(target_os = "macos"))] + #[cfg(target_os = "linux")] let base = std::env::var_os("XDG_STATE_HOME") .map(PathBuf::from) .filter(|p| p.is_absolute()) .or_else(|| std::env::var_os("HOME").map(|p| PathBuf::from(p).join(".local/state"))) .ok_or("history state directory unavailable")?; + #[cfg(windows)] + let base = cache_location::windows_local_data()?; Self::at( root, base.join("sidequery/sqlnb-history") - .join(revision(root.as_os_str().as_bytes())), + .join(revision(root.as_os_str().as_encoded_bytes())), ) } fn at(root: &Path, directory: PathBuf) -> Result { fs::create_dir_all(&directory).map_err(|e| e.to_string())?; - if fs::symlink_metadata(&directory) - .map_err(|e| e.to_string())? - .file_type() - .is_symlink() - { + if is_reparse(&fs::symlink_metadata(&directory).map_err(|e| e.to_string())?) { return Err("history directory must not be a symlink".into()); } + #[cfg(unix)] fs::set_permissions(&directory, fs::Permissions::from_mode(0o700)) .map_err(|e| e.to_string())?; let history = Self { @@ -127,7 +121,7 @@ impl History { root_fd: open_directory(root).map_err(|error| error.to_string())?, }; let lock = open_lock_file(&history.directory.join("lock"))?; - lock_file(&lock, libc::LOCK_EX, "notebook history")?; + lock_file(&lock, LOCK_EX, "notebook history")?; let repo = Repository::init_bare(history.directory.join("snapshots.git")) .map_err(|e| e.to_string())?; repo.config() @@ -142,24 +136,20 @@ impl History { return Err(invalid_path()); }; let name = cstring_component(name)?; - let duplicate = unsafe { libc::dup(self.root_fd.as_raw_fd()) }; - if duplicate < 0 { - return Err(file_io(std::io::Error::last_os_error())); - } - let mut directory = unsafe { OwnedFd::from_raw_fd(duplicate) }; + let mut directory = self.root_fd.try_clone().map_err(file_io)?; for component in components { let Component::Normal(component) = component else { return Err(invalid_path()); }; - match open_directory_at(directory.as_raw_fd(), &cstring_component(component)?) { + match open_directory_at(directory.platform_handle(), &cstring_component(component)?) { Ok(next) => directory = next, - Err(error) if error.raw_os_error() == Some(libc::ENOENT) => return Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(file_io(error)), } } - let file = match open_file_at(directory.as_raw_fd(), &name) { + let file = match open_file_at(directory.platform_handle(), &name) { Ok(file) => file, - Err(error) if error.raw_os_error() == Some(libc::ENOENT) => return Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(file_io(error)), }; let metadata = file.metadata().map_err(file_io)?; @@ -177,7 +167,7 @@ impl History { } pub fn lock(&self) -> Result, ApiError> { let lock = open_lock_file(&self.directory.join("lock")).map_err(err)?; - lock_file(&lock, libc::LOCK_EX, "notebook history").map_err(err)?; + lock_file(&lock, LOCK_EX, "notebook history").map_err(err)?; let path = self.directory.join("catalog.json"); let catalog = if path.exists() { serde_json::from_slice(&fs::read(path).map_err(err)?).map_err(err)? @@ -295,12 +285,8 @@ impl Transaction<'_> { let directory = objects.join(&hex[..2]); let object = directory.join(&hex[2..]); if object.exists() { - File::open(object) - .and_then(|file| file.sync_all()) - .map_err(err)?; - File::open(directory) - .and_then(|file| file.sync_all()) - .map_err(err)?; + sync_history_path(&object).map_err(err)?; + sync_history_path(&directory).map_err(err)?; } } for path in [ @@ -310,9 +296,7 @@ impl Transaction<'_> { self.repo.path().join("refs"), self.repo.path().to_owned(), ] { - File::open(path) - .and_then(|file| file.sync_all()) - .map_err(err)?; + sync_history_path(&path).map_err(err)?; } Ok(Revision { id: id.to_string(), @@ -429,9 +413,7 @@ impl Transaction<'_> { self.save()?; } fs::remove_file(journal).map_err(err)?; - File::open(&self.history.directory) - .and_then(|f| f.sync_all()) - .map_err(err)?; + sync_history_path(&self.history.directory).map_err(err)?; self.prune() } pub fn cancel(&mut self) -> Result<(), ApiError> { @@ -536,11 +518,9 @@ impl Transaction<'_> { builder.write(&packs, 0o600).map_err(err)?; } for entry in fs::read_dir(&packs).map_err(err)? { - File::open(entry.map_err(err)?.path()) - .and_then(|f| f.sync_all()) - .map_err(err)?; + sync_history_path(&entry.map_err(err)?.path()).map_err(err)?; } - File::open(&packs).and_then(|f| f.sync_all()).map_err(err)?; + sync_history_path(&packs).map_err(err)?; // All live objects are durably packed. Remove loose objects and superseded packs only. for entry in fs::read_dir(&objects).map_err(err)? { let entry = entry.map_err(err)?; @@ -937,7 +917,7 @@ mod tests { let (temp, store) = fixture(); let outside = temp.path().join("outside.sql"); fs::write(&outside, b"private").unwrap(); - std::os::unix::fs::symlink(outside, store.root.join("linked.sql")).unwrap(); + crate::tests::symlink(outside, store.root.join("linked.sql")).unwrap(); store.reconcile().unwrap(); assert!(store.history.lock().unwrap().list("linked.sql").is_empty()); assert!(store diff --git a/http/src/lib.rs b/http/src/lib.rs index 5631283..8620158 100644 --- a/http/src/lib.rs +++ b/http/src/lib.rs @@ -12,6 +12,11 @@ use axum::{ #[cfg(test)] use bytes::Bytes; mod cache_location; +mod platform; +#[cfg(windows)] +mod windows_process; +use platform::*; +mod duckdb_ui; mod history; mod mcp; mod project_git; @@ -22,9 +27,6 @@ use std::{ fs::{self, File, OpenOptions}, io::{Read, Write}, net::{Ipv4Addr, SocketAddrV4, TcpListener}, - os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd}, - os::unix::ffi::OsStringExt, - os::unix::fs::{MetadataExt, OpenOptionsExt}, path::{Component, Path, PathBuf}, ptr, sync::{mpsc, Arc, Mutex}, @@ -35,8 +37,8 @@ use subtle::ConstantTimeEq; use tokio::sync::watch; use url::Url; -#[cfg(not(unix))] -compile_error!("sidequery-ui-http currently supports macOS and Linux"); +#[cfg(not(any(unix, windows)))] +compile_error!("sidequery-ui-http supports macOS, Linux, and Windows"); const MAX_QUACK_MESSAGE_BYTES: usize = 64 * 1024 * 1024; const MAX_NOTEBOOK_BYTES: usize = 16 * 1024 * 1024; @@ -133,13 +135,13 @@ struct CacheTransaction { impl CacheTransaction { fn lock(cache_scope: &Path) -> Result { let file = open_lock_file(&cache_scope.join(ASSET_CACHE_LOCK))?; - lock_file(&file, libc::LOCK_EX, "UI cache transaction")?; + lock_file(&file, LOCK_EX, "UI cache transaction")?; Ok(Self { file }) } fn try_lock(cache_scope: &Path) -> Result, String> { let file = open_lock_file(&cache_scope.join(ASSET_CACHE_LOCK))?; - match try_lock_file(&file, libc::LOCK_EX, "UI cache transaction")? { + match try_lock_file(&file, LOCK_EX, "UI cache transaction")? { true => Ok(Some(Self { file })), false => Ok(None), } @@ -171,7 +173,7 @@ struct AssetManifestFile { struct NotebookStore { root: PathBuf, - root_fd: OwnedFd, + root_fd: File, writes: Mutex<()>, recent_path: Option, history: history::History, @@ -295,6 +297,21 @@ struct MoveFileResponse { revision: String, } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct DuplicateFileRequest { + path: String, + destination_path: String, + expected_revision: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct DeleteFileRequest { + path: String, + expected_revision: String, +} + #[derive(Deserialize)] struct CreateFolderRequest { path: String, @@ -465,51 +482,6 @@ fn resolve_remote_assets_with_budget( }) } -fn open_lock_file(path: &Path) -> Result { - OpenOptions::new() - .read(true) - .write(true) - .create(true) - .mode(0o600) - .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) - .open(path) - .map_err(|error| format!("failed to open UI cache lock {}: {error}", path.display())) -} - -fn lock_file(file: &File, operation: libc::c_int, label: &str) -> Result<(), String> { - loop { - if unsafe { libc::flock(file.as_raw_fd(), operation) } == 0 { - return Ok(()); - } - let error = std::io::Error::last_os_error(); - if error.kind() != std::io::ErrorKind::Interrupted { - return Err(format!("failed to lock {label}: {error}")); - } - } -} - -fn try_lock_file(file: &File, operation: libc::c_int, label: &str) -> Result { - loop { - if unsafe { libc::flock(file.as_raw_fd(), operation | libc::LOCK_NB) } == 0 { - return Ok(true); - } - let error = std::io::Error::last_os_error(); - if error - .raw_os_error() - .is_some_and(|code| code == libc::EWOULDBLOCK || code == libc::EAGAIN) - { - return Ok(false); - } - if error.kind() != std::io::ErrorKind::Interrupted { - return Err(format!("failed to lock {label}: {error}")); - } - } -} - -fn unlock_file(file: &File) -> Result<(), String> { - lock_file(file, libc::LOCK_UN, "UI cache file") -} - fn asset_client() -> Result { reqwest::blocking::Client::builder() .redirect(reqwest::redirect::Policy::none()) @@ -890,7 +862,7 @@ fn select_valid_fallback(cache_scope: &Path) -> Result { let Ok(metadata) = fs::symlink_metadata(&path) else { continue; }; - if metadata.file_type().is_symlink() || !metadata.is_dir() { + if is_reparse(&metadata) || !metadata.is_dir() { continue; } if validate_cached_generation(&path, None).is_ok() { @@ -917,17 +889,13 @@ fn acquire_generation_lease( match fs::symlink_metadata(&lease_path) { Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(error) => Err(format!("failed to inspect UI generation lease: {error}")), - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + Ok(metadata) if is_reparse(&metadata) || !metadata.is_file() => { Err("UI generation lease marker is not a regular file".into()) } Ok(_) => { - let file = OpenOptions::new() - .read(true) - .write(true) - .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) - .open(&lease_path) + let file = open_lease_file(&lease_path) .map_err(|error| format!("failed to open UI generation lease: {error}"))?; - lock_file(&file, libc::LOCK_SH, "UI generation lease")?; + lock_file(&file, LOCK_SH, "UI generation lease")?; Ok(Some(GenerationLease { file: Some(file), cache_scope: cache_scope.to_path_buf(), @@ -953,7 +921,7 @@ fn prune_generations_locked(cache_scope: &Path) -> Result<(), String> { let Ok(metadata) = fs::symlink_metadata(&path) else { continue; }; - if metadata.file_type().is_symlink() || !metadata.is_dir() { + if is_reparse(&metadata) || !metadata.is_dir() { continue; } let Some(cache_id) = entry.file_name().to_str().map(str::to_owned) else { @@ -985,18 +953,13 @@ fn prune_generations_locked(cache_scope: &Path) -> Result<(), String> { // an older process may still have the directory pinned. continue; }; - if metadata.file_type().is_symlink() || !metadata.is_file() { + if is_reparse(&metadata) || !metadata.is_file() { continue; } - let Ok(file) = OpenOptions::new() - .read(true) - .write(true) - .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) - .open(&lease_path) - else { + let Ok(file) = open_lease_file(&lease_path) else { continue; }; - if try_lock_file(&file, libc::LOCK_EX, "UI generation lease")? { + if try_lock_file(&file, LOCK_EX, "UI generation lease")? { fs::remove_dir_all(&path).map_err(|error| { format!("failed to remove obsolete UI generation {cache_id}: {error}") })?; @@ -1036,7 +999,7 @@ fn validate_cached_generation( let path = generation.join(relative); let metadata = fs::symlink_metadata(&path) .map_err(|error| format!("failed to inspect cached UI asset {}: {error}", file.path))?; - if metadata.file_type().is_symlink() || !metadata.is_file() || metadata.len() != file.size { + if is_reparse(&metadata) || !metadata.is_file() || metadata.len() != file.size { return Err(format!( "cached UI asset {} has invalid metadata", file.path @@ -1097,8 +1060,8 @@ fn notebook_root(configured: &str) -> Result { configured }; let path = if configured == "~" || configured.starts_with("~/") { - let home = - std::env::var_os("HOME").ok_or("HOME is unavailable; notebookRoot must be absolute")?; + let home = home_directory() + .ok_or("home directory is unavailable; notebookRoot must be absolute")?; let suffix = configured.strip_prefix("~/").unwrap_or(""); PathBuf::from(home).join(suffix) } else { @@ -1111,7 +1074,7 @@ fn notebook_root(configured: &str) -> Result { .map_err(|error| format!("failed to create notebook root: {error}"))?; let metadata = fs::symlink_metadata(&path) .map_err(|error| format!("failed to inspect notebook root: {error}"))?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { + if is_reparse(&metadata) || !metadata.is_dir() { return Err("notebookRoot must be a real directory, not a symlink".into()); } path.canonicalize() @@ -1158,7 +1121,7 @@ impl NotebookStore { let mut files = Vec::new(); let mut directories = Vec::new(); list_directory_fd( - self.root_fd.as_raw_fd(), + self.root_fd.platform_handle(), Path::new(""), &mut files, &mut directories, @@ -1173,14 +1136,29 @@ impl NotebookStore { // Preferences are optional and untrusted. Bound reads and revalidate // the notebook through the same descriptor-based file boundary. let recent: RecentNotebook = serde_json::from_reader(file.take(8192)).ok()?; - let path = validate_relative_path(recent.path.as_deref()?).ok()?; + let identifier = recent.path.as_deref()?; + if identifier.contains("://") { + return validate_duckdb_ui_identifier(identifier).ok(); + } + let path = validate_relative_path(identifier).ok()?; self.read(&path).ok()?; slash_path(&path).ok() } + #[cfg(test)] fn remember(&self, path: &Path) -> Result { self.read(path)?; - let path = slash_path(path)?; + self.remember_identifier(&slash_path(path)?) + } + + fn remember_identifier(&self, identifier: &str) -> Result { + let path = if identifier.contains("://") { + validate_duckdb_ui_identifier(identifier)? + } else { + let path = validate_relative_path(identifier)?; + self.read(&path)?; + slash_path(&path)? + }; let destination = self.recent_path.as_ref().ok_or_else(|| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, @@ -1202,12 +1180,7 @@ impl NotebookStore { ) })?; let result = (|| { - let mut file = OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(&temporary) - .map_err(file_io)?; + let mut file = private_new_file(&temporary).map_err(file_io)?; file.write_all(&bytes).map_err(file_io)?; file.sync_all().map_err(file_io)?; fs::rename(&temporary, destination).map_err(file_io) @@ -1276,15 +1249,26 @@ impl NotebookStore { "file lock is poisoned", ) })?; + self.write_locked(relative, source, expected, reason) + } + + // Caller holds writes, including when copying a revision into a new file. + fn write_locked( + &self, + relative: &Path, + source: &[u8], + expected: &str, + reason: &str, + ) -> Result { let mut history = self.history.lock()?; history.finish(true)?; let history_path = slash_path(relative)?; let mut baseline = None; let (parent, name) = self.open_parent(relative, true)?; - let current = open_file_at(parent.as_raw_fd(), &name); + let current = open_file_at(parent.platform_handle(), &name); match (expected.is_empty(), current) { (true, Ok(_)) => return Err(conflict("file already exists")), - (true, Err(error)) if error.raw_os_error() == Some(libc::ENOENT) => {} + (true, Err(error)) if error.kind() == std::io::ErrorKind::NotFound => {} (true, Err(error)) => return Err(file_io(error)), (false, Ok(file)) => { let metadata = file.metadata().map_err(file_io)?; @@ -1300,7 +1284,7 @@ impl NotebookStore { } baseline = Some(bytes); } - (false, Err(error)) if error.raw_os_error() == Some(libc::ENOENT) => { + (false, Err(error)) if error.kind() == std::io::ErrorKind::NotFound => { return Err(conflict("file no longer exists")) } (false, Err(error)) => return Err(file_io(error)), @@ -1313,10 +1297,10 @@ impl NotebookStore { } history.prepare(&history_path, source, reason, None)?; let written = - atomic_write_at(parent.as_raw_fd(), &name, source, expected).map_err(|error| { - if error.raw_os_error() == Some(libc::EEXIST) { + atomic_write_at(parent.platform_handle(), &name, source, expected).map_err(|error| { + if error.kind() == std::io::ErrorKind::AlreadyExists { conflict("file already exists") - } else if error.raw_os_error() == Some(libc::EBUSY) { + } else if error.kind() == std::io::ErrorKind::ResourceBusy { conflict("file changed since it was opened") } else { file_io(error) @@ -1338,6 +1322,47 @@ impl NotebookStore { Ok(revision(source)) } + fn duplicate_file( + &self, + source: &Path, + destination: &Path, + expected: &str, + ) -> Result { + let _guard = self + .writes + .lock() + .map_err(|_| file_io(std::io::Error::other("file lock poisoned")))?; + let (content, current) = self.read(source)?; + if expected.is_empty() || current != expected { + return Err(conflict("file changed since it was opened")); + } + self.write_locked(destination, content.as_bytes(), "", "duplicate") + } + + fn delete_file(&self, relative: &Path, expected: &str) -> Result<(), ApiError> { + let _guard = self + .writes + .lock() + .map_err(|_| file_io(std::io::Error::other("file lock poisoned")))?; + let mut history = self.history.lock()?; + history.finish(true)?; + let (content, current) = self.read(relative)?; + if expected.is_empty() || current != expected { + return Err(conflict("file changed since it was opened")); + } + let (parent, name) = self.open_parent(relative, false)?; + history.capture(&slash_path(relative)?, content.as_bytes(), "baseline")?; + // As with save, recheck immediately before the filesystem mutation. + // Portable filesystems cannot CAS against non-cooperating external writers. + if self.read(relative)?.1 != expected { + return Err(conflict("file changed since it was opened")); + } + remove_file_at(parent.platform_handle(), &name).map_err(file_io)?; + // Deletion has committed; don't report it as failed on directory fsync. + let _ = sync_directory(parent.platform_handle()); + Ok(()) + } + fn create_folder(&self, relative: &Path) -> Result<(), ApiError> { let _guard = self.writes.lock().map_err(|_| { ApiError::new( @@ -1347,10 +1372,8 @@ impl NotebookStore { ) })?; let (parent, name) = self.open_parent(relative, true)?; - let created = unsafe { libc::mkdirat(parent.as_raw_fd(), name.as_ptr(), 0o755) }; - if created != 0 { - let error = std::io::Error::last_os_error(); - if error.raw_os_error() == Some(libc::EEXIST) { + if let Err(error) = create_directory_at(parent.platform_handle(), &name) { + if error.kind() == std::io::ErrorKind::AlreadyExists { return Err(ApiError::new( StatusCode::CONFLICT, "already_exists", @@ -1359,9 +1382,7 @@ impl NotebookStore { } return Err(file_io(error)); } - if unsafe { libc::fsync(parent.as_raw_fd()) } != 0 { - return Err(file_io(std::io::Error::last_os_error())); - } + sync_directory(parent.platform_handle()).map_err(file_io)?; Ok(()) } @@ -1382,8 +1403,8 @@ impl NotebookStore { history.finish(true)?; let (source_parent, source_name) = self.open_parent(source, false)?; let source_file = - open_file_at(source_parent.as_raw_fd(), &source_name).map_err(|error| { - if error.raw_os_error() == Some(libc::ENOENT) { + open_file_at(source_parent.platform_handle(), &source_name).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { ApiError::new( StatusCode::NOT_FOUND, "not_found", @@ -1394,6 +1415,7 @@ impl NotebookStore { } })?; let source_metadata = source_file.metadata().map_err(file_io)?; + let source_identity = file_identity(&source_file).map_err(file_io)?; if !source_metadata.is_file() || source_metadata.len() > MAX_NOTEBOOK_BYTES as u64 { return Err(ApiError::new( StatusCode::PAYLOAD_TOO_LARGE, @@ -1425,11 +1447,8 @@ impl NotebookStore { // parent to exist. The move endpoint never creates directory state. let (destination_parent, destination_name) = self.open_parent(destination, false)?; let current_metadata = - metadata_at(source_parent.as_raw_fd(), &source_name).map_err(file_io)?; - if current_metadata.st_dev as u64 != source_metadata.dev() - || current_metadata.st_ino as u64 != source_metadata.ino() - || (current_metadata.st_mode & libc::S_IFMT) != libc::S_IFREG - { + metadata_at(source_parent.platform_handle(), &source_name).map_err(file_io)?; + if current_metadata != source_identity || !current_metadata.regular { return Err(conflict("file changed while it was being moved")); } let source_path = slash_path(source)?; @@ -1437,18 +1456,20 @@ impl NotebookStore { history.capture(&source_path, &bytes, "baseline")?; history.prepare(&source_path, &bytes, "rename", Some(&destination_path))?; let moved = rename_no_replace( - source_parent.as_raw_fd(), + source_parent.platform_handle(), &source_name, - destination_parent.as_raw_fd(), + destination_parent.platform_handle(), &destination_name, ) - .map_err(|error| match error.raw_os_error() { - Some(libc::EEXIST) | Some(libc::ENOTEMPTY) => ApiError::new( - StatusCode::CONFLICT, - "already_exists", - "destination already exists", - ), - Some(libc::ENOENT) => conflict("file changed while it was being moved"), + .map_err(|error| match error.kind() { + std::io::ErrorKind::AlreadyExists | std::io::ErrorKind::DirectoryNotEmpty => { + ApiError::new( + StatusCode::CONFLICT, + "already_exists", + "destination already exists", + ) + } + std::io::ErrorKind::NotFound => conflict("file changed while it was being moved"), _ => file_io(error), }); if let Err(error) = moved { @@ -1459,15 +1480,15 @@ impl NotebookStore { // The rename is already committed at this point. Directory fsync is // best-effort: reporting an error would make the client believe the // source path still exists and leave its active document stale. - let _ = unsafe { libc::fsync(source_parent.as_raw_fd()) }; - let _ = unsafe { libc::fsync(destination_parent.as_raw_fd()) }; + let _ = sync_directory(source_parent.platform_handle()); + let _ = sync_directory(destination_parent.platform_handle()); Ok(source_revision) } fn open_file(&self, relative: &Path) -> Result { let (parent, name) = self.open_parent(relative, false)?; - open_file_at(parent.as_raw_fd(), &name).map_err(|error| { - if error.raw_os_error() == Some(libc::ENOENT) { + open_file_at(parent.platform_handle(), &name).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { ApiError::new( StatusCode::NOT_FOUND, "not_found", @@ -1479,27 +1500,24 @@ impl NotebookStore { }) } - fn open_parent(&self, relative: &Path, create: bool) -> Result<(OwnedFd, CString), ApiError> { + fn open_parent(&self, relative: &Path, create: bool) -> Result<(File, CString), ApiError> { let mut components: Vec<_> = relative.components().collect(); let name = match components.pop() { Some(Component::Normal(name)) => cstring_component(name)?, _ => return Err(invalid_path()), }; - let duplicated = unsafe { libc::dup(self.root_fd.as_raw_fd()) }; - if duplicated < 0 { - return Err(file_io(std::io::Error::last_os_error())); - } - let mut directory = unsafe { OwnedFd::from_raw_fd(duplicated) }; + let mut directory = self.root_fd.try_clone().map_err(file_io)?; for component in components { let Component::Normal(name) = component else { return Err(invalid_path()); }; let name = cstring_component(name)?; - match open_directory_at(directory.as_raw_fd(), &name) { + match open_directory_at(directory.platform_handle(), &name) { Ok(next) => directory = next, - Err(error) if create && error.raw_os_error() == Some(libc::ENOENT) => { - mkdir_at(directory.as_raw_fd(), &name).map_err(file_io)?; - directory = open_directory_at(directory.as_raw_fd(), &name).map_err(file_io)?; + Err(error) if create && error.kind() == std::io::ErrorKind::NotFound => { + mkdir_at(directory.platform_handle(), &name).map_err(file_io)?; + directory = + open_directory_at(directory.platform_handle(), &name).map_err(file_io)?; } Err(error) => return Err(file_io(error)), } @@ -1508,6 +1526,19 @@ impl NotebookStore { } } +// Source identifiers are permitted only in the recent preference, never in +// filesystem operations. Require the canonical hyphenated UUID spelling. +fn validate_duckdb_ui_identifier(value: &str) -> Result { + let id = value + .strip_prefix("duckdb-ui://") + .ok_or_else(invalid_path)?; + let parsed = uuid::Uuid::parse_str(id).map_err(|_| invalid_path())?; + if parsed.hyphenated().to_string() != id { + return Err(invalid_path()); + } + Ok(value.to_owned()) +} + fn validate_relative_path(value: &str) -> Result { let path = validate_relative_components(value)?; if !allowed_extension(&path) { @@ -1534,7 +1565,7 @@ fn validate_relative_components(value: &str) -> Result { if path.is_absolute() || !path .components() - .all(|component| matches!(component, Component::Normal(name) if !name.as_encoded_bytes().eq_ignore_ascii_case(b".git"))) + .all(|component| matches!(component, Component::Normal(name) if valid_component(name))) { return Err(invalid_path()); } @@ -1589,295 +1620,6 @@ fn slash_path(path: &Path) -> Result { Ok(parts?.join("/")) } -fn cstring_component(value: &std::ffi::OsStr) -> Result { - use std::os::unix::ffi::OsStrExt; - CString::new(value.as_bytes()).map_err(|_| invalid_path()) -} - -struct DirectoryStream(*mut libc::DIR); - -impl Drop for DirectoryStream { - fn drop(&mut self) { - unsafe { - libc::closedir(self.0); - } - } -} - -fn list_directory_fd( - directory: libc::c_int, - relative: &Path, - files: &mut Vec, - directories: &mut Vec, -) -> Result<(), ApiError> { - let dot = c"."; - // openat(".") creates an independent open file description. dup would - // share the directory offset and make a second listing appear empty. - let reopened = open_directory_at(directory, dot).map_err(file_io)?; - let reopened = reopened.into_raw_fd(); - let stream = unsafe { libc::fdopendir(reopened) }; - if stream.is_null() { - unsafe { libc::close(reopened) }; - return Err(file_io(std::io::Error::last_os_error())); - } - let stream = DirectoryStream(stream); - loop { - clear_errno(); - let entry = unsafe { libc::readdir(stream.0) }; - if entry.is_null() { - let error = current_errno(); - if error != 0 { - return Err(file_io(std::io::Error::from_raw_os_error(error))); - } - break; - } - let name = unsafe { CStr::from_ptr((*entry).d_name.as_ptr()) }; - if name.to_bytes() == b"." - || name.to_bytes() == b".." - || name.to_bytes().eq_ignore_ascii_case(b".git") - { - continue; - } - let mut metadata: libc::stat = unsafe { std::mem::zeroed() }; - if unsafe { - libc::fstatat( - directory, - name.as_ptr(), - &mut metadata, - libc::AT_SYMLINK_NOFOLLOW, - ) - } != 0 - { - return Err(file_io(std::io::Error::last_os_error())); - } - let kind = metadata.st_mode & libc::S_IFMT; - if kind == libc::S_IFLNK { - continue; - } - let file_name = std::ffi::OsString::from_vec(name.to_bytes().to_vec()); - let child_relative = relative.join(file_name); - if kind == libc::S_IFDIR { - let child_name = CString::new(name.to_bytes()).map_err(|_| invalid_path())?; - let child = open_directory_at(directory, &child_name).map_err(file_io)?; - if let Ok(path) = slash_path(&child_relative) { - directories.push(path); - } - list_directory_fd(child.as_raw_fd(), &child_relative, files, directories)?; - } else if kind == libc::S_IFREG && allowed_extension(&child_relative) { - if let Ok(path) = slash_path(&child_relative) { - files.push(FileEntry { path }); - } - } - } - Ok(()) -} - -#[cfg(target_os = "macos")] -fn clear_errno() { - unsafe { *libc::__error() = 0 } -} - -#[cfg(target_os = "macos")] -fn current_errno() -> i32 { - unsafe { *libc::__error() } -} - -#[cfg(target_os = "linux")] -fn clear_errno() { - unsafe { *libc::__errno_location() = 0 } -} - -#[cfg(target_os = "linux")] -fn current_errno() -> i32 { - unsafe { *libc::__errno_location() } -} - -fn open_directory(path: &Path) -> std::io::Result { - use std::os::unix::ffi::OsStrExt; - let path = CString::new(path.as_os_str().as_bytes()) - .map_err(|_| std::io::Error::from_raw_os_error(libc::EINVAL))?; - let fd = unsafe { - libc::open( - path.as_ptr(), - libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC, - ) - }; - if fd < 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(unsafe { OwnedFd::from_raw_fd(fd) }) - } -} - -fn open_directory_at(parent: libc::c_int, name: &CStr) -> std::io::Result { - let fd = unsafe { - libc::openat( - parent, - name.as_ptr(), - libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC, - ) - }; - if fd < 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(unsafe { OwnedFd::from_raw_fd(fd) }) - } -} - -fn mkdir_at(parent: libc::c_int, name: &CStr) -> std::io::Result<()> { - let result = unsafe { libc::mkdirat(parent, name.as_ptr(), 0o755) }; - if result == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EEXIST) { - Ok(()) - } else { - Err(std::io::Error::last_os_error()) - } -} - -fn open_file_at(parent: libc::c_int, name: &CStr) -> std::io::Result { - let fd = unsafe { - libc::openat( - parent, - name.as_ptr(), - libc::O_RDONLY | libc::O_NONBLOCK | libc::O_NOFOLLOW | libc::O_CLOEXEC, - ) - }; - if fd < 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(unsafe { File::from_raw_fd(fd) }) - } -} - -fn metadata_at(parent: libc::c_int, name: &CStr) -> std::io::Result { - let mut metadata: libc::stat = unsafe { std::mem::zeroed() }; - let result = unsafe { - libc::fstatat( - parent, - name.as_ptr(), - &mut metadata, - libc::AT_SYMLINK_NOFOLLOW, - ) - }; - if result == 0 { - Ok(metadata) - } else { - Err(std::io::Error::last_os_error()) - } -} - -#[cfg(target_os = "macos")] -fn rename_no_replace( - source_parent: libc::c_int, - source_name: &CStr, - destination_parent: libc::c_int, - destination_name: &CStr, -) -> std::io::Result<()> { - let result = unsafe { - libc::renameatx_np( - source_parent, - source_name.as_ptr(), - destination_parent, - destination_name.as_ptr(), - libc::RENAME_EXCL, - ) - }; - if result == 0 { - Ok(()) - } else { - Err(std::io::Error::last_os_error()) - } -} - -#[cfg(target_os = "linux")] -fn rename_no_replace( - source_parent: libc::c_int, - source_name: &CStr, - destination_parent: libc::c_int, - destination_name: &CStr, -) -> std::io::Result<()> { - const RENAME_NOREPLACE: libc::c_uint = 1; - let result = unsafe { - libc::syscall( - libc::SYS_renameat2, - source_parent, - source_name.as_ptr(), - destination_parent, - destination_name.as_ptr(), - RENAME_NOREPLACE, - ) - }; - if result == 0 { - Ok(()) - } else { - Err(std::io::Error::last_os_error()) - } -} - -fn atomic_write_at( - parent: libc::c_int, - name: &CStr, - source: &[u8], - expected_revision: &str, -) -> std::io::Result<()> { - let temporary = CString::new(format!(".sidequery-{}.tmp", uuid::Uuid::new_v4())) - .expect("UUID contains no NUL"); - let fd = unsafe { - libc::openat( - parent, - temporary.as_ptr(), - libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_NOFOLLOW | libc::O_CLOEXEC, - 0o600, - ) - }; - if fd < 0 { - return Err(std::io::Error::last_os_error()); - } - let mut file = unsafe { File::from_raw_fd(fd) }; - let result = (|| { - file.write_all(source)?; - file.sync_all()?; - if expected_revision.is_empty() { - let linked = - unsafe { libc::linkat(parent, temporary.as_ptr(), parent, name.as_ptr(), 0) }; - if linked != 0 { - return Err(std::io::Error::last_os_error()); - } - let _ = unsafe { libc::unlinkat(parent, temporary.as_ptr(), 0) }; - } else { - // Recheck after the temporary file is durable, immediately before - // replacement. Portable filesystems do not expose a cross-process - // compare-and-swap rename, so a non-cooperating writer can still - // race the final rename after this check. - let current = open_file_at(parent, name)?; - let metadata = current.metadata()?; - if !metadata.is_file() || metadata.len() > MAX_NOTEBOOK_BYTES as u64 { - return Err(std::io::Error::from_raw_os_error(libc::EBUSY)); - } - let mut bytes = Vec::with_capacity(metadata.len() as usize); - current - .take((MAX_NOTEBOOK_BYTES + 1) as u64) - .read_to_end(&mut bytes)?; - if bytes.len() > MAX_NOTEBOOK_BYTES || revision(&bytes) != expected_revision { - return Err(std::io::Error::from_raw_os_error(libc::EBUSY)); - } - let renamed = - unsafe { libc::renameat(parent, temporary.as_ptr(), parent, name.as_ptr()) }; - if renamed != 0 { - return Err(std::io::Error::last_os_error()); - } - } - let synced = unsafe { libc::fsync(parent) }; - if synced != 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) - })(); - if result.is_err() { - let _ = unsafe { libc::unlinkat(parent, temporary.as_ptr(), 0) }; - } - result -} - async fn security_middleware( State(state): State, request: Request, @@ -2273,9 +2015,8 @@ async fn put_recent_notebook( Json(body): Json, ) -> Result { authenticate(&headers, &state)?; - let path = validate_relative_path(&body.path)?; let notebooks = state.notebooks.clone(); - let path = tokio::task::spawn_blocking(move || notebooks.remember(&path)) + let path = tokio::task::spawn_blocking(move || notebooks.remember_identifier(&body.path)) .await .map_err(|_| { ApiError::new( @@ -2365,6 +2106,39 @@ async fn put_file( })) } +async fn duplicate_file( + State(state): State, + headers: HeaderMap, + Json(body): Json, +) -> Result { + authenticate(&headers, &state)?; + let source = validate_relative_path(&body.path)?; + let destination = validate_relative_path(&body.destination_path)?; + let path = slash_path(&destination)?; + let notebooks = state.notebooks.clone(); + let revision = tokio::task::spawn_blocking(move || { + notebooks.duplicate_file(&source, &destination, &body.expected_revision) + }) + .await + .map_err(|_| file_io(std::io::Error::other("file task failed")))??; + Ok(Json(PutFileResponse { path, revision })) +} + +async fn delete_file( + State(state): State, + headers: HeaderMap, + Json(body): Json, +) -> Result { + authenticate(&headers, &state)?; + let relative = validate_relative_path(&body.path)?; + let path = slash_path(&relative)?; + let notebooks = state.notebooks.clone(); + tokio::task::spawn_blocking(move || notebooks.delete_file(&relative, &body.expected_revision)) + .await + .map_err(|_| file_io(std::io::Error::other("file task failed")))??; + Ok(Json(CreateFolderResponse { path })) +} + async fn move_file( State(state): State, headers: HeaderMap, @@ -2520,8 +2294,12 @@ fn router(state: AppState) -> Router { "/api/v1/recent-notebook", get(get_recent_notebook).put(put_recent_notebook), ) - .route("/api/v1/file", get(get_file).put(put_file)) + .route( + "/api/v1/file", + get(get_file).put(put_file).delete(delete_file), + ) .route("/api/v1/file/move", post(move_file)) + .route("/api/v1/file/duplicate", post(duplicate_file)) .route("/api/v1/folder", post(create_folder)) .route("/quack", post(quack).options(options_quack)); if !state.mcp_token.is_empty() { @@ -2666,6 +2444,50 @@ fn set_error(error_out: *mut *mut c_char, message: String) { } } +#[no_mangle] +pub extern "C" fn sidequery_duckdb_ui_next_version( + previous: *const c_char, + edited: *const c_char, + error_out: *mut *mut c_char, +) -> *mut c_char { + let result = std::panic::catch_unwind(|| -> Result { + if edited.is_null() { + return Err("DuckDB UI notebook JSON must not be null".into()); + } + let previous = if previous.is_null() { + None + } else { + Some( + unsafe { CStr::from_ptr(previous) } + .to_str() + .map_err(|_| "DuckDB UI notebook JSON must be UTF-8")?, + ) + }; + let edited = unsafe { CStr::from_ptr(edited) } + .to_str() + .map_err(|_| "DuckDB UI notebook JSON must be UTF-8")?; + CString::new(match previous { + Some(previous) => duckdb_ui::next_version(previous, edited)?, + None => duckdb_ui::initial_version(edited)?, + }) + .map_err(|_| "DuckDB UI notebook JSON contains NUL".into()) + }); + match result { + Ok(Ok(json)) => json.into_raw(), + Ok(Err(error)) => { + set_error(error_out, error); + ptr::null_mut() + } + Err(_) => { + set_error( + error_out, + "Unexpected error validating a DuckDB UI notebook".into(), + ); + ptr::null_mut() + } + } +} + #[no_mangle] pub extern "C" fn sidequery_http_start_with_query( config_json: *const c_char, diff --git a/http/src/platform.rs b/http/src/platform.rs new file mode 100644 index 0000000..138c4ad --- /dev/null +++ b/http/src/platform.rs @@ -0,0 +1,87 @@ +//! OS-specific file operations behind the notebook root capability. +#[cfg(unix)] +mod unix; +#[cfg(windows)] +mod windows; +#[cfg(unix)] +pub(crate) use unix::*; +#[cfg(windows)] +pub(crate) use windows::*; + +#[derive(PartialEq, Eq)] +pub(crate) struct FileIdentity { + pub volume: u64, + pub index: u64, + pub regular: bool, +} + +pub(crate) fn home_directory() -> Option { + #[cfg(windows)] + let home = std::env::var_os("USERPROFILE"); + #[cfg(unix)] + let home = std::env::var_os("HOME"); + home.filter(|value| !value.is_empty()) +} + +pub(crate) fn valid_component(name: &std::ffi::OsStr) -> bool { + if name.as_encoded_bytes().eq_ignore_ascii_case(b".git") { + return false; + } + #[cfg(windows)] + { + let Some(name) = name.to_str() else { + return false; + }; + if name.ends_with(['.', ' ']) || name.chars().any(|c| c < ' ' || ":<>\"|?*".contains(c)) { + return false; + } + let stem = name.split('.').next().unwrap_or("").to_ascii_uppercase(); + if stem + .strip_prefix("GIT~") + .is_some_and(|suffix| !suffix.is_empty() && suffix.bytes().all(|c| c.is_ascii_digit())) + || matches!( + stem.as_str(), + "CON" | "PRN" | "AUX" | "NUL" | "CONIN$" | "CONOUT$" + ) + || ["COM", "LPT"].iter().any(|prefix| { + stem.strip_prefix(prefix).is_some_and(|suffix| { + matches!( + suffix, + "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "¹" | "²" | "³" + ) + }) + }) + { + return false; + } + } + true +} + +pub(crate) fn sync_history_path(path: &std::path::Path) -> std::io::Result<()> { + #[cfg(unix)] + { + std::fs::File::open(path)?.sync_all() + } + #[cfg(windows)] + { + let metadata = std::fs::symlink_metadata(path)?; + if is_reparse(&metadata) { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "reparse history path", + )); + } + if metadata.is_dir() || metadata.permissions().readonly() { + // Windows cannot flush directories or reopen immutable Git objects + // for GENERIC_WRITE. libgit2 flushes these objects before making + // them readonly (core.fsyncObjectFiles is enabled by History). + return Ok(()); + } + std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path)? + .sync_all() + } +} diff --git a/http/src/platform/unix.rs b/http/src/platform/unix.rs new file mode 100644 index 0000000..dc7cfc6 --- /dev/null +++ b/http/src/platform/unix.rs @@ -0,0 +1,402 @@ +//! Unix descriptor-relative notebook operations. +use crate::*; +pub(crate) use libc::{LOCK_EX, LOCK_SH}; +use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd}; +use std::os::unix::ffi::OsStringExt; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + +pub(crate) trait PlatformHandle { + fn platform_handle(&self) -> libc::c_int; +} +impl PlatformHandle for File { + fn platform_handle(&self) -> libc::c_int { + self.as_raw_fd() + } +} + +pub(crate) fn open_lock_file(path: &Path) -> Result { + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .mode(0o600) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .open(path) + .map_err(|error| format!("failed to open UI cache lock {}: {error}", path.display())) +} + +pub(crate) fn lock_file(file: &File, operation: libc::c_int, label: &str) -> Result<(), String> { + loop { + if unsafe { libc::flock(file.platform_handle(), operation) } == 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if error.kind() != std::io::ErrorKind::Interrupted { + return Err(format!("failed to lock {label}: {error}")); + } + } +} + +pub(crate) fn try_lock_file( + file: &File, + operation: libc::c_int, + label: &str, +) -> Result { + loop { + if unsafe { libc::flock(file.platform_handle(), operation | libc::LOCK_NB) } == 0 { + return Ok(true); + } + let error = std::io::Error::last_os_error(); + if error + .raw_os_error() + .is_some_and(|code| code == libc::EWOULDBLOCK || code == libc::EAGAIN) + { + return Ok(false); + } + if error.kind() != std::io::ErrorKind::Interrupted { + return Err(format!("failed to lock {label}: {error}")); + } + } +} + +pub(crate) fn unlock_file(file: &File) -> Result<(), String> { + lock_file(file, libc::LOCK_UN, "UI cache file") +} + +pub(crate) fn cstring_component(value: &std::ffi::OsStr) -> Result { + use std::os::unix::ffi::OsStrExt; + CString::new(value.as_bytes()).map_err(|_| invalid_path()) +} + +struct DirectoryStream(*mut libc::DIR); + +impl Drop for DirectoryStream { + fn drop(&mut self) { + unsafe { + libc::closedir(self.0); + } + } +} + +pub(crate) fn list_directory_fd( + directory: libc::c_int, + relative: &Path, + files: &mut Vec, + directories: &mut Vec, +) -> Result<(), ApiError> { + let dot = c"."; + // openat(".") creates an independent open file description. dup would + // share the directory offset and make a second listing appear empty. + let reopened = open_directory_at(directory, dot).map_err(file_io)?; + let reopened = reopened.into_raw_fd(); + let stream = unsafe { libc::fdopendir(reopened) }; + if stream.is_null() { + unsafe { libc::close(reopened) }; + return Err(file_io(std::io::Error::last_os_error())); + } + let stream = DirectoryStream(stream); + loop { + clear_errno(); + let entry = unsafe { libc::readdir(stream.0) }; + if entry.is_null() { + let error = current_errno(); + if error != 0 { + return Err(file_io(std::io::Error::from_raw_os_error(error))); + } + break; + } + let name = unsafe { CStr::from_ptr((*entry).d_name.as_ptr()) }; + if name.to_bytes() == b"." + || name.to_bytes() == b".." + || name.to_bytes().eq_ignore_ascii_case(b".git") + { + continue; + } + let mut metadata: libc::stat = unsafe { std::mem::zeroed() }; + if unsafe { + libc::fstatat( + directory, + name.as_ptr(), + &mut metadata, + libc::AT_SYMLINK_NOFOLLOW, + ) + } != 0 + { + return Err(file_io(std::io::Error::last_os_error())); + } + let kind = metadata.st_mode & libc::S_IFMT; + if kind == libc::S_IFLNK { + continue; + } + let file_name = std::ffi::OsString::from_vec(name.to_bytes().to_vec()); + let child_relative = relative.join(file_name); + if kind == libc::S_IFDIR { + let child_name = CString::new(name.to_bytes()).map_err(|_| invalid_path())?; + let child = open_directory_at(directory, &child_name).map_err(file_io)?; + if let Ok(path) = slash_path(&child_relative) { + directories.push(path); + } + list_directory_fd(child.as_raw_fd(), &child_relative, files, directories)?; + } else if kind == libc::S_IFREG && allowed_extension(&child_relative) { + if let Ok(path) = slash_path(&child_relative) { + files.push(FileEntry { path }); + } + } + } + Ok(()) +} + +#[cfg(target_os = "macos")] +pub(crate) fn clear_errno() { + unsafe { *libc::__error() = 0 } +} + +#[cfg(target_os = "macos")] +pub(crate) fn current_errno() -> i32 { + unsafe { *libc::__error() } +} + +#[cfg(target_os = "linux")] +pub(crate) fn clear_errno() { + unsafe { *libc::__errno_location() = 0 } +} + +#[cfg(target_os = "linux")] +pub(crate) fn current_errno() -> i32 { + unsafe { *libc::__errno_location() } +} + +pub(crate) fn open_directory(path: &Path) -> std::io::Result { + use std::os::unix::ffi::OsStrExt; + let path = CString::new(path.as_os_str().as_bytes()) + .map_err(|_| std::io::Error::from_raw_os_error(libc::EINVAL))?; + let fd = unsafe { + libc::open( + path.as_ptr(), + libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC, + ) + }; + if fd < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(unsafe { File::from_raw_fd(fd) }) + } +} + +pub(crate) fn open_directory_at(parent: libc::c_int, name: &CStr) -> std::io::Result { + let fd = unsafe { + libc::openat( + parent, + name.as_ptr(), + libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC, + ) + }; + if fd < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(unsafe { File::from_raw_fd(fd) }) + } +} + +pub(crate) fn mkdir_at(parent: libc::c_int, name: &CStr) -> std::io::Result<()> { + let result = unsafe { libc::mkdirat(parent, name.as_ptr(), 0o755) }; + if result == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EEXIST) { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +pub(crate) fn open_file_at(parent: libc::c_int, name: &CStr) -> std::io::Result { + let fd = unsafe { + libc::openat( + parent, + name.as_ptr(), + libc::O_RDONLY | libc::O_NONBLOCK | libc::O_NOFOLLOW | libc::O_CLOEXEC, + ) + }; + if fd < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(unsafe { File::from_raw_fd(fd) }) + } +} + +pub(crate) fn metadata_at(parent: libc::c_int, name: &CStr) -> std::io::Result { + let mut metadata: libc::stat = unsafe { std::mem::zeroed() }; + if unsafe { + libc::fstatat( + parent, + name.as_ptr(), + &mut metadata, + libc::AT_SYMLINK_NOFOLLOW, + ) + } != 0 + { + return Err(std::io::Error::last_os_error()); + } + Ok(FileIdentity { + volume: metadata.st_dev as u64, + index: metadata.st_ino as u64, + regular: metadata.st_mode & libc::S_IFMT == libc::S_IFREG, + }) +} +pub(crate) fn file_identity(file: &File) -> std::io::Result { + let meta = file.metadata()?; + Ok(FileIdentity { + volume: meta.dev(), + index: meta.ino(), + regular: meta.is_file(), + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn rename_no_replace( + source_parent: libc::c_int, + source_name: &CStr, + destination_parent: libc::c_int, + destination_name: &CStr, +) -> std::io::Result<()> { + let result = unsafe { + libc::renameatx_np( + source_parent, + source_name.as_ptr(), + destination_parent, + destination_name.as_ptr(), + libc::RENAME_EXCL, + ) + }; + if result == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn rename_no_replace( + source_parent: libc::c_int, + source_name: &CStr, + destination_parent: libc::c_int, + destination_name: &CStr, +) -> std::io::Result<()> { + const RENAME_NOREPLACE: libc::c_uint = 1; + let result = unsafe { + libc::syscall( + libc::SYS_renameat2, + source_parent, + source_name.as_ptr(), + destination_parent, + destination_name.as_ptr(), + RENAME_NOREPLACE, + ) + }; + if result == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +pub(crate) fn atomic_write_at( + parent: libc::c_int, + name: &CStr, + source: &[u8], + expected_revision: &str, +) -> std::io::Result<()> { + let temporary = CString::new(format!(".sidequery-{}.tmp", uuid::Uuid::new_v4())) + .expect("UUID contains no NUL"); + let fd = unsafe { + libc::openat( + parent, + temporary.as_ptr(), + libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0o600, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error()); + } + let mut file = unsafe { File::from_raw_fd(fd) }; + let result = (|| { + file.write_all(source)?; + file.sync_all()?; + if expected_revision.is_empty() { + let linked = + unsafe { libc::linkat(parent, temporary.as_ptr(), parent, name.as_ptr(), 0) }; + if linked != 0 { + return Err(std::io::Error::last_os_error()); + } + let _ = unsafe { libc::unlinkat(parent, temporary.as_ptr(), 0) }; + } else { + // Recheck after the temporary file is durable, immediately before + // replacement. Portable filesystems do not expose a cross-process + // compare-and-swap rename, so a non-cooperating writer can still + // race the final rename after this check. + let current = open_file_at(parent, name)?; + let metadata = current.metadata()?; + if !metadata.is_file() || metadata.len() > MAX_NOTEBOOK_BYTES as u64 { + return Err(std::io::Error::from_raw_os_error(libc::EBUSY)); + } + let mut bytes = Vec::with_capacity(metadata.len() as usize); + current + .take((MAX_NOTEBOOK_BYTES + 1) as u64) + .read_to_end(&mut bytes)?; + if bytes.len() > MAX_NOTEBOOK_BYTES || revision(&bytes) != expected_revision { + return Err(std::io::Error::from_raw_os_error(libc::EBUSY)); + } + let renamed = + unsafe { libc::renameat(parent, temporary.as_ptr(), parent, name.as_ptr()) }; + if renamed != 0 { + return Err(std::io::Error::last_os_error()); + } + } + let synced = unsafe { libc::fsync(parent) }; + if synced != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + })(); + if result.is_err() { + let _ = unsafe { libc::unlinkat(parent, temporary.as_ptr(), 0) }; + } + result +} + +pub(crate) fn create_directory_at(parent: libc::c_int, name: &CStr) -> std::io::Result<()> { + if unsafe { libc::mkdirat(parent, name.as_ptr(), 0o755) } == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} +pub(crate) fn remove_file_at(parent: libc::c_int, name: &CStr) -> std::io::Result<()> { + if unsafe { libc::unlinkat(parent, name.as_ptr(), 0) } == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} +pub(crate) fn sync_directory(parent: libc::c_int) -> std::io::Result<()> { + if unsafe { libc::fsync(parent) } == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} +pub(crate) fn private_new_file(path: &Path) -> std::io::Result { + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) +} +pub(crate) fn open_lease_file(path: &Path) -> std::io::Result { + OpenOptions::new() + .read(true) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .open(path) +} +pub(crate) fn is_reparse(metadata: &fs::Metadata) -> bool { + metadata.file_type().is_symlink() +} diff --git a/http/src/platform/windows.rs b/http/src/platform/windows.rs new file mode 100644 index 0000000..a047c16 --- /dev/null +++ b/http/src/platform/windows.rs @@ -0,0 +1,586 @@ +//! Root-relative Windows file operations. No child path is resolved through a reparse point. +use crate::*; +use std::io::{Error, ErrorKind, Result}; +use std::os::windows::{ + ffi::OsStringExt, + fs::{MetadataExt, OpenOptionsExt}, + io::{AsRawHandle, FromRawHandle}, +}; +use windows_sys::{ + Wdk::{ + Foundation::OBJECT_ATTRIBUTES, + Storage::FileSystem::{ + NtCreateFile, FILE_CREATE, FILE_DIRECTORY_FILE, FILE_NON_DIRECTORY_FILE, FILE_OPEN, + FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT, + }, + }, + Win32::{ + Foundation::*, + Storage::FileSystem::*, + System::IO::{IO_STATUS_BLOCK, OVERLAPPED}, + }, +}; + +pub(crate) const LOCK_EX: i32 = 2; +pub(crate) const LOCK_SH: i32 = 1; +pub(crate) trait PlatformHandle { + fn platform_handle(&self) -> HANDLE; +} +impl PlatformHandle for File { + fn platform_handle(&self) -> HANDLE { + self.as_raw_handle() + } +} + +fn reparse_error() -> Error { + Error::new( + ErrorKind::PermissionDenied, + "reparse points are not allowed", + ) +} +pub(crate) fn is_reparse(metadata: &fs::Metadata) -> bool { + metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT != 0 +} +fn checked(file: File) -> Result { + if is_reparse(&file.metadata()?) { + Err(reparse_error()) + } else { + Ok(file) + } +} + +fn checked_component(file: File) -> Result { + let file = checked(file)?; + let mut buffer = vec![0u16; 512]; + loop { + let length = unsafe { + GetFinalPathNameByHandleW( + file.platform_handle(), + buffer.as_mut_ptr(), + buffer.len() as u32, + FILE_NAME_NORMALIZED | VOLUME_NAME_NT, + ) + } as usize; + if length == 0 { + return Err(Error::last_os_error()); + } + if length >= buffer.len() { + if length > 32768 { + return Err(Error::new( + ErrorKind::InvalidData, + "resolved filename too long", + )); + } + buffer.resize(length + 1, 0); + continue; + } + let path = PathBuf::from(std::ffi::OsString::from_wide(&buffer[..length])); + // NTFS short aliases are not reparse points. Check the actual component + // on the opened handle as well as the caller's spelling (e.g. GIT~1). + if path.file_name().is_none_or(|name| !valid_component(name)) { + return Err(Error::new( + ErrorKind::PermissionDenied, + "reserved notebook component", + )); + } + return Ok(file); + } +} + +pub(crate) fn validate_existing_component(path: &Path) -> Result<()> { + checked_component( + OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?, + ) + .map(drop) +} +pub(crate) fn cstring_component(value: &std::ffi::OsStr) -> std::result::Result { + CString::new(value.to_str().ok_or_else(invalid_path)?).map_err(|_| invalid_path()) +} +fn wide_component(name: &CStr) -> Result> { + let text = name + .to_str() + .map_err(|_| Error::new(ErrorKind::InvalidInput, "invalid UTF-8 filename"))?; + if text.is_empty() || text.contains(['/', '\\', ':']) || text == ".." { + return Err(Error::new( + ErrorKind::InvalidInput, + "invalid filename component", + )); + } + let wide: Vec = text.encode_utf16().collect(); + if wide.len() > (u16::MAX as usize / 2) { + return Err(Error::new(ErrorKind::InvalidInput, "filename too long")); + } + Ok(wide) +} + +// NtCreateFile is the Windows openat equivalent: ObjectAttributes.RootDirectory +// binds one component to a retained directory handle. OPEN_REPARSE_POINT opens +// the point itself, and checking that handle rejects junctions and every other tag. +fn open_at(parent: HANDLE, name: &CStr, access: u32, disposition: u32, kind: u32) -> Result { + let mut wide = wide_component(name)?; + let mut unicode = UNICODE_STRING { + Length: (wide.len() * 2) as u16, + MaximumLength: (wide.len() * 2) as u16, + Buffer: wide.as_mut_ptr(), + }; + let attributes = OBJECT_ATTRIBUTES { + Length: std::mem::size_of::() as u32, + RootDirectory: parent, + ObjectName: &mut unicode, + Attributes: OBJ_CASE_INSENSITIVE, + SecurityDescriptor: ptr::null_mut(), + SecurityQualityOfService: ptr::null_mut(), + }; + let mut status = IO_STATUS_BLOCK::default(); + let mut handle = ptr::null_mut(); + let result = unsafe { + NtCreateFile( + &mut handle, + access | SYNCHRONIZE, + &attributes, + &mut status, + ptr::null(), + FILE_ATTRIBUTE_NORMAL, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + disposition, + kind | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT, + ptr::null(), + 0, + ) + }; + if result < 0 { + return Err(Error::from_raw_os_error( + unsafe { RtlNtStatusToDosError(result) } as i32, + )); + } + checked_component(unsafe { File::from_raw_handle(handle) }) +} +pub(crate) fn open_directory(path: &Path) -> Result { + checked( + OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?, + ) +} +pub(crate) fn open_directory_at(parent: HANDLE, name: &CStr) -> Result { + open_at( + parent, + name, + FILE_LIST_DIRECTORY | FILE_READ_ATTRIBUTES, + FILE_OPEN, + FILE_DIRECTORY_FILE, + ) +} +pub(crate) fn open_file_at(parent: HANDLE, name: &CStr) -> Result { + open_at( + parent, + name, + FILE_READ_DATA | FILE_READ_ATTRIBUTES, + FILE_OPEN, + FILE_NON_DIRECTORY_FILE, + ) +} +pub(crate) fn create_directory_at(parent: HANDLE, name: &CStr) -> Result<()> { + open_at( + parent, + name, + FILE_LIST_DIRECTORY | FILE_READ_ATTRIBUTES, + FILE_CREATE, + FILE_DIRECTORY_FILE, + ) + .map(drop) +} +pub(crate) fn mkdir_at(parent: HANDLE, name: &CStr) -> Result<()> { + match create_directory_at(parent, name) { + Err(error) if error.kind() == ErrorKind::AlreadyExists => Ok(()), + result => result, + } +} +pub(crate) fn sync_directory(_parent: HANDLE) -> Result<()> { + // Windows has no supported directory fsync equivalent. File contents are + // flushed before publishing; directory metadata durability is OS-managed. + Ok(()) +} +pub(crate) fn file_identity(file: &File) -> Result { + let mut info = BY_HANDLE_FILE_INFORMATION::default(); + if unsafe { GetFileInformationByHandle(file.platform_handle(), &mut info) } == 0 { + return Err(Error::last_os_error()); + } + Ok(FileIdentity { + volume: info.dwVolumeSerialNumber as u64, + index: ((info.nFileIndexHigh as u64) << 32) | info.nFileIndexLow as u64, + regular: info.dwFileAttributes & (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT) + == 0, + }) +} +pub(crate) fn metadata_at(parent: HANDLE, name: &CStr) -> Result { + file_identity(&open_file_at(parent, name)?) +} +fn rename_handle(file: &File, destination: HANDLE, name: &CStr, replace: bool) -> Result<()> { + let wide = wide_component(name)?; + let offset = std::mem::offset_of!(FILE_RENAME_INFO, FileName); + let size = offset + wide.len() * 2; + // A u64 allocation supplies the structure's required alignment. + let mut storage = vec![0u64; size.div_ceil(8)]; + let info = storage.as_mut_ptr().cast::(); + unsafe { + (*info).Anonymous.ReplaceIfExists = replace; + (*info).RootDirectory = destination; + (*info).FileNameLength = (wide.len() * 2) as u32; + ptr::copy_nonoverlapping(wide.as_ptr(), (*info).FileName.as_mut_ptr(), wide.len()); + if SetFileInformationByHandle( + file.platform_handle(), + FileRenameInfo, + info.cast(), + size as u32, + ) == 0 + { + return Err(Error::last_os_error()); + } + } + Ok(()) +} +pub(crate) fn rename_no_replace( + source: HANDLE, + source_name: &CStr, + destination: HANDLE, + destination_name: &CStr, +) -> Result<()> { + let file = open_at( + source, + source_name, + DELETE | FILE_READ_ATTRIBUTES, + FILE_OPEN, + FILE_NON_DIRECTORY_FILE, + )?; + rename_handle(&file, destination, destination_name, false) +} +fn delete_handle(file: &File) -> Result<()> { + let info = FILE_DISPOSITION_INFO { DeleteFile: true }; + if unsafe { + SetFileInformationByHandle( + file.platform_handle(), + FileDispositionInfo, + (&info as *const FILE_DISPOSITION_INFO).cast(), + std::mem::size_of_val(&info) as u32, + ) + } == 0 + { + return Err(Error::last_os_error()); + } + Ok(()) +} +pub(crate) fn remove_file_at(parent: HANDLE, name: &CStr) -> Result<()> { + let file = open_at( + parent, + name, + DELETE | FILE_READ_ATTRIBUTES, + FILE_OPEN, + FILE_NON_DIRECTORY_FILE, + )?; + delete_handle(&file) +} +pub(crate) fn atomic_write_at( + parent: HANDLE, + name: &CStr, + source: &[u8], + expected: &str, +) -> Result<()> { + let temporary = CString::new(format!(".sidequery-{}.tmp", uuid::Uuid::new_v4())).unwrap(); + let mut file = open_at( + parent, + &temporary, + FILE_WRITE_DATA | FILE_READ_ATTRIBUTES | DELETE, + FILE_CREATE, + FILE_NON_DIRECTORY_FILE, + )?; + let result = (|| { + file.write_all(source)?; + file.sync_all()?; + if !expected.is_empty() { + let current = open_file_at(parent, name)?; + let metadata = current.metadata()?; + if !metadata.is_file() || metadata.len() > MAX_NOTEBOOK_BYTES as u64 { + return Err(Error::new(ErrorKind::ResourceBusy, "notebook changed")); + } + let mut bytes = Vec::new(); + current + .take((MAX_NOTEBOOK_BYTES + 1) as u64) + .read_to_end(&mut bytes)?; + if bytes.len() > MAX_NOTEBOOK_BYTES || revision(&bytes) != expected { + return Err(Error::new(ErrorKind::ResourceBusy, "notebook changed")); + } + } + // As on Unix, a non-cooperating editor can race the last revision check. + // The destination is nevertheless resolved relative to the pinned parent. + rename_handle(&file, parent, name, !expected.is_empty()) + })(); + if result.is_err() { + let _ = delete_handle(&file); + } + result +} +pub(crate) fn list_directory_fd( + directory: HANDLE, + relative: &Path, + files: &mut Vec, + directories: &mut Vec, +) -> std::result::Result<(), ApiError> { + let directory = open_directory_at(directory, c".").map_err(file_io)?; + let mut storage = vec![0u64; 8192]; + let mut first = true; + loop { + let class = if first { + FileIdBothDirectoryRestartInfo + } else { + FileIdBothDirectoryInfo + }; + first = false; + if unsafe { + GetFileInformationByHandleEx( + directory.platform_handle(), + class, + storage.as_mut_ptr().cast(), + (storage.len() * 8) as u32, + ) + } == 0 + { + let error = Error::last_os_error(); + if error.raw_os_error() == Some(ERROR_NO_MORE_FILES as i32) { + return Ok(()); + } + return Err(file_io(error)); + } + let mut offset = 0usize; + loop { + let capacity = storage.len() * 8; + let name_offset = std::mem::offset_of!(FILE_ID_BOTH_DIR_INFO, FileName); + if offset % std::mem::align_of::() != 0 + || offset + std::mem::size_of::() > capacity + { + return Err(file_io(Error::new( + ErrorKind::InvalidData, + "invalid directory entry", + ))); + } + let entry = unsafe { + &*storage + .as_ptr() + .cast::() + .add(offset) + .cast::() + }; + if entry.FileNameLength % 2 != 0 + || offset + name_offset + entry.FileNameLength as usize > capacity + { + return Err(file_io(Error::new( + ErrorKind::InvalidData, + "invalid directory filename", + ))); + } + let name = unsafe { + std::slice::from_raw_parts( + entry.FileName.as_ptr(), + entry.FileNameLength as usize / 2, + ) + }; + if let Ok(name) = String::from_utf16(name) { + if name != "." + && name != ".." + && valid_component(std::ffi::OsStr::new(&name)) + && entry.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT == 0 + { + let child_relative = relative.join(&name); + let child_name = CString::new(name).map_err(|_| invalid_path())?; + if entry.FileAttributes & FILE_ATTRIBUTE_DIRECTORY != 0 { + let child = open_directory_at(directory.platform_handle(), &child_name) + .map_err(file_io)?; + directories.push(slash_path(&child_relative)?); + list_directory_fd( + child.platform_handle(), + &child_relative, + files, + directories, + )?; + } else if allowed_extension(&child_relative) { + // Reopen through the secured boundary: enumeration metadata can race. + if open_file_at(directory.platform_handle(), &child_name).is_ok() { + files.push(FileEntry { + path: slash_path(&child_relative)?, + }); + } + } + } + } + if entry.NextEntryOffset == 0 { + break; + } + if (entry.NextEntryOffset as usize) < name_offset + entry.FileNameLength as usize { + return Err(file_io(Error::new( + ErrorKind::InvalidData, + "invalid directory offset", + ))); + } + offset += entry.NextEntryOffset as usize; + } + } +} +pub(crate) fn private_new_file(path: &Path) -> Result { + OpenOptions::new() + .write(true) + .create_new(true) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(path) +} +pub(crate) fn open_lease_file(path: &Path) -> Result { + checked( + OpenOptions::new() + .read(true) + .write(true) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?, + ) +} +pub(crate) fn open_lock_file(path: &Path) -> std::result::Result { + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(path) + .and_then(checked) + .map_err(|error| format!("failed to open lock: {error}")) +} +fn lock(file: &File, exclusive: bool, immediate: bool) -> Result { + let mut overlapped = OVERLAPPED::default(); + let flags = if exclusive { + LOCKFILE_EXCLUSIVE_LOCK + } else { + 0 + } | if immediate { + LOCKFILE_FAIL_IMMEDIATELY + } else { + 0 + }; + if unsafe { LockFileEx(file.platform_handle(), flags, 0, 1, 0, &mut overlapped) } != 0 { + return Ok(true); + } + let error = Error::last_os_error(); + if immediate && error.raw_os_error() == Some(ERROR_LOCK_VIOLATION as i32) { + return Ok(false); + } + Err(error) +} +pub(crate) fn lock_file( + file: &File, + operation: i32, + label: &str, +) -> std::result::Result<(), String> { + lock(file, operation == LOCK_EX, false) + .map(|_| ()) + .map_err(|error| format!("failed to lock {label}: {error}")) +} +pub(crate) fn try_lock_file( + file: &File, + operation: i32, + label: &str, +) -> std::result::Result { + lock(file, operation == LOCK_EX, true) + .map_err(|error| format!("failed to lock {label}: {error}")) +} +pub(crate) fn unlock_file(file: &File) -> std::result::Result<(), String> { + let mut overlapped = OVERLAPPED::default(); + if unsafe { UnlockFileEx(file.platform_handle(), 0, 1, 0, &mut overlapped) } == 0 { + return Err(format!("failed to unlock file: {}", Error::last_os_error())); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn junction(target: &Path, link: &Path) { + let output = std::process::Command::new("cmd") + .args(["/d", "/c", "mklink", "/J"]) + .arg(link) + .arg(target) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn junctions_are_rejected_and_pinned_parent_cannot_be_redirected() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("root"); + let outside = temp.path().join("outside"); + fs::create_dir(&root).unwrap(); + fs::create_dir(&outside).unwrap(); + fs::create_dir(root.join("folder")).unwrap(); + let root_handle = open_directory(&root).unwrap(); + let parent = open_directory_at(root_handle.platform_handle(), c"folder").unwrap(); + fs::rename(root.join("folder"), root.join("original")).unwrap(); + junction(&outside, &root.join("folder")); + assert!(open_directory_at(root_handle.platform_handle(), c"folder").is_err()); + atomic_write_at(parent.platform_handle(), c"test.sql", b"select 1", "").unwrap(); + assert_eq!( + fs::read(root.join("original/test.sql")).unwrap(), + b"select 1" + ); + assert!(!outside.join("test.sql").exists()); + } + + #[test] + fn windows_aliases_and_streams_are_invalid_notebook_paths() { + for path in [ + "name.sql:stream", + "CON.sql", + "aux.sql", + "LPT1/query.sql", + "COM¹.sql", + "git~1/config.sql", + "GIT~2/config.sql", + "folder./test.sql", + "folder /test.sql", + "C:/outside.sql", + "\\\\server\\share\\a.sql", + ] { + assert!(validate_relative_path(path).is_err(), "accepted {path}"); + } + assert!(validate_relative_path("Unicode/π.sql").is_ok()); + } + + #[test] + fn handle_boundary_rejects_git_even_without_lexical_validation() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir(temp.path().join(".git")).unwrap(); + let root = open_directory(temp.path()).unwrap(); + assert!(open_directory_at(root.platform_handle(), c".git").is_err()); + assert!(validate_existing_component(&temp.path().join(".git")).is_err()); + } + + #[test] + fn shared_leases_exclude_writers_until_last_handle_closes() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("lock"); + let first = open_lock_file(&path).unwrap(); + let second = open_lock_file(&path).unwrap(); + let writer = open_lock_file(&path).unwrap(); + lock_file(&first, LOCK_SH, "first").unwrap(); + lock_file(&second, LOCK_SH, "second").unwrap(); + assert!(!try_lock_file(&writer, LOCK_EX, "writer").unwrap()); + drop(first); + assert!(!try_lock_file(&writer, LOCK_EX, "writer").unwrap()); + drop(second); + assert!(try_lock_file(&writer, LOCK_EX, "writer").unwrap()); + } +} diff --git a/http/src/project_git.rs b/http/src/project_git.rs index bb100c8..f30eaa6 100644 --- a/http/src/project_git.rs +++ b/http/src/project_git.rs @@ -447,7 +447,7 @@ impl ProjectGit { ); } let meta = fs::symlink_metadata(&self.root).map_err(|_| "Cannot inspect notebook root")?; - if !meta.is_dir() || meta.file_type().is_symlink() { + if !meta.is_dir() || crate::platform::is_reparse(&meta) { return Err("Notebook root must be a real directory".into()); } checked(&self.root, None, &["init", "--", "."], None)?; @@ -475,9 +475,9 @@ impl ProjectGit { || path .split('/') .any(|part| matches!(part, "" | "." | ".." | ".git")) - || relative - .components() - .any(|c| !matches!(c, Component::Normal(_))) + || relative.components().any( + |c| !matches!(c, Component::Normal(name) if crate::platform::valid_component(name)), + ) || !matches!( relative.extension().and_then(|s| s.to_str()), Some("sql" | "sqlnb") @@ -489,7 +489,7 @@ impl ProjectGit { for component in relative.components() { current.push(component); match fs::symlink_metadata(¤t) { - Ok(meta) if meta.file_type().is_symlink() => { + Ok(meta) if crate::platform::is_reparse(&meta) => { return Err("Symlink notebook paths are not allowed".into()) } Ok(meta) if current == self.root.join(relative) && !meta.is_file() => { @@ -498,7 +498,11 @@ impl ProjectGit { Ok(meta) if current != self.root.join(relative) && !meta.is_dir() => { return Err("Notebook parent must be a directory".into()) } - Ok(_) => {} + Ok(_) => { + #[cfg(windows)] + crate::platform::validate_existing_component(¤t) + .map_err(|_| "Notebook path resolves to a reserved component")?; + } Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(_) => return Err("Cannot inspect notebook path".into()), } @@ -508,12 +512,9 @@ impl ProjectGit { } fn repo_path(repo: &Repository, path: &str) -> Result { - repo.prefix - .join(path) - .to_str() - .map(str::to_owned) - .ok_or_else(|| "Git path must be UTF-8".into()) + crate::slash_path(&repo.prefix.join(path)).map_err(|_| "Git path must be UTF-8".into()) } + fn line(bytes: &[u8]) -> Result { Ok(std::str::from_utf8(bytes) .map_err(|_| "Git response must be UTF-8")? @@ -705,6 +706,7 @@ fn bounded_read( continue; } Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(_) if finished.load(Ordering::Acquire) => break, Err(error) => return Err(error), }; if count == 0 { @@ -785,6 +787,11 @@ fn run_with_timeout( }); } } + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + command.creation_flags(windows_sys::Win32::System::Threading::CREATE_SUSPENDED); + } let mut child = command.spawn().map_err(|error| { if error.kind() == std::io::ErrorKind::NotFound { "Git executable is unavailable".to_owned() @@ -792,6 +799,9 @@ fn run_with_timeout( "Cannot start Git".to_owned() } })?; + #[cfg(windows)] + let job = crate::windows_process::Job::contain_and_resume(&mut child) + .map_err(|_| "Cannot contain Git subprocess")?; let stdout = child.stdout.take().ok_or("Cannot read Git output")?; let stderr = child.stderr.take().ok_or("Cannot read Git output")?; let mut stdin = child.stdin.take().ok_or("Cannot write Git input")?; @@ -838,6 +848,8 @@ fn run_with_timeout( unsafe { libc::kill(-(child.id() as i32), libc::SIGKILL); } + #[cfg(windows)] + job.terminate(); let _ = child.kill(); let _ = child.wait(); break None; @@ -851,7 +863,15 @@ fn run_with_timeout( } // A daemonized helper can escape the process group. Nonblocking pipes and // this completion flag keep its inherited descriptors from hanging joins. + #[cfg(windows)] + job.terminate(); finished.store(true, Ordering::Release); + #[cfg(windows)] + { + crate::windows_process::cancel_pipe_worker(&out_thread); + crate::windows_process::cancel_pipe_worker(&err_thread); + crate::windows_process::cancel_pipe_worker(&input_thread); + } let bytes = out_thread .join() .map_err(|_| "Cannot read Git output")? diff --git a/http/src/tests.rs b/http/src/tests.rs index a0b4fae..84ad4e4 100644 --- a/http/src/tests.rs +++ b/http/src/tests.rs @@ -5,7 +5,19 @@ use std::collections::HashMap; use std::ffi::CString; use std::io::{BufRead, BufReader}; use std::net::{Shutdown, SocketAddr, TcpStream}; -use std::os::unix::fs::symlink; +#[cfg(unix)] +pub(crate) use std::os::unix::fs::symlink; +#[cfg(windows)] +pub(crate) fn symlink( + source: impl AsRef, + destination: impl AsRef, +) -> std::io::Result<()> { + if source.as_ref().is_dir() { + std::os::windows::fs::symlink_dir(source, destination) + } else { + std::os::windows::fs::symlink_file(source, destination) + } +} use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; @@ -518,7 +530,7 @@ fn remote_asset_cache_leases_bound_gc_without_deleting_pinned_generations() { .write(true) .open(v1_path.join(ASSET_LEASE_FILE)) .unwrap(); - assert!(!try_lock_file(&lease_probe, libc::LOCK_EX, "test lease").unwrap()); + assert!(!try_lock_file(&lease_probe, LOCK_EX, "test lease").unwrap()); assert!(v1_path.exists() && v2_path.exists() && v3_path.exists()); drop(lease_probe); @@ -1631,3 +1643,166 @@ async fn mcp_contract_rejects_legacy_names_invalid_arguments_and_ambiguous_targe assert_eq!(response.status(), StatusCode::OK); assert_eq!(task.await.unwrap().unwrap(), acknowledgment); } + +#[tokio::test] +async fn notebook_lifecycle_preserves_whole_source_and_checks_revisions() { + let fixture = Fixture::new(); + let source = "-- notebook metadata\n-- @a [First]\nselect 1;\n-- @b [Second]\nselect 2;\n"; + fs::write(fixture.root.join("original.sqlnb"), source).unwrap(); + let app = router(fixture.state.clone()); + let expected = revision(source.as_bytes()); + let response = app + .clone() + .oneshot(fixture.request( + "POST", + "/api/v1/file/duplicate", + Some(json!({ + "path":"original.sqlnb", "destinationPath":"copy.sqlnb", "expectedRevision":expected + })), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + fs::read_to_string(fixture.root.join("copy.sqlnb")).unwrap(), + source + ); + for revision in ["stale", ""] { + let response = app + .clone() + .oneshot(fixture.request( + "DELETE", + "/api/v1/file", + Some(json!({ + "path":"copy.sqlnb", "expectedRevision":revision + })), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CONFLICT); + } + let response = app + .clone() + .oneshot(fixture.request( + "POST", + "/api/v1/file/duplicate", + Some(json!({ + "path":"original.sqlnb", "destinationPath":"copy.sqlnb", "expectedRevision":expected + })), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CONFLICT); + let response = app + .oneshot(fixture.request( + "DELETE", + "/api/v1/file", + Some(json!({ + "path":"copy.sqlnb", "expectedRevision":expected + })), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert!(!fixture.root.join("copy.sqlnb").exists()); + assert_eq!( + fs::read_to_string(fixture.root.join("original.sqlnb")).unwrap(), + source + ); + assert!(!fixture + .state + .notebooks + .history + .lock() + .unwrap() + .list("copy.sqlnb") + .is_empty()); +} + +#[tokio::test] +async fn notebook_lifecycle_rejects_symlinks_traversal_and_unauthenticated_deletes() { + let fixture = Fixture::new(); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("protected.sql"), "select 1;").unwrap(); + symlink( + outside.path().join("protected.sql"), + fixture.root.join("link.sql"), + ) + .unwrap(); + let app = router(fixture.state.clone()); + for path in ["link.sql", "../protected.sql"] { + let response = app + .clone() + .oneshot(fixture.request( + "DELETE", + "/api/v1/file", + Some(json!({ + "path":path,"expectedRevision":revision(b"select 1;") + })), + )) + .await + .unwrap(); + assert_ne!(response.status(), StatusCode::OK); + let response = app.clone().oneshot(fixture.request("POST", "/api/v1/file/duplicate", Some(json!({ + "path":path,"destinationPath":"copy.sql","expectedRevision":revision(b"select 1;") + })))).await.unwrap(); + assert_ne!(response.status(), StatusCode::OK); + } + fs::write(fixture.root.join("protected.sql"), "select 1;").unwrap(); + let mut request = fixture.request( + "DELETE", + "/api/v1/file", + Some(json!({"path":"protected.sql","expectedRevision":revision(b"select 1;")})), + ); + request.headers_mut().remove(header::AUTHORIZATION); + let response = app.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert!(fixture.root.join("protected.sql").exists()); + assert_eq!( + fs::read_to_string(outside.path().join("protected.sql")).unwrap(), + "select 1;" + ); +} + +#[tokio::test] +async fn recent_duckdb_ui_identifier_survives_reopen_without_filesystem_access() { + let fixture = Fixture::new(); + let identifier = "duckdb-ui://b7af9310-8cb8-4224-8e84-7db9548362fb"; + let app = router(fixture.state.clone()); + let response = app + .clone() + .oneshot(fixture.request( + "PUT", + "/api/v1/recent-notebook", + Some(json!({"path":identifier})), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let mut reopened = NotebookStore::open(fixture.root.clone()).unwrap(); + reopened.recent_path = fixture.state.notebooks.recent_path.clone(); + assert_eq!(reopened.recent().as_deref(), Some(identifier)); + for invalid in [ + "https://example.com", + "duckdb-ui://../file.sql", + "duckdb-ui://b7af9310-8cb8-4224-8e84-7db9548362fb/extra", + "duckdb-ui://b7af93108cb842248e847db9548362fb", + ] { + let response = app + .clone() + .oneshot(fixture.request( + "PUT", + "/api/v1/recent-notebook", + Some(json!({"path":invalid})), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!(reopened.recent().as_deref(), Some(identifier)); + } + let response = app + .oneshot(fixture.request("GET", &format!("/api/v1/file?path={identifier}"), None)) + .await + .unwrap(); + assert_ne!(response.status(), StatusCode::OK); +} diff --git a/http/src/windows_process.rs b/http/src/windows_process.rs new file mode 100644 index 0000000..b6125ac --- /dev/null +++ b/http/src/windows_process.rs @@ -0,0 +1,81 @@ +//! A Git child enters its kill-on-close job before its first instruction runs. +use std::io; +use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; +use std::process::Child; +use windows_sys::Win32::{ + Foundation::*, + System::{Diagnostics::ToolHelp::*, JobObjects::*, Threading::*, IO::CancelSynchronousIo}, +}; + +pub(crate) struct Job(OwnedHandle); + +impl Job { + pub(crate) fn contain_and_resume(child: &mut Child) -> io::Result { + let result = (|| unsafe { + let handle = CreateJobObjectW(std::ptr::null(), std::ptr::null()); + if handle.is_null() { + return Err(io::Error::last_os_error()); + } + let job = Self(OwnedHandle::from_raw_handle(handle)); + let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if SetInformationJobObject( + handle, + JobObjectExtendedLimitInformation, + (&limits as *const JOBOBJECT_EXTENDED_LIMIT_INFORMATION).cast(), + std::mem::size_of_val(&limits) as u32, + ) == 0 + || AssignProcessToJobObject(handle, child.as_raw_handle()) == 0 + { + return Err(io::Error::last_os_error()); + } + // std::process owns the process handle but not its primary thread. + // The suspended process has exactly its initial thread; find it + // through the documented Toolhelp API and resume only after assignment. + let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + if snapshot == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + let snapshot = OwnedHandle::from_raw_handle(snapshot); + let mut entry = THREADENTRY32 { + dwSize: std::mem::size_of::() as u32, + ..Default::default() + }; + let mut exists = Thread32First(snapshot.as_raw_handle(), &mut entry); + while exists != 0 { + if entry.th32OwnerProcessID == child.id() { + let thread = OpenThread(THREAD_SUSPEND_RESUME, 0, entry.th32ThreadID); + if thread.is_null() { + return Err(io::Error::last_os_error()); + } + let thread = OwnedHandle::from_raw_handle(thread); + if ResumeThread(thread.as_raw_handle()) == u32::MAX { + return Err(io::Error::last_os_error()); + } + return Ok(job); + } + exists = Thread32Next(snapshot.as_raw_handle(), &mut entry); + } + Err(io::Error::other("cannot find suspended Git thread")) + })(); + if result.is_err() { + let _ = child.kill(); + let _ = child.wait(); + } + result + } + + pub(crate) fn terminate(&self) { + unsafe { + TerminateJobObject(self.0.as_raw_handle(), 1); + } + } +} + +pub(crate) fn cancel_pipe_worker(worker: &std::thread::JoinHandle) { + // Completion is set before this call; a pending synchronous read/write + // returns ERROR_OPERATION_ABORTED and the worker exits without retrying. + unsafe { + CancelSynchronousIo(worker.as_raw_handle()); + } +} diff --git a/scripts/duckdb-ui-smoke.mjs b/scripts/duckdb-ui-smoke.mjs new file mode 100644 index 0000000..6894995 --- /dev/null +++ b/scripts/duckdb-ui-smoke.mjs @@ -0,0 +1,91 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +const [duckdb, extension] = process.argv.slice(2).map((path) => resolve(path)); +assert(duckdb && extension, 'usage: bun scripts/duckdb-ui-smoke.mjs DUCKDB SQLNB_EXTENSION'); +const home = mkdtempSync(join(tmpdir(), 'sqlnb-duckdb-ui-test-')); +const directory = join(home, '.duckdb/extension_data/ui'); +const database = join(directory, 'ui.db'); +const id = 'b7af9310-8cb8-4224-8e84-7db9548362fb'; +const literal = (value) => `'${value.replaceAll("'", "''")}'`; +const original = { notebookSerializationFormat: 3, version: 2, currentDatabase: 'analytics', + viewMode: { mode: 'default' }, future: { preserved: true }, cells: [ + { cellId: 2, query: "-- ordinary SQL comment\nselect 'π; quoted' as value;", useDatabase: 'analytics', future: 42 }, + ] }; +function run(sql, error) { + const result = spawnSync(duckdb, ['-unsigned', '-init', '/dev/null', '-json', ':memory:', '-c', + `load ${literal(extension)}; ${sql}`], { env: { ...process.env, HOME: home }, encoding: 'utf8', timeout: 15_000 }); + assert.ifError(result.error); + if (error) { + assert.notEqual(result.status, 0, 'expected a database error'); + assert.match(result.stderr, error); + return; + } + assert.equal(result.status, 0, result.stderr); + return result.stdout.trim() ? JSON.parse(result.stdout) : undefined; +} +try { + assert.equal(run('select sqlnb_duckdb_ui_notebooks() as notebooks')[0].notebooks, '[]'); + assert.equal(existsSync(database), false, 'browsing an absent source must not create it'); + mkdirSync(directory, { recursive: true }); + run(`attach ${literal(database)} as _duckdb_ui; + create table _duckdb_ui.main.notebooks(id uuid primary key, name varchar, created timestamp); + create table _duckdb_ui.main.notebook_versions(notebook_id uuid, version integer, title varchar, json varchar, created timestamp, expires timestamp, primary key(notebook_id,version)); + create table _duckdb_ui.main.current_notebook_id(id uuid); + insert into _duckdb_ui.main.notebooks values (${literal(id)},'original-internal-name',current_timestamp); + insert into _duckdb_ui.main.notebook_versions values (${literal(id)},1,'Official fixture',${literal(JSON.stringify(original))},current_timestamp,null); + insert into _duckdb_ui.main.current_notebook_id values (${literal(id)}); + select sqlnb_duckdb_ui_notebooks() as notebooks`); + assert.deepEqual(JSON.parse(run('select sqlnb_duckdb_ui_notebooks() as notebooks')[0].notebooks), [ + { id, version: 1, title: 'Official fixture' }, + ]); + const opened = JSON.parse(run(`select sqlnb_duckdb_ui_notebook(${literal(id)}) as notebook`)[0].notebook); + assert.deepEqual(JSON.parse(opened.json), original); + const edited = structuredClone(original); + edited.cells[0].query = "-- @answer [An edited answer]\n-- ordinary SQL comment\nselect 'edited π; quoted' as value;"; + // Prefer a catalog already attached to this instance, including a nonstandard alias. + const saved = JSON.parse(run(`attach ${literal(database)} as custom_ui; + select sqlnb_duckdb_ui_save(${literal(id)},1,'Edited fixture',${literal(JSON.stringify(edited))}) as notebook`)[0].notebook); + assert.equal(saved.version, 2); + assert.deepEqual(JSON.parse(saved.json), { ...edited, version: 3 }); + const old = run(`attach ${literal(database)} as ui; + select count(*) as versions, count(*) filter(where expires is null) as current, + count(*) filter(where expires between current_timestamp + interval 6 days and current_timestamp + interval 8 days) as retained + from ui.main.notebook_versions`)[0]; + assert.deepEqual(old, { versions: 2, current: 1, retained: 1 }); + run(`select sqlnb_duckdb_ui_save(${literal(id)},1,'Stale edit',${literal(JSON.stringify(original))})`, /REVISION_CONFLICT/); + run(`select sqlnb_duckdb_ui_save(${literal(id)},2,'Malformed','{')`, /Invalid DuckDB UI notebook JSON/); + run(`set enable_external_access=false; select sqlnb_duckdb_ui_notebooks()`, /enable_external_access/); + const final = run(`attach ${literal(database)} as ui; + select (select count(*) from ui.main.notebook_versions) as versions, + (select name from ui.main.notebooks) as name, + (select id::varchar from ui.main.current_notebook_id) as current`)[0]; + assert.deepEqual(final, { versions: 2, name: 'original-internal-name', current: id }); + const created = JSON.parse(run(`select sqlnb_duckdb_ui_create('Created',${literal(JSON.stringify(original))}) as notebook`)[0].notebook); + assert.notEqual(created.id, id); + assert.equal(created.version, 1); + assert.deepEqual(JSON.parse(created.json), { ...original, version: 1 }); + const copied = JSON.parse(run(`select sqlnb_duckdb_ui_duplicate(${literal(id)},2,'Copied') as notebook`)[0].notebook); + assert.notEqual(copied.id, id); + assert.equal(copied.version, 1); + assert.deepEqual(JSON.parse(copied.json), { ...edited, version: 1 }); + run(`select sqlnb_duckdb_ui_duplicate(${literal(id)},1,'Stale')`, /REVISION_CONFLICT/); + run(`select sqlnb_duckdb_ui_delete(${literal(id)},1)`, /REVISION_CONFLICT/); + run(`select sqlnb_duckdb_ui_create('Malformed','{}')`, /Unsupported DuckDB UI notebook format/); + run(`attach ${literal(database)} as ui; create table ui.main.protected_data as select 42 as answer;`); + assert.deepEqual(JSON.parse(run(`select sqlnb_duckdb_ui_delete(${literal(id)},2) as deleted`)[0].deleted), { id }); + const remaining = run(`attach ${literal(database)} as ui; + select (select count(*) from ui.main.notebooks) as notebooks, + (select count(*) from ui.main.notebook_versions where notebook_id=${literal(id)}::uuid) as deleted_versions, + (select count(*) from ui.main.current_notebook_id) as selected, + (select answer from ui.main.protected_data) as answer`)[0]; + assert.deepEqual(remaining, { notebooks: 2, deleted_versions: 0, selected: 0, answer: 42 }); + run(`select sqlnb_duckdb_ui_delete(${literal(id)},2)`, /REVISION_CONFLICT/); + assert.equal(JSON.parse(run(`select sqlnb_duckdb_ui_notebook(${literal(copied.id)}) as notebook`)[0].notebook).title, 'Copied'); + console.log('DuckDB UI source: read/save/create/duplicate/delete/conflict/rollback/metadata/access checks passed'); +} finally { + rmSync(home, { recursive: true, force: true }); +} diff --git a/scripts/qualify-windows.ps1 b/scripts/qualify-windows.ps1 new file mode 100644 index 0000000..6e120e6 --- /dev/null +++ b/scripts/qualify-windows.ps1 @@ -0,0 +1,29 @@ +param([string]$BuildDirectory = (Join-Path $env:TEMP ('sqlnb-windows-' + [guid]::NewGuid()))) +$ErrorActionPreference = 'Stop' +$PSNativeCommandUseErrorActionPreference = $true +$Repo = Split-Path -Parent $PSScriptRoot +$DuckDBRevision = 'd54cbbe3df17e8e5ce68312f483b4155836112aa' +$VcpkgRevision = '84bab45d415d22042bd0b9081aea57f362da3f35' +$DuckDB = Join-Path $BuildDirectory 'duckdb' +$Vcpkg = Join-Path $BuildDirectory 'vcpkg' +$Build = Join-Path $BuildDirectory 'build' +New-Item -ItemType Directory -Path $BuildDirectory -Force | Out-Null +foreach ($Checkout in @(@('https://github.com/duckdb/duckdb.git', $DuckDBRevision, $DuckDB), @('https://github.com/microsoft/vcpkg.git', $VcpkgRevision, $Vcpkg))) { + git init --quiet $Checkout[2] + git -C $Checkout[2] remote add origin $Checkout[0] + git -C $Checkout[2] fetch --quiet --depth 1 origin $Checkout[1] + git -C $Checkout[2] checkout --quiet --detach FETCH_HEAD + if ((git -C $Checkout[2] rev-parse HEAD) -ne $Checkout[1]) { throw 'Dependency revision mismatch' } +} +& (Join-Path $Vcpkg 'bootstrap-vcpkg.bat') -disableMetrics +& (Join-Path $Vcpkg 'vcpkg.exe') install openssl:x64-windows-static zlib:x64-windows-static +cmake -S $DuckDB -B $Build -G Ninja -DCMAKE_BUILD_TYPE=Release ` + "-DCMAKE_TOOLCHAIN_FILE=$Vcpkg/scripts/buildsystems/vcpkg.cmake" ` + -DVCPKG_MANIFEST_MODE=OFF -DVCPKG_TARGET_TRIPLET=x64-windows-static ` + "-DDUCKDB_EXTENSION_CONFIGS=$Repo/extension_config_release.cmake" ` + -DDUCKDB_EXPLICIT_VERSION=v2.0.0 -DDUCKDB_EXPLICIT_PLATFORM=windows_amd64 ` + -DBUILD_UNITTESTS=OFF -DBUILD_SHELL=ON -DEXTENSION_STATIC_BUILD=OFF +cmake --build $Build --target shell quack_loadable_extension sqlnb_loadable_extension --parallel 2 +bun "$Repo/scripts/windows-runtime-smoke.mjs" ` + --duckdb "$Build/duckdb.exe" --extension "$Build/extension/sqlnb/sqlnb.duckdb_extension" ` + --quack-extension "$Build/extension/quack/quack.duckdb_extension" diff --git a/scripts/runtime-smoke.mjs b/scripts/runtime-smoke.mjs index 720eea4..840dadd 100644 --- a/scripts/runtime-smoke.mjs +++ b/scripts/runtime-smoke.mjs @@ -198,6 +198,27 @@ assert.equal( firstValue(await runSQL('create temp table caller_temp as select 9::integer as value; select true')), true, ); +// Instant SQL is available on the final 2.0 binding API. An AT expression is +// evaluated at bind time, before DuckDB rejects time travel on native tables. +const hasReadPolicy = firstValue(await runSQL( + "select count(*) from duckdb_functions() where function_name = 'sqlnb_read_query'", +)) > 0; +if (hasReadPolicy) { + assert.equal(firstValue(await runSQL( + "create sequence instant_policy_sequence; select nextval('instant_policy_sequence')", + )), 1); + const unsafeRead = "select * from shared_table at (version => nextval('instant_policy_sequence'))"; + assert.equal(firstValue(await runSQL(`select sqlnb_is_read_only(${sqlLiteral(unsafeRead)})`)), false); + await expectSQLError( + `select * from sqlnb_read_query(${sqlLiteral(unsafeRead)})`, + /SQLNB_READ_ONLY/, + ); + assert.equal(firstValue(await runSQL("select currval('instant_policy_sequence')")), 1); + assert.deepEqual( + await runSQL("select * from sqlnb_read_query('select * from shared_table')"), + [{ value: 7 }], + ); +} const startRows = await runSQL( `select sqlnb_start(assets := ${sqlLiteral(fixture)}, open_browser := false, notebook_root := ${sqlLiteral(notebookRoot)}, mcp := true) as status`, ); diff --git a/scripts/windows-runtime-smoke.mjs b/scripts/windows-runtime-smoke.mjs new file mode 100644 index 0000000..e4b3c07 --- /dev/null +++ b/scripts/windows-runtime-smoke.mjs @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createInterface } from 'node:readline'; + +const args = new Map(); +for (let i = 2; i < process.argv.length; i += 2) args.set(process.argv[i], process.argv[i + 1]); +for (const key of ['--duckdb', '--extension', '--quack-extension']) assert(args.get(key), `missing ${key}`); +const fixture = mkdtempSync(join(tmpdir(), 'sqlnb-windows-runtime-')); +const assets = join(fixture, 'assets'); +const notebooks = join(fixture, 'notebooks'); +mkdirSync(assets); +mkdirSync(notebooks); +writeFileSync(join(assets, 'index.html'), 'Windows native bridge fixture'); +const init = join(fixture, 'init.sql'); +writeFileSync(init, ''); +const child = spawn(args.get('--duckdb'), ['-unsigned', '-init', init, '-json', ':memory:'], { + env: { ...process.env, HOME: fixture, USERPROFILE: fixture, LOCALAPPDATA: join(fixture, 'AppData', 'Local') }, + stdio: ['pipe', 'pipe', 'pipe'], +}); +let errors = ''; +child.stderr.on('data', chunk => { errors += chunk.toString(); }); +const lines = createInterface({ input: child.stdout }); +let pending; +lines.on('line', line => { + if (pending && line.trim()) { + const { resolve, reject, timer } = pending; + pending = undefined; + clearTimeout(timer); + try { resolve(JSON.parse(line)); } catch (error) { reject(error); } + } +}); +child.on('exit', code => { + if (pending) { clearTimeout(pending.timer); pending.reject(new Error(`DuckDB exited ${code}: ${errors}`)); pending = undefined; } +}); +const literal = text => `'${text.replaceAll("'", "''")}'`; +function query(sql) { + assert(!pending, 'queries must be serialized'); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { pending = undefined; reject(new Error(`SQL timed out: ${errors}`)); }, 120_000); + pending = { resolve, reject, timer }; + child.stdin.write(`${sql};\n`); + }); +} +try { + const rows = await query(`load ${literal(args.get('--quack-extension'))}; load ${literal(args.get('--extension'))}; select sqlnb_start(assets := ${literal(assets)}, notebook_root := ${literal(notebooks)}, open_browser := false) as status`); + const state = JSON.parse(rows[0].status); + const token = new URL(state.launchUrl).hash.match(/^#sidequery_token=([0-9a-f]{64})$/)?.[1]; + assert(token); + const headers = { authorization: `Bearer ${token}`, origin: state.baseUrl, 'content-type': 'application/json' }; + async function request(path, method = 'GET', body, status = 200) { + const response = await fetch(`${state.baseUrl}/api/v1/${path}`, { method, headers, body: body === undefined ? undefined : JSON.stringify(body) }); + const result = await response.json(); + assert.equal(response.status, status, JSON.stringify(result)); + return result; + } + assert.equal((await request('bootstrap')).protocol, 'quack'); + assert.equal((await fetch(`${state.baseUrl}/api/v1/files`)).status, 401); + const created = await request('file', 'PUT', { path: 'Unicode/π.sql', source: 'select 1;', expectedRevision: '' }); + const opened = await request(`file?${new URLSearchParams({ path: 'Unicode/π.sql' })}`); + assert.equal(opened.source, 'select 1;'); + const saved = await request('file', 'PUT', { path: opened.path, source: 'select 2;', expectedRevision: created.revision }); + await request('file', 'PUT', { path: opened.path, source: 'stale', expectedRevision: created.revision }, 409); + assert.equal(readFileSync(join(notebooks, 'Unicode', 'π.sql'), 'utf8'), 'select 2;'); + const copy = await request('file/duplicate', 'POST', { path: opened.path, destinationPath: 'copy.sql', expectedRevision: saved.revision }); + await request('file/move', 'POST', { path: 'copy.sql', destinationPath: 'moved.sql', expectedRevision: copy.revision }); + await request('file', 'DELETE', { path: 'moved.sql', expectedRevision: copy.revision }); + assert.deepEqual((await request('files')).files, [{ path: opened.path }]); + assert((await request(`history?${new URLSearchParams({ path: opened.path })}`)).revisions.length >= 2); + for (const path of ['CON.sql', 'file.sql:stream', '../escape.sql', 'folder./escape.sql']) { + await request('file', 'PUT', { path, source: 'forbidden', expectedRevision: '' }, 400); + } + await query('select sqlnb_stop() as stopped'); + console.log('Windows native startup, authenticated HTTP, Unicode files, revisions, lifecycle, history, and path security passed'); +} finally { + child.stdin.end(); + child.kill(); + lines.close(); +} diff --git a/src/include/duckdb_ui_notebooks.hpp b/src/include/duckdb_ui_notebooks.hpp new file mode 100644 index 0000000..f5ee83b --- /dev/null +++ b/src/include/duckdb_ui_notebooks.hpp @@ -0,0 +1,280 @@ +#pragma once + +// Included inside sqlnb's implementation namespace, after JSONEscape. +struct DuckDBUINotebooks { + unique_ptr external_database; + unique_ptr connection; + string catalog; + + DuckDBUINotebooks(ClientContext &context, bool writable) { + if (!Settings::Get(context)) { + throw PermissionException("DuckDB UI notebooks require enable_external_access=true"); + } + auto &fs = FileSystem::GetFileSystem(context); + auto path = fs.ExpandPath("~/.duckdb/extension_data/ui/ui.db"); + connection = make_uniq(*context.db); + auto databases = Checked(connection->Query( + "select database_name from duckdb_databases() where type = 'duckdb' " + "and (database_name = '_duckdb_ui' or path = ?) order by database_name = '_duckdb_ui' desc", Value(path))); + if (auto chunk = databases->Fetch()) { + catalog = chunk->GetValue(0, 0).GetValue(); + return; + } + if (!fs.FileExists(path)) { + connection.reset(); + return; // Browsing must never create or migrate the official database. + } + DBConfig config; + config.options.access_mode = writable ? AccessMode::READ_WRITE : AccessMode::READ_ONLY; + external_database = make_uniq(path, &config); + connection = make_uniq(*external_database); + auto current = Checked(connection->Query("select current_database()"))->Fetch(); + catalog = current->GetValue(0, 0).GetValue(); + } + + static unique_ptr Checked(unique_ptr result) { + if (result->HasError()) throw IOException("DuckDB UI notebooks: %s", result->GetError()); + return result; + } + + string Table(const string &name) const { + return "\"" + StringUtil::Replace(catalog, "\"", "\"\"") + "\".main." + name; + } + + string Read(const string &id) { + if (!connection) throw InvalidInputException("DuckDB UI notebook database does not exist"); + auto result = Checked(connection->Query( + "select n.id::varchar, v.version, v.title, v.json from " + Table("notebooks") + + " n join " + Table("notebook_versions") + " v on n.id=v.notebook_id " + "where n.id=?::uuid and v.expires is null", Value::UUID(id))); + auto chunk = result->Fetch(); + if (!chunk || chunk->size() != 1 || result->Fetch()) { + throw InvalidInputException("DuckDB UI notebook is missing or has multiple current versions"); + } + return Record(*chunk, 0); + } + + static string Record(DataChunk &chunk, idx_t row) { + return "{\"id\":" + JSONEscape(chunk.GetValue(0, row).GetValue()) + + ",\"version\":" + chunk.GetValue(1, row).ToString() + + ",\"title\":" + JSONEscape(chunk.GetValue(2, row).GetValue()) + + ",\"json\":" + JSONEscape(chunk.GetValue(3, row).GetValue()) + "}"; + } +}; + +static void DuckDBUIList(DataChunk &, ExpressionState &state, Vector &result) { + DuckDBUINotebooks store(state.GetContext(), false); + string json = "["; + if (store.connection) { + auto records = DuckDBUINotebooks::Checked(store.connection->Query( + "select n.id::varchar, v.version, v.title from " + store.Table("notebooks") + + " n join " + store.Table("notebook_versions") + " v on n.id=v.notebook_id " + "where v.expires is null order by v.title, n.id")); + bool first = true; + while (auto chunk = records->Fetch()) { + for (idx_t row = 0; row < chunk->size(); row++) { + if (!first) json += ","; + first = false; + json += "{\"id\":" + JSONEscape(chunk->GetValue(0, row).GetValue()) + + ",\"version\":" + chunk->GetValue(1, row).ToString() + + ",\"title\":" + JSONEscape(chunk->GetValue(2, row).GetValue()) + "}"; + } + } + } + result.SetValue(0, Value(json + "]")); + result.SetVectorType(VectorType::CONSTANT_VECTOR); +} + +static void RequireDuckDBUIConstants(DataChunk &args) { + if (!args.AllConstant()) throw InvalidInputException("DuckDB UI notebook arguments must be constant"); + for (idx_t i = 0; i < args.ColumnCount(); i++) { + if (args.data[i].GetValue(0).IsNull()) throw InvalidInputException("DuckDB UI notebook arguments must not be NULL"); + } +} + +static void DuckDBUIRead(DataChunk &args, ExpressionState &state, Vector &result) { + RequireDuckDBUIConstants(args); + DuckDBUINotebooks store(state.GetContext(), false); + result.SetValue(0, Value(store.Read(args.data[0].GetValue(0).GetValue()))); + result.SetVectorType(VectorType::CONSTANT_VECTOR); +} + +static void DuckDBUISave(DataChunk &args, ExpressionState &state, Vector &result) { + RequireDuckDBUIConstants(args); + auto id = args.data[0].GetValue(0).GetValue(); + auto expected = args.data[1].GetValue(0).GetValue(); + auto title = args.data[2].GetValue(0).GetValue(); + auto edited = args.data[3].GetValue(0).GetValue(); + if (expected < 1 || expected == NumericLimits::Maximum()) { + throw InvalidInputException("Invalid DuckDB UI notebook version"); + } + DuckDBUINotebooks store(state.GetContext(), true); + if (!store.connection) throw InvalidInputException("DuckDB UI notebook database does not exist"); + auto &connection = *store.connection; + DuckDBUINotebooks::Checked(connection.Query("begin transaction")); + try { + auto current = DuckDBUINotebooks::Checked(connection.Query( + "select v.version, v.json from " + store.Table("notebooks") + " n join " + + store.Table("notebook_versions") + " v on n.id=v.notebook_id " + "where n.id=?::uuid and v.expires is null", Value::UUID(id))); + auto chunk = current->Fetch(); + if (!chunk || chunk->size() != 1 || current->Fetch() || + chunk->GetValue(0, 0).GetValue() != expected) { + throw InvalidInputException("REVISION_CONFLICT: DuckDB UI notebook changed; reopen it before saving. Your editor changes are preserved."); + } + auto previous = chunk->GetValue(1, 0).GetValue(); + char *error = nullptr; + auto encoded = sidequery_duckdb_ui_next_version(previous.c_str(), edited.c_str(), &error); + if (!encoded) { + string message = error ? error : "Invalid DuckDB UI notebook"; + if (error) sidequery_http_string_free(error); + throw InvalidInputException("%s", message); + } + string json(encoded); + sidequery_http_string_free(encoded); + DuckDBUINotebooks::Checked(connection.Query( + "insert into " + store.Table("notebook_versions") + + " (notebook_id,version,title,json,created,expires) values (?::uuid,?,?,?,timezone('UTC',current_timestamp),NULL)", + Value::UUID(id), expected + 1, Value(title), Value(json))); + DuckDBUINotebooks::Checked(connection.Query( + "update " + store.Table("notebook_versions") + + " set expires=timezone('UTC',current_timestamp)+interval 7 days where notebook_id=?::uuid and version=? and expires is null", + Value::UUID(id), expected)); + auto saved = store.Read(id); + DuckDBUINotebooks::Checked(connection.Query("commit")); + result.SetValue(0, Value(saved)); + result.SetVectorType(VectorType::CONSTANT_VECTOR); + } catch (...) { + connection.Query("rollback"); + throw; + } +} + +// Caller owns the transaction. New identities start fresh version sequences; +// the validator preserves every unknown notebook and cell metadata field. +static string InsertDuckDBUINotebook(DuckDBUINotebooks &store, const string &title, const string &source) { + char *error = nullptr; + auto encoded = sidequery_duckdb_ui_next_version(nullptr, source.c_str(), &error); + if (!encoded) { + string message = error ? error : "Invalid DuckDB UI notebook"; + if (error) sidequery_http_string_free(error); + throw InvalidInputException("%s", message); + } + string json(encoded); + sidequery_http_string_free(encoded); + auto inserted = DuckDBUINotebooks::Checked(store.connection->Query( + "insert into " + store.Table("notebooks") + + " (id,name,created) values (uuid(),?,timezone('UTC',current_timestamp)) returning id::varchar", Value(title))); + auto chunk = inserted->Fetch(); + auto id = chunk->GetValue(0, 0).GetValue(); + DuckDBUINotebooks::Checked(store.connection->Query( + "insert into " + store.Table("notebook_versions") + + " (notebook_id,version,title,json,created,expires) values (?::uuid,1,?,?,timezone('UTC',current_timestamp),NULL)", + Value::UUID(id), Value(title), Value(json))); + return store.Read(id); +} + +static string CheckDuckDBUIRevision(DuckDBUINotebooks &store, const string &id, int32_t expected) { + auto current = DuckDBUINotebooks::Checked(store.connection->Query( + "select v.version,v.json from " + store.Table("notebooks") + " n join " + + store.Table("notebook_versions") + " v on n.id=v.notebook_id where n.id=?::uuid and v.expires is null", + Value::UUID(id))); + auto chunk = current->Fetch(); + if (expected < 1 || !chunk || chunk->size() != 1 || current->Fetch() || + chunk->GetValue(0, 0).GetValue() != expected) { + throw InvalidInputException("REVISION_CONFLICT: DuckDB UI notebook changed; reopen it before continuing."); + } + // Establish a write conflict with another transaction changing this version. + DuckDBUINotebooks::Checked(store.connection->Query( + "update " + store.Table("notebook_versions") + + " set json=json where notebook_id=?::uuid and version=? and expires is null", Value::UUID(id), expected)); + return chunk->GetValue(1, 0).GetValue(); +} + +static void DuckDBUICreate(DataChunk &args, ExpressionState &state, Vector &result) { + RequireDuckDBUIConstants(args); + DuckDBUINotebooks store(state.GetContext(), true); + if (!store.connection) throw InvalidInputException("DuckDB UI notebook database does not exist"); + DuckDBUINotebooks::Checked(store.connection->Query("begin transaction")); + try { + auto record = InsertDuckDBUINotebook(store, args.data[0].GetValue(0).GetValue(), + args.data[1].GetValue(0).GetValue()); + DuckDBUINotebooks::Checked(store.connection->Query("commit")); + result.SetValue(0, Value(record)); + result.SetVectorType(VectorType::CONSTANT_VECTOR); + } catch (...) { + store.connection->Query("rollback"); + throw; + } +} + +static void DuckDBUIDuplicate(DataChunk &args, ExpressionState &state, Vector &result) { + RequireDuckDBUIConstants(args); + DuckDBUINotebooks store(state.GetContext(), true); + if (!store.connection) throw InvalidInputException("DuckDB UI notebook database does not exist"); + DuckDBUINotebooks::Checked(store.connection->Query("begin transaction")); + try { + auto source = CheckDuckDBUIRevision(store, args.data[0].GetValue(0).GetValue(), + args.data[1].GetValue(0).GetValue()); + auto record = InsertDuckDBUINotebook(store, args.data[2].GetValue(0).GetValue(), source); + DuckDBUINotebooks::Checked(store.connection->Query("commit")); + result.SetValue(0, Value(record)); + result.SetVectorType(VectorType::CONSTANT_VECTOR); + } catch (...) { + store.connection->Query("rollback"); + throw; + } +} + +static void DuckDBUIDelete(DataChunk &args, ExpressionState &state, Vector &result) { + RequireDuckDBUIConstants(args); + auto id = args.data[0].GetValue(0).GetValue(); + DuckDBUINotebooks store(state.GetContext(), true); + if (!store.connection) throw InvalidInputException("DuckDB UI notebook database does not exist"); + DuckDBUINotebooks::Checked(store.connection->Query("begin transaction")); + try { + CheckDuckDBUIRevision(store, id, args.data[1].GetValue(0).GetValue()); + // Delete only this notebook's records and its selected-ID preference. + // Never execute notebook SQL or touch the databases referenced by cells. + DuckDBUINotebooks::Checked(store.connection->Query( + "delete from " + store.Table("notebook_versions") + " where notebook_id=?::uuid", Value::UUID(id))); + DuckDBUINotebooks::Checked(store.connection->Query( + "delete from " + store.Table("notebooks") + " where id=?::uuid", Value::UUID(id))); + DuckDBUINotebooks::Checked(store.connection->Query( + "delete from " + store.Table("current_notebook_id") + " where id=?::uuid", Value::UUID(id))); + DuckDBUINotebooks::Checked(store.connection->Query("commit")); + result.SetValue(0, Value("{\"id\":" + JSONEscape(id) + "}")); + result.SetVectorType(VectorType::CONSTANT_VECTOR); + } catch (...) { + store.connection->Query("rollback"); + throw; + } +} + +static void RegisterDuckDBUINotebooks(ExtensionLoader &loader) { + ScalarFunction list("sqlnb_duckdb_ui_notebooks", {}, LogicalType::VARCHAR, DuckDBUIList); + list.SetVolatile(); + list.SetFallible(); + loader.RegisterFunction(list); + ScalarFunction read("sqlnb_duckdb_ui_notebook", {LogicalType::VARCHAR}, LogicalType::VARCHAR, DuckDBUIRead); + read.SetVolatile(); + read.SetFallible(); + loader.RegisterFunction(read); + ScalarFunction save("sqlnb_duckdb_ui_save", {LogicalType::VARCHAR, LogicalType::INTEGER, LogicalType::VARCHAR, LogicalType::VARCHAR}, + LogicalType::VARCHAR, DuckDBUISave); + save.SetVolatile(); + save.SetFallible(); + loader.RegisterFunction(save); + ScalarFunction create("sqlnb_duckdb_ui_create", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, DuckDBUICreate); + create.SetVolatile(); + create.SetFallible(); + loader.RegisterFunction(create); + ScalarFunction duplicate("sqlnb_duckdb_ui_duplicate", {LogicalType::VARCHAR, LogicalType::INTEGER, LogicalType::VARCHAR}, LogicalType::VARCHAR, DuckDBUIDuplicate); + duplicate.SetVolatile(); + duplicate.SetFallible(); + loader.RegisterFunction(duplicate); + ScalarFunction remove("sqlnb_duckdb_ui_delete", {LogicalType::VARCHAR, LogicalType::INTEGER}, LogicalType::VARCHAR, DuckDBUIDelete); + remove.SetVolatile(); + remove.SetFallible(); + loader.RegisterFunction(remove); +} diff --git a/src/include/sidequery_http.h b/src/include/sidequery_http.h index 0043b60..99d3dca 100644 --- a/src/include/sidequery_http.h +++ b/src/include/sidequery_http.h @@ -19,6 +19,8 @@ uint16_t sidequery_http_port(void *server); void sidequery_http_stop(void *server); void sidequery_http_string_free(char *value); uint16_t sidequery_http_unused_port(void); +// A null previous validates a new notebook and starts its content version at 1. +char *sidequery_duckdb_ui_next_version(const char *previous, const char *edited, char **error_out); #ifdef __cplusplus } diff --git a/src/include/sqlnb_read_policy.hpp b/src/include/sqlnb_read_policy.hpp new file mode 100644 index 0000000..9f8c32c --- /dev/null +++ b/src/include/sqlnb_read_policy.hpp @@ -0,0 +1,277 @@ +#pragma once + +// This policy deliberately accepts a subset of SQL. It parses without extension +// hooks and never PREPAREs, EXPLAINs or executes unverified SQL to classify it. +// Unknown constructs remain available through explicit Run, never Instant SQL. +#if __has_include("duckdb/main/query_result_stream.hpp") +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/catalog/entry_lookup_info.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/function/table_function.hpp" +#include "duckdb/main/extension/extension_loader.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/parser/column_list.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/common_table_expression_info.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/expression/subquery_expression.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/query_node/set_operation_node.hpp" +#include "duckdb/parser/statement/select_statement.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/parser/tableref/expressionlistref.hpp" +#include "duckdb/parser/tableref/joinref.hpp" +#include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/parser/tableref/table_function_ref.hpp" + +namespace duckdb { +namespace sqlnb_read_policy { + +class Policy { +public: + explicit Policy(ClientContext &context_p) : context(context_p) {} + + unique_ptr Parse(const string &sql) { + auto parser = Parser::GetBuiltinParser(); + parser.ParseQuery(sql); + if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { + Reject(); + } + auto statement = unique_ptr_cast(std::move(parser.statements[0])); + CheckNode(*statement->node); + return statement; + } + +private: + [[noreturn]] static void Reject() { + throw InvalidInputException("SQLNB_READ_ONLY: this statement cannot be verified for Instant SQL; use Run explicitly"); + } + + void CheckFunction(FunctionExpression &function, bool table_function) { + const auto name = StringUtil::Lower(function.FunctionName().GetIdentifierName()); + // Names are only a first filter. Resolve the actual entry to exclude user + // macros/UDFs shadowing an allowed built-in and extension-provided overloads. + static const unordered_set scans = {"range", "generate_series", "unnest", "repeat"}; + static const unordered_set scalars = { + "+", "-", "*", "/", "//", "%", "**", "||", "abs", "ceil", "ceiling", "floor", "round", "trunc", + "sqrt", "pow", "power", "exp", "ln", "log", "log10", "sign", "greatest", "least", + "lower", "upper", "length", "char_length", "octet_length", "trim", "ltrim", "rtrim", "substring", + "substr", "left", "right", "replace", "repeat", "concat", "concat_ws", "starts_with", "ends_with", "contains", + "like_escape", "not_like_escape", "ilike_escape", "not_ilike_escape", "regexp_matches", "regexp_full_match", + "count", "count_star", "sum", "avg", "min", "max", "median", "stddev", "stddev_samp", "stddev_pop", + "variance", "var_samp", "var_pop", "bool_and", "bool_or", "first", "last", "any_value", "string_agg", + "list", "array_agg", "list_value", "struct_pack", "struct_extract", "array_extract", "list_extract", + "array_slice", "list_slice", "array_length", "list_count", "list_sum", "list_min", "list_max", + "date_part", "date_trunc", "date_diff", "datediff", "strftime", "strptime", "try_strptime" + }; + if (!(table_function ? scans : scalars).count(name)) Reject(); + auto entry = Catalog::GetEntry(context, + EntryLookupInfo(table_function ? CatalogType::TABLE_FUNCTION_ENTRY : CatalogType::SCALAR_FUNCTION_ENTRY, + function.GetQualifiedName()), OnEntryNotFound::RETURN_NULL); + if (!entry || !entry->internal || + (entry->type != CatalogType::SCALAR_FUNCTION_ENTRY && entry->type != CatalogType::AGGREGATE_FUNCTION_ENTRY && + entry->type != CatalogType::TABLE_FUNCTION_ENTRY)) Reject(); + auto extension = entry->extension_name.GetIdentifierName(); + if (!extension.empty() && extension != "core_functions") Reject(); + ParsedExpressionIterator::EnumerateChildren(function, [this](ParsedExpression &child) { CheckExpression(child); }); + } + + void CheckExpression(ParsedExpression &expression) { + switch (expression.GetExpressionClass()) { + case ExpressionClass::FUNCTION: + CheckFunction(expression.Cast(), false); + return; + case ExpressionClass::SUBQUERY: + CheckNode(*expression.Cast().SubqueryMutable()->node); + break; + case ExpressionClass::COLUMN_REF: + // A dotted reference can be a relation-qualified column or an implicit + // struct/JSON extraction that binds a scalar through the search path. + // Without binding we cannot distinguish them safely. + if (expression.Cast().ColumnNames().size() != 1) Reject(); + break; + case ExpressionClass::OPERATOR: + switch (expression.GetExpressionType()) { + case ExpressionType::OPERATOR_IS_NULL: + case ExpressionType::OPERATOR_IS_NOT_NULL: + case ExpressionType::OPERATOR_COALESCE: + case ExpressionType::OPERATOR_NOT: + case ExpressionType::OPERATOR_TRY: + case ExpressionType::COMPARE_IN: + case ExpressionType::COMPARE_NOT_IN: + break; + default: + // Array/struct extraction and other operators can dispatch to hidden + // scalar calls. Require an explicitly checked function call instead. + Reject(); + } + break; + case ExpressionClass::CASE: + case ExpressionClass::COMPARISON: + case ExpressionClass::CONJUNCTION: + case ExpressionClass::CONSTANT: + case ExpressionClass::STAR: + case ExpressionClass::POSITIONAL_REFERENCE: + case ExpressionClass::BETWEEN: + break; + default: + // Casts, collations, lambdas and window functions can resolve additional + // user-defined code at bind time. No permissive default for new AST kinds. + Reject(); + } + ParsedExpressionIterator::EnumerateChildren(expression, [this](ParsedExpression &child) { CheckExpression(child); }); + } + + static void CheckType(const LogicalType &type) { + // Aliased/custom types can introduce user-defined implicit casts, even when + // the SQL itself contains no explicit function or CAST expression. + if (type.HasAlias()) Reject(); + switch (type.id()) { + case LogicalTypeId::BOOLEAN: + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::HUGEINT: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::UHUGEINT: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::DECIMAL: + case LogicalTypeId::DATE: + case LogicalTypeId::TIME: + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_SEC: + case LogicalTypeId::TIMESTAMP_MS: + case LogicalTypeId::TIMESTAMP_NS: + case LogicalTypeId::TIMESTAMP_TZ: + case LogicalTypeId::TIME_TZ: + case LogicalTypeId::INTERVAL: + case LogicalTypeId::BLOB: + case LogicalTypeId::UUID: + return; + case LogicalTypeId::VARCHAR: + if (!StringType::GetCollation(type).empty()) Reject(); + return; + case LogicalTypeId::LIST: + CheckType(ListType::GetChildType(type)); + return; + case LogicalTypeId::ARRAY: + CheckType(ArrayType::GetChildType(type)); + return; + case LogicalTypeId::STRUCT: + for (const auto &child : StructType::GetChildTypes(type)) CheckType(child.second); + return; + case LogicalTypeId::MAP: + CheckType(MapType::KeyType(type)); + CheckType(MapType::ValueType(type)); + return; + default: + Reject(); + } + } + + void CheckTable(TableRef &reference) { + switch (reference.type) { + case TableReferenceType::EMPTY_FROM: + return; + case TableReferenceType::BASE_TABLE: { + auto &table = reference.Cast(); + // DuckDB evaluates AT expressions during binding, even when the catalog + // subsequently rejects time travel. Native catalogs admitted here do not + // support it, so reject before any hidden expression can execute. + if (table.at_clause) Reject(); + // Missing tables are rejected before replacement scans (including files) + // can run. Views and generated columns can conceal calls: fail closed. + auto entry = Catalog::GetEntry(context, EntryLookupInfo(CatalogType::TABLE_ENTRY, table.GetQualifiedName()), + OnEntryNotFound::RETURN_NULL); + if (!entry || entry->type != CatalogType::TABLE_ENTRY || entry->ParentCatalog().GetCatalogType() != "duckdb" || + entry->Cast().HasGeneratedColumns()) Reject(); + for (const auto &column : entry->Cast().GetColumns().Logical()) CheckType(column.Type()); + return; + } + case TableReferenceType::SUBQUERY: + CheckNode(*reference.Cast().subquery->node); + return; + case TableReferenceType::JOIN: { + auto &join = reference.Cast(); + CheckTable(*join.left); + CheckTable(*join.right); + if (join.condition) CheckExpression(*join.condition); + if (join.ranking_expression) CheckExpression(*join.ranking_expression); + for (auto &expression : join.duplicate_eliminated_columns) CheckExpression(*expression); + return; + } + case TableReferenceType::EXPRESSION_LIST: + for (auto &row : reference.Cast().values) { + for (auto &expression : row) CheckExpression(*expression); + } + return; + case TableReferenceType::TABLE_FUNCTION: { + auto &table = reference.Cast(); + if (!table.function || table.function->GetExpressionClass() != ExpressionClass::FUNCTION || table.subquery) Reject(); + CheckFunction(table.function->Cast(), true); + return; + } + default: + Reject(); + } + } + + void CheckNode(QueryNode &node) { + // CTEs can contain modifying query nodes in DuckDB 2. Keep them explicit + // until their lexical scope and every node kind have a dedicated policy. + if (!node.cte_map.map.empty()) Reject(); + if (node.type == QueryNodeType::SELECT_NODE) { + auto &select = node.Cast(); + for (auto &expression : select.select_list) CheckExpression(*expression); + for (auto &expression : select.groups.group_expressions) CheckExpression(*expression); + if (select.where_clause) CheckExpression(*select.where_clause); + if (select.having) CheckExpression(*select.having); + if (select.qualify) CheckExpression(*select.qualify); + if (select.from_table) CheckTable(*select.from_table); + } else if (node.type == QueryNodeType::SET_OPERATION_NODE) { + for (auto &child : node.Cast().children) CheckNode(*child); + } else Reject(); + ParsedExpressionIterator::EnumerateQueryNodeModifiers(node, + [this](unique_ptr &expression) { CheckExpression(*expression); }); + } + + ClientContext &context; +}; + +static void IsReadOnly(DataChunk &input, ExpressionState &state, Vector &result) { + UnaryExecutor::Execute(input.data[0], result, input.size(), [&](string_t sql) { + try { + Policy(state.GetContext()).Parse(sql.GetString()); + return true; + } catch (const Exception &) { + return false; + } + }); +} + +static unique_ptr ReadQuery(ClientContext &context, TableFunctionBindInput &input) { + if (input.inputs[0].IsNull()) throw InvalidInputException("SQLNB_READ_ONLY: SQL cannot be null"); + // Validate and return the same parsed tree, in the outer query's binding + // transaction. A separate prior classification cannot authorize this path. + return make_uniq(Policy(context).Parse(input.inputs[0].GetValue())); +} + +} // namespace sqlnb_read_policy + +static void RegisterSQLNBReadPolicy(ExtensionLoader &loader) { + ScalarFunction policy("sqlnb_is_read_only", {LogicalType::VARCHAR}, LogicalType::BOOLEAN, sqlnb_read_policy::IsReadOnly); + policy.SetVolatile(); + loader.RegisterFunction(policy); + TableFunction read("sqlnb_read_query", {LogicalType::VARCHAR}, nullptr, nullptr); + read.bind_replace = sqlnb_read_policy::ReadQuery; + loader.RegisterFunction(read); +} +} // namespace duckdb +#endif diff --git a/src/sidequery_ui_extension.cpp b/src/sidequery_ui_extension.cpp index 8f3f4c2..be0c75f 100644 --- a/src/sidequery_ui_extension.cpp +++ b/src/sidequery_ui_extension.cpp @@ -2,6 +2,12 @@ #ifndef _WIN32 #include #include +#else +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include #endif #include "sidequery_ui_extension.hpp" @@ -13,6 +19,7 @@ #include "duckdb/main/extension/extension_loader.hpp" #include "duckdb/main/extension_helper.hpp" #include "duckdb/main/settings.hpp" +#include "duckdb/main/config.hpp" #include "duckdb/main/prepared_statement.hpp" #include "duckdb/catalog/catalog.hpp" #include "duckdb/catalog/entry_lookup_info.hpp" @@ -29,6 +36,7 @@ #if __has_include("duckdb/main/query_result_stream.hpp") #include "duckdb/main/query_result_stream.hpp" #define SQLNB_HAS_RESULT_STREAM 1 +#include "sqlnb_read_policy.hpp" #endif #if __has_include("duckdb/main/extension_repository_manager.hpp") @@ -50,11 +58,6 @@ #ifndef _WIN32 extern char **environ; #endif -#ifdef _WIN32 -#include -#include -#endif - namespace duckdb { namespace { @@ -88,6 +91,8 @@ static string JSONEscape(const string &value) { return out.str(); } +#include "duckdb_ui_notebooks.hpp" + static string RandomHex(idx_t bytes) { vector random(bytes); if (RAND_bytes(random.data(), NumericCast(random.size())) != 1) { @@ -762,6 +767,10 @@ static void StopFunction(DataChunk &, ExpressionState &state, Vector &result) { } static void LoadInternal(ExtensionLoader &loader) { + RegisterDuckDBUINotebooks(loader); +#ifdef SQLNB_HAS_RESULT_STREAM + RegisterSQLNBReadPolicy(loader); +#endif auto state_extension = make_shared_ptr(); state_extension->storage_info = make_uniq(); StorageExtension::Register(loader.GetDatabaseInstance().config, SQLNB_STATE_KEY,