Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
c447c1e
fix: keep URL passwords and SAS queries out of Azure store errors and…
dwsmith1983 Oct 20, 2026
65b5ad2
fix: keep ABFS URL passwords out of store keys, resolution errors and…
dwsmith1983 Oct 20, 2026
5fbb8fd
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 21, 2026
a3275e6
fix: name only the scheme when an Azure store gets a non-Azure URL
dwsmith1983 Oct 21, 2026
4c8da96
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 21, 2026
8acc4a2
docs: keep only the comments that explain the URL redaction rules
dwsmith1983 Oct 22, 2026
a1b30cf
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 22, 2026
bb33309
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 23, 2026
27946a9
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 23, 2026
0e6ec24
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 24, 2026
27e66d9
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 25, 2026
71ec9f5
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 26, 2026
2532764
fix: key encrypted ABFS reads by the password-free authority
dwsmith1983 Oct 26, 2026
c487ef8
fix: mask overlapping secrets as one span in ABFS diagnostics
dwsmith1983 Oct 26, 2026
5c61d9c
perf: reuse the raw ABFS authority when it has no password
dwsmith1983 Oct 26, 2026
6176c20
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 27, 2026
7f85460
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 27, 2026
8c519a9
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 28, 2026
1aa8c51
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 28, 2026
55fb54d
refactor: render the ABFS native fallback reason's location with disp…
dwsmith1983 Oct 28, 2026
1756d73
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 28, 2026
e23cfac
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 29, 2026
5ba2ac7
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 30, 2026
95b81f6
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 30, 2026
0b8687e
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 31, 2026
bc001bf
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Nov 5, 2026
dbcaa8a
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Nov 7, 2026
79b5b4f
fix: hide short URI passwords in ABFS driver warnings
dwsmith1983 Nov 7, 2026
e660eb7
fix: keep URL query strings and passwords out of native Delta errors
dwsmith1983 Nov 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 89 additions & 1 deletion native/core/src/execution/delta_dv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ use roaring::{RoaringBitmap, RoaringTreemap};

use crate::execution::operators::ExecutionError;
use crate::execution::operators::ExecutionError::GeneralError;
use crate::execution::planner::redacted_url_display;
use datafusion_comet_proto::spark_operator::DeltaSparkDvDescriptor;

const NATIVE_MAGIC: i32 = 1681511376;
Expand Down Expand Up @@ -630,6 +631,8 @@ async fn attach_access_plan(
if dv.cardinality == 0 && dv.size_in_bytes == 0 {
return Ok(file);
}
// Only messages print the file's URL; reads go through `file.object_meta` and `data_store`.
let file_path = redacted_url_display(&file_path);
if dv.size_in_bytes < 0 {
return Err(GeneralError(format!(
"Deletion vector for {file_path} has negative size {}",
Expand Down Expand Up @@ -666,7 +669,12 @@ async fn attach_access_plan(
let blob = store
.get_range(&dv_store_path, offset..offset + framed_len)
.await
.map_err(|e| GeneralError(format!("Failed to read deletion vector {dv_path}: {e}")))?;
.map_err(|e| {
GeneralError(format!(
"Failed to read deletion vector {}: {e}",
redacted_url_display(dv_path)
))
})?;
unframe_dv_blob(&blob, dv.size_in_bytes as usize)?.to_vec()
} else {
return Err(GeneralError(
Expand Down Expand Up @@ -2751,4 +2759,84 @@ mod tests {
"every single-bit flip of the inline payload must have been exercised"
);
}

/// Userinfo and query markers the test URLs carry, none of which may be shown.
const URL_MARKERS: [&str; 3] = ["alice", "SECRET", "SAS-MARKER"];

/// A file whose deletion vector fails before any parquet footer is read.
fn failing_dv_file(file_path: &str, dv: DeltaSparkDvDescriptor) -> DvScanFile {
let store: Arc<dyn ObjectStore> = Arc::new(object_store::memory::InMemory::new());
DvScanFile {
file: PartitionedFile::new("t/f0.parquet", 100),
file_path: file_path.to_string(),
dv: Some(dv),
data_store: Arc::clone(&store),
dv_store: Some((store, Path::from("t/deletion_vector_x.bin"))),
}
}

async fn attach_error(file: DvScanFile) -> String {
match attach_access_plans(Arc::new(RuntimeEnv::default()), vec![file]).await {
Ok(_) => panic!("the deletion vector must be rejected"),
Err(e) => e.to_string(),
}
}

#[tokio::test]
async fn dv_errors_do_not_echo_the_file_url() {
let file_path = "s3://alice:SECRET@bucket/t/f0.parquet?sig=SAS-MARKER";
let negative_size = DeltaSparkDvDescriptor {
storage_type: "i".to_string(),
absolute_path: None,
inline_data: Some(vec![0; 4]),
offset: None,
size_in_bytes: -1,
cardinality: 1,
};
let wrong_inline_size = DeltaSparkDvDescriptor {
size_in_bytes: 8,
..negative_size.clone()
};
for (dv, expected) in [
(
negative_size,
"Deletion vector for s3://***@bucket/t/f0.parquet has negative size -1",
),
(
wrong_inline_size,
"Inline deletion vector for s3://***@bucket/t/f0.parquet has 4 bytes",
),
] {
let msg = attach_error(failing_dv_file(file_path, dv)).await;
for marker in URL_MARKERS {
assert!(!msg.contains(marker), "{marker} echoed: {msg}");
}
assert!(
msg.contains(expected),
"expected the file without its credentials or query: {msg}"
);
}
}

#[tokio::test]
async fn dv_read_error_does_not_echo_the_dv_url() {
let dv = DeltaSparkDvDescriptor {
storage_type: "p".to_string(),
absolute_path: Some(
"s3://alice:SECRET@bucket/t/deletion_vector_x.bin?sig=SAS-MARKER".to_string(),
),
inline_data: None,
offset: Some(1),
size_in_bytes: 8,
cardinality: 1,
};
let msg = attach_error(failing_dv_file("s3://bucket/t/f0.parquet", dv)).await;
for marker in URL_MARKERS {
assert!(!msg.contains(marker), "{marker} echoed: {msg}");
}
assert!(
msg.contains("Failed to read deletion vector s3://***@bucket/t/deletion_vector_x.bin"),
"expected the deletion vector without its credentials or query: {msg}"
);
}
}
2 changes: 2 additions & 0 deletions native/core/src/execution/planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ mod delta_scan;
// JVM-planned Delta sibling of the kernel handler above; see delta_spark_scan.rs.
#[cfg(feature = "delta")]
mod delta_spark_scan;
#[cfg(feature = "delta")]
pub(crate) use delta_spark_scan::redacted_url_display;
#[cfg(feature = "contrib-lance")]
mod lance_scan;

Expand Down
Loading