diff --git a/Cargo.lock b/Cargo.lock
index 7498487f..03ab7f4c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -79,6 +79,21 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "alloc-no-stdlib"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a"
+
+[[package]]
+name = "alloc-stdlib"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1"
+dependencies = [
+ "alloc-no-stdlib",
+]
+
[[package]]
name = "allocator-api2"
version = "0.2.21"
@@ -177,6 +192,18 @@ version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
+[[package]]
+name = "async-compression"
+version = "0.4.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ad47a740fb906fffdd891e8b6d5fde2511600f7bfaebd68939fa08a14999459"
+dependencies = [
+ "compression-codecs",
+ "compression-core",
+ "pin-project-lite",
+ "tokio",
+]
+
[[package]]
name = "async-stream"
version = "0.3.6"
@@ -520,6 +547,27 @@ dependencies = [
"hybrid-array",
]
+[[package]]
+name = "brotli"
+version = "9.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "6.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75c74c9c51b9140663e196b5fb88bc94c334d7a497c8311cb759d8557874747f"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.3"
@@ -709,6 +757,26 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "compression-codecs"
+version = "0.4.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84f840cff84d10085260e460023934d0af699e71212b09b2a41452e8bd9cda43"
+dependencies = [
+ "brotli",
+ "compression-core",
+ "flate2",
+ "memchr",
+ "zstd",
+ "zstd-safe",
+]
+
+[[package]]
+name = "compression-core"
+version = "0.4.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a"
+
[[package]]
name = "const-oid"
version = "0.10.2"
@@ -721,6 +789,17 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
+[[package]]
+name = "cookie"
+version = "0.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87"
+dependencies = [
+ "percent-encoding",
+ "time",
+ "version_check",
+]
+
[[package]]
name = "cookie-factory"
version = "0.3.3"
@@ -730,6 +809,24 @@ dependencies = [
"futures",
]
+[[package]]
+name = "cookie_store"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206"
+dependencies = [
+ "cookie",
+ "document-features",
+ "idna",
+ "log",
+ "publicsuffix",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "time",
+ "url",
+]
+
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -1107,6 +1204,15 @@ dependencies = [
"syn 3.0.5",
]
+[[package]]
+name = "document-features"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"
+dependencies = [
+ "litrs",
+]
+
[[package]]
name = "dtoa"
version = "1.0.11"
@@ -2104,6 +2210,12 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
+[[package]]
+name = "litrs"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4744e383959f0db86ede514b809b1c53251889093803c05267acc7d4e7030d70"
+
[[package]]
name = "lock_api"
version = "0.4.14"
@@ -2597,6 +2709,22 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "psl-types"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
+
+[[package]]
+name = "publicsuffix"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf"
+dependencies = [
+ "idna",
+ "psl-types",
+]
+
[[package]]
name = "pulley-interpreter"
version = "49.0.5"
@@ -2873,6 +3001,8 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
dependencies = [
"base64 0.22.1",
"bytes",
+ "cookie",
+ "cookie_store",
"futures-core",
"futures-util",
"h2",
@@ -3864,12 +3994,17 @@ version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
dependencies = [
+ "async-compression",
"bitflags",
"bytes",
+ "futures-core",
"futures-util",
"http",
"http-body",
+ "http-body-util",
"pin-project-lite",
+ "tokio",
+ "tokio-util",
"tower",
"tower-layer",
"tower-service",
@@ -4013,7 +4148,7 @@ dependencies = [
[[package]]
name = "tw-api"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"serde",
"serde_json",
@@ -4024,7 +4159,7 @@ dependencies = [
[[package]]
name = "tw-bedrock"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -4045,7 +4180,7 @@ dependencies = [
[[package]]
name = "tw-breaker"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"serde",
"serde_json",
@@ -4053,7 +4188,7 @@ dependencies = [
[[package]]
name = "tw-config"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"blake3",
"icu_locale_core",
@@ -4081,7 +4216,7 @@ dependencies = [
[[package]]
name = "tw-control"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"axum",
"base64 0.22.1",
@@ -4127,7 +4262,7 @@ dependencies = [
[[package]]
name = "tw-dialect"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"memchr",
"serde",
@@ -4136,7 +4271,7 @@ dependencies = [
[[package]]
name = "tw-engine"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"chrono",
"serde",
@@ -4150,7 +4285,7 @@ dependencies = [
[[package]]
name = "tw-gateway"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"arc-swap",
"async-stream",
@@ -4198,13 +4333,14 @@ dependencies = [
"tw-pricing",
"tw-secret",
"tw-types",
+ "url",
"uuid",
"zstd",
]
[[package]]
name = "tw-guard"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"aho-corasick",
"base64 0.22.1",
@@ -4225,7 +4361,7 @@ dependencies = [
[[package]]
name = "tw-handover"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"libc",
"serde",
@@ -4238,7 +4374,7 @@ dependencies = [
[[package]]
name = "tw-link"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"serde",
"serde_json",
@@ -4252,7 +4388,7 @@ dependencies = [
[[package]]
name = "tw-observe"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"tokio",
"tracing",
@@ -4261,7 +4397,7 @@ dependencies = [
[[package]]
name = "tw-pack"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"age",
"blake3",
@@ -4285,7 +4421,7 @@ dependencies = [
[[package]]
name = "tw-plugin"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"libc",
"rand 0.10.2",
@@ -4301,7 +4437,7 @@ dependencies = [
[[package]]
name = "tw-pricing"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"arc-swap",
"flate2",
@@ -4314,14 +4450,14 @@ dependencies = [
[[package]]
name = "tw-secret"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"thiserror 2.0.20",
]
[[package]]
name = "tw-store"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"blake3",
"bytes",
@@ -4341,7 +4477,7 @@ dependencies = [
[[package]]
name = "tw-sync"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"age",
"axum",
@@ -4369,7 +4505,7 @@ dependencies = [
[[package]]
name = "tw-types"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"serde",
"serde_json",
@@ -4378,7 +4514,7 @@ dependencies = [
[[package]]
name = "tw-watch"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"libc",
"notify",
@@ -4389,7 +4525,7 @@ dependencies = [
[[package]]
name = "tw-yaml"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"saphyr-parser",
"serde_yaml_ng",
@@ -4399,7 +4535,7 @@ dependencies = [
[[package]]
name = "twcore"
-version = "0.97.0"
+version = "0.98.0"
dependencies = [
"anyhow",
"axum",
diff --git a/Cargo.toml b/Cargo.toml
index 2fd657c0..13067cf5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -61,7 +61,7 @@ categories = ["network-programming", "web-programming::http-server"]
# 二进制走 CalVer,crate 走 SemVer —— 两者是两回事,卖给不同的人。
# 这里是 crate 的版本。
-version = "0.97.0"
+version = "0.98.0"
[workspace.dependencies]
# ── 内部 crate ───────────────────────────────────────────
diff --git a/bin/twcore/src/upgrade.rs b/bin/twcore/src/upgrade.rs
index c2dcb45a..c6415730 100644
--- a/bin/twcore/src/upgrade.rs
+++ b/bin/twcore/src/upgrade.rs
@@ -285,6 +285,12 @@ fn client() -> Result {
/// 走环境变量里的代理(`HTTPS_PROXY`):服务器上出网常常只有这一条路。
fn builder() -> reqwest::ClientBuilder {
reqwest::Client::builder()
+ // Search opts into content decoding; release downloads retain their
+ // existing behavior when those reqwest features are unified.
+ .no_gzip()
+ .no_brotli()
+ .no_deflate()
+ .no_zstd()
.user_agent(concat!("twcore/", env!("CARGO_PKG_VERSION")))
.connect_timeout(Duration::from_secs(15))
.timeout(Duration::from_secs(300))
diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt
index 08b1a660..a8e64872 100644
--- a/crates/tw-api/msg-codes.txt
+++ b/crates/tw-api/msg-codes.txt
@@ -476,6 +476,7 @@ gw.wait.rate_limited
gw.wait.used_up
gw.wait.window_full
gw.web_tools.failed
+gw.web_tools.query_mismatch
gw.ws.alias_unserved
gw.ws.alias_unserved_rewritten
gw.ws.bad_header
diff --git a/crates/tw-api/src/web_tools.rs b/crates/tw-api/src/web_tools.rs
index 96684c01..0f7ea7ad 100644
--- a/crates/tw-api/src/web_tools.rs
+++ b/crates/tw-api/src/web_tools.rs
@@ -17,6 +17,7 @@ pub enum WebToolsMode {
pub enum SearchEngine {
Bing,
Duckduckgo,
+ GoogleDocs,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -77,6 +78,18 @@ pub struct WebToolsView {
pub config_version: String,
/// Detected on the Core host, which may be remote. Empty fields mean unavailable.
pub system_locale: WebSearchLocale,
+ /// Engine capabilities advertised by the connected Core.
+ #[serde(default)]
+ pub available_engines: Vec,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+pub struct WebSearchEngine {
+ pub id: SearchEngine,
+ pub name: String,
+ /// The engine's minimum applies even if the user's configured interval is lower.
+ pub min_interval_ms: u32,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs
index 31e5167f..31c31549 100644
--- a/crates/tw-config/tests/manual/schema.rs
+++ b/crates/tw-config/tests/manual/schema.rs
@@ -1310,8 +1310,8 @@ pub fn sections() -> Vec {
Kind::Strs,
Def::Is("[]"),
t(
- "Empty selects available engines automatically. Otherwise list bing and/or duckduckgo in priority order.",
- "空列表自动选择可用引擎;手动填写 bing、duckduckgo,按顺序尝试。",
+ "Empty tries google_docs, duckduckgo, then bing. Otherwise list these engine IDs in priority order.",
+ "空列表依次尝试 google_docs、duckduckgo、bing;也可手动按优先顺序填写这些引擎 ID。",
),
),
row(
@@ -1403,8 +1403,8 @@ pub fn sections() -> Vec {
Kind::Int,
Def::Is("1000"),
t(
- "Minimum interval per search engine, 250–60000 milliseconds.",
- "同一搜索引擎的最小请求间隔,250–60000 毫秒。",
+ "Minimum interval per search engine, 250–60000 milliseconds. A source may enforce a higher minimum (google_docs: 6000).",
+ "同一搜索引擎的最小请求间隔,250–60000 毫秒。搜索源可要求更长间隔(google_docs:6000)。",
),
),
row(
diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs
index f3790f88..ec055a2e 100644
--- a/crates/tw-control/src/sync.rs
+++ b/crates/tw-control/src/sync.rs
@@ -218,6 +218,12 @@ impl Host for CoreHost {
/// 同步位置的密码就跟着去了),证书照常校验
fn http(cfg: &tw_config::Config, proxy: &str) -> Result {
let mut b = reqwest::Client::builder()
+ // Search enables reqwest's decoding features workspace-wide. Keep sync
+ // downloads' existing byte semantics independent of that feature set.
+ .no_gzip()
+ .no_brotli()
+ .no_deflate()
+ .no_zstd()
.connect_timeout(Duration::from_secs(10))
// 一轮里的请求是接连发的;两轮之间的空闲连接不留:很多服务端几秒就关掉空闲连接,
// 留着的下一轮一发就断(发请求那一处另有一次重发)
diff --git a/crates/tw-control/src/web_tools.rs b/crates/tw-control/src/web_tools.rs
index ea5eac71..f504ffd5 100644
--- a/crates/tw-control/src/web_tools.rs
+++ b/crates/tw-control/src/web_tools.rs
@@ -15,6 +15,7 @@ async fn view(State(s): State) -> Json {
config: s.config().web_tools.clone(),
config_version: s.cfg.current().map(|c| c.version()).unwrap_or_default(),
system_locale: tw_config::web_tools::system_locale(),
+ available_engines: tw_gateway::web_tools::available_engines(),
})
}
diff --git a/crates/tw-gateway/Cargo.toml b/crates/tw-gateway/Cargo.toml
index 0785b080..2217796d 100644
--- a/crates/tw-gateway/Cargo.toml
+++ b/crates/tw-gateway/Cargo.toml
@@ -34,7 +34,8 @@ bytes = { workspace = true }
chrono = { workspace = true }
futures = { workspace = true }
http-body-util = { workspace = true }
-reqwest = { workspace = true }
+url = { workspace = true }
+reqwest = { workspace = true, features = ["cookies", "gzip", "brotli", "deflate", "zstd"] }
rustls = { workspace = true }
rustls-platform-verifier = { workspace = true }
tokio-rustls = { workspace = true }
diff --git a/crates/tw-gateway/src/outbound.rs b/crates/tw-gateway/src/outbound.rs
index 64e39411..ff5bba36 100644
--- a/crates/tw-gateway/src/outbound.rs
+++ b/crates/tw-gateway/src/outbound.rs
@@ -7,6 +7,12 @@ use tw_types::msg;
/// 的那批和不走代理的那批会慢慢长出不同的超时行为。
pub(crate) fn base_client_builder() -> reqwest::ClientBuilder {
reqwest::Client::builder()
+ // Web search opts into browser content decoding. Other outbound paths
+ // preserve their existing wire-body semantics despite feature unification.
+ .no_gzip()
+ .no_brotli()
+ .no_deflate()
+ .no_zstd()
// 分段超时。**没有整体超时** —— 一个跑了六分钟的
// Opus 任务不该被中间层掐断,让客户端自己决定何时放弃。
.connect_timeout(std::time::Duration::from_secs(10))
diff --git a/crates/tw-gateway/src/web_tools/cache.rs b/crates/tw-gateway/src/web_tools/cache.rs
new file mode 100644
index 00000000..440407cf
--- /dev/null
+++ b/crates/tw-gateway/src/web_tools/cache.rs
@@ -0,0 +1,89 @@
+//! Bounded, caller-keyed cache slots. Holding a slot coalesces concurrent work;
+//! eviction only removes one idle slot and cannot split an in-flight request.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::time::Instant;
+use tokio::sync::Mutex as AsyncMutex;
+use tw_types::Msg;
+
+type Slot = Arc>>;
+struct Entry {
+ used: Instant,
+ slot: Slot,
+}
+
+pub(super) struct Cache {
+ capacity: usize,
+ entries: Mutex>>,
+}
+
+impl Cache {
+ pub fn new(capacity: usize) -> Self {
+ Self {
+ capacity,
+ entries: Mutex::new(HashMap::new()),
+ }
+ }
+
+ pub fn get(&self, key: String) -> Result, Msg> {
+ let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner());
+ if let Some(entry) = entries.get_mut(&key) {
+ entry.used = Instant::now();
+ return Ok(entry.slot.clone());
+ }
+ if entries.len() >= self.capacity {
+ let oldest = entries
+ .iter()
+ .filter(|(_, e)| Arc::strong_count(&e.slot) == 1)
+ .min_by_key(|(_, e)| e.used)
+ .map(|(key, _)| key.clone());
+ if let Some(oldest) = oldest {
+ entries.remove(&oldest);
+ } else {
+ return Err(super::error("too many pending web operations"));
+ }
+ }
+ let slot = Arc::new(AsyncMutex::new(None));
+ entries.insert(
+ key,
+ Entry {
+ used: Instant::now(),
+ slot: slot.clone(),
+ },
+ );
+ Ok(slot)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[tokio::test]
+ async fn eviction_retains_hot_values_and_coalesces_in_flight_work() {
+ let cache = Cache::new(2);
+ let first = cache.get("first".into()).unwrap();
+ *first.lock().await = Some((Instant::now(), "saved"));
+ let pending = cache.get("pending".into()).unwrap();
+ let guard = pending.lock().await;
+ let joined = cache.get("pending".into()).unwrap();
+ assert!(Arc::ptr_eq(&pending, &joined));
+ assert!(joined.try_lock().is_err());
+ assert!(
+ cache.get("new".into()).is_err(),
+ "active work cannot be evicted"
+ );
+ drop(guard);
+ drop(joined);
+ drop(pending);
+ drop(first);
+ // Touch the completed result: inserting another key must evict the older
+ // idle slot, not flush all successfully cached results.
+ drop(cache.get("first".into()).unwrap());
+ drop(cache.get("new".into()).unwrap());
+ let first = cache.get("first".into()).unwrap();
+ assert_eq!(first.lock().await.as_ref().unwrap().1, "saved");
+ assert_eq!(cache.entries.lock().unwrap().len(), 2);
+ }
+}
diff --git a/crates/tw-gateway/src/web_tools/collect.rs b/crates/tw-gateway/src/web_tools/collect.rs
index 561a8593..b1819ac5 100644
--- a/crates/tw-gateway/src/web_tools/collect.rs
+++ b/crates/tw-gateway/src/web_tools/collect.rs
@@ -11,18 +11,12 @@ fn clean(s: &str) -> String {
s.split_whitespace().collect::>().join(" ")
}
-pub(super) fn search(engine: SearchEngine, body: &str) -> Result, Msg> {
- if body.contains("anomaly.js") || body.contains("challenge-form") || body.contains("b_captcha")
- {
- return Err(error("search engine requires a verification challenge"));
- }
- match engine {
- SearchEngine::Bing => rss(body),
- SearchEngine::Duckduckgo => duckduckgo(body),
- }
-}
-
-fn source(engine: SearchEngine, url: &str, title: &str, text: &str) -> Option {
+pub(super) fn source(
+ engine: SearchEngine,
+ url: &str,
+ title: &str,
+ text: &str,
+) -> Option {
let mut url = super::public::parse(url).ok()?;
url.set_fragment(None);
let params: Vec<_> = url
@@ -44,126 +38,17 @@ fn source(engine: SearchEngine, url: &str, title: &str, text: &str) -> Option Result, Msg> {
- use quick_xml::events::Event;
- let mut reader = quick_xml::Reader::from_str(body);
- reader.config_mut().trim_text(false);
- let mut items = Vec::new();
- let mut in_item = false;
- let mut rss = false;
- let (mut tag, mut title, mut url, mut text, mut date) = (
- Vec::new(),
- String::new(),
- String::new(),
- String::new(),
- String::new(),
- );
- loop {
- match reader.read_event() {
- Ok(Event::Start(e)) => {
- tag = e.name().as_ref().as_bytes().to_vec();
- if tag == b"rss" {
- rss = true;
- }
- if tag == b"item" {
- in_item = true;
- title.clear();
- url.clear();
- text.clear();
- date.clear();
- }
- }
- Ok(event @ (Event::Text(_) | Event::CData(_) | Event::GeneralRef(_))) if in_item => {
- let value = match &event {
- Event::Text(e) => e.as_ref().to_string(),
- Event::CData(e) => e.as_ref().to_string(),
- Event::GeneralRef(e) => {
- quick_xml::escape::unescape(&format!("&{};", e.as_ref()))
- .map_err(|_| error("invalid search XML entity"))?
- .into_owned()
- }
- _ => unreachable!(),
- };
- match tag.as_slice() {
- b"title" => title.push_str(&value),
- b"link" => url.push_str(&value),
- b"description" => text.push_str(&value),
- b"pubDate" => date.push_str(&value),
- _ => {}
- }
- }
- Ok(Event::End(e)) if e.name().as_ref() == "item" => {
- if let Some(mut s) = source(
- SearchEngine::Bing,
- &url,
- &title,
- &clean(
- &Html::parse_fragment(&text)
- .root_element()
- .text()
- .collect::>()
- .join(" "),
- ),
- ) {
- if !date.is_empty() {
- s.published_at = Some(date.clone());
- }
- items.push(s);
- }
- in_item = false;
- }
- Ok(Event::Eof) => break,
- Err(_) => return Err(error("invalid search XML")),
- _ => {}
- }
- }
- if !rss {
- return Err(error("search engine did not return RSS results"));
- }
- Ok(items)
-}
-
-fn duckduckgo(body: &str) -> Result, Msg> {
- let doc = Html::parse_document(body);
- let mut result = Vec::new();
- for row in doc.select(&selector(".result")) {
- let Some(link) = row.select(&selector(".result__a")).next() else {
- continue;
- };
- let Some(href) = link.value().attr("href") else {
- continue;
- };
- let absolute = if href.starts_with("//") {
- format!("https:{href}")
- } else {
- href.to_string()
- };
- let Ok(url) = reqwest::Url::parse(&absolute) else {
- continue;
- };
- let destination = url
- .query_pairs()
- .find(|(k, _)| k == "uddg")
- .map(|(_, v)| v.into_owned())
- .unwrap_or(absolute);
- let title = link.text().collect::>().join(" ");
- let text = row
- .select(&selector(".result__snippet"))
- .next()
- .map(|n| n.text().collect::>().join(" "))
- .unwrap_or_default();
- if let Some(s) = source(SearchEngine::Duckduckgo, &destination, &title, &text) {
- result.push(s);
- }
- }
- if result.is_empty() && !body.contains("no-results") {
- return Err(error("search page layout was not recognized"));
- }
- Ok(result)
-}
-
-pub(super) fn page(body: &str) -> (String, String) {
+pub(super) fn page(body: &str) -> Result<(String, String), Msg> {
let mut doc = Html::parse_document(body);
+ if doc
+ .select(&selector(
+ "#challenge-form,#cf-challenge-running,#challenge-running",
+ ))
+ .next()
+ .is_some()
+ {
+ return Err(error("page requires a verification challenge"));
+ }
let remove: Vec<_> = doc
.select(&selector(
"script,style,noscript,nav,footer,header,aside,form,svg,template",
@@ -181,16 +66,66 @@ pub(super) fn page(body: &str) -> (String, String) {
.map(|n| clean(&n.text().collect::>().join(" ")))
.unwrap_or_default();
let root = doc
- .select(&selector("article,main,[role=main]"))
+ .select(&selector("main,[role=main]"))
.next()
+ .or_else(|| doc.select(&selector("article")).next())
+ .or_else(|| doc.select(&selector("body")).next())
.unwrap_or_else(|| doc.root_element());
- let text = root
- .text()
+ // Text nodes are not paragraphs: inline , and elements must
+ // not split a sentence into unrelated lines. Use an explicit stack so a
+ // deeply nested public page cannot overflow the call stack.
+ let mut text = String::new();
+ let mut stack: Vec<_> = root.children().rev().map(|node| (node, false)).collect();
+ while let Some((node, closing)) = stack.pop() {
+ if let Some(element) = node.value().as_element() {
+ let block = matches!(
+ element.name(),
+ "p" | "div"
+ | "section"
+ | "article"
+ | "main"
+ | "li"
+ | "ul"
+ | "ol"
+ | "pre"
+ | "blockquote"
+ | "h1"
+ | "h2"
+ | "h3"
+ | "h4"
+ | "h5"
+ | "h6"
+ | "br"
+ | "tr"
+ | "dt"
+ | "dd"
+ );
+ if block && !text.ends_with('\n') {
+ text.push('\n');
+ }
+ if closing && matches!(element.name(), "td" | "th") {
+ text.push(' ');
+ }
+ if !closing {
+ stack.push((node, true));
+ stack.extend(node.children().rev().map(|child| (child, false)));
+ }
+ } else if let Some(value) = node.value().as_text() {
+ text.push_str(value);
+ }
+ }
+ let text = text
+ .lines()
.map(clean)
.filter(|s| !s.is_empty())
.collect::>()
.join("\n");
- (title.chars().take(500).collect(), text)
+ if text.is_empty() {
+ return Err(error(
+ "page contained no readable text; it may require JavaScript",
+ ));
+ }
+ Ok((title.chars().take(500).collect(), text))
}
pub(super) fn excerpt(text: &str, query: &str, limit: usize) -> String {
@@ -199,19 +134,20 @@ pub(super) fn excerpt(text: &str, query: &str, limit: usize) -> String {
.filter(|s| s.chars().count() > 2)
.map(str::to_lowercase)
.collect();
- let mut lines: Vec<_> = text.lines().enumerate().collect();
- lines.sort_by_key(|(i, line)| {
+ let lines: Vec<_> = text.lines().collect();
+ let score = |line: &str| {
let l = line.to_lowercase();
- (
- std::cmp::Reverse(words.iter().filter(|w| l.contains(w.as_str())).count()),
- *i,
- )
- });
- lines
+ words.iter().filter(|w| l.contains(w.as_str())).count()
+ };
+ let best = lines
.iter()
- .take(12)
- .map(|(_, s)| *s)
- .collect::>()
+ .enumerate()
+ .max_by_key(|(i, line)| (score(line), std::cmp::Reverse(*i)))
+ .map(|(i, _)| i)
+ .unwrap_or(0);
+ // Preserve source order and surrounding context instead of sorting sentences
+ // by keyword count, which can change what the page actually says.
+ lines[best.saturating_sub(1)..lines.len().min(best + 11)]
.join("\n")
.chars()
.take(limit)
@@ -244,38 +180,52 @@ pub(super) fn find(text: &str, pattern: &str, limit: usize) -> String {
mod tests {
use super::*;
#[test]
- fn parses_search_sources_and_rejects_challenges() {
- let r = search(SearchEngine::Bing, "- Releasehttps://example.org/?utm_source=xVersion 3
").unwrap();
- assert_eq!(r[0].url, "https://example.org/");
- let html = r#""#;
- assert_eq!(
- search(SearchEngine::Duckduckgo, html).unwrap()[0].text,
- "Version 3"
- );
- assert!(search(SearchEngine::Duckduckgo, "
]]>"#).unwrap();
- assert_eq!(result[0].title, "Rust & Web 中文");
- assert_eq!(result[0].url, "https://example.org/?a=1&b=2");
- assert_eq!(result[0].text, "A & B");
+ fn inline_markup_does_not_break_sentences_or_phrase_searches() {
+ let (_, text) = page(
+ "DocUse asyncio.TaskGroup to await all tasks.
Errors cancel the remaining tasks.
",
+ ).unwrap();
+ assert_eq!(
+ text,
+ "Use asyncio.TaskGroup to await all tasks.\nErrors cancel the remaining tasks."
+ );
+ assert!(find(&text, "await all tasks", 1000).contains("Use asyncio.TaskGroup"));
+ }
+
+ #[test]
+ fn excerpts_keep_context_in_the_original_order() {
+ let text = "Introduction\nDo not use this on older versions.\nTaskGroup first appeared in Python 3.11.\nIn earlier versions, use gather.\nTaskGroup handles child tasks together.";
+ let extracted = excerpt(text, "TaskGroup Python", 1000);
+ assert!(extracted.starts_with("Do not use this on older versions."));
+ assert!(
+ extracted.find("first appeared").unwrap()
+ < extracted.find("In earlier versions").unwrap()
+ );
+ assert!(
+ extracted.find("In earlier versions").unwrap()
+ < extracted.find("handles child").unwrap()
+ );
+ assert_eq!(excerpt("", "anything", 100), "");
+ }
+
+ #[test]
+ fn empty_pages_and_verification_are_not_returned_as_content() {
+ assert!(page("").is_err());
+ assert!(page("").is_err());
+ // Cloudflare also injects scripts into successfully loaded normal pages.
+ assert!(page("Article text").is_ok());
+ let (_, text) = page("Advertisement").unwrap();
+ assert_eq!(text, "Version 3.11");
}
}
diff --git a/crates/tw-gateway/src/web_tools/mod.rs b/crates/tw-gateway/src/web_tools/mod.rs
index 379545e2..2430ba72 100644
--- a/crates/tw-gateway/src/web_tools/mod.rs
+++ b/crates/tw-gateway/src/web_tools/mod.rs
@@ -1,38 +1,107 @@
//! Search and public-page tools, shared by the gateway and the control-plane tester.
+mod cache;
mod collect;
mod history;
-mod locale;
pub(crate) mod protocol;
+mod providers;
mod public;
+mod quality;
+mod session;
pub(crate) mod wire;
use std::collections::HashMap;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{Mutex as AsyncMutex, Semaphore};
use tw_api::{SearchEngine, WebToolRequest, WebToolResult, WebToolsConfig};
use tw_types::{Msg, msg};
-type PageCached = Arc>>;
-
-type Cached = Arc>>;
-
pub struct WebTools {
pub(crate) history: Arc,
config: WebToolsConfig,
client: public::ClientSettings,
slots: Semaphore,
- cache: Mutex>,
- pages: Mutex>,
+ cache: cache::Cache,
+ pages: cache::Cache<(String, String)>,
engines: AsyncMutex>,
+ sessions: session::Pool,
}
#[derive(Default)]
struct EngineState {
next: Option,
failed_until: Option,
+ last_failure: Option,
+}
+
+/// Capabilities come from the same registry that executes requests. Desktop
+/// clients can offer new engines without hard-coding an exhaustive enum list.
+pub fn available_engines() -> Vec {
+ providers::AUTOMATIC
+ .iter()
+ .map(|&engine| {
+ let spec = providers::get(engine).spec();
+ tw_api::WebSearchEngine {
+ id: engine,
+ name: spec.name.into(),
+ min_interval_ms: spec.min_interval_ms,
+ }
+ })
+ .collect()
+}
+
+fn engine_name(engine: SearchEngine) -> &'static str {
+ providers::get(engine).spec().name
+}
+
+fn failure_detail(e: &Msg) -> &str {
+ if e.code == "gw.web_tools.failed" {
+ e.args.get("detail").map(String::as_str).unwrap_or(&e.text)
+ } else {
+ &e.text
+ }
+}
+
+fn http_failure(status: reqwest::StatusCode, headers: &http::HeaderMap) -> Msg {
+ let mut failure = error(format!("source returned HTTP {}", status.as_u16()));
+ failure
+ .args
+ .insert("http_status".into(), status.as_u16().to_string());
+ if matches!(status.as_u16(), 429 | 503)
+ && let Some(value) = headers.get("retry-after").and_then(|v| v.to_str().ok())
+ {
+ let seconds = value.trim().parse::().ok().or_else(|| {
+ chrono::DateTime::parse_from_rfc2822(value)
+ .ok()
+ .map(|at| (at.timestamp() - chrono::Utc::now().timestamp()).max(0) as u64)
+ });
+ if let Some(seconds) = seconds {
+ failure
+ .args
+ .insert("retry_after_secs".into(), seconds.to_string());
+ }
+ }
+ failure
+}
+
+impl EngineState {
+ fn cooling_down(&self, engine: SearchEngine) -> Msg {
+ let seconds = self
+ .failed_until
+ .map(|until| until.saturating_duration_since(Instant::now()).as_secs() + 1)
+ .unwrap_or(0);
+ let cause = self
+ .last_failure
+ .as_ref()
+ .map(failure_detail)
+ .unwrap_or("previous search failed");
+ error(format!(
+ "{} is cooling down; retry in {seconds}s. Last failure: {cause}",
+ engine_name(engine)
+ ))
+ }
}
pub(crate) fn error(detail: impl Into) -> Msg {
@@ -113,10 +182,7 @@ impl WebTools {
result: WebToolResult,
) {
let key = Self::cache_key(identity, request, &self.locale()).unwrap();
- self.cache.lock().unwrap().insert(
- key,
- Arc::new(AsyncMutex::new(Some((Instant::now(), result)))),
- );
+ *self.cache.get(key).unwrap().lock().await = Some((Instant::now(), result));
}
pub fn new(config: &tw_config::Config) -> Result {
Ok(Self {
@@ -124,9 +190,10 @@ impl WebTools {
config: config.web_tools.clone(),
client: public::ClientSettings::new(config)?,
slots: Semaphore::new(config.web_tools.max_concurrent as usize),
- cache: Mutex::new(HashMap::new()),
- pages: Mutex::new(HashMap::new()),
+ cache: cache::Cache::new(256),
+ pages: cache::Cache::new(32),
engines: AsyncMutex::new(HashMap::new()),
+ sessions: session::Pool::default(),
})
}
@@ -155,19 +222,7 @@ impl WebTools {
} else {
format!("{key}\n{}", serde_json::json!([allowed, blocked]))
};
- let entry = {
- let mut cache = self.cache.lock().unwrap_or_else(|e| e.into_inner());
- if cache.len() >= 256 {
- cache.retain(|_, e| Arc::strong_count(e) > 1);
- }
- if cache.len() >= 256 && !cache.contains_key(&key) {
- return Err(error("too many pending web operations"));
- }
- cache
- .entry(key)
- .or_insert_with(|| Arc::new(AsyncMutex::new(None)))
- .clone()
- };
+ let entry = self.cache.get(key)?;
// Holding this lock coalesces concurrent requests without leaving a background
// operation behind when the last caller is cancelled.
let mut cached = entry.lock().await;
@@ -229,7 +284,7 @@ impl WebTools {
public::validate_domains(allowed_domains)?;
public::validate_domains(blocked_domains)?;
let engines = if self.config.engines.is_empty() {
- vec![SearchEngine::Bing, SearchEngine::Duckduckgo]
+ providers::AUTOMATIC.to_vec()
} else {
self.config.engines.clone()
};
@@ -243,49 +298,102 @@ impl WebTools {
.saturating_sub(Duration::from_millis(20));
let budget = remaining
/ (engine_count - index + usize::from(self.config.read_pages > 0)) as u32;
+ let mut searching = false;
let attempt = tokio::time::timeout(budget, async {
if !self.reserve(engine).await {
return Ok(None);
}
- self.search(engine, query, allowed_domains, blocked_domains, locale)
- .await
- .map(Some)
+ searching = true;
+ self.search(
+ identity,
+ engine,
+ query,
+ allowed_domains,
+ blocked_domains,
+ locale,
+ )
+ .await
+ .map(Some)
})
.await
- .unwrap_or_else(|_| Err(error("search engine timed out")));
+ .unwrap_or_else(|_| {
+ Err(error(if searching {
+ "search engine timed out"
+ } else {
+ "search rate-limit wait timed out"
+ }))
+ });
match attempt {
- Ok(None) => continue,
+ Ok(None) => {
+ let engines = self.engines.lock().await;
+ if let Some(state) = engines.get(&engine) {
+ result.warnings.push(state.cooling_down(engine));
+ }
+ }
Ok(Some(sources)) => {
+ if let Some(state) = self.engines.lock().await.get_mut(&engine) {
+ state.failed_until = None;
+ state.last_failure = None;
+ }
succeeded = true;
for source in sources {
if !result.sources.iter().any(|s| s.url == source.url) {
result.sources.push(source);
}
}
- if result.sources.len() >= self.config.max_results as usize {
+ // Engine order is failover order. Filling a healthy
+ // engine's shorter result set from a worse fallback
+ // can reintroduce unrelated pages.
+ if !result.sources.is_empty() {
break;
}
}
Err(e) => {
- self.engines
- .lock()
- .await
- .entry(engine)
- .or_default()
- .failed_until = Some(Instant::now() + Duration::from_secs(60));
+ let query_mismatch = e.code == "gw.web_tools.query_mismatch";
+ let cooldown = e
+ .args
+ .get("retry_after_secs")
+ .and_then(|v| v.parse::().ok())
+ .and_then(|seconds| {
+ Instant::now().checked_add(Duration::from_secs(seconds.max(1)))
+ })
+ .unwrap_or_else(|| Instant::now() + Duration::from_secs(60));
+ let e =
+ error(format!("{}: {}", engine_name(engine), failure_detail(&e)));
+ // A poor match for one query does not make the engine
+ // unavailable for every other caller for a minute.
+ if searching && !query_mismatch {
+ let mut engines = self.engines.lock().await;
+ let state = engines.entry(engine).or_default();
+ state.failed_until = Some(cooldown);
+ state.last_failure = Some(e.clone());
+ }
result.warnings.push(e);
}
}
}
if !succeeded {
- return Err(error(
- "all configured search engines failed or are cooling down",
- ));
+ // Do not discard the engine errors: a challenge, a broken proxy,
+ // and a timeout need different remedies. Retries during cooldown
+ // retain the original cause and say when another attempt is due.
+ let reasons = result
+ .warnings
+ .iter()
+ .map(failure_detail)
+ .collect::>()
+ .join("; ");
+ return Err(error(format!(
+ "all configured search engines failed or are cooling down. {reasons}"
+ )));
}
result.sources.truncate(self.config.max_results as usize);
let mut read = 0;
for source in &mut result.sources {
- if read >= self.config.read_pages || source.text.chars().count() >= 300 {
+ if read >= self.config.read_pages
+ || source.text.chars().count() >= 300
+ || reqwest::Url::parse(&source.url)
+ .is_ok_and(|url| url.path().to_ascii_lowercase().ends_with(".pdf"))
+ {
continue;
}
read += 1;
@@ -303,7 +411,19 @@ impl WebTools {
.await
.unwrap_or_else(|_| Err(error("optional page reading timed out")))
{
- Ok((_, text)) => source.text = collect::excerpt(&text, query, 3000),
+ Ok((_, text)) => {
+ let excerpt = collect::excerpt(&text, query, 3000);
+ if !excerpt.is_empty() && !source.text.contains(&excerpt) {
+ // The engine snippet may be more focused than the
+ // landing page. Keep it instead of replacing it
+ // with navigation text or unrelated paragraphs.
+ source.text = format!("{}\n\n{excerpt}", source.text)
+ .trim()
+ .chars()
+ .take(4000)
+ .collect();
+ }
+ }
Err(e) => result.warnings.push(e),
}
}
@@ -344,62 +464,101 @@ impl WebTools {
}
async fn reserve(&self, engine: SearchEngine) -> bool {
- let delay = {
- let mut engines = self.engines.lock().await;
- let state = engines.entry(engine).or_default();
- let now = Instant::now();
- if state.failed_until.is_some_and(|until| until > now) {
- return false;
- }
- let at = state.next.unwrap_or(now).max(now);
- state.next = Some(at + Duration::from_millis(u64::from(self.config.interval_ms)));
- at.saturating_duration_since(now)
- };
- tokio::time::sleep(delay).await;
- true
+ loop {
+ let delay = {
+ let mut engines = self.engines.lock().await;
+ let state = engines.entry(engine).or_default();
+ let now = Instant::now();
+ if state.failed_until.is_some_and(|until| until > now) {
+ return false;
+ }
+ let at = state.next.unwrap_or(now);
+ if at <= now {
+ state.next = Some(
+ now + Duration::from_millis(u64::from(
+ self.config
+ .interval_ms
+ .max(providers::get(engine).spec().min_interval_ms),
+ )),
+ );
+ return true;
+ }
+ at.saturating_duration_since(now)
+ };
+ // Reserve only when a request can actually start. Cancelled waiters
+ // must not accumulate future slots and delay unrelated searches.
+ tokio::time::sleep(delay).await;
+ }
}
async fn search(
&self,
+ identity: &str,
engine: SearchEngine,
query: &str,
allowed: &[String],
blocked: &[String],
locale: &tw_api::WebSearchLocale,
) -> Result, Msg> {
- let mut q = query.to_string();
- if !allowed.is_empty() {
- q.push_str(&format!(
- " ({})",
- allowed
- .iter()
- .map(|d| format!("site:{d}"))
- .collect::>()
- .join(" OR ")
- ));
- } else if !self.config.allowed_domains.is_empty() {
- q.push_str(&format!(
- " ({})",
- self.config
- .allowed_domains
- .iter()
- .map(|d| format!("site:{d}"))
- .collect::>()
- .join(" OR ")
- ));
+ let provider = providers::get(engine);
+ let spec = provider.spec();
+ let session = self.sessions.get(identity, spec, locale)?;
+ let context = session::Request {
+ spec,
+ kind: providers::Kind::Document,
+ session: session.as_ref().map(|s| &s.session),
+ locale,
+ };
+ if let Some(session) = &session {
+ session
+ .session
+ .ready
+ .get_or_try_init(|| async {
+ let home = spec
+ .home
+ .expect("session requires a homepage")
+ .parse()
+ .unwrap();
+ let (_, body) = self.fetch(home, None, Some(context), None).await?;
+ provider.validate_home(&body)
+ })
+ .await?;
}
- let mut url = reqwest::Url::parse(match engine {
- SearchEngine::Bing => "https://www.bing.com/search",
- SearchEngine::Duckduckgo => "https://html.duckduckgo.com/html/",
+ let query_context = providers::Query {
+ text: query.trim(),
+ allowed: if allowed.is_empty() {
+ &self.config.allowed_domains
+ } else {
+ allowed
+ },
+ locale,
+ };
+ let mut requests = 0;
+ let mut sources = providers::run(provider, &query_context, |request| {
+ let retry = requests > 0;
+ requests += 1;
+ async move {
+ // The first reservation is made by the scheduler. Further exchange
+ // steps with a source minimum must obey that same engine-wide pace.
+ if retry && spec.min_interval_ms > 0 && !self.reserve(engine).await {
+ return Err(error("search engine entered cooldown"));
+ }
+ let context = session::Request {
+ kind: request.kind,
+ ..context
+ };
+ self.fetch(request.url, None, Some(context), request.form.as_deref())
+ .await
+ .map(|(_, body)| body)
+ }
})
- .expect("built-in URL");
- url.query_pairs_mut().append_pair("q", &q);
- if engine == SearchEngine::Bing {
- url.query_pairs_mut().append_pair("format", "rss");
+ .await?;
+ let had_results = !sources.is_empty();
+ let usable = quality::filter(query, &mut sources);
+ if had_results && !usable {
+ return Err(msg!("gw.web_tools.query_mismatch" =>
+ "The search engine returned results that did not match the query."));
}
- locale::apply(&mut url, engine, locale);
- let (_, body) = self.fetch(url, None).await?;
- let mut sources = collect::search(engine, &body)?;
sources.retain(|s| {
reqwest::Url::parse(&s.url).ok().is_some_and(|url| {
self.allowed(&url) && public::domains_allow(&url, allowed, blocked)
@@ -427,19 +586,7 @@ impl WebTools {
"{identity}\n{url}\n{}",
serde_json::json!([allowed, blocked])
);
- let entry = {
- let mut pages = self.pages.lock().unwrap_or_else(|e| e.into_inner());
- if pages.len() >= 32 {
- pages.retain(|_, p| Arc::strong_count(p) > 1);
- }
- if pages.len() >= 32 && !pages.contains_key(&key) {
- return Err(error("too many pending page reads"));
- }
- pages
- .entry(key)
- .or_insert_with(|| Arc::new(AsyncMutex::new(None)))
- .clone()
- };
+ let entry = self.pages.get(key)?;
let mut cache = entry.lock().await;
if let Some((at, page)) = &*cache
&& at.elapsed() < Duration::from_secs(u64::from(self.config.cache_secs))
@@ -463,9 +610,11 @@ impl WebTools {
if !self.allowed(&url) {
return Err(error("page is excluded by the configured domain rules"));
}
- let (kind, body) = self.fetch(url, Some((allowed, blocked))).await?;
+ let (kind, body) = self
+ .fetch(url, Some((allowed, blocked)), None, None)
+ .await?;
if kind.contains("html") {
- Ok(collect::page(&body))
+ collect::page(&body)
} else if kind.starts_with("text/") {
Ok((String::new(), body))
} else {
@@ -477,17 +626,36 @@ impl WebTools {
&self,
mut url: reqwest::Url,
filters: Option<(&[String], &[String])>,
+ search: Option>,
+ form: Option<&str>,
) -> Result<(String, String), Msg> {
use futures::StreamExt;
for _ in 0..6 {
- let addresses = public::check(&url).await?;
if let Some((allowed, blocked)) = filters
&& (!self.allowed(&url) || !public::domains_allow(&url, allowed, blocked))
{
return Err(error("redirect is excluded by the configured domain rules"));
}
- let response = self.client.get(&url, &addresses).await?;
+ let response = match search {
+ Some(context) => {
+ let response = self
+ .client
+ .get_search(context.spec, &url, context.headers(&url)?, form)
+ .await?;
+ context.remember(&url, response.headers())?;
+ response
+ }
+ None => {
+ let addresses = public::check(&url).await?;
+ self.client.get(&url, &addresses).await?
+ }
+ };
if response.status().is_redirection() {
+ // Form queries never follow redirects: do not replay their body
+ // into a login, verification, or different document endpoint.
+ if form.is_some() {
+ return Err(http_failure(response.status(), response.headers()));
+ }
let to = response
.headers()
.get("location")
@@ -501,10 +669,7 @@ impl WebTools {
continue;
}
if !response.status().is_success() {
- return Err(error(format!(
- "source returned HTTP {}",
- response.status().as_u16()
- )));
+ return Err(http_failure(response.status(), response.headers()));
}
let kind = response
.headers()
@@ -512,6 +677,10 @@ impl WebTools {
.and_then(|v| v.to_str().ok())
.unwrap_or("application/octet-stream")
.to_lowercase();
+ // Reject binary documents before trying to decode them as UTF-8.
+ if search.is_none() && !kind.contains("html") && !kind.starts_with("text/") {
+ return Err(error("page must be HTML or plain text"));
+ }
let mut bytes = Vec::new();
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
@@ -532,11 +701,358 @@ impl WebTools {
#[cfg(test)]
mod tests {
use super::*;
+
+ #[test]
+ fn rate_limits_keep_server_retry_after_seconds_and_dates() {
+ let mut headers = http::HeaderMap::new();
+ headers.insert("retry-after", "120".parse().unwrap());
+ assert_eq!(
+ http_failure(reqwest::StatusCode::TOO_MANY_REQUESTS, &headers).arg("retry_after_secs"),
+ "120"
+ );
+ let future = chrono::Utc::now() + chrono::Duration::minutes(2);
+ headers.insert("retry-after", future.to_rfc2822().parse().unwrap());
+ let seconds: u64 = http_failure(reqwest::StatusCode::SERVICE_UNAVAILABLE, &headers)
+ .arg("retry_after_secs")
+ .parse()
+ .unwrap();
+ assert!((118..=120).contains(&seconds));
+ headers.insert("retry-after", "invalid".parse().unwrap());
+ assert!(
+ http_failure(reqwest::StatusCode::TOO_MANY_REQUESTS, &headers)
+ .arg("retry_after_secs")
+ .is_empty()
+ );
+ }
+
+ #[tokio::test]
+ #[ignore = "live traditional-engine relevance, filters, and concurrent cache checks"]
+ async fn live_direct_search_matrix() {
+ live_engine_matrix(&[SearchEngine::Duckduckgo]).await;
+ }
+
+ #[tokio::test]
+ #[ignore = "live Bing relevance with a configured Chinese locale, filters, and cache"]
+ async fn live_bing_search_matrix() {
+ live_engine_matrix(&[SearchEngine::Bing]).await;
+ }
+
+ #[tokio::test]
+ #[ignore = "live Bing-first search with DuckDuckGo failover, filters, and cache"]
+ async fn live_bing_failover_matrix() {
+ live_engine_matrix(&[SearchEngine::Bing, SearchEngine::Duckduckgo]).await;
+ }
+
+ #[tokio::test]
+ #[ignore = "live Google Docs relevance, domain filters, and concurrent cache checks"]
+ async fn live_google_docs_search_matrix() {
+ live_engine_matrix(&[SearchEngine::GoogleDocs]).await;
+ }
+
+ #[tokio::test]
+ #[ignore = "live automatic search with Google Docs and traditional-engine failover"]
+ async fn live_automatic_search_matrix() {
+ live_engine_matrix(providers::AUTOMATIC).await;
+ }
+
+ async fn live_engine_matrix(engines: &[SearchEngine]) {
+ let mut config = tw_config::Config::default();
+ config.web_tools.engines = engines.to_vec();
+ config.web_tools.language = "zh-Hans".into();
+ config.web_tools.region = "CN".into();
+ config.web_tools.read_pages = 0;
+ let tools = WebTools::new(&config).unwrap();
+ let request = |query: &str| WebToolRequest::Search {
+ query: query.into(),
+ allowed_domains: vec![],
+ blocked_domains: vec![],
+ };
+ let mut failures = Vec::new();
+ for (query, domain, path) in [
+ (
+ "Python asyncio TaskGroup documentation",
+ "python.org",
+ "asyncio-task",
+ ),
+ ("Rust OnceLock documentation", "rust-lang.org", "OnceLock"),
+ ("SQLite release notes", "sqlite.org", "changes"),
+ ("劳动合同法 第三十八条", "gov.cn", ""),
+ (
+ "2025 Nobel physics prize winners",
+ "nobelprize.org",
+ "physics/2025",
+ ),
+ (
+ "Apple Mac mini M4 technical specifications",
+ "apple.com",
+ "121555",
+ ),
+ ("中华人民共和国民法典 第一千零六十二条", "gov.cn", ""),
+ ("OpenAI Responses API reference", "openai.com", "responses"),
+ ] {
+ let result = match tools.execute("matrix", &request(query)).await {
+ Ok(result) => result,
+ Err(error) => {
+ failures.push(format!("{query}: {error}"));
+ continue;
+ }
+ };
+ eprintln!(
+ "{query}: {} sources, {} ms, {:?}, {:?}",
+ result.sources.len(),
+ result.elapsed_ms,
+ result.sources.first().and_then(|s| s.engine),
+ result
+ .sources
+ .iter()
+ .take(3)
+ .map(|s| &s.url)
+ .collect::>()
+ );
+ if engines.len() == 1 {
+ assert!(result.warnings.is_empty(), "{query}: {:?}", result.warnings);
+ }
+ if !result.sources.iter().any(|source| {
+ reqwest::Url::parse(&source.url).is_ok_and(|url| {
+ public::domains_allow(&url, &[domain.into()], &[]) && url.path().contains(path)
+ })
+ }) {
+ failures.push(format!("{query}: missing {domain}/{path}"));
+ }
+ assert!(
+ result
+ .sources
+ .iter()
+ .all(|s| s.engine.is_some_and(|e| engines.contains(&e)))
+ );
+ }
+ assert!(
+ failures.is_empty(),
+ "{engines:?} failed relevance checks: {failures:?}"
+ );
+ let req = request("PostgreSQL transaction isolation documentation");
+ let results =
+ futures::future::join_all((0..4).map(|_| tools.execute("matrix", &req))).await;
+ let results: Vec<_> = results.into_iter().collect::>().unwrap();
+ assert_eq!(results.iter().filter(|r| r.cached).count(), 3);
+ assert!(results.iter().all(|r| !r.sources.is_empty()));
+ for (allowed_domains, blocked_domains) in [
+ (vec!["sqlite.org".into()], vec![]),
+ (vec![], vec!["sqlite.org".into()]),
+ ] {
+ let result = tools
+ .execute(
+ "matrix",
+ &WebToolRequest::Search {
+ query: "SQLite release notes".into(),
+ allowed_domains: allowed_domains.clone(),
+ blocked_domains: blocked_domains.clone(),
+ },
+ )
+ .await
+ .unwrap();
+ assert!(!result.sources.is_empty());
+ assert!(result.sources.iter().all(|s| public::domains_allow(
+ &reqwest::Url::parse(&s.url).unwrap(),
+ &allowed_domains,
+ &blocked_domains
+ )));
+ }
+ }
+
+ #[tokio::test]
+ async fn source_minimum_interval_and_capabilities_share_the_registry() {
+ let mut config = tw_config::Config::default();
+ config.web_tools.interval_ms = 250;
+ let tools = WebTools::new(&config).unwrap();
+ assert!(tools.reserve(SearchEngine::GoogleDocs).await);
+ let next = tools.engines.lock().await[&SearchEngine::GoogleDocs]
+ .next
+ .unwrap();
+ assert!(next.saturating_duration_since(Instant::now()) > Duration::from_millis(5900));
+ assert!(
+ tokio::time::timeout(
+ Duration::from_millis(10),
+ tools.reserve(SearchEngine::GoogleDocs)
+ )
+ .await
+ .is_err()
+ );
+ assert_eq!(
+ tools.engines.lock().await[&SearchEngine::GoogleDocs].next,
+ Some(next)
+ );
+ let available = available_engines();
+ assert_eq!(available[0].id, SearchEngine::GoogleDocs);
+ assert_eq!(available[0].min_interval_ms, 6000);
+ }
+
+ #[tokio::test]
+ async fn cancelled_waiters_do_not_reserve_more_engine_slots() {
+ let tools = WebTools::new(&tw_config::Config::default()).unwrap();
+ assert!(tools.reserve(SearchEngine::Bing).await);
+ let next = tools.engines.lock().await[&SearchEngine::Bing].next;
+ for _ in 0..3 {
+ assert!(
+ tokio::time::timeout(Duration::from_millis(1), tools.reserve(SearchEngine::Bing))
+ .await
+ .is_err()
+ );
+ assert_eq!(tools.engines.lock().await[&SearchEngine::Bing].next, next);
+ }
+ }
+
+ #[tokio::test]
+ async fn rate_limit_wait_timeout_does_not_mark_the_engine_unavailable() {
+ let mut config = tw_config::Config::default();
+ config.web_tools.timeout_secs = 1;
+ config.web_tools.read_pages = 0;
+ config.web_tools.engines = vec![SearchEngine::Bing];
+ let tools = WebTools::new(&config).unwrap();
+ tools.engines.lock().await.insert(
+ SearchEngine::Bing,
+ EngineState {
+ next: Some(Instant::now() + Duration::from_secs(60)),
+ ..Default::default()
+ },
+ );
+ let failure = tools
+ .execute(
+ "test",
+ &WebToolRequest::Search {
+ query: "SQLite release notes".into(),
+ allowed_domains: vec![],
+ blocked_domains: vec![],
+ },
+ )
+ .await
+ .unwrap_err();
+ assert!(failure.text.contains("rate-limit wait"), "{failure}");
+ assert!(
+ tools.engines.lock().await[&SearchEngine::Bing]
+ .failed_until
+ .is_none()
+ );
+ }
+
+ #[tokio::test]
+ async fn unavailable_engines_keep_their_causes_and_retry_deadline() {
+ let mut config = tw_config::Config::default();
+ config.web_tools.engines = vec![SearchEngine::Bing, SearchEngine::Duckduckgo];
+ let tools = WebTools::new(&config).unwrap();
+ let until = Instant::now() + Duration::from_secs(60);
+ for (engine, reason) in [
+ (SearchEngine::Bing, "source returned HTTP 429"),
+ (
+ SearchEngine::Duckduckgo,
+ "search engine requires a verification challenge",
+ ),
+ ] {
+ tools.engines.lock().await.insert(
+ engine,
+ EngineState {
+ next: None,
+ failed_until: Some(until),
+ last_failure: Some(error(format!("{}: {reason}", engine_name(engine)))),
+ },
+ );
+ }
+ for query in ["SQLite release notes", "Rust documentation"] {
+ let failure = tools
+ .execute(
+ "test",
+ &WebToolRequest::Search {
+ query: query.into(),
+ allowed_domains: vec![],
+ blocked_domains: vec![],
+ },
+ )
+ .await
+ .unwrap_err();
+ assert!(failure.text.contains("Bing"), "{failure}");
+ assert!(failure.text.contains("429"), "{failure}");
+ assert!(failure.text.contains("DuckDuckGo"), "{failure}");
+ assert!(failure.text.contains("verification challenge"), "{failure}");
+ assert!(failure.text.contains("retry in"), "{failure}");
+ assert_eq!(failure.text.matches("Web tools:").count(), 1, "{failure}");
+ for state in tools.engines.lock().await.values() {
+ assert_eq!(
+ state.failed_until,
+ Some(until),
+ "retry must not extend cooldown"
+ );
+ }
+ }
+ }
+
+ #[tokio::test]
+ #[ignore = "uses the public DuckDuckGo search engine"]
+ async fn live_search_duckduckgo() {
+ live_duckduckgo_after_cooldown(false).await;
+ }
+
+ #[tokio::test]
+ #[ignore = "live DuckDuckGo fallback while Google Docs is cooling down"]
+ async fn live_search_google_cooldown_falls_back() {
+ live_duckduckgo_after_cooldown(true).await;
+ }
+
+ async fn live_duckduckgo_after_cooldown(google_failed: bool) {
+ let mut config = tw_config::Config::default();
+ config.web_tools.engines = vec![SearchEngine::Duckduckgo];
+ if google_failed {
+ config.web_tools.engines.insert(0, SearchEngine::GoogleDocs);
+ }
+ config.web_tools.read_pages = 0;
+ let tools = WebTools::new(&config).unwrap();
+ if google_failed {
+ tools.engines.lock().await.insert(
+ SearchEngine::GoogleDocs,
+ EngineState {
+ failed_until: Some(Instant::now() + Duration::from_secs(60)),
+ last_failure: Some(error("Google (Docs): source returned HTTP 500")),
+ ..Default::default()
+ },
+ );
+ }
+ let result = tools
+ .execute(
+ "live-test",
+ &WebToolRequest::Search {
+ query: "SQLite release notes".into(),
+ allowed_domains: vec![],
+ blocked_domains: vec![],
+ },
+ )
+ .await
+ .unwrap();
+ if google_failed {
+ assert!(result.warnings.iter().any(|w| w.text.contains("500")));
+ }
+ assert!(
+ result.sources.iter().any(|s| {
+ reqwest::Url::parse(&s.url).is_ok_and(|url| {
+ url.host_str()
+ .is_some_and(|host| host == "sqlite.org" || host.ends_with(".sqlite.org"))
+ && url.path().contains("changes")
+ })
+ }),
+ "release notes query must find SQLite's release history: {:?}",
+ result.sources
+ );
+ assert!(
+ result
+ .sources
+ .iter()
+ .all(|s| s.engine == Some(SearchEngine::Duckduckgo))
+ );
+ }
+
#[tokio::test]
#[ignore = "uses public search engines and public webpages"]
async fn live_search_read_and_find() {
let mut config = tw_config::Config::default();
- config.web_tools.engines = vec![SearchEngine::Bing];
+ config.web_tools.engines = vec![SearchEngine::Duckduckgo];
config.web_tools.read_pages = 0;
let tools = WebTools::new(&config).unwrap();
let search = tools
@@ -550,7 +1066,17 @@ mod tests {
)
.await
.unwrap();
- assert!(!search.sources.is_empty());
+ assert!(
+ search.sources.iter().any(|s| {
+ reqwest::Url::parse(&s.url).is_ok_and(|url| {
+ url.host_str()
+ .is_some_and(|host| host == "sqlite.org" || host.ends_with(".sqlite.org"))
+ && url.path().contains("changes")
+ })
+ }),
+ "release notes query must find SQLite's release history: {:?}",
+ search.sources
+ );
let page = tools
.execute(
"live-test",
diff --git a/crates/tw-gateway/src/web_tools/providers/bing.rs b/crates/tw-gateway/src/web_tools/providers/bing.rs
new file mode 100644
index 00000000..c22c042d
--- /dev/null
+++ b/crates/tw-gateway/src/web_tools/providers/bing.rs
@@ -0,0 +1,282 @@
+use super::super::{collect::source, error};
+use scraper::{Html, Selector};
+use std::sync::LazyLock;
+use tw_api::{SearchEngine, WebSource};
+use tw_types::Msg;
+
+static CHALLENGE: LazyLock = LazyLock::new(|| {
+ Selector::parse("#challenge-form, #anomaly-form, #b_captcha, .b_captcha, #turnstile-widget")
+ .unwrap()
+});
+static ROWS: LazyLock = LazyLock::new(|| Selector::parse("#b_results .b_algo").unwrap());
+static LINK: LazyLock = LazyLock::new(|| Selector::parse("h2 a[href]").unwrap());
+static SNIPPET: LazyLock =
+ LazyLock::new(|| Selector::parse(".b_caption p, .b_snippet").unwrap());
+static EMPTY: LazyLock = LazyLock::new(|| Selector::parse("#b_results .b_no").unwrap());
+fn clean(s: &str) -> String {
+ s.split_whitespace().collect::>().join(" ")
+}
+
+fn verification(doc: &Html) -> Result<(), Msg> {
+ if doc.select(&CHALLENGE).next().is_some() {
+ Err(error("search engine requires a verification challenge"))
+ } else {
+ Ok(())
+ }
+}
+
+fn html_results(body: &str) -> Result, Msg> {
+ use base64::Engine;
+ let doc = Html::parse_document(body);
+ verification(&doc)?;
+ let mut sources = Vec::new();
+ let base = reqwest::Url::parse(URL).unwrap();
+ for row in doc.select(&ROWS) {
+ let Some(link) = row.select(&LINK).next() else {
+ continue;
+ };
+ let Some(href) = link.attr("href") else {
+ continue;
+ };
+ let Ok(url) = base.join(href) else { continue };
+ let destination = if matches!(
+ url.host_str(),
+ Some("www.bing.com" | "cn.bing.com" | "bing.com")
+ ) {
+ if !matches!(url.path(), "/ck/a" | "/ck/a/") {
+ continue;
+ }
+ let Some(encoded) = url
+ .query_pairs()
+ .find_map(|(key, value)| (key == "u").then(|| value.into_owned()))
+ else {
+ continue;
+ };
+ let Some(encoded) = encoded.strip_prefix("a1") else {
+ continue;
+ };
+ let Ok(bytes) = base64::engine::general_purpose::URL_SAFE_NO_PAD
+ .decode(encoded.trim_end_matches('='))
+ else {
+ continue;
+ };
+ let Ok(destination) = String::from_utf8(bytes) else {
+ continue;
+ };
+ destination
+ } else {
+ url.to_string()
+ };
+ let title = link.text().collect::();
+ let snippet = row
+ .select(&SNIPPET)
+ .next()
+ .map(|node| node.text().collect::())
+ .unwrap_or_default();
+ if let Some(source) = source(SearchEngine::Bing, &destination, &title, &snippet) {
+ sources.push(source);
+ }
+ }
+ if sources.is_empty() && doc.select(&EMPTY).next().is_none() {
+ return Err(error("Bing search page layout was not recognized"));
+ }
+ Ok(sources)
+}
+
+fn rss(body: &str) -> Result, Msg> {
+ use quick_xml::events::Event;
+ let mut reader = quick_xml::Reader::from_str(body);
+ reader.config_mut().trim_text(false);
+ let mut items = Vec::new();
+ let mut in_item = false;
+ let mut rss = false;
+ let (mut tag, mut title, mut url, mut text) =
+ (Vec::new(), String::new(), String::new(), String::new());
+ loop {
+ match reader.read_event() {
+ Ok(Event::Start(e)) => {
+ tag = e.name().as_ref().as_bytes().to_vec();
+ if tag == b"rss" {
+ rss = true;
+ }
+ if tag == b"item" {
+ in_item = true;
+ title.clear();
+ url.clear();
+ text.clear();
+ }
+ }
+ Ok(event @ (Event::Text(_) | Event::CData(_) | Event::GeneralRef(_))) if in_item => {
+ let value = match &event {
+ Event::Text(e) => e.as_ref().to_string(),
+ Event::CData(e) => e.as_ref().to_string(),
+ Event::GeneralRef(e) => {
+ quick_xml::escape::unescape(&format!("&{};", e.as_ref()))
+ .map_err(|_| error("invalid search XML entity"))?
+ .into_owned()
+ }
+ _ => unreachable!(),
+ };
+ match tag.as_slice() {
+ b"title" => title.push_str(&value),
+ b"link" => url.push_str(&value),
+ b"description" => text.push_str(&value),
+ _ => {}
+ }
+ }
+ Ok(Event::End(e)) if e.name().as_ref() == "item" => {
+ if let Some(s) = source(
+ SearchEngine::Bing,
+ &url,
+ &title,
+ &clean(
+ &Html::parse_fragment(&text)
+ .root_element()
+ .text()
+ .collect::>()
+ .join(" "),
+ ),
+ ) {
+ // Bing RSS dates describe the search item, not a verified
+ // publication date of the linked page. Do not invent recency.
+ items.push(s);
+ }
+ in_item = false;
+ }
+ Ok(Event::Eof) => break,
+ Err(_) => return Err(error("invalid search XML")),
+ _ => {}
+ }
+ }
+ if !rss {
+ return Err(error("search engine did not return RSS results"));
+ }
+ Ok(items)
+}
+
+fn search(body: &str) -> Result, Msg> {
+ if body.trim_start().starts_with(" &'static Spec {
+ &SPEC
+ }
+ fn start(&self, query: &Query<'_>) -> Result {
+ let mut q = query.text.to_owned();
+ if !query.allowed.is_empty() {
+ q.push_str(&format!(
+ " ({})",
+ query
+ .allowed
+ .iter()
+ .map(|d| format!("site:{d}"))
+ .collect::>()
+ .join(" OR ")
+ ));
+ }
+ // Language is supplied by the browser headers. Forced cc/setlang URL
+ // parameters can select a regional backend that broadens the query.
+ let mut url = reqwest::Url::parse(URL).unwrap();
+ url.query_pairs_mut().append_pair("q", &q);
+ Ok(Request {
+ url,
+ kind: Kind::Document,
+ form: None,
+ })
+ }
+ fn parse(&self, _: &Request, body: &str, _: &Query<'_>) -> Result {
+ search(body).map(Step::Done)
+ }
+ fn validate_home(&self, body: &str) -> Result<(), Msg> {
+ verification(&Html::parse_document(body))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn bing_web_results_exclude_ads_and_decode_only_known_tracking_links() {
+ use base64::Engine;
+ let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD
+ .encode("https://docs.example.org/guide?a=1&b=2#section");
+ let body = format!(
+ r#"
+
+ Use TaskGroup & await tasks.
+ Plain snippet
+
+
+
"#
+ );
+ let rows = search(&body).unwrap();
+ assert_eq!(rows.len(), 2);
+ assert_eq!(rows[0].url, "https://docs.example.org/guide?a=1&b=2");
+ assert_eq!(rows[0].title, "The official guide");
+ assert_eq!(rows[0].text, "Use TaskGroup & await tasks.");
+ assert!(rows[0].published_at.is_none());
+ assert_eq!(rows[1].text, "Plain snippet");
+ assert!(
+ search(r#"- No results
"#)
+ .unwrap()
+ .is_empty()
+ );
+ assert!(search(r#"Verify
"#).is_err());
+ assert!(
+ search(r#""#)
+ .unwrap_err()
+ .text
+ .contains("verification")
+ );
+ // Merely mentioning a challenge in scripts or an article is not one.
+ assert!(
+ verification(&Html::parse_document(
+ r#""#
+ ))
+ .is_ok()
+ );
+ assert!(search("Unexpected layout").is_err());
+ }
+}
+#[cfg(test)]
+mod entity_tests {
+ use super::*;
+ #[test]
+ fn rss_preserves_entities_cdata_and_query_parameters() {
+ let result = search( r#"- Rust & Web 中文https://example.org/?a=1&b=2A & B]]>
"#).unwrap();
+ assert_eq!(result[0].title, "Rust & Web 中文");
+ assert_eq!(result[0].url, "https://example.org/?a=1&b=2");
+ assert_eq!(result[0].text, "A & B");
+ }
+
+ #[test]
+ fn search_item_dates_are_not_page_publication_dates() {
+ let result = search( "- Old pagehttps://example.org/Mon, 16 Nov 2026 00:00:00 GMT
").unwrap();
+ assert!(result[0].published_at.is_none());
+ }
+}
diff --git a/crates/tw-gateway/src/web_tools/providers/duckduckgo.rs b/crates/tw-gateway/src/web_tools/providers/duckduckgo.rs
new file mode 100644
index 00000000..3214f3c8
--- /dev/null
+++ b/crates/tw-gateway/src/web_tools/providers/duckduckgo.rs
@@ -0,0 +1,257 @@
+//! Read the same first-party organic results used by DuckDuckGo's regular site.
+//! The HTML endpoint frequently challenges even when this normal flow works.
+//! Never execute returned JavaScript or consume ads / generated answer text.
+
+use scraper::{Html, Selector};
+use serde::Deserialize;
+use std::sync::LazyLock;
+use tw_api::{SearchEngine, WebSource};
+use tw_types::Msg;
+
+use super::super::{collect, error};
+use super::{Endpoint, Kind, Profile, Provider, Query, Request, Spec, Step};
+
+static CHALLENGE: LazyLock = LazyLock::new(|| {
+ Selector::parse("#challenge-form, #anomaly-form, script[src*='anomaly.js']").unwrap()
+});
+static PRELOAD: LazyLock = LazyLock::new(|| {
+ Selector::parse("script#deep_preload_script[src], link#deep_preload_link[href]").unwrap()
+});
+
+pub(super) fn is_data_url(url: &reqwest::Url) -> bool {
+ url.scheme() == "https"
+ && matches!(
+ url.host_str(),
+ Some("links.duckduckgo.com" | "duckduckgo.com")
+ )
+ && url.port_or_known_default() == Some(443)
+ && url.path() == "/d.js"
+ && url.username().is_empty()
+ && url.password().is_none()
+ && url.fragment().is_none()
+}
+
+pub(super) fn data_url(body: &str, query: &str) -> Result {
+ let doc = Html::parse_document(body);
+ if doc.select(&CHALLENGE).next().is_some() {
+ return Err(error("search engine requires a verification challenge"));
+ }
+ let link = doc
+ .select(&PRELOAD)
+ .next()
+ .and_then(|node| node.attr("src").or_else(|| node.attr("href")))
+ .ok_or_else(|| error("DuckDuckGo search page layout was not recognized"))?;
+ let base = reqwest::Url::parse(URL).unwrap();
+ let url = base
+ .join(link)
+ .map_err(|_| error("invalid search data URL"))?;
+ // This URL comes from untrusted HTML. Do not grant its host proxy remote DNS
+ // unless it is exactly a known first-party data endpoint for this query.
+ let queries: Vec<_> = url.query_pairs().filter(|(key, _)| key == "q").collect();
+ if !is_data_url(&url) || queries.len() != 1 || queries[0].1 != query {
+ return Err(error("DuckDuckGo returned an unexpected search data URL"));
+ }
+ Ok(url)
+}
+
+#[derive(Deserialize)]
+struct Row {
+ #[serde(default)]
+ u: Option,
+ #[serde(default)]
+ t: Option,
+ #[serde(default)]
+ a: Option,
+ #[serde(default)]
+ n: Option,
+}
+
+fn text(html: &str) -> String {
+ Html::parse_fragment(html).root_element().text().collect()
+}
+
+pub(super) fn results(body: &str) -> Result, Msg> {
+ // Only the 'd' (organic) array. The 'a' array and deepPayload can contain
+ // advertisements and AI answers. Parse JSON as data, never evaluate code.
+ let data = ["DDG.pageLayout.load('d',", "DDG.pageLayout.load(\"d\","]
+ .iter()
+ .find_map(|marker| body.split_once(marker).map(|(_, tail)| tail))
+ .ok_or_else(|| {
+ error(
+ if body.contains("isJsaChallenge") || body.contains("challenge-form") {
+ "search engine requires browser verification"
+ } else {
+ "DuckDuckGo did not return organic search results"
+ },
+ )
+ })?;
+ let mut stream = serde_json::Deserializer::from_str(data).into_iter::>();
+ let rows = stream
+ .next()
+ .ok_or_else(|| error("DuckDuckGo search results are missing"))?
+ .map_err(|_| error("invalid DuckDuckGo search results"))?;
+ if !data[stream.byte_offset()..].trim_start().starts_with(')') {
+ return Err(error("invalid DuckDuckGo search result wrapper"));
+ }
+ let mut sources = Vec::new();
+ let mut malformed = false;
+ for row in rows {
+ match (row.u, row.t) {
+ (Some(url), Some(title)) if !title.trim().is_empty() => {
+ if let Some(source) = collect::source(
+ SearchEngine::Duckduckgo,
+ &url,
+ &text(&title),
+ &text(&row.a.unwrap_or_default()),
+ ) {
+ sources.push(source);
+ } else {
+ malformed = true;
+ }
+ }
+ // A final pagination object is not a result; never follow it here.
+ (None, None) if row.n.is_some() => {}
+ _ => malformed = true,
+ }
+ }
+ if malformed && sources.is_empty() {
+ return Err(error("DuckDuckGo search result layout was not recognized"));
+ }
+ Ok(sources)
+}
+
+const URL: &str = "https://duckduckgo.com/";
+static SPEC: Spec = Spec {
+ id: SearchEngine::Duckduckgo,
+ name: "DuckDuckGo",
+ profile: Profile::Basic,
+ min_interval_ms: 0,
+ home: None,
+ endpoints: &[
+ Endpoint {
+ host: "duckduckgo.com",
+ paths: &["/", "/d.js"],
+ },
+ Endpoint {
+ host: "links.duckduckgo.com",
+ paths: &["/d.js"],
+ },
+ ],
+};
+pub(super) struct Duckduckgo;
+impl Provider for Duckduckgo {
+ fn spec(&self) -> &'static Spec {
+ &SPEC
+ }
+ fn start(&self, query: &Query<'_>) -> Result {
+ let mut url = reqwest::Url::parse(URL).unwrap();
+ url.query_pairs_mut()
+ .append_pair("q", query.text)
+ .append_pair("ia", "web");
+ if let Some(market) = super::locale::duckduckgo_market(query.locale) {
+ url.query_pairs_mut().append_pair("kl", market);
+ }
+ Ok(Request {
+ url,
+ kind: Kind::Document,
+ form: None,
+ })
+ }
+ fn parse(&self, request: &Request, body: &str, query: &Query<'_>) -> Result {
+ match request.kind {
+ Kind::Document => data_url(body, query.text).map(|url| {
+ Step::Fetch(Request {
+ url,
+ kind: Kind::Data,
+ form: None,
+ })
+ }),
+ Kind::Data => results(body).map(Step::Done),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn discovers_first_party_data_without_executing_scripts() {
+ let url = data_url(
+ r#""#,
+ "Rust OnceLock",
+ ).unwrap();
+ assert_eq!(url.query_pairs().count(), 2);
+ assert!(is_data_url(&url));
+ assert!(
+ data_url(
+ r#""#,
+ "中文"
+ )
+ .is_ok()
+ );
+ }
+
+ #[test]
+ fn rejects_changed_queries_and_untrusted_data_destinations() {
+ for url in [
+ "https://evil.example/d.js?q=Rust",
+ "https://links.duckduckgo.com.evil.example/d.js?q=Rust",
+ "https://127.0.0.1/d.js?q=Rust",
+ "http://links.duckduckgo.com/d.js?q=Rust",
+ "https://links.duckduckgo.com:8443/d.js?q=Rust",
+ "https://user@links.duckduckgo.com/d.js?q=Rust",
+ "https://links.duckduckgo.com/redirect?q=Rust",
+ "https://links.duckduckgo.com/d.js?q=Python",
+ "https://links.duckduckgo.com/d.js?q=Rust&q=Python",
+ ] {
+ assert!(
+ data_url(
+ &format!(r#""#),
+ "Rust"
+ )
+ .is_err(),
+ "{url}"
+ );
+ }
+ assert!(
+ data_url("", "Rust")
+ .unwrap_err()
+ .text
+ .contains("verification")
+ );
+ }
+
+ #[test]
+ fn reads_only_organic_json_and_preserves_markup_text_and_escaped_punctuation() {
+ let body = r#"window.execDeep = function() { DDG.pageLayout.load('a',[{"u":"https://ads.example/","t":"Ad"}]); DDG.deep.deepPayload={"answer":"invented answer"}; DDG.pageLayout.load('d',[{"u":"https://docs.example/a?x=1&y=2&utm_source=search#top","t":"Rust & 中文","a":"Use OnceLock with \"quoted ], text\"."},{"n":"/d.js?next=1"}]); }"#;
+ let found = results(body).unwrap();
+ assert_eq!(found.len(), 1);
+ assert_eq!(found[0].url, "https://docs.example/a?x=1&y=2");
+ assert_eq!(found[0].title, "Rust & 中文");
+ assert_eq!(found[0].text, "Use OnceLock with \"quoted ], text\".");
+ assert!(found[0].published_at.is_none());
+ assert!(results("DDG.pageLayout.load('d',[]);").unwrap().is_empty());
+ }
+
+ #[test]
+ fn malformed_data_and_challenges_are_failures_not_empty_successes() {
+ for body in [
+ "DDG.pageLayout.load('a',[]);",
+ "DDG.pageLayout.load('d',[{bad}]);",
+ "DDG.pageLayout.load('d',[{\"new_field\":1}]);",
+ "DDG.pageLayout.load('d',[] + injected());",
+ "