Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 158 additions & 22 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ categories = ["network-programming", "web-programming::http-server"]

# 二进制走 CalVer,crate 走 SemVer —— 两者是两回事,卖给不同的人。
# 这里是 crate 的版本。
version = "0.97.0"
version = "0.98.0"

[workspace.dependencies]
# ── 内部 crate ───────────────────────────────────────────
Expand Down
6 changes: 6 additions & 0 deletions bin/twcore/src/upgrade.rs
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,12 @@ fn client() -> Result<reqwest::Client> {
/// 走环境变量里的代理(`HTTPS_PROXY`):服务器上出网常常只有这一条路。
fn builder() -> reqwest::ClientBuilder {
reqwest::Client::builder()
// Search opts into content decoding; release downloads retain their
// existing behavior when those reqwest features are unified.
.no_gzip()
.no_brotli()
.no_deflate()
.no_zstd()
.user_agent(concat!("twcore/", env!("CARGO_PKG_VERSION")))
.connect_timeout(Duration::from_secs(15))
.timeout(Duration::from_secs(300))
Expand Down
1 change: 1 addition & 0 deletions crates/tw-api/msg-codes.txt
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,7 @@ gw.wait.rate_limited
gw.wait.used_up
gw.wait.window_full
gw.web_tools.failed
gw.web_tools.query_mismatch
gw.ws.alias_unserved
gw.ws.alias_unserved_rewritten
gw.ws.bad_header
Expand Down
13 changes: 13 additions & 0 deletions crates/tw-api/src/web_tools.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ pub enum WebToolsMode {
pub enum SearchEngine {
Bing,
Duckduckgo,
GoogleDocs,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
Expand Down Expand Up @@ -77,6 +78,18 @@ pub struct WebToolsView {
pub config_version: String,
/// Detected on the Core host, which may be remote. Empty fields mean unavailable.
pub system_locale: WebSearchLocale,
/// Engine capabilities advertised by the connected Core.
#[serde(default)]
pub available_engines: Vec<WebSearchEngine>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
pub struct WebSearchEngine {
pub id: SearchEngine,
pub name: String,
/// The engine's minimum applies even if the user's configured interval is lower.
pub min_interval_ms: u32,
}

#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
Expand Down
8 changes: 4 additions & 4 deletions crates/tw-config/tests/manual/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1310,8 +1310,8 @@ pub fn sections() -> Vec<Section> {
Kind::Strs,
Def::Is("[]"),
t(
"Empty selects available engines automatically. Otherwise list bing and/or duckduckgo in priority order.",
"空列表自动选择可用引擎;手动填写 bing、duckduckgo,按顺序尝试。",
"Empty tries google_docs, duckduckgo, then bing. Otherwise list these engine IDs in priority order.",
"空列表依次尝试 google_docs、duckduckgo、bing;也可手动按优先顺序填写这些引擎 ID。",
),
),
row(
Expand Down Expand Up @@ -1403,8 +1403,8 @@ pub fn sections() -> Vec<Section> {
Kind::Int,
Def::Is("1000"),
t(
"Minimum interval per search engine, 250–60000 milliseconds.",
"同一搜索引擎的最小请求间隔,250–60000 毫秒。",
"Minimum interval per search engine, 250–60000 milliseconds. A source may enforce a higher minimum (google_docs: 6000).",
"同一搜索引擎的最小请求间隔,250–60000 毫秒。搜索源可要求更长间隔(google_docs:6000)。",
),
),
row(
Expand Down
6 changes: 6 additions & 0 deletions crates/tw-control/src/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,12 @@ impl Host for CoreHost {
/// 同步位置的密码就跟着去了),证书照常校验
fn http(cfg: &tw_config::Config, proxy: &str) -> Result<reqwest::Client, Msg> {
let mut b = reqwest::Client::builder()
// Search enables reqwest's decoding features workspace-wide. Keep sync
// downloads' existing byte semantics independent of that feature set.
.no_gzip()
.no_brotli()
.no_deflate()
.no_zstd()
.connect_timeout(Duration::from_secs(10))
// 一轮里的请求是接连发的;两轮之间的空闲连接不留:很多服务端几秒就关掉空闲连接,
// 留着的下一轮一发就断(发请求那一处另有一次重发)
Expand Down
1 change: 1 addition & 0 deletions crates/tw-control/src/web_tools.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ async fn view(State(s): State<ControlState>) -> Json<tw_api::WebToolsView> {
config: s.config().web_tools.clone(),
config_version: s.cfg.current().map(|c| c.version()).unwrap_or_default(),
system_locale: tw_config::web_tools::system_locale(),
available_engines: tw_gateway::web_tools::available_engines(),
})
}

Expand Down
3 changes: 2 additions & 1 deletion crates/tw-gateway/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ bytes = { workspace = true }
chrono = { workspace = true }
futures = { workspace = true }
http-body-util = { workspace = true }
reqwest = { workspace = true }
url = { workspace = true }
reqwest = { workspace = true, features = ["cookies", "gzip", "brotli", "deflate", "zstd"] }
rustls = { workspace = true }
rustls-platform-verifier = { workspace = true }
tokio-rustls = { workspace = true }
Expand Down
6 changes: 6 additions & 0 deletions crates/tw-gateway/src/outbound.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ use tw_types::msg;
/// 的那批和不走代理的那批会慢慢长出不同的超时行为。
pub(crate) fn base_client_builder() -> reqwest::ClientBuilder {
reqwest::Client::builder()
// Web search opts into browser content decoding. Other outbound paths
// preserve their existing wire-body semantics despite feature unification.
.no_gzip()
.no_brotli()
.no_deflate()
.no_zstd()
// 分段超时。**没有整体超时** —— 一个跑了六分钟的
// Opus 任务不该被中间层掐断,让客户端自己决定何时放弃。
.connect_timeout(std::time::Duration::from_secs(10))
Expand Down
89 changes: 89 additions & 0 deletions crates/tw-gateway/src/web_tools/cache.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
//! Bounded, caller-keyed cache slots. Holding a slot coalesces concurrent work;
//! eviction only removes one idle slot and cannot split an in-flight request.

use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::Instant;
use tokio::sync::Mutex as AsyncMutex;
use tw_types::Msg;

type Slot<T> = Arc<AsyncMutex<Option<(Instant, T)>>>;
struct Entry<T> {
used: Instant,
slot: Slot<T>,
}

pub(super) struct Cache<T> {
capacity: usize,
entries: Mutex<HashMap<String, Entry<T>>>,
}

impl<T> Cache<T> {
pub fn new(capacity: usize) -> Self {
Self {
capacity,
entries: Mutex::new(HashMap::new()),
}
}

pub fn get(&self, key: String) -> Result<Slot<T>, Msg> {
let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner());
if let Some(entry) = entries.get_mut(&key) {
entry.used = Instant::now();
return Ok(entry.slot.clone());
}
if entries.len() >= self.capacity {
let oldest = entries
.iter()
.filter(|(_, e)| Arc::strong_count(&e.slot) == 1)
.min_by_key(|(_, e)| e.used)
.map(|(key, _)| key.clone());
if let Some(oldest) = oldest {
entries.remove(&oldest);
} else {
return Err(super::error("too many pending web operations"));
}
}
let slot = Arc::new(AsyncMutex::new(None));
entries.insert(
key,
Entry {
used: Instant::now(),
slot: slot.clone(),
},
);
Ok(slot)
}
}

#[cfg(test)]
mod tests {
use super::*;

#[tokio::test]
async fn eviction_retains_hot_values_and_coalesces_in_flight_work() {
let cache = Cache::new(2);
let first = cache.get("first".into()).unwrap();
*first.lock().await = Some((Instant::now(), "saved"));
let pending = cache.get("pending".into()).unwrap();
let guard = pending.lock().await;
let joined = cache.get("pending".into()).unwrap();
assert!(Arc::ptr_eq(&pending, &joined));
assert!(joined.try_lock().is_err());
assert!(
cache.get("new".into()).is_err(),
"active work cannot be evicted"
);
drop(guard);
drop(joined);
drop(pending);
drop(first);
// Touch the completed result: inserting another key must evict the older
// idle slot, not flush all successfully cached results.
drop(cache.get("first".into()).unwrap());
drop(cache.get("new".into()).unwrap());
let first = cache.get("first".into()).unwrap();
assert_eq!(first.lock().await.as_ref().unwrap().1, "saved");
assert_eq!(cache.entries.lock().unwrap().len(), 2);
}
}
Loading
Loading