From c29492882c80e8673b4451fb7e3fd7e9da93650e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 09:00:31 +0000 Subject: [PATCH] Traffic: keep the statistics 14 days by default, IPv6 by its /64 alone Requests and client addresses stay recorded by default, but an install that never set a retention now keeps the day files, and the addresses in them, for 14 days instead of 90. An IPv6 client is noted by its /64 alone, so the full address is neither written to the files nor returned by GET /admin/traffic (clients[].address is gone; the network says it all). An upgrade note says that recording is on from the upgrade on and where to turn it off; the deployment docs name the address column. Fixes #758 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012e6DJamFAQZiLpD3FjPBYJ --- crates/server/src/traffic/mod.rs | 23 +++++++++++-- crates/traffic/src/layout.rs | 34 ++++++++++++++++--- crates/traffic/src/lib.rs | 10 +++--- crates/traffic/src/query.rs | 6 +--- crates/traffic/src/rows.rs | 4 ++- crates/traffic/src/settings.rs | 9 +++-- crates/traffic/src/slot.rs | 2 +- docs/changes/2.2.0/709.feature.md | 2 +- docs/changes/2.2.0/758.upgrade.md | 9 +++++ docs/deployment.md | 21 ++++++++---- web/src/api/traffic.ts | 2 +- web/src/routes/admin/traffic/Settings.tsx | 2 +- web/src/routes/admin/traffic/Traffic.tsx | 3 -- web/src/routes/admin/traffic/traffic.test.tsx | 3 +- 14 files changed, 94 insertions(+), 36 deletions(-) create mode 100644 docs/changes/2.2.0/758.upgrade.md diff --git a/crates/server/src/traffic/mod.rs b/crates/server/src/traffic/mod.rs index 69f70c14..496a55fd 100644 --- a/crates/server/src/traffic/mod.rs +++ b/crates/server/src/traffic/mod.rs @@ -6,7 +6,8 @@ //! * The client address is the one the principal layer resolves (the socket //! peer, or what a trusted proxy forwarded), counted under its network as //! the login lockout counts it: an IPv6 address by its /64 -//! ([`client_network`]). +//! ([`client_network`]), the rest of the address dropped before it is +//! noted, so that it is neither written nor shown. //! * The settings ([`TrafficSettings`]) are kept in `forge_setting` under //! [`SETTINGS_KEY`]; every instance follows a change with its next flush. //! * The worker flushes once a minute, a couple of seconds after the minute @@ -53,9 +54,9 @@ pub fn network_label(ip: IpAddr) -> String { } } -/// Note the request's client address. +/// Note the request's client address: an IPv6 address by its /64 alone. pub fn note_client(ip: IpAddr) { - forge_traffic::note_client(ip.to_canonical(), network_label(ip)); + forge_traffic::note_client(client_network(ip), network_label(ip)); } /// The stored [`TrafficSettings`], or the defaults when never saved. @@ -200,6 +201,22 @@ mod tests { ); } + #[tokio::test] + async fn an_ipv6_client_is_noted_by_its_64_alone() { + let slot = forge_traffic::Slot::new(); + slot.scope(async { note_client("2001:db8:1:2:3:4:5:6".parse().unwrap()) }) + .await; + let noted = slot.take(); + assert_eq!(noted.client, "2001:db8:1:2::".parse().ok()); + assert_eq!(noted.network.as_deref(), Some("2001:db8:1:2::/64")); + + slot.scope(async { note_client("::ffff:192.0.2.7".parse().unwrap()) }) + .await; + let noted = slot.take(); + assert_eq!(noted.client, "192.0.2.7".parse().ok()); + assert_eq!(noted.network.as_deref(), Some("192.0.2.7")); + } + #[test] fn the_flush_waits_for_the_minute_to_turn() { let at = |s: &str| DateTime::parse_from_rfc3339(s).unwrap().to_utc(); diff --git a/crates/traffic/src/layout.rs b/crates/traffic/src/layout.rs index 6c431621..63fc760b 100644 --- a/crates/traffic/src/layout.rs +++ b/crates/traffic/src/layout.rs @@ -5,7 +5,8 @@ //! * `days/{date}/{instance}.parquet`: one instance's requests of a day //! (UTC), a row per repository and client network with their country and //! autonomous system; rewritten every minute while the day lasts. Kept as -//! long as the settings say ([`TrafficSettings::retention_days`]). +//! long as the settings say ([`TrafficSettings::retention_days`], two +//! weeks by default). //! * `geoip/{country,asn}.mmdb`: the geolocation databases //! ([`crate::geo`]). //! @@ -132,7 +133,8 @@ struct RowKey { struct RowCounts { requests: u64, bytes: u64, - /// The last address seen; `""` without addresses. + /// The last address seen (an IPv6 address's /64 alone, as noted); `""` + /// without addresses. address: String, /// The autonomous system's organisation, for a row without a network. as_org: String, @@ -509,6 +511,8 @@ mod tests { day_key(day("2026-07-31"), "a"), day_key(day("2026-08-01"), "a"), day_key(day("2026-08-01"), "b"), + day_key(day("2026-10-15"), "a"), + day_key(day("2026-10-16"), "a"), day_key(day("2026-10-30"), "a"), "traffic/v1/days/not-a-date/x.parquet".to_string(), "traffic/v1/geoip/country.mmdb".to_string(), @@ -516,6 +520,27 @@ mod tests { objects.put(&key, vec![1]).await.unwrap(); } // 90 days up to 2026-10-29: from 2026-08-01 on. + let ninety = TrafficSettings { + retention_days: 90, + ..TrafficSettings::default() + }; + let deleted = prune(&objects, &ninety, t("2026-10-29T12:00:00Z")) + .await + .unwrap(); + assert_eq!(deleted, 1); + assert_eq!( + objects.keys(), + vec![ + "traffic/v1/days/2026-08-01/a.parquet", + "traffic/v1/days/2026-08-01/b.parquet", + "traffic/v1/days/2026-10-15/a.parquet", + "traffic/v1/days/2026-10-16/a.parquet", + "traffic/v1/days/2026-10-30/a.parquet", + "traffic/v1/days/not-a-date/x.parquet", + "traffic/v1/geoip/country.mmdb", + ] + ); + // The default, two weeks up to 2026-10-29: from 2026-10-16 on. let deleted = prune( &objects, &TrafficSettings::default(), @@ -523,12 +548,11 @@ mod tests { ) .await .unwrap(); - assert_eq!(deleted, 1); + assert_eq!(deleted, 3); assert_eq!( objects.keys(), vec![ - "traffic/v1/days/2026-08-01/a.parquet", - "traffic/v1/days/2026-08-01/b.parquet", + "traffic/v1/days/2026-10-16/a.parquet", "traffic/v1/days/2026-10-30/a.parquet", "traffic/v1/days/not-a-date/x.parquet", "traffic/v1/geoip/country.mmdb", diff --git a/crates/traffic/src/lib.rs b/crates/traffic/src/lib.rs index cc09af48..908c6522 100644 --- a/crates/traffic/src/lib.rs +++ b/crates/traffic/src/lib.rs @@ -20,9 +20,11 @@ //! * **Pruning** ([`layout::prune`]) deletes the days past their retention //! ([`TrafficSettings`]); it is idempotent and runs on every instance. //! -//! Client addresses are personal data: they can be switched off, and then -//! requests are still counted by repository, country and autonomous system, -//! the address only looked up in memory before it is dropped. +//! Client addresses are personal data: an IPv6 address is cut to its /64 +//! before it is noted, and the files are kept two weeks by default +//! ([`DEFAULT_RETENTION_DAYS`]). The addresses can be switched off: requests +//! are then still counted by repository, country and autonomous system, the +//! address only looked up in memory before it is dropped. pub mod geo; pub mod layout; @@ -41,7 +43,7 @@ pub use geo::{DbInfo, Geo, GeoKind, Location}; pub use layout::{instance_id, FlushReport, Flusher}; pub use objects::{Listed, MemoryObjects, Objects}; pub use record::{Drained, Event, PendingCounts, PendingKey, Recorder, OTHER}; -pub use settings::{TrafficSettings, MAX_RETENTION_DAYS}; +pub use settings::{TrafficSettings, DEFAULT_RETENTION_DAYS, MAX_RETENTION_DAYS}; pub use slot::{note_client, note_repository, Noted, Slot}; /// What the server holds of the traffic statistics: this instance's diff --git a/crates/traffic/src/query.rs b/crates/traffic/src/query.rs index ad3ea771..148a4676 100644 --- a/crates/traffic/src/query.rs +++ b/crates/traffic/src/query.rs @@ -195,9 +195,8 @@ pub struct AsnShare { /// A client network's requests. #[derive(Debug, Clone, Default, Serialize, PartialEq, Eq)] pub struct ClientShare { + /// The address, or an IPv6 address's /64: no address is kept whole. pub network: String, - /// The last address of the network seen. - pub address: String, pub country: String, pub asn: u32, pub as_org: String, @@ -354,9 +353,6 @@ pub async fn overview( tally.days.insert(row.day); if tally.latest.is_none_or(|latest| row.day >= latest) { tally.latest = Some(row.day); - if !row.address.is_empty() { - client.address.clone_from(&row.address); - } if !row.country.is_empty() || row.asn != 0 { client.country.clone_from(&row.country); client.asn = u32::try_from(row.asn).unwrap_or(0); diff --git a/crates/traffic/src/rows.rs b/crates/traffic/src/rows.rs index 2d91faab..37a1d253 100644 --- a/crates/traffic/src/rows.rs +++ b/crates/traffic/src/rows.rs @@ -26,7 +26,9 @@ pub(crate) struct DayRow { /// The client network: the address, or an IPv6 address's /64; `""` with /// client addresses switched off, `other` past the day's bound. pub network: String, - /// The last address of the network seen; `""` with addresses off. + /// The address the network was located by: the IPv4 address, the first + /// address of an IPv6 /64 (an IPv6 address is never kept whole); `""` + /// with addresses off. pub address: String, /// The ISO 3166-1 code of the country the network is in; `""` unknown. pub country: String, diff --git a/crates/traffic/src/settings.rs b/crates/traffic/src/settings.rs index 1ece4299..97ba294e 100644 --- a/crates/traffic/src/settings.rs +++ b/crates/traffic/src/settings.rs @@ -3,6 +3,10 @@ /// The longest retention accepted, in days (100 years). pub const MAX_RETENTION_DAYS: u32 = 36_500; +/// The retention of an install that never set one, in days: two weeks, so +/// that the client addresses recorded by default are not kept longer. +pub const DEFAULT_RETENTION_DAYS: u32 = 14; + /// The traffic settings of every instance, set in the web UI (Admin → /// Traffic → Settings) or through `/admin/settings/traffic` and kept in the /// database. An install that never saved them uses @@ -19,7 +23,8 @@ pub struct TrafficSettings { /// Whether client addresses are recorded. Off, requests are still /// counted by repository, country and autonomous system. pub client_addresses: bool, - /// How long the day files are kept, in days. + /// How long the day files are kept, in days; with them the client + /// addresses they hold. pub retention_days: u32, } @@ -28,7 +33,7 @@ impl Default for TrafficSettings { TrafficSettings { enabled: true, client_addresses: true, - retention_days: 90, + retention_days: DEFAULT_RETENTION_DAYS, } } } diff --git a/crates/traffic/src/slot.rs b/crates/traffic/src/slot.rs index b8d0dba6..826fccaa 100644 --- a/crates/traffic/src/slot.rs +++ b/crates/traffic/src/slot.rs @@ -19,7 +19,7 @@ tokio::task_local! { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Noted { /// The client address: the socket peer, or the one a trusted proxy - /// forwarded. + /// forwarded. The server notes an IPv6 address cut to its /64. pub client: Option, /// The network the address is counted under (the address itself, an /// IPv6 address's /64), as text. diff --git a/docs/changes/2.2.0/709.feature.md b/docs/changes/2.2.0/709.feature.md index ab279c17..13f7b619 100644 --- a/docs/changes/2.2.0/709.feature.md +++ b/docs/changes/2.2.0/709.feature.md @@ -8,4 +8,4 @@ so the page covers all instances without Redis. Countries and networks come from MaxMind's GeoLite2 databases, which Forge downloads every three days with a MaxMind account ID and license key set under **Admin → Traffic → Settings**, or which can be put in the bucket by hand. Recording, client -addresses and the retention (90 days by default) are set there too. +addresses and the retention (14 days by default) are set there too. diff --git a/docs/changes/2.2.0/758.upgrade.md b/docs/changes/2.2.0/758.upgrade.md new file mode 100644 index 00000000..dc45d565 --- /dev/null +++ b/docs/changes/2.2.0/758.upgrade.md @@ -0,0 +1,9 @@ +From the upgrade on, every instance counts the requests it serves for +**Admin → Traffic** and records the client address of each, without an +administrator turning anything on. The addresses are personal data (under +the GDPR, for one): an IPv6 client is kept by its /64 alone, and the +statistics, addresses included, are kept 14 days by default. Turn +**Client addresses** or **Count requests** off, or set another retention, +under **Admin → Traffic → Settings** (`/admin/settings/traffic`); with +addresses off, requests are still counted by repository, country and +autonomous system. diff --git a/docs/deployment.md b/docs/deployment.md index 0b93978e..4359ab75 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1512,17 +1512,24 @@ them from the bucket and sums them: it needs no Redis and no call between instances. The files are plain Parquet, so DuckDB, pandas or Spark read them too. -What is recorded and how long it is kept (90 days by default) is set under -**Admin → Traffic → Settings** (`/admin/settings/traffic`), for every -instance at once. Each instance deletes the days past the retention about -once an hour, whichever instance wrote them. +Requests and client addresses are recorded unless an administrator turns +them off. What is recorded and how long it is kept (14 days by default) is +set under **Admin → Traffic → Settings** (`/admin/settings/traffic`), for +every instance at once. Each instance deletes the days past the retention, +and the client addresses they hold, about once an hour, whichever instance +wrote them. A longer retention keeps the addresses as long: to keep the +counts for months, consider turning **Client addresses** off. - **Client addresses are personal data.** The address is the one Forge resolves for the login lockout: the socket peer, or the `X-Forwarded-For` of a [trusted proxy](#reverse-proxy). An IPv6 client is counted by its - /64. With **Client addresses** off, requests are still counted by - repository, country and autonomous system: the address is looked up in - memory and dropped before anything is written. + /64, and only the /64 is kept: the rest of the address is dropped before + the request is counted. The files' `network` column holds the IPv4 + address or the IPv6 /64, and their `address` column the address the + network was located by (the IPv4 address, the first address of the /64). + With **Client addresses** off, requests are still counted by repository, + country and autonomous system: the address is looked up in memory and + dropped before anything is written, and both columns stay empty. - **Accuracy.** Today fills in as the instances write it, once a minute. An instance that stops writes what it counted on a graceful shutdown; one that is killed loses its last minute. Past 500,000 clients a day on one diff --git a/web/src/api/traffic.ts b/web/src/api/traffic.ts index f541a123..e4939d88 100644 --- a/web/src/api/traffic.ts +++ b/web/src/api/traffic.ts @@ -58,8 +58,8 @@ export interface TrafficShare { /** A client network's requests. */ export interface TrafficClient { + /** The address, or an IPv6 address's /64. */ network: string - address: string /** ISO 3166-1 alpha-2; `""` unknown. */ country: string /** `0` unknown. */ diff --git a/web/src/routes/admin/traffic/Settings.tsx b/web/src/routes/admin/traffic/Settings.tsx index 26870f64..87937d24 100644 --- a/web/src/routes/admin/traffic/Settings.tsx +++ b/web/src/routes/admin/traffic/Settings.tsx @@ -125,7 +125,7 @@ function RecordingForm({ settings }: { settings: TrafficSettings }) { label="Client addresses" checked={addresses} onChange={setAddresses} - description="Addresses are personal data. Off, requests are still counted by repository, country and network: the address is looked up in memory and dropped." + description="Addresses are personal data, kept as long as the statistics; an IPv6 address by its /64 alone. Off, requests are still counted by repository, country and network: the address is looked up in memory and dropped." /> {c.network} - {c.address && c.address !== c.network && ( -
{c.address}
- )} {count.format(c.requests)} {formatBytes(c.bytes)} diff --git a/web/src/routes/admin/traffic/traffic.test.tsx b/web/src/routes/admin/traffic/traffic.test.tsx index 99ec4167..88caf5ab 100644 --- a/web/src/routes/admin/traffic/traffic.test.tsx +++ b/web/src/routes/admin/traffic/traffic.test.tsx @@ -50,7 +50,6 @@ const overview: TrafficOverview = { clients: [ { network: '2001:db8:1:2::/64', - address: '2001:db8:1:2::7', country: 'DE', asn: 3320, as_org: 'Deutsche Telekom AG', @@ -65,7 +64,7 @@ const overview: TrafficOverview = { recording, } -const settings: TrafficSettings = { enabled: true, client_addresses: true, retention_days: 90 } +const settings: TrafficSettings = { enabled: true, client_addresses: true, retention_days: 14 } const geoip = (over: Partial = {}): GeoipStatus => ({ country: { database_type: 'GeoLite2-Country', built_at: '2026-10-28T00:00:00Z', size: 9_000_000 },