From 2d00a30528a62ee1f2d3e41eacdcba576a8b3f4f Mon Sep 17 00:00:00 2001 From: Hildegar Escalona Date: Fri, 13 Nov 2026 00:15:47 -0400 Subject: [PATCH 1/2] feat(web): fill view locations from a GeoIP database when self-hosted Views only get a country and city from Vercel's x-vercel-ip-* headers, so self-hosted instances record every view with no location and the analytics map stays empty. When those headers are missing and CAP_GEOIP_DATABASE_PATH points to a GeoIP2/GeoLite2 City .mmdb file (MaxMind, DB-IP Lite), look up the client address instead. Covers the share page tracker and the mobile view event. Without the variable nothing changes; a missing or unreadable file logs one warning and views keep no location. --- .../__tests__/unit/visitor-location.test.ts | 91 +++++++++++++++++++ apps/web/app/api/analytics/track/route.ts | 13 ++- apps/web/app/api/mobile/[...route]/route.ts | 20 +++- apps/web/lib/visitor-location.ts | 72 +++++++++++++++ apps/web/package.json | 1 + bun.lock | 3 + packages/env/server.ts | 6 ++ 7 files changed, 199 insertions(+), 7 deletions(-) create mode 100644 apps/web/__tests__/unit/visitor-location.test.ts create mode 100644 apps/web/lib/visitor-location.ts diff --git a/apps/web/__tests__/unit/visitor-location.test.ts b/apps/web/__tests__/unit/visitor-location.test.ts new file mode 100644 index 00000000000..f3732d6927a --- /dev/null +++ b/apps/web/__tests__/unit/visitor-location.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it, vi } from "vitest"; +import { + type GeoIpReader, + geoIpLocation, + lookupLocation, +} from "@/lib/visitor-location"; + +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + CAP_GEOIP_DATABASE_PATH: "/nonexistent/dbip-city-lite.mmdb", + }), +})); + +describe("geoIpLocation", () => { + it("returns null and warns once when the database file is missing", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(geoIpLocation("8.8.8.8")).toBeNull(); + expect(geoIpLocation("1.1.1.1")).toBeNull(); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); + }); +}); + +const readerWith = (records: Record): GeoIpReader => ({ + get: (ip) => { + if (ip === "not-an-ip") throw new Error("Invalid IP address"); + return (records[ip] ?? null) as ReturnType; + }, +}); + +describe("lookupLocation", () => { + it("reads country, region and city from a GeoIP2 City record", () => { + const reader = readerWith({ + "81.2.69.142": { + country: { iso_code: "GB" }, + subdivisions: [{ iso_code: "ENG", names: { en: "England" } }], + city: { names: { en: "London" } }, + }, + }); + expect(lookupLocation(reader, "81.2.69.142")).toEqual({ + country: "GB", + region: "ENG", + city: "London", + }); + }); + + // DB-IP Lite has subdivision names but no subdivision codes. + it("falls back to the subdivision name when it has no code", () => { + const reader = readerWith({ + "190.202.0.1": { + country: { iso_code: "VE" }, + subdivisions: [{ names: { en: "Zulia" } }], + city: { names: { en: "San Carlos del Zulia" } }, + }, + }); + expect(lookupLocation(reader, "190.202.0.1")).toEqual({ + country: "VE", + region: "Zulia", + city: "San Carlos del Zulia", + }); + }); + + it("keeps the country when the record has no region or city", () => { + const reader = readerWith({ + "2001:db8::1": { country: { iso_code: "us" } }, + }); + expect(lookupLocation(reader, "2001:db8::1")).toEqual({ + country: "US", + region: "", + city: "", + }); + }); + + it("returns null for missing, unknown, private and invalid addresses", () => { + const reader = readerWith({ + "10.0.0.1": { country: {} }, + "203.0.113.9": { country: { iso_code: "EU1" } }, + }); + for (const ip of [ + null, + undefined, + "", + "127.0.0.1", + "10.0.0.1", + "203.0.113.9", + "not-an-ip", + ]) { + expect(lookupLocation(reader, ip)).toBeNull(); + } + }); +}); diff --git a/apps/web/app/api/analytics/track/route.ts b/apps/web/app/api/analytics/track/route.ts index 58730d43381..aa455e1479e 100644 --- a/apps/web/app/api/analytics/track/route.ts +++ b/apps/web/app/api/analytics/track/route.ts @@ -14,8 +14,9 @@ import { createAnonymousViewNotification, sendFirstViewEmail, } from "@/lib/Notification"; -import { isRateLimited, RATE_LIMIT_IDS } from "@/lib/rate-limit"; +import { clientAddress, isRateLimited, RATE_LIMIT_IDS } from "@/lib/rate-limit"; import { runPromise } from "@/lib/server"; +import { geoIpLocation } from "@/lib/visitor-location"; const trackPayloadSchema = z.object({ videoId: z.string().min(1).max(64), @@ -86,14 +87,18 @@ export async function POST(request: NextRequest) { const timestamp = viewTimestamp(body.occurredAt, Date.now()); - const country = + let country = sanitizeString(request.headers.get("x-vercel-ip-country")) || ""; - const region = + let region = sanitizeString(request.headers.get("x-vercel-ip-country-region")) || ""; - const city = + let city = sanitizeString( decodeUrlEncodedHeaderValue(request.headers.get("x-vercel-ip-city")), ) || ""; + if (!country) { + const location = geoIpLocation(clientAddress(request.headers)); + if (location) ({ country, region, city } = location); + } await runPromise( Effect.gen(function* () { diff --git a/apps/web/app/api/mobile/[...route]/route.ts b/apps/web/app/api/mobile/[...route]/route.ts index 66fecd95f0e..d5017b1e89f 100644 --- a/apps/web/app/api/mobile/[...route]/route.ts +++ b/apps/web/app/api/mobile/[...route]/route.ts @@ -107,6 +107,7 @@ import { createIdentityEditSpec, } from "@/lib/video-edits"; import { startVideoProcessingWorkflow } from "@/lib/video-processing"; +import { geoIpLocation } from "@/lib/visitor-location"; export const dynamic = "force-dynamic"; @@ -2965,6 +2966,19 @@ const trackMobileCapView = Effect.fn("Mobile.trackCapView")(function* ( return { success: true as const }; } + const vercelCountry = readHeader(request, "x-vercel-ip-country"); + const location = vercelCountry + ? { + country: vercelCountry, + region: readHeader(request, "x-vercel-ip-country-region"), + city: readHeader(request, "x-vercel-ip-city"), + } + : (geoIpLocation(clientAddress(toHeaders(request.headers))) ?? { + country: "", + region: "", + city: "", + }); + const tinybird = yield* Tinybird; yield* tinybird .appendEvents([ @@ -2976,9 +2990,9 @@ const trackMobileCapView = Effect.fn("Mobile.trackCapView")(function* ( tenant_id: video.orgId || video.ownerId, video_id: videoId, pathname: `/s/${videoId}`, - country: readHeader(request, "x-vercel-ip-country"), - region: readHeader(request, "x-vercel-ip-country-region"), - city: readHeader(request, "x-vercel-ip-city"), + country: location.country, + region: location.region, + city: location.city, browser: "Cap for iOS", device: "mobile", os: "iOS", diff --git a/apps/web/lib/visitor-location.ts b/apps/web/lib/visitor-location.ts new file mode 100644 index 00000000000..9ea3b3d509b --- /dev/null +++ b/apps/web/lib/visitor-location.ts @@ -0,0 +1,72 @@ +import { readFileSync } from "node:fs"; +import { serverEnv } from "@cap/env"; +import { type CityResponse, Reader } from "mmdb-lib"; + +export type VisitorLocation = { + country: string; + region: string; + city: string; +}; + +type GeoIpRecord = { + country?: { iso_code?: string }; + subdivisions?: Array<{ iso_code?: string; names?: { en?: string } }>; + city?: { names?: { en?: string } }; +}; + +export type GeoIpReader = { get(ip: string): GeoIpRecord | null }; + +const MAX_LENGTH = 256; + +const clean = (value: string | undefined) => + value?.trim().slice(0, MAX_LENGTH) ?? ""; + +export function lookupLocation( + reader: GeoIpReader, + ip: string | null | undefined, +): VisitorLocation | null { + if (!ip) return null; + let record: GeoIpRecord | null; + try { + record = reader.get(ip); + } catch { + return null; + } + const country = clean(record?.country?.iso_code).toUpperCase(); + if (!/^[A-Z]{2}$/.test(country)) return null; + const subdivision = record?.subdivisions?.[0]; + return { + country, + region: clean(subdivision?.iso_code || subdivision?.names?.en), + city: clean(record?.city?.names?.en), + }; +} + +let reader: GeoIpReader | null | undefined; + +function geoIpReader(): GeoIpReader | null { + if (reader !== undefined) return reader; + reader = null; + const path = serverEnv().CAP_GEOIP_DATABASE_PATH?.trim(); + if (!path) return reader; + try { + reader = new Reader(readFileSync(path)); + } catch (error) { + console.warn( + `[geoip] Could not open ${path}; views keep no location`, + error, + ); + } + return reader; +} + +/** + * Location of an IP from the GeoIP database in CAP_GEOIP_DATABASE_PATH, for + * hosts other than Vercel, which sends no x-vercel-ip-* headers. + */ +export function geoIpLocation( + ip: string | null | undefined, +): VisitorLocation | null { + const current = geoIpReader(); + return current ? lookupLocation(current, ip) : null; +} diff --git a/apps/web/package.json b/apps/web/package.json index 4cf91c10076..617c0b032df 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -119,6 +119,7 @@ "lucide-react": "^0.525.0", "media-chrome": "^4.12.0", "mediabunny": "^1.45.2", + "mmdb-lib": "3.0.3", "moment": "^2.31.0", "motion": "^12.18.1", "next": "16.4.5", diff --git a/bun.lock b/bun.lock index 45323cb08d5..8fd660e149b 100644 --- a/bun.lock +++ b/bun.lock @@ -218,6 +218,7 @@ "lucide-react": "^0.525.0", "media-chrome": "^4.12.0", "mediabunny": "^1.45.2", + "mmdb-lib": "3.0.3", "moment": "^2.31.0", "motion": "^12.18.1", "next": "16.4.5", @@ -3629,6 +3630,8 @@ "mlly": ["mlly@1.8.2", "", { "dependencies": { "acorn": "8.17.0", "pathe": "2.0.3", "pkg-types": "1.3.1", "ufo": "1.6.4" } }, "sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA=="], + "mmdb-lib": ["mmdb-lib@3.0.3", "", {}, "sha512-xQPoBXcNjjHiOvOraFBKtA++uNWF6aCVHL9dRKFXEov8eI3QJwtgiw3qApsonFT5SpoqsEVISUTg3HIDs2DiXw=="], + "module-details-from-path": ["module-details-from-path@1.0.4", "", {}, "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w=="], "moment": ["moment@2.31.0", "", {}, "sha512-0acOTfMiWOheYS4eoWb80yYMb/JLvVv9SHbs2PehaDzfUG0Bw855SKyk0IKTnPGa5+U2bmi3W68l1+sGLX/pvw=="], diff --git a/packages/env/server.ts b/packages/env/server.ts index 7e764bb5036..17230a00ab0 100644 --- a/packages/env/server.ts +++ b/packages/env/server.ts @@ -224,6 +224,12 @@ function createServerEnv() { /// Tinybird analytics TINYBIRD_HOST: z.string().optional(), TINYBIRD_TOKEN: z.string().optional(), + CAP_GEOIP_DATABASE_PATH: z + .string() + .optional() + .describe( + "Path to a GeoIP2/GeoLite2 City .mmdb file. Fills the country and city of views when the host does not send Vercel's location headers", + ), VERCEL_ENV: z .union([ z.literal("production"), From a1afacb01943660ca2f6086483dacc715b56dc1e Mon Sep 17 00:00:00 2001 From: Hildegar Escalona Date: Fri, 13 Nov 2026 00:23:32 -0400 Subject: [PATCH 2/2] test(web): cover the GeoIP fallback in the analytics track route The route test mocks @/lib/rate-limit as a whole, so it now provides clientAddress, and mocks the GeoIP lookup to check that Vercel's headers still win and the database fills the location without them. --- .../unit/analytics-track-route.test.ts | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/apps/web/__tests__/unit/analytics-track-route.test.ts b/apps/web/__tests__/unit/analytics-track-route.test.ts index 7878d9bd438..4b3011a5060 100644 --- a/apps/web/__tests__/unit/analytics-track-route.test.ts +++ b/apps/web/__tests__/unit/analytics-track-route.test.ts @@ -31,6 +31,15 @@ vi.mock("@/lib/rate-limit", () => ({ return count > REQUESTS_PER_IP_LIMIT; }, ), + clientAddress: (headers: Headers) => headers.get("x-real-ip") ?? undefined, +})); + +vi.mock("@/lib/visitor-location", () => ({ + geoIpLocation: vi.fn((ip: string | null | undefined) => + ip === "198.51.100.7" + ? { country: "VE", region: "Zulia", city: "Maracaibo" } + : null, + ), })); vi.mock("server-only", () => ({})); @@ -128,12 +137,21 @@ vi.mock("@/lib/Notification", () => ({ const { POST } = await import("@/app/api/analytics/track/route"); const { createAnonymousViewNotification } = await import("@/lib/Notification"); +const { geoIpLocation } = await import("@/lib/visitor-location"); -const track = (body: Record, ip = "203.0.113.1") => +const track = ( + body: Record, + ip = "203.0.113.1", + headers: Record = {}, +) => POST( new Request("https://cap.so/api/analytics/track", { method: "POST", - headers: { "content-type": "application/json", "x-real-ip": ip }, + headers: { + "content-type": "application/json", + "x-real-ip": ip, + ...headers, + }, body: JSON.stringify(body), }) as unknown as NextRequest, ); @@ -150,6 +168,7 @@ describe("POST /api/analytics/track", () => { mocks.requestsPerIp.clear(); mocks.rateLimitCalls = []; vi.mocked(createAnonymousViewNotification).mockClear(); + vi.mocked(geoIpLocation).mockClear(); mocks.userId = null; mocks.orgMember = false; mocks.verifiedPasswords = []; @@ -188,6 +207,34 @@ describe("POST /api/analytics/track", () => { ]); }); + it("takes the location from Vercel's headers when present", async () => { + await track({ videoId: VIDEO_ID }, "198.51.100.7", { + "x-vercel-ip-country": "GB", + "x-vercel-ip-country-region": "ENG", + "x-vercel-ip-city": "London", + }); + + expect(geoIpLocation).not.toHaveBeenCalled(); + expect(recordedEvents()).toEqual([ + expect.objectContaining({ country: "GB", region: "ENG", city: "London" }), + ]); + }); + + it("falls back to the GeoIP database without Vercel's headers", async () => { + await track({ videoId: VIDEO_ID }, "198.51.100.7"); + await track({ videoId: VIDEO_ID }, "203.0.113.50"); + + expect(geoIpLocation).toHaveBeenCalledWith("198.51.100.7"); + expect(recordedEvents()).toEqual([ + expect.objectContaining({ + country: "VE", + region: "Zulia", + city: "Maracaibo", + }), + expect.objectContaining({ country: "", region: "", city: "" }), + ]); + }); + it("does not record views for videos that do not exist", async () => { mocks.video = undefined;