From c62dddcfa3dbe16af3e82632c99ed77793cb7725 Mon Sep 17 00:00:00 2001 From: Jay Goldman Date: Sat, 19 Sep 2026 18:40:30 -0400 Subject: [PATCH 1/2] feat(connectors): add Readwise Reader (reading-state) connector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `readwise-reader` connector that carries reading *state* via the Readwise Reader API v3 — distinct from the existing highlights-only `readwise` connector (classic /api/v2, hidden). - Archive-on-finish: when a document is finished on the device, marks the matching Reader document archived + seen (PATCH /api/v3/bulk_update/). - Fan-in (pullProgress): pulls a Reader document's reading_progress so a part-read article resumes on the device, mirroring the Audiobookshelf fan-in path. Readwise exposes only a 0-1 percentage, so exact-line seek is best-effort (a prior device sample fills the position when present). - Matches by title/author metadata using the framework's decideMatch(), gated at a high confidence (0.85) so it never archives the wrong doc. - Approximates rate-limit backoff (Readwise is 20 req/min) with a cooldown, since the HttpTransport exposes no response headers. Tests: full connector unit coverage (validate/match/push/fan-in/207 retry) and updates the connector-list assertion. Full suite green (269 tests). Co-Authored-By: Claude --- README.md | 2 +- src/connectors/readwise-reader.ts | 231 ++++++++++++++++++++++++++++++ src/connectors/registry.ts | 2 + test/connectors.test.ts | 5 +- test/readwise-reader.test.ts | 132 +++++++++++++++++ 5 files changed, 369 insertions(+), 3 deletions(-) create mode 100644 src/connectors/readwise-reader.ts create mode 100644 test/readwise-reader.test.ts diff --git a/README.md b/README.md index fae9da5..3fd18b6 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ and any KOReader device. only the sync-server URL. Same accounts, same auth, same endpoints as `sync.koreader.rocks`. - **Better multi-device sync.** Progress is stored per device and the newest position wins, fixing the ping-pong you get with stock kosync servers. -- **Server-side service connectors.** Link services such as Hardcover, Micro.blog, and Audiobookshelf once; readers continue speaking standard KOSync while the server updates external reading state. +- **Server-side service connectors.** Link services such as Hardcover, Micro.blog, Audiobookshelf, and Readwise Reader once; readers continue speaking standard KOSync while the server updates external reading state. - **Lossless CrossPoint sync.** An extended API carries the full CrossPoint position (spine, paragraph, anchor, page hints), not just a lossy xpath + percentage. - **Bookmarks, clippings, and reading stats.** Delta sync with tombstones for bookmarks and diff --git a/src/connectors/readwise-reader.ts b/src/connectors/readwise-reader.ts new file mode 100644 index 0000000..c97ff90 --- /dev/null +++ b/src/connectors/readwise-reader.ts @@ -0,0 +1,231 @@ +import { decideMatch, extractTitleAuthor, type Candidate } from './matching.js'; +import type { + Connector, + Credential, + DocumentMeta, + ExternalBook, + HttpTransport, + InboundChange, + Match, + OutboundEvent, + PushResult, + ValidateResult, +} from './types.js'; + +/** + * Readwise Reader connector (Tier 1) — archive-on-finish. + * + * Distinct from the highlights-only `readwise` connector (classic /api/v2): this + * one carries reading *state*. When a document is finished on the device, it + * marks the matching Readwise Reader document archived + seen via the Reader + * API v3 (`PATCH /api/v3/bulk_update/`). No highlights. + * + * Matching is by title/author metadata (the firmware sends the EPUB's title). + * It works with any source whose EPUB `dc:title` equals the Reader document's + * title; when the article is delivered by an OPDS bridge that copies the Readwise + * title verbatim the match is effectively exact — but we still gate on a high + * confidence so we never archive the wrong document. + * + * Reader API v3 (docs: readwise.io/reader_api): `GET /api/v3/list/` (paged, + * 20 req/min, `location`/`pageCursor`/`nextPageCursor`) and + * `PATCH /api/v3/bulk_update/` with `{ updates: [{ id, location, seen }] }` + * (returns 200, or 207 when some items failed — see push()). + */ + +const BASE = 'https://readwise.io/api/v3'; + +// Non-archived locations = the pool of docs that could still be "finished". +const CANDIDATE_LOCATIONS = ['new', 'later', 'shortlist', 'feed'] as const; +// One page (100 docs) per location keeps us well under the 20 req/min limit. +const MAX_CANDIDATE_PAGES = 1; +// Archiving the wrong doc is worse than not archiving; our titles match ~1.0. +const MATCH_THRESHOLD = 0.85; + +// Rate-limit backoff. Readwise is 20 req/min; on a 429 we can't read the exact +// Retry-After (crosspoint-sync's HttpTransport exposes no response headers), so +// we approximate it: after a 429, skip all Readwise calls for this window. The +// fan-in worker and the queue retry after it clears. +const COOLDOWN_MS = Number(process.env.READWISE_RATE_COOLDOWN_MS ?? 60_000); +let rateLimitedUntil = 0; +function rateLimited(): boolean { + return Date.now() < rateLimitedUntil; +} +function noteRateLimit(status: number): void { + if (status === 429) rateLimitedUntil = Date.now() + COOLDOWN_MS; +} + +interface ReadwiseCred extends Credential { + token: string; +} +interface ReaderDoc { + id: string; + title?: string; + author?: string; + site_name?: string; + location?: string; +} +interface ReaderList { + results?: ReaderDoc[]; + nextPageCursor?: string | null; +} + +function tokenOf(cred: Credential): string { + const t = (cred as ReadwiseCred).token; + if (typeof t !== 'string' || t.length === 0) throw new Error('missing readwise token'); + return t; +} + +function authHeaders(token: string): Record { + return { authorization: `Token ${token}`, 'content-type': 'application/json' }; +} + +async function listDocs(token: string, http: HttpTransport, location: string): Promise { + const out: ReaderDoc[] = []; + let cursor: string | undefined; + for (let page = 0; page < MAX_CANDIDATE_PAGES; page++) { + const params = new URLSearchParams({ location, withHtmlContent: 'false' }); + if (cursor) params.set('pageCursor', cursor); + const res = await http(`${BASE}/list/?${params.toString()}`, { + method: 'GET', + headers: authHeaders(token), + }); + if (res.status === 429) { + noteRateLimit(res.status); + break; + } + if (res.status !== 200) break; + const body = (await res.json()) as ReaderList; + out.push(...(body.results ?? [])); + cursor = body.nextPageCursor ?? undefined; + if (!cursor) break; + } + return out; +} + +async function candidatePool(token: string, http: HttpTransport): Promise { + const seen = new Set(); + const pool: Candidate[] = []; + for (const loc of CANDIDATE_LOCATIONS) { + for (const d of await listDocs(token, http, loc)) { + if (!d.title || seen.has(d.id)) continue; + seen.add(d.id); + pool.push({ externalId: d.id, title: d.title, author: d.author ?? d.site_name ?? null }); + } + } + return pool; +} + +async function validate(cred: Credential, http: HttpTransport): Promise { + try { + const res = await http(`${BASE}/list/?withHtmlContent=false`, { + method: 'GET', + headers: authHeaders(tokenOf(cred)), + }); + if (res.status === 200) return { ok: true }; + if (res.status === 401) return { ok: false, error: 'invalid token' }; + return { ok: false, error: `unexpected status ${res.status}` }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +/** Only act on finish — ignore ongoing progress so we don't churn the API. */ +function shouldPush(ev: OutboundEvent): boolean { + return ev.kind === 'finished'; +} + +async function match(cred: Credential, doc: DocumentMeta, http: HttpTransport): Promise { + if (rateLimited()) return null; + const ta = extractTitleAuthor(doc); + if (!ta) return null; + const candidates = await candidatePool(tokenOf(cred), http); + const decision = decideMatch(ta.title, ta.author, candidates, { threshold: MATCH_THRESHOLD }); + if (!decision.accepted || !decision.best) return null; + return { + externalId: decision.best.externalId, + confidence: decision.best.score, + queryUsed: ta.title, + title: decision.best.title, + author: decision.best.author ?? null, + }; +} + +/** The Reader "in progress" pool (non-archived), used as the match candidate set. */ +async function listCurrentlyReading(cred: Credential, http: HttpTransport): Promise { + const pool = await candidatePool(tokenOf(cred), http); + return pool.map((c) => ({ externalId: c.externalId, title: c.title, author: c.author ?? null })); +} + +/** + * Fan-in: pull the current Readwise reading position for a matched document, so + * a half-read-in-Readwise article resumes on the device. Readwise exposes only a + * 0–1 `reading_progress` (no KOReader xpath), so we return the percentage; the + * exact-line seek is best-effort (crosspoint-sync fills a position from a prior + * device sample when it has one). + */ +async function pullProgress( + cred: Credential, + match: Match, + http: HttpTransport, + sinceMs: number +): Promise { + if (rateLimited()) return null; + const res = await http( + `${BASE}/list/?id=${encodeURIComponent(match.externalId)}&withHtmlContent=false`, + { method: 'GET', headers: authHeaders(tokenOf(cred)) } + ); + noteRateLimit(res.status); + if (res.status !== 200) return null; + const body = (await res.json()) as { + results?: { reading_progress?: number; updated_at?: string }[]; + }; + const doc = body.results?.[0]; + if (!doc || typeof doc.reading_progress !== 'number') return null; + const pct = Math.max(0, Math.min(1, doc.reading_progress)); + const updatedAtMs = doc.updated_at ? Date.parse(doc.updated_at) : Date.now(); + if (sinceMs && Number.isFinite(updatedAtMs) && updatedAtMs <= sinceMs) return null; + return { externalId: match.externalId, percentage: pct, finished: pct >= 0.98, updatedAtMs }; +} + +async function push( + cred: Credential, + m: Match, + ev: OutboundEvent, + http: HttpTransport +): Promise { + if (ev.kind !== 'finished') return { ok: true }; + const res = await http(`${BASE}/bulk_update/`, { + method: 'PATCH', + headers: authHeaders(tokenOf(cred)), + body: JSON.stringify({ updates: [{ id: m.externalId, location: 'archive', seen: true }] }), + }); + if (res.status === 401) return { ok: false, retryable: false, needsReauth: true, error: 'unauthorized' }; + if (res.status === 429) { + noteRateLimit(res.status); + return { ok: false, retryable: true, error: 'rate limited' }; + } + if (res.status >= 500) return { ok: false, retryable: true, error: `server ${res.status}` }; + if (res.status === 207) { + // Partial failure: the single item we sent didn't apply. Retry. + return { ok: false, retryable: true, error: 'bulk_update partial failure' }; + } + if (res.status >= 200 && res.status < 300) return { ok: true }; + return { ok: false, retryable: false, error: `unexpected status ${res.status}` }; +} + +export const readwiseReaderConnector: Connector = { + id: 'readwise-reader', + displayName: 'Readwise Reader (archive on finish)', + tier: 1, + capabilities: { read: true, write: true }, + carries: ['finished'], + credentialKind: 'token', + experimental: false, + matchBy: 'metadata', + validate, + shouldPush, + match, + push, + pullProgress, + listCurrentlyReading, +}; diff --git a/src/connectors/registry.ts b/src/connectors/registry.ts index 5a80c4f..93714f4 100644 --- a/src/connectors/registry.ts +++ b/src/connectors/registry.ts @@ -1,6 +1,7 @@ import type { Connector, HttpTransport } from './types.js'; import { hardcoverConnector } from './hardcover.js'; import { readwiseConnector } from './readwise.js'; +import { readwiseReaderConnector } from './readwise-reader.js'; import { kosyncConnector } from './kosync.js'; import { bookfusionConnector } from './bookfusion.js'; import { audiobookshelfConnector } from './audiobookshelf.js'; @@ -11,6 +12,7 @@ const CONNECTORS: Connector[] = [ kosyncConnector, hardcoverConnector, readwiseConnector, + readwiseReaderConnector, bookfusionConnector, audiobookshelfConnector, microblogConnector, diff --git a/test/connectors.test.ts b/test/connectors.test.ts index 270c9a7..4b9fef2 100644 --- a/test/connectors.test.ts +++ b/test/connectors.test.ts @@ -54,8 +54,9 @@ describe('connector management API', () => { const body = await res.json(); expect(body.encryption).toBe('enabled'); const ids = body.connectors.map((c: { id: string }) => c.id).sort(); - // Readwise is hidden for now; still registered but not listed. - expect(ids).toEqual(['audiobookshelf', 'bookfusion', 'hardcover', 'kosync', 'microblog']); + // The classic (highlights-only) readwise connector is hidden; still + // registered but not listed. readwise-reader (reading-state) is listed. + expect(ids).toEqual(['audiobookshelf', 'bookfusion', 'hardcover', 'kosync', 'microblog', 'readwise-reader']); expect(body.connectors.every((c: { linked: boolean }) => !c.linked)).toBe(true); }); diff --git a/test/readwise-reader.test.ts b/test/readwise-reader.test.ts new file mode 100644 index 0000000..fd87983 --- /dev/null +++ b/test/readwise-reader.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from 'vitest'; +import type { HttpTransport } from '../src/connectors/types.js'; +import { readwiseReaderConnector } from '../src/connectors/readwise-reader.js'; + +// Minimal fake transport (same shape as the one in connectors-more.test.ts): +// records calls, and returns the last-registered handler whose match string +// appears in the URL or body. +function fakeTransport() { + const calls: { url: string; method: string; body?: string }[] = []; + const handlers: { match: string; status: number; body: unknown }[] = []; + const t: HttpTransport = async (url, init) => { + calls.push({ url, method: init.method, body: init.body }); + const h = [...handlers].reverse().find((x) => url.includes(x.match) || (init.body ?? '').includes(x.match)); + const status = h?.status ?? 200; + const body = h?.body ?? {}; + return { status, text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), json: async () => body }; + }; + return { transport: t, calls, on: (m: string, s: number, b: unknown) => handlers.push({ match: m, status: s, body: b }) }; +} + +const CRED = { token: 'rw_token' }; + +describe('readwise-reader connector', () => { + it('validates via GET /api/v3/list/', async () => { + const fake = fakeTransport(); + fake.on('/api/v3/list/', 200, { results: [] }); + const v = await readwiseReaderConnector.validate(CRED, fake.transport); + expect(v.ok).toBe(true); + expect(fake.calls[0].url).toContain('/api/v3/list/'); + expect(fake.calls[0].url).toContain('withHtmlContent=false'); + }); + + it('reports an invalid token on 401', async () => { + const fake = fakeTransport(); + fake.on('/api/v3/list/', 401, {}); + const v = await readwiseReaderConnector.validate(CRED, fake.transport); + expect(v.ok).toBe(false); + }); + + it('matches a document by title against the non-archived pool', async () => { + const fake = fakeTransport(); + fake.on('/list/', 200, { + results: [ + { id: '01aaa', title: 'Some Other Article', author: 'Nobody' }, + { id: '01bbb', title: 'Stop Eating the Oreos', author: 'A. Writer' }, + ], + }); + const m = await readwiseReaderConnector.match!( + CRED, + { document: 'hash', title: 'Stop Eating the Oreos', author: 'A. Writer', filename: null }, + fake.transport + ); + expect(m?.externalId).toBe('01bbb'); + // Sends the Token auth header, never a query token. + expect(fake.calls[0].url).not.toContain('rw_token'); + }); + + it('returns no match when nothing crosses the confidence threshold', async () => { + const fake = fakeTransport(); + fake.on('/list/', 200, { results: [{ id: '01ccc', title: 'Completely Unrelated', author: 'X' }] }); + const m = await readwiseReaderConnector.match!( + CRED, + { document: 'hash', title: 'Stop Eating the Oreos', author: 'A. Writer', filename: null }, + fake.transport + ); + expect(m).toBeNull(); + }); + + it('archives on finish via PATCH /bulk_update/', async () => { + const fake = fakeTransport(); + fake.on('/bulk_update/', 200, {}); + const r = await readwiseReaderConnector.push( + CRED, + { externalId: '01bbb', confidence: 1 }, + { kind: 'finished', document: 'hash', percentage: 1, timestamp: 1 }, + fake.transport + ); + expect(r.ok).toBe(true); + const call = fake.calls.find((c) => c.url.includes('/bulk_update/')); + expect(call?.method).toBe('PATCH'); + expect(JSON.parse(call!.body!)).toEqual({ updates: [{ id: '01bbb', location: 'archive', seen: true }] }); + }); + + it('ignores in-progress events (only finish archives)', async () => { + const fake = fakeTransport(); + const r = await readwiseReaderConnector.push( + CRED, + { externalId: '01bbb', confidence: 1 }, + { kind: 'progress', document: 'hash', percentage: 0.5, timestamp: 1 }, + fake.transport + ); + expect(r.ok).toBe(true); + expect(fake.calls).toHaveLength(0); + }); + + it('retries a 207 partial failure', async () => { + const fake = fakeTransport(); + fake.on('/bulk_update/', 207, {}); + const r = await readwiseReaderConnector.push( + CRED, + { externalId: '01bbb', confidence: 1 }, + { kind: 'finished', document: 'hash', percentage: 1, timestamp: 1 }, + fake.transport + ); + expect(r.ok).toBe(false); + expect(r.retryable).toBe(true); + }); + + it('fan-in: pullProgress returns the Reader reading_progress newer than the cursor', async () => { + const fake = fakeTransport(); + fake.on('id=', 200, { results: [{ reading_progress: 0.42, updated_at: '2026-01-02T00:00:00Z' }] }); + const change = await readwiseReaderConnector.pullProgress!( + CRED, + { externalId: '01bbb', confidence: 1 }, + fake.transport, + Date.parse('2026-01-01T00:00:00Z') + ); + expect(change).toMatchObject({ externalId: '01bbb', percentage: 0.42, finished: false }); + }); + + it('fan-in: skips a document not updated since the cursor', async () => { + const fake = fakeTransport(); + fake.on('id=', 200, { results: [{ reading_progress: 0.42, updated_at: '2026-01-01T00:00:00Z' }] }); + const change = await readwiseReaderConnector.pullProgress!( + CRED, + { externalId: '01bbb', confidence: 1 }, + fake.transport, + Date.parse('2026-06-01T00:00:00Z') + ); + expect(change).toBeNull(); + }); +}); From bb494dc9ee3ad93e8265e265a444195d0165d599 Mon Sep 17 00:00:00 2001 From: Jay Goldman Date: Sat, 19 Sep 2026 18:56:48 -0400 Subject: [PATCH 2/2] Address review: full pagination, per-token cooldown, finite fan-in timestamp CodeRabbit findings on the readwise-reader connector: - Candidate pool now follows nextPageCursor to the end of each location (with a repeated-cursor loop guard) instead of reading only the first 100 docs, so a finished document on a later page still matches and archives. - Rate-limit cooldown is keyed per access token, not a module global, so one account's 429 no longer suppresses Readwise calls for other users on a multi-user server. - pullProgress always returns a finite updatedAtMs (falls back to now when updated_at is missing/unparseable) so it never leaks NaN into an InboundChange. Adds regression tests for each (second-page match, per-token isolation, missing updated_at). Full suite green (272 tests). Co-Authored-By: Claude --- src/connectors/readwise-reader.ts | 63 +++++++++++++++++---------- test/readwise-reader.test.ts | 72 +++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 23 deletions(-) diff --git a/src/connectors/readwise-reader.ts b/src/connectors/readwise-reader.ts index c97ff90..a9bbf72 100644 --- a/src/connectors/readwise-reader.ts +++ b/src/connectors/readwise-reader.ts @@ -36,22 +36,29 @@ const BASE = 'https://readwise.io/api/v3'; // Non-archived locations = the pool of docs that could still be "finished". const CANDIDATE_LOCATIONS = ['new', 'later', 'shortlist', 'feed'] as const; -// One page (100 docs) per location keeps us well under the 20 req/min limit. -const MAX_CANDIDATE_PAGES = 1; +// Follow nextPageCursor to the end of each location; this is just a safety bound +// against a pathological cursor loop (a 429 mid-walk stops us far sooner and the +// backfill/fan-in worker resumes after the cooldown). +const MAX_CANDIDATE_PAGES = 50; // Archiving the wrong doc is worse than not archiving; our titles match ~1.0. const MATCH_THRESHOLD = 0.85; -// Rate-limit backoff. Readwise is 20 req/min; on a 429 we can't read the exact -// Retry-After (crosspoint-sync's HttpTransport exposes no response headers), so -// we approximate it: after a 429, skip all Readwise calls for this window. The -// fan-in worker and the queue retry after it clears. +// Rate-limit backoff. Readwise is 20 req/min *per access token*; on a 429 we +// can't read the exact Retry-After (crosspoint-sync's HttpTransport exposes no +// response headers), so we approximate it: after a 429, skip that token's +// Readwise calls for this window. Keyed per token so one account's 429 never +// suppresses another's (crosspoint-sync is multi-user). The fan-in worker and +// the queue retry after it clears. const COOLDOWN_MS = Number(process.env.READWISE_RATE_COOLDOWN_MS ?? 60_000); -let rateLimitedUntil = 0; -function rateLimited(): boolean { - return Date.now() < rateLimitedUntil; +const rateLimitedUntil = new Map(); +function rateLimited(token: string): boolean { + const until = rateLimitedUntil.get(token) ?? 0; + if (Date.now() < until) return true; + if (until) rateLimitedUntil.delete(token); // expired; keep the map small + return false; } -function noteRateLimit(status: number): void { - if (status === 429) rateLimitedUntil = Date.now() + COOLDOWN_MS; +function noteRateLimit(token: string, status: number): void { + if (status === 429) rateLimitedUntil.set(token, Date.now() + COOLDOWN_MS); } interface ReadwiseCred extends Credential { @@ -82,6 +89,7 @@ function authHeaders(token: string): Record { async function listDocs(token: string, http: HttpTransport, location: string): Promise { const out: ReaderDoc[] = []; let cursor: string | undefined; + const seenCursors = new Set(); for (let page = 0; page < MAX_CANDIDATE_PAGES; page++) { const params = new URLSearchParams({ location, withHtmlContent: 'false' }); if (cursor) params.set('pageCursor', cursor); @@ -90,14 +98,17 @@ async function listDocs(token: string, http: HttpTransport, location: string): P headers: authHeaders(token), }); if (res.status === 429) { - noteRateLimit(res.status); + noteRateLimit(token, res.status); break; } if (res.status !== 200) break; const body = (await res.json()) as ReaderList; out.push(...(body.results ?? [])); - cursor = body.nextPageCursor ?? undefined; - if (!cursor) break; + const next = body.nextPageCursor ?? undefined; + // Stop at the last page, or if the API ever repeats a cursor (loop guard). + if (!next || seenCursors.has(next)) break; + seenCursors.add(next); + cursor = next; } return out; } @@ -135,10 +146,11 @@ function shouldPush(ev: OutboundEvent): boolean { } async function match(cred: Credential, doc: DocumentMeta, http: HttpTransport): Promise { - if (rateLimited()) return null; + const token = tokenOf(cred); + if (rateLimited(token)) return null; const ta = extractTitleAuthor(doc); if (!ta) return null; - const candidates = await candidatePool(tokenOf(cred), http); + const candidates = await candidatePool(token, http); const decision = decideMatch(ta.title, ta.author, candidates, { threshold: MATCH_THRESHOLD }); if (!decision.accepted || !decision.best) return null; return { @@ -169,12 +181,13 @@ async function pullProgress( http: HttpTransport, sinceMs: number ): Promise { - if (rateLimited()) return null; + const token = tokenOf(cred); + if (rateLimited(token)) return null; const res = await http( `${BASE}/list/?id=${encodeURIComponent(match.externalId)}&withHtmlContent=false`, - { method: 'GET', headers: authHeaders(tokenOf(cred)) } + { method: 'GET', headers: authHeaders(token) } ); - noteRateLimit(res.status); + noteRateLimit(token, res.status); if (res.status !== 200) return null; const body = (await res.json()) as { results?: { reading_progress?: number; updated_at?: string }[]; @@ -182,8 +195,11 @@ async function pullProgress( const doc = body.results?.[0]; if (!doc || typeof doc.reading_progress !== 'number') return null; const pct = Math.max(0, Math.min(1, doc.reading_progress)); - const updatedAtMs = doc.updated_at ? Date.parse(doc.updated_at) : Date.now(); - if (sinceMs && Number.isFinite(updatedAtMs) && updatedAtMs <= sinceMs) return null; + // Always return a finite timestamp: a missing/unparseable updated_at falls + // back to "now" so the cursor comparison works and we never leak NaN. + const parsed = doc.updated_at ? Date.parse(doc.updated_at) : NaN; + const updatedAtMs = Number.isFinite(parsed) ? parsed : Date.now(); + if (sinceMs && updatedAtMs <= sinceMs) return null; return { externalId: match.externalId, percentage: pct, finished: pct >= 0.98, updatedAtMs }; } @@ -194,14 +210,15 @@ async function push( http: HttpTransport ): Promise { if (ev.kind !== 'finished') return { ok: true }; + const token = tokenOf(cred); const res = await http(`${BASE}/bulk_update/`, { method: 'PATCH', - headers: authHeaders(tokenOf(cred)), + headers: authHeaders(token), body: JSON.stringify({ updates: [{ id: m.externalId, location: 'archive', seen: true }] }), }); if (res.status === 401) return { ok: false, retryable: false, needsReauth: true, error: 'unauthorized' }; if (res.status === 429) { - noteRateLimit(res.status); + noteRateLimit(token, res.status); return { ok: false, retryable: true, error: 'rate limited' }; } if (res.status >= 500) return { ok: false, retryable: true, error: `server ${res.status}` }; diff --git a/test/readwise-reader.test.ts b/test/readwise-reader.test.ts index fd87983..303516e 100644 --- a/test/readwise-reader.test.ts +++ b/test/readwise-reader.test.ts @@ -129,4 +129,76 @@ describe('readwise-reader connector', () => { ); expect(change).toBeNull(); }); + + it('fan-in: a missing updated_at yields a finite timestamp, never NaN', async () => { + const fake = fakeTransport(); + fake.on('id=', 200, { results: [{ reading_progress: 0.3 }] }); // no updated_at + const change = await readwiseReaderConnector.pullProgress!( + CRED, + { externalId: '01bbb', confidence: 1 }, + fake.transport, + Date.parse('2026-01-01T00:00:00Z') + ); + expect(change).not.toBeNull(); + expect(change!.percentage).toBe(0.3); + expect(Number.isFinite(change!.updatedAtMs)).toBe(true); + }); + + it('matches a document that only appears on a later page (pagination)', async () => { + const fake = fakeTransport(); + // First page of each location: one unrelated doc + a nextPageCursor. + fake.on('/list/', 200, { + results: [{ id: 'p1', title: 'First Page Filler', author: 'Q' }], + nextPageCursor: 'CURSOR2', + }); + // Second page (request carries pageCursor=) holds the target, no more pages. + fake.on('pageCursor=', 200, { + results: [{ id: 'p2', title: 'Buried On Page Two', author: 'Z' }], + nextPageCursor: null, + }); + const m = await readwiseReaderConnector.match!( + CRED, + { document: 'hash', title: 'Buried On Page Two', author: 'Z', filename: null }, + fake.transport + ); + expect(m?.externalId).toBe('p2'); + expect(fake.calls.some((c) => c.url.includes('pageCursor=CURSOR2'))).toBe(true); + }); + + it('rate-limit cooldown is per-token, not global', async () => { + // Token A trips a 429 -> its cooldown is set. + const a1 = fakeTransport(); + a1.on('id=', 429, {}); + const blocked = await readwiseReaderConnector.pullProgress!( + { token: 'rw_A' }, + { externalId: 'x', confidence: 1 }, + a1.transport, + 0 + ); + expect(blocked).toBeNull(); + + // A is now in cooldown: a follow-up makes no HTTP call at all. + const a2 = fakeTransport(); + a2.on('id=', 200, { results: [{ reading_progress: 0.9, updated_at: '2026-01-01T00:00:00Z' }] }); + const stillBlocked = await readwiseReaderConnector.pullProgress!( + { token: 'rw_A' }, + { externalId: 'x', confidence: 1 }, + a2.transport, + 0 + ); + expect(stillBlocked).toBeNull(); + expect(a2.calls).toHaveLength(0); + + // A different token B is unaffected and goes through. + const b = fakeTransport(); + b.on('id=', 200, { results: [{ reading_progress: 0.5, updated_at: '2026-01-01T00:00:00Z' }] }); + const ok = await readwiseReaderConnector.pullProgress!( + { token: 'rw_B' }, + { externalId: 'x', confidence: 1 }, + b.transport, + 0 + ); + expect(ok?.percentage).toBe(0.5); + expect(b.calls.length).toBeGreaterThan(0); + }); });