From e61f04ad6afdf9b2e32dba848cbbafb49e703832 Mon Sep 17 00:00:00 2001 From: Rafa Cardenas <253999660+rafa-stacks@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:53:43 -0600 Subject: [PATCH 1/5] dynamic token cache ttl --- README.md | 1 + src/api/util/cache.ts | 46 ++++-- src/env.ts | 7 + src/pg/pg-store.ts | 39 ++++- src/pg/stacks-core-pg-store.ts | 32 +++- src/pg/types.ts | 11 ++ tests/api/cache.test.ts | 189 ++++++++++++++++++++++ tests/helpers.ts | 31 +++- tests/stacks-core/block-processor.test.ts | 111 ++++++++++++- 9 files changed, 441 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index fbc908e4..a94cd6be 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,7 @@ All configuration is done via environment variables. Defaults are shown in paren | `METADATA_MAX_PAYLOAD_BYTE_SIZE` | Max metadata JSON payload size (bytes) | `1000000` | | `METADATA_MAX_NFT_CONTRACT_TOKEN_COUNT` | Max tokens to index per NFT contract | `50000` | | `METADATA_DYNAMIC_TOKEN_REFRESH_INTERVAL` | Interval for dynamic token refreshes (seconds) | `86400` | +| `METADATA_DYNAMIC_TOKEN_MAX_CACHE_AGE` | Max `max-age` advertised for dynamic tokens with a TTL (seconds) | `86400` | | `METADATA_RATE_LIMITED_HOST_RETRY_AFTER` | Wait time after a 429 response (seconds) | `60` | | `METADATA_FETCH_MAX_REDIRECTIONS` | Max HTTP redirects to follow | `5` | diff --git a/src/api/util/cache.ts b/src/api/util/cache.ts index 54d2c508..e6bb1936 100644 --- a/src/api/util/cache.ts +++ b/src/api/util/cache.ts @@ -2,6 +2,7 @@ import { FastifyReply, FastifyRequest } from 'fastify'; import { SmartContractRegEx } from '../schemas.js'; import { CACHE_CONTROL_MUST_REVALIDATE, parseIfNoneMatchHeader } from '@stacks/api-toolkit'; import { parseContractIdentifiers } from './helpers.js'; +import { DbTokenCacheInfo } from '../../pg/types.js'; enum ETagType { chainTip = 'chain_tip', @@ -11,29 +12,46 @@ enum ETagType { async function handleCache(type: ETagType, request: FastifyRequest, reply: FastifyReply) { const ifNoneMatch = parseIfNoneMatchHeader(request.headers['if-none-match']); - let etag: string | undefined; + let cache: DbTokenCacheInfo | undefined; switch (type) { case ETagType.chainTip: { const chainTip = await request.server.db.core.getChainTip(request.server.db.sql); - etag = chainTip?.index_block_hash; + if (chainTip?.index_block_hash) cache = { etag: chainTip.index_block_hash }; break; } case ETagType.token: - etag = await getTokenEtag(request); + cache = await getTokenCacheInfo(request); break; - case ETagType.bulkToken: - etag = await getBulkTokenEtag(request); + case ETagType.bulkToken: { + const etag = await getBulkTokenEtag(request); + if (etag) cache = { etag }; break; + } } - if (etag) { - if (ifNoneMatch && ifNoneMatch.includes(etag)) { - await reply.header('Cache-Control', CACHE_CONTROL_MUST_REVALIDATE).code(304).send(); + if (cache?.etag) { + const headers = cacheControlHeaders(cache); + if (ifNoneMatch && ifNoneMatch.includes(cache.etag)) { + await reply.headers(headers).code(304).send(); } else { - void reply.headers({ 'Cache-Control': CACHE_CONTROL_MUST_REVALIDATE, ETag: `"${etag}"` }); + void reply.headers({ ...headers, ETag: `"${cache.etag}"` }); } } } +/** + * Builds the freshness headers for a token response. Tokens marked as `dynamic` with an explicit + * TTL (see SIP-019) can't change until that TTL elapses, so we advertise that lifetime to clients + * in order to avoid revalidation requests that are not necessary. Everything else must be + * revalidated on every request because it can change at any block. + */ +function cacheControlHeaders(cache: DbTokenCacheInfo): Record { + if (cache.maxAge === undefined) return { 'Cache-Control': CACHE_CONTROL_MUST_REVALIDATE }; + return { + 'Cache-Control': `public, max-age=${cache.maxAge}, must-revalidate`, + Expires: new Date(Date.now() + cache.maxAge * 1000).toUTCString(), + }; +} + export async function handleTokenCache(request: FastifyRequest, reply: FastifyReply) { return handleCache(ETagType.token, request, reply); } @@ -48,14 +66,16 @@ export async function handleBulkTokenCache(request: FastifyRequest, reply: Fasti export function setReplyNonCacheable(reply: FastifyReply): void { void reply.removeHeader('Cache-Control'); + void reply.removeHeader('Expires'); void reply.removeHeader('Etag'); } /** - * Retrieve the token's last modified date as a UNIX epoch so we can use it as the response ETag. - * @returns Etag string + * Retrieve the token's cache information, including its last modified date as a UNIX epoch so we + * can use it as the response ETag. + * @returns `DbTokenCacheInfo` */ -async function getTokenEtag(request: FastifyRequest): Promise { +async function getTokenCacheInfo(request: FastifyRequest): Promise { try { const components = request.url.split('/'); let tokenNumber: bigint = 1n; @@ -71,7 +91,7 @@ async function getTokenEtag(request: FastifyRequest): Promise { - const result = await this.sql<{ etag: string }[]>` - SELECT date_part('epoch', t.updated_at)::text AS etag + }): Promise { + // Cap the TTL before turning it into an `interval` so an absurd value declared by a contract + // can't overflow postgres' interval type. + const maxAge = ENV.METADATA_DYNAMIC_TOKEN_MAX_CACHE_AGE; + const result = await this.sql<{ etag: string; max_age: number | null }[]>` + SELECT + date_part('epoch', t.updated_at)::text AS etag, + CASE WHEN n.update_mode = 'dynamic' AND n.ttl IS NOT NULL THEN + CEIL(EXTRACT(EPOCH FROM ( + COALESCE(t.updated_at, t.created_at) + + INTERVAL '1 seconds' * LEAST(n.ttl, ${maxAge}) - NOW() + )))::int + END AS max_age FROM tokens AS t INNER JOIN smart_contracts AS s ON s.id = t.smart_contract_id + LEFT JOIN LATERAL ( + SELECT update_mode, ttl + FROM update_notifications + WHERE token_id = t.id + ORDER BY block_height DESC, tx_index DESC, event_index DESC + LIMIT 1 + ) AS n ON TRUE WHERE s.principal = ${args.contractPrincipal} AND t.token_number = ${args.tokenNumber} `; if (result.count === 0) { return undefined; } - return result[0].etag; + const cache: DbTokenCacheInfo = { etag: result[0].etag }; + // Only advertise a freshness lifetime if the token isn't already due for a refresh. + if (result[0].max_age !== null && result[0].max_age > 0) { + cache.maxAge = Math.min(result[0].max_age, maxAge); + } + return cache; } async getJobStatusCounts(): Promise<{ count: number; status: string }[]> { diff --git a/src/pg/stacks-core-pg-store.ts b/src/pg/stacks-core-pg-store.ts index f5aeabb5..17939db8 100644 --- a/src/pg/stacks-core-pg-store.ts +++ b/src/pg/stacks-core-pg-store.ts @@ -25,6 +25,13 @@ import { import { dbSipNumberToDbTokenType } from '../token-processor/util/helpers.js'; import { DecodedStacksBlock } from '../stacks-core/stacks-core-block-processor.js'; +/** + * Upper bound (seconds, ~100 years) for a SIP-019 TTL declared by a contract. TTLs are `uint`s so + * they can be arbitrarily large, and postgres throws `interval out of range` when converting them, + * which would abort block ingestion. + */ +const MAX_TOKEN_TTL_SECONDS = 3_153_600_000; + export class StacksCorePgStore extends BasePgStoreModule { /** * Writes a processed Stacks Core block to the database. @@ -435,18 +442,35 @@ export class StacksCorePgStore extends BasePgStoreModule { const interval = ENV.METADATA_DYNAMIC_TOKEN_REFRESH_INTERVAL.toString(); await this.sql` WITH dynamic_tokens AS ( - SELECT DISTINCT ON (token_id) token_id, ttl + SELECT DISTINCT ON (token_id) + token_id, ttl, block_height, tx_index, COALESCE(event_index, -1) AS notif_event_index FROM update_notifications WHERE update_mode = 'dynamic' ORDER BY token_id, block_height DESC, tx_index DESC, event_index DESC ), + current_dynamic_tokens AS ( + -- A token is only dynamic if its latest notification says so. Any later notification with + -- a different update mode (e.g. 'frozen' or 'standard') supersedes the 'dynamic' one. + SELECT d.token_id, d.ttl + FROM dynamic_tokens AS d + WHERE NOT EXISTS ( + SELECT 1 + FROM update_notifications AS n + WHERE n.token_id = d.token_id + AND n.update_mode <> 'dynamic' + AND (n.block_height, n.tx_index, COALESCE(n.event_index, -1)) + > (d.block_height, d.tx_index, d.notif_event_index) + ) + ), due_for_refresh AS ( SELECT d.token_id - FROM dynamic_tokens AS d + FROM current_dynamic_tokens AS d INNER JOIN tokens AS t ON t.id = d.token_id WHERE CASE WHEN d.ttl IS NOT NULL THEN - COALESCE(t.updated_at, t.created_at) < (NOW() - INTERVAL '1 seconds' * ttl) + -- Cap the TTL so an absurd value declared by a contract can't overflow the interval. + COALESCE(t.updated_at, t.created_at) < + (NOW() - INTERVAL '1 seconds' * LEAST(d.ttl, ${MAX_TOKEN_TTL_SECONDS})) ELSE COALESCE(t.updated_at, t.created_at) < (NOW() - INTERVAL '${this.sql(interval)} seconds') @@ -454,7 +478,7 @@ export class StacksCorePgStore extends BasePgStoreModule { ) UPDATE jobs SET status = 'pending', updated_at = NOW() - WHERE status IN ('done', 'failed') AND token_id = ( + WHERE status IN ('done', 'failed') AND token_id IN ( SELECT token_id FROM due_for_refresh ) `; diff --git a/src/pg/types.ts b/src/pg/types.ts index 06f02f5a..a493e4d8 100644 --- a/src/pg/types.ts +++ b/src/pg/types.ts @@ -110,6 +110,17 @@ export type DbJob = { retry_after?: string; }; +/** Cache information for a single token response. */ +export type DbTokenCacheInfo = { + /** Token ETag, based on its last updated date. */ + etag: string; + /** + * Seconds this response may be considered fresh by clients without revalidating. Only set for + * `dynamic` tokens that declare an explicit TTL via SIP-019. + */ + maxAge?: number; +}; + export type DbUpdateNotification = { token_id: number; block_height: number; diff --git a/tests/api/cache.test.ts b/tests/api/cache.test.ts index f69aef82..32e3f04a 100644 --- a/tests/api/cache.test.ts +++ b/tests/api/cache.test.ts @@ -5,10 +5,13 @@ import { DbSipNumber } from '../../src/pg/types.js'; import { TestFastifyServer, insertAndEnqueueTestContractWithTokens, + insertTestUpdateNotification, setupEnv, startTestApiServer, } from '../helpers.js'; import { afterEach, beforeEach, describe, test } from 'node:test'; +import { DbTokenUpdateMode } from '../../src/pg/types.js'; +import { ENV } from '../../src/env.js'; describe('ETag cache', () => { let db: PgStore; @@ -355,3 +358,189 @@ describe('ETag cache', () => { assert.strictEqual(cached2.statusCode, 200); }); }); + +describe('Dynamic token cache control', () => { + const contract = 'SP2SYHR84SDJJDK8M09HFS4KBFXPPCX9H7RZ9YVTS.hello-world'; + const url = `/metadata/v1/nft/${contract}/1`; + let db: PgStore; + let fastify: TestFastifyServer; + let maxCacheAge: number; + + async function insertProcessedNft() { + await insertAndEnqueueTestContractWithTokens(db, contract, DbSipNumber.sip009, 1n); + await db.core.updateProcessedTokenWithMetadata({ + id: 1, + values: { + token: { + name: 'hello-world', + symbol: null, + decimals: null, + total_supply: '1', + uri: 'http://test.com/uri.json', + }, + metadataLocales: [ + { + metadata: { + sip: 16, + token_id: 1, + name: 'hello-world', + l10n_locale: 'en', + l10n_uri: null, + l10n_default: true, + description: 'test', + image: null, + cached_image: null, + cached_thumbnail_image: null, + }, + }, + ], + }, + }); + } + + /** Reads the `max-age` directive from a `Cache-Control` header. */ + function maxAge(response: { headers: Record }): number | undefined { + const header = response.headers['cache-control'] as string | undefined; + const match = header?.match(/max-age=(\d+)/); + return match ? parseInt(match[1]) : undefined; + } + + beforeEach(async () => { + setupEnv(); + maxCacheAge = ENV.METADATA_DYNAMIC_TOKEN_MAX_CACHE_AGE; + db = await PgStore.connect({ skipMigrations: true }); + fastify = await startTestApiServer(db); + await cycleMigrations(MIGRATIONS_DIR); + await insertProcessedNft(); + }); + + afterEach(async () => { + ENV.METADATA_DYNAMIC_TOKEN_MAX_CACHE_AGE = maxCacheAge; + await fastify.close(); + await db.close(); + }); + + test('dynamic token with ttl advertises its freshness lifetime', async () => { + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() WHERE id = 1`; + + const response = await fastify.inject({ method: 'GET', url }); + assert.strictEqual(response.statusCode, 200); + // Allow for a small delta between the DB write and the request. + const age = maxAge(response); + assert.ok(age !== undefined && age > 3590 && age <= 3600, `unexpected max-age: ${age}`); + assert.match(response.headers['cache-control'] as string, /^public, max-age=/); + assert.notStrictEqual(response.headers.etag, undefined); + const expires = Date.parse(response.headers.expires as string); + assert.ok(!isNaN(expires), 'Expires header is not a valid HTTP date'); + assert.ok(expires > Date.now(), 'Expires header is in the past'); + }); + + test('304 responses keep the freshness lifetime', async () => { + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() WHERE id = 1`; + + const response = await fastify.inject({ method: 'GET', url }); + const cached = await fastify.inject({ + method: 'GET', + url, + headers: { 'if-none-match': response.headers.etag as string }, + }); + assert.strictEqual(cached.statusCode, 304); + const age = maxAge(cached); + assert.ok(age !== undefined && age > 3590 && age <= 3600, `unexpected max-age: ${age}`); + assert.notStrictEqual(cached.headers.expires, undefined); + }); + + test('freshness lifetime is capped', async () => { + ENV.METADATA_DYNAMIC_TOKEN_MAX_CACHE_AGE = 300; + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + // A ttl large enough to overflow postgres' interval type if it weren't capped. + ttl: 99999999999999, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() WHERE id = 1`; + + const response = await fastify.inject({ method: 'GET', url }); + assert.strictEqual(response.statusCode, 200); + const age = maxAge(response); + assert.ok(age !== undefined && age > 290 && age <= 300, `unexpected max-age: ${age}`); + }); + + test('dynamic token without a ttl must revalidate', async () => { + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + }); + + const response = await fastify.inject({ method: 'GET', url }); + assert.strictEqual(response.statusCode, 200); + assert.strictEqual(response.headers['cache-control'], 'public, no-cache, must-revalidate'); + assert.strictEqual(response.headers.expires, undefined); + }); + + test('standard token must revalidate', async () => { + const response = await fastify.inject({ method: 'GET', url }); + assert.strictEqual(response.statusCode, 200); + assert.strictEqual(response.headers['cache-control'], 'public, no-cache, must-revalidate'); + assert.strictEqual(response.headers.expires, undefined); + }); + + test('dynamic token that is already due for refresh must revalidate', async () => { + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() - INTERVAL '2 hours' WHERE id = 1`; + + const response = await fastify.inject({ method: 'GET', url }); + assert.strictEqual(response.statusCode, 200); + assert.strictEqual(response.headers['cache-control'], 'public, no-cache, must-revalidate'); + assert.strictEqual(response.headers.expires, undefined); + }); + + test('a later update mode supersedes a dynamic ttl', async () => { + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + event_index: 0, + }); + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.frozen, + event_index: 1, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() WHERE id = 1`; + + const response = await fastify.inject({ method: 'GET', url }); + assert.strictEqual(response.statusCode, 200); + assert.strictEqual(response.headers['cache-control'], 'public, no-cache, must-revalidate'); + assert.strictEqual(response.headers.expires, undefined); + }); + + test('errors are not cacheable', async () => { + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() WHERE id = 1`; + + const response = await fastify.inject({ method: 'GET', url: `/metadata/v1/nft/${contract}/2` }); + assert.strictEqual(response.statusCode, 404); + assert.strictEqual(response.headers['cache-control'], undefined); + assert.strictEqual(response.headers.expires, undefined); + assert.strictEqual(response.headers.etag, undefined); + }); +}); diff --git a/tests/helpers.ts b/tests/helpers.ts index 308f4f26..ac7621d3 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -5,7 +5,13 @@ import { FastifyBaseLogger, FastifyInstance } from 'fastify'; import { IncomingMessage, ServerResponse } from 'http'; import { TypeBoxTypeProvider } from '@fastify/type-provider-typebox'; import { SmartContractDeployment } from '../src/token-processor/util/sip-validation.js'; -import { DbJob, DbSipNumber, DbSmartContract, DbUpdateNotification } from '../src/pg/types.js'; +import { + DbJob, + DbSipNumber, + DbSmartContract, + DbTokenUpdateMode, + DbUpdateNotification, +} from '../src/pg/types.js'; import { waiter } from '@stacks/api-toolkit'; import { DecodedStacksBlock, @@ -1441,6 +1447,29 @@ export async function insertAndEnqueueTestContractWithTokens( }); } +export async function insertTestUpdateNotification( + db: PgStore, + args: { + token_id: number; + update_mode: DbTokenUpdateMode; + ttl?: number; + block_height?: number; + index_block_hash?: string; + tx_index?: number; + event_index?: number; + } +): Promise { + await db.sql` + INSERT INTO update_notifications + (token_id, update_mode, ttl, block_height, index_block_hash, tx_id, tx_index, event_index) + VALUES ( + ${args.token_id}, ${args.update_mode}, ${args.ttl ?? null}, ${args.block_height ?? 1}, + ${args.index_block_hash ?? '0x000001'}, '0x123456', ${args.tx_index ?? 0}, + ${args.event_index ?? 0} + ) + `; +} + export async function markAllJobsAsDone(db: PgStore): Promise { await db.sql`UPDATE jobs SET status = 'done' WHERE TRUE`; } diff --git a/tests/stacks-core/block-processor.test.ts b/tests/stacks-core/block-processor.test.ts index 9028389b..523e8f03 100644 --- a/tests/stacks-core/block-processor.test.ts +++ b/tests/stacks-core/block-processor.test.ts @@ -1,11 +1,12 @@ import { strict as assert } from 'node:assert'; import { cvToHex, tupleCV, bufferCV, uintCV, stringUtf8CV } from '@stacks/transactions'; -import { DbSipNumber } from '../../src/pg/types.js'; +import { DbSipNumber, DbTokenUpdateMode } from '../../src/pg/types.js'; import { cycleMigrations } from '@stacks/api-toolkit'; import { ENV } from '../../src/env.js'; import { PgStore, MIGRATIONS_DIR } from '../../src/pg/pg-store.js'; import { insertAndEnqueueTestContractWithTokens, + insertTestUpdateNotification, markAllJobsAsDone, TestTransactionBuilder, TestBlockBuilder, @@ -206,5 +207,113 @@ describe('block processor', () => { const job = await db.getJob({ id: 2 }); assert.strictEqual(job?.status, 'pending'); }); + + describe('dynamic token refresh', () => { + const address = 'SP1K1A1PMGW2ZJCNF46NWZWHG8TS1D23EGH1KNK60'; + const contractId = `${address}.friedger-pool-nft`; + + /** Processes a block with an unrelated event so the refresh scheduler runs. */ + async function processNextBlock() { + await processor.processBlock( + new TestBlockBuilder({ + block_height: 2, + index_block_hash: '0x000002', + parent_index_block_hash: '0x000001', + }) + .addTransaction( + new TestTransactionBuilder({ tx_id: '0x01', sender: address }) + .addContractEvent(contractId, cvToHex(stringUtf8CV('test'))) + .build() + ) + .build() + ); + } + + async function getTokenJobStatuses(): Promise { + const result = await db.sql<{ status: string }[]>` + SELECT status FROM jobs WHERE token_id IS NOT NULL ORDER BY token_id ASC + `; + return result.map(r => r.status); + } + + test('enqueues every dynamic token that is due for refresh', async () => { + ENV.METADATA_DYNAMIC_TOKEN_REFRESH_INTERVAL = 99999; + await insertAndEnqueueTestContractWithTokens(db, contractId, DbSipNumber.sip009, 3n); + for (const token_id of [1, 2, 3]) + await insertTestUpdateNotification(db, { + token_id, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() - INTERVAL '2 hours'`; + await markAllJobsAsDone(db); + + await processNextBlock(); + + assert.deepStrictEqual(await getTokenJobStatuses(), ['pending', 'pending', 'pending']); + }); + + test('does not refresh tokens whose latest update mode is no longer dynamic', async () => { + ENV.METADATA_DYNAMIC_TOKEN_REFRESH_INTERVAL = 99999; + await insertAndEnqueueTestContractWithTokens(db, contractId, DbSipNumber.sip009, 2n); + for (const token_id of [1, 2]) + await insertTestUpdateNotification(db, { + token_id, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + event_index: 0, + }); + // Token 1 is frozen afterwards, token 2 stays dynamic. + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.frozen, + event_index: 1, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() - INTERVAL '2 hours'`; + await markAllJobsAsDone(db); + + await processNextBlock(); + + assert.deepStrictEqual(await getTokenJobStatuses(), ['done', 'pending']); + }); + + test('refreshes tokens that became dynamic after another update mode', async () => { + ENV.METADATA_DYNAMIC_TOKEN_REFRESH_INTERVAL = 99999; + await insertAndEnqueueTestContractWithTokens(db, contractId, DbSipNumber.sip009, 1n); + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.standard, + event_index: 0, + }); + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + event_index: 1, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() - INTERVAL '2 hours'`; + await markAllJobsAsDone(db); + + await processNextBlock(); + + assert.deepStrictEqual(await getTokenJobStatuses(), ['pending']); + }); + + test('tolerates a ttl large enough to overflow an interval', async () => { + ENV.METADATA_DYNAMIC_TOKEN_REFRESH_INTERVAL = 99999; + await insertAndEnqueueTestContractWithTokens(db, contractId, DbSipNumber.sip009, 1n); + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 99999999999999, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() - INTERVAL '2 hours'`; + await markAllJobsAsDone(db); + + await processNextBlock(); + + assert.deepStrictEqual(await getTokenJobStatuses(), ['done']); + }); + }); }); }); From b9b027058f798223b78df9fbe65c31d696aa729a Mon Sep 17 00:00:00 2001 From: Rafa Cardenas <253999660+rafa-stacks@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:17:22 -0600 Subject: [PATCH 2/5] pr fixes --- src/env.ts | 12 +++-- src/pg/pg-store.ts | 8 +++- src/pg/stacks-core-pg-store.ts | 9 ++-- tests/api/cache.test.ts | 57 +++++++++++++++++++++++ tests/helpers.ts | 6 ++- tests/stacks-core/block-processor.test.ts | 28 +++++++++++ 6 files changed, 110 insertions(+), 10 deletions(-) diff --git a/src/env.ts b/src/env.ts index ef76c5ba..3569656d 100644 --- a/src/env.ts +++ b/src/env.ts @@ -110,9 +110,15 @@ const schema = Type.Object({ * Maximum `max-age` (seconds) this API will advertise to clients for `dynamic` token metadata * that declares an explicit TTL via SIP-019. Tokens declaring a TTL longer than this will be * advertised with this value instead, so clients are never told to cache a response for an - * unreasonable amount of time. Defaults to 86400 seconds (24 hours). - */ - METADATA_DYNAMIC_TOKEN_MAX_CACHE_AGE: Type.Number({ default: 86_400 }), // 24 hours + * unreasonable amount of time. Must be a non-negative integer because it is emitted as an HTTP + * `delta-seconds` directive. Set it to `0` to always require revalidation. Defaults to 86400 + * seconds (24 hours). + */ + METADATA_DYNAMIC_TOKEN_MAX_CACHE_AGE: Type.Integer({ + default: 86_400, // 24 hours + minimum: 0, + maximum: 2_147_483_647, + }), /** * Time that must elapse between a 429 'Too many requests' response returned by a hostname and the * next request that is sent to it (seconds). This value will be overridden by the `Retry-After` diff --git a/src/pg/pg-store.ts b/src/pg/pg-store.ts index 37e83e69..73055c33 100644 --- a/src/pg/pg-store.ts +++ b/src/pg/pg-store.ts @@ -214,7 +214,10 @@ export class PgStore extends BasePgStore { const result = await this.sql<{ etag: string; max_age: number | null }[]>` SELECT date_part('epoch', t.updated_at)::text AS etag, - CASE WHEN n.update_mode = 'dynamic' AND n.ttl IS NOT NULL THEN + -- A job that is not done means a refresh is already in flight (e.g. a notification just + -- arrived), and we keep serving the previous metadata until it completes. Advertising + -- freshness then would cache metadata we already know is about to change. + CASE WHEN n.update_mode = 'dynamic' AND n.ttl IS NOT NULL AND j.status = 'done' THEN CEIL(EXTRACT(EPOCH FROM ( COALESCE(t.updated_at, t.created_at) + INTERVAL '1 seconds' * LEAST(n.ttl, ${maxAge}) - NOW() @@ -222,10 +225,11 @@ export class PgStore extends BasePgStore { END AS max_age FROM tokens AS t INNER JOIN smart_contracts AS s ON s.id = t.smart_contract_id + LEFT JOIN jobs AS j ON j.token_id = t.id LEFT JOIN LATERAL ( SELECT update_mode, ttl FROM update_notifications - WHERE token_id = t.id + WHERE token_id = t.id AND canonical = TRUE ORDER BY block_height DESC, tx_index DESC, event_index DESC LIMIT 1 ) AS n ON TRUE diff --git a/src/pg/stacks-core-pg-store.ts b/src/pg/stacks-core-pg-store.ts index 17939db8..65a51621 100644 --- a/src/pg/stacks-core-pg-store.ts +++ b/src/pg/stacks-core-pg-store.ts @@ -445,12 +445,14 @@ export class StacksCorePgStore extends BasePgStoreModule { SELECT DISTINCT ON (token_id) token_id, ttl, block_height, tx_index, COALESCE(event_index, -1) AS notif_event_index FROM update_notifications - WHERE update_mode = 'dynamic' + WHERE update_mode = 'dynamic' AND canonical = TRUE ORDER BY token_id, block_height DESC, tx_index DESC, event_index DESC ), current_dynamic_tokens AS ( - -- A token is only dynamic if its latest notification says so. Any later notification with - -- a different update mode (e.g. 'frozen' or 'standard') supersedes the 'dynamic' one. + -- A token is only dynamic if its latest canonical notification says so. Any later + -- notification with a different update mode (e.g. 'frozen' or 'standard') supersedes the + -- 'dynamic' one. Re-orgs keep notification rows around and only flip their canonical + -- flag, so an orphaned event must not stop a canonical dynamic token from being refreshed. SELECT d.token_id, d.ttl FROM dynamic_tokens AS d WHERE NOT EXISTS ( @@ -458,6 +460,7 @@ export class StacksCorePgStore extends BasePgStoreModule { FROM update_notifications AS n WHERE n.token_id = d.token_id AND n.update_mode <> 'dynamic' + AND n.canonical = TRUE AND (n.block_height, n.tx_index, COALESCE(n.event_index, -1)) > (d.block_height, d.tx_index, d.notif_event_index) ) diff --git a/tests/api/cache.test.ts b/tests/api/cache.test.ts index 32e3f04a..d006ac31 100644 --- a/tests/api/cache.test.ts +++ b/tests/api/cache.test.ts @@ -6,6 +6,7 @@ import { TestFastifyServer, insertAndEnqueueTestContractWithTokens, insertTestUpdateNotification, + markAllJobsAsDone, setupEnv, startTestApiServer, } from '../helpers.js'; @@ -396,6 +397,7 @@ describe('Dynamic token cache control', () => { ], }, }); + await markAllJobsAsDone(db); } /** Reads the `max-age` directive from a `Cache-Control` header. */ @@ -529,6 +531,61 @@ describe('Dynamic token cache control', () => { assert.strictEqual(response.headers.expires, undefined); }); + test('a pending refresh suppresses the freshness lifetime', async () => { + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() WHERE id = 1`; + // A notification that just arrived enqueues a refresh while the previous metadata is still + // being served, so the response must not be cached for the full TTL. + await db.sql`UPDATE jobs SET status = 'pending' WHERE token_id = 1`; + + const response = await fastify.inject({ method: 'GET', url }); + assert.strictEqual(response.statusCode, 200); + assert.strictEqual(response.headers['cache-control'], 'public, no-cache, must-revalidate'); + assert.strictEqual(response.headers.expires, undefined); + }); + + test('non-canonical notifications are ignored', async () => { + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + canonical: false, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() WHERE id = 1`; + + const response = await fastify.inject({ method: 'GET', url }); + assert.strictEqual(response.statusCode, 200); + assert.strictEqual(response.headers['cache-control'], 'public, no-cache, must-revalidate'); + assert.strictEqual(response.headers.expires, undefined); + }); + + test('a non-canonical update mode does not supersede a dynamic ttl', async () => { + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + event_index: 0, + }); + // Re-orgs keep notification rows and only flip `canonical`, so this orphaned event must not + // affect the token's update mode. + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.frozen, + event_index: 1, + canonical: false, + }); + await db.sql`UPDATE tokens SET updated_at = NOW() WHERE id = 1`; + + const response = await fastify.inject({ method: 'GET', url }); + assert.strictEqual(response.statusCode, 200); + const age = maxAge(response); + assert.ok(age !== undefined && age > 3590 && age <= 3600, `unexpected max-age: ${age}`); + }); + test('errors are not cacheable', async () => { await insertTestUpdateNotification(db, { token_id: 1, diff --git a/tests/helpers.ts b/tests/helpers.ts index ac7621d3..d54d2709 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -1457,15 +1457,17 @@ export async function insertTestUpdateNotification( index_block_hash?: string; tx_index?: number; event_index?: number; + canonical?: boolean; } ): Promise { await db.sql` INSERT INTO update_notifications - (token_id, update_mode, ttl, block_height, index_block_hash, tx_id, tx_index, event_index) + (token_id, update_mode, ttl, block_height, index_block_hash, tx_id, tx_index, event_index, + canonical) VALUES ( ${args.token_id}, ${args.update_mode}, ${args.ttl ?? null}, ${args.block_height ?? 1}, ${args.index_block_hash ?? '0x000001'}, '0x123456', ${args.tx_index ?? 0}, - ${args.event_index ?? 0} + ${args.event_index ?? 0}, ${args.canonical ?? true} ) `; } diff --git a/tests/stacks-core/block-processor.test.ts b/tests/stacks-core/block-processor.test.ts index 523e8f03..0ebf355c 100644 --- a/tests/stacks-core/block-processor.test.ts +++ b/tests/stacks-core/block-processor.test.ts @@ -299,6 +299,34 @@ describe('block processor', () => { assert.deepStrictEqual(await getTokenJobStatuses(), ['pending']); }); + test('ignores non-canonical notifications', async () => { + ENV.METADATA_DYNAMIC_TOKEN_REFRESH_INTERVAL = 99999; + await insertAndEnqueueTestContractWithTokens(db, contractId, DbSipNumber.sip009, 2n); + for (const token_id of [1, 2]) + await insertTestUpdateNotification(db, { + token_id, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + event_index: 0, + }); + // Re-orgs keep notification rows and only flip `canonical`. An orphaned 'frozen' event + // must not stop token 1 from being refreshed, and an orphaned 'dynamic' event must not + // make token 3 eligible. + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.frozen, + event_index: 1, + canonical: false, + }); + await db.sql`UPDATE update_notifications SET canonical = false WHERE token_id = 2`; + await db.sql`UPDATE tokens SET updated_at = NOW() - INTERVAL '2 hours'`; + await markAllJobsAsDone(db); + + await processNextBlock(); + + assert.deepStrictEqual(await getTokenJobStatuses(), ['pending', 'done']); + }); + test('tolerates a ttl large enough to overflow an interval', async () => { ENV.METADATA_DYNAMIC_TOKEN_REFRESH_INTERVAL = 99999; await insertAndEnqueueTestContractWithTokens(db, contractId, DbSipNumber.sip009, 1n); From 8c80fb5b7e0ecca9641563978a97f67f703b75ef Mon Sep 17 00:00:00 2001 From: Rafa Cardenas <253999660+rafa-stacks@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:38:25 -0600 Subject: [PATCH 3/5] ttl precision --- src/pg/pg-store.ts | 20 ++++++++++++-------- src/pg/stacks-core-pg-store.ts | 2 +- tests/api/cache.test.ts | 16 ++++++++++++++++ tests/stacks-core/block-processor.test.ts | 4 ++-- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/pg/pg-store.ts b/src/pg/pg-store.ts index 73055c33..3809f745 100644 --- a/src/pg/pg-store.ts +++ b/src/pg/pg-store.ts @@ -38,7 +38,7 @@ import { } from '@stacks/api-toolkit'; import * as path from 'path'; import { fileURLToPath } from 'url'; -import { StacksCorePgStore } from './stacks-core-pg-store.js'; +import { MAX_TOKEN_TTL_SECONDS, StacksCorePgStore } from './stacks-core-pg-store.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -208,8 +208,6 @@ export class PgStore extends BasePgStore { contractPrincipal: string; tokenNumber: bigint; }): Promise { - // Cap the TTL before turning it into an `interval` so an absurd value declared by a contract - // can't overflow postgres' interval type. const maxAge = ENV.METADATA_DYNAMIC_TOKEN_MAX_CACHE_AGE; const result = await this.sql<{ etag: string; max_age: number | null }[]>` SELECT @@ -217,11 +215,17 @@ export class PgStore extends BasePgStore { -- A job that is not done means a refresh is already in flight (e.g. a notification just -- arrived), and we keep serving the previous metadata until it completes. Advertising -- freshness then would cache metadata we already know is about to change. + -- Seconds left until the token may change, capped by the configured max. The TTL itself + -- is only clamped to keep an absurd value from overflowing the interval type, so a TTL + -- longer than the cap keeps advertising the cap instead of expiring along with it. CASE WHEN n.update_mode = 'dynamic' AND n.ttl IS NOT NULL AND j.status = 'done' THEN - CEIL(EXTRACT(EPOCH FROM ( - COALESCE(t.updated_at, t.created_at) - + INTERVAL '1 seconds' * LEAST(n.ttl, ${maxAge}) - NOW() - )))::int + LEAST( + CEIL(EXTRACT(EPOCH FROM ( + COALESCE(t.updated_at, t.created_at) + + INTERVAL '1 seconds' * LEAST(n.ttl, ${MAX_TOKEN_TTL_SECONDS}) - NOW() + ))), + ${maxAge} + )::int END AS max_age FROM tokens AS t INNER JOIN smart_contracts AS s ON s.id = t.smart_contract_id @@ -242,7 +246,7 @@ export class PgStore extends BasePgStore { const cache: DbTokenCacheInfo = { etag: result[0].etag }; // Only advertise a freshness lifetime if the token isn't already due for a refresh. if (result[0].max_age !== null && result[0].max_age > 0) { - cache.maxAge = Math.min(result[0].max_age, maxAge); + cache.maxAge = result[0].max_age; } return cache; } diff --git a/src/pg/stacks-core-pg-store.ts b/src/pg/stacks-core-pg-store.ts index 65a51621..0ac476bf 100644 --- a/src/pg/stacks-core-pg-store.ts +++ b/src/pg/stacks-core-pg-store.ts @@ -30,7 +30,7 @@ import { DecodedStacksBlock } from '../stacks-core/stacks-core-block-processor.j * they can be arbitrarily large, and postgres throws `interval out of range` when converting them, * which would abort block ingestion. */ -const MAX_TOKEN_TTL_SECONDS = 3_153_600_000; +export const MAX_TOKEN_TTL_SECONDS = 3_153_600_000; export class StacksCorePgStore extends BasePgStoreModule { /** diff --git a/tests/api/cache.test.ts b/tests/api/cache.test.ts index d006ac31..ca39c796 100644 --- a/tests/api/cache.test.ts +++ b/tests/api/cache.test.ts @@ -478,6 +478,22 @@ describe('Dynamic token cache control', () => { assert.ok(age !== undefined && age > 290 && age <= 300, `unexpected max-age: ${age}`); }); + test('a ttl longer than the cap keeps advertising the cap', async () => { + ENV.METADATA_DYNAMIC_TOKEN_MAX_CACHE_AGE = 300; + await insertTestUpdateNotification(db, { + token_id: 1, + update_mode: DbTokenUpdateMode.dynamic, + ttl: 3600, + }); + // Well past the cap but still an hour short of the token's own TTL, so the metadata provably + // can't change yet and we should keep advertising the capped lifetime. + await db.sql`UPDATE tokens SET updated_at = NOW() - INTERVAL '20 minutes' WHERE id = 1`; + + const response = await fastify.inject({ method: 'GET', url }); + assert.strictEqual(response.statusCode, 200); + assert.strictEqual(maxAge(response), 300); + }); + test('dynamic token without a ttl must revalidate', async () => { await insertTestUpdateNotification(db, { token_id: 1, diff --git a/tests/stacks-core/block-processor.test.ts b/tests/stacks-core/block-processor.test.ts index 0ebf355c..7637d809 100644 --- a/tests/stacks-core/block-processor.test.ts +++ b/tests/stacks-core/block-processor.test.ts @@ -310,8 +310,8 @@ describe('block processor', () => { event_index: 0, }); // Re-orgs keep notification rows and only flip `canonical`. An orphaned 'frozen' event - // must not stop token 1 from being refreshed, and an orphaned 'dynamic' event must not - // make token 3 eligible. + // must not stop token 1 from being refreshed, and token 2's only notification is orphaned + // so it must not be treated as dynamic at all. await insertTestUpdateNotification(db, { token_id: 1, update_mode: DbTokenUpdateMode.frozen, From aa118e77c18291719a25cbe2e31c8b1b66b4945b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:46:55 +0000 Subject: [PATCH 4/5] Fix same-origin redirect header normalization Co-authored-by: rafa-stacks <253999660+rafa-stacks@users.noreply.github.com> --- .../util/fetch-header-policy.ts | 7 +++++- .../fetch-destination-policy.test.ts | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/token-processor/util/fetch-header-policy.ts b/src/token-processor/util/fetch-header-policy.ts index f4152c75..8b0aa268 100644 --- a/src/token-processor/util/fetch-header-policy.ts +++ b/src/token-processor/util/fetch-header-policy.ts @@ -1,5 +1,9 @@ import { Dispatcher } from 'undici'; +function normalizeOrigin(origin: string): string { + return new URL(origin).origin; +} + /** * Drops the named headers from a dispatch, whatever shape undici is carrying them in: the first * dispatch gets the object the caller passed, while a redirected one gets the flat @@ -47,8 +51,9 @@ export function stripHeadersOffOrigin( headerNames: string[] ): Dispatcher.DispatcherComposeInterceptor { const drop = new Set(headerNames.map(name => name.toLowerCase())); + const normalizedOrigin = normalizeOrigin(origin); return dispatch => (opts, handler) => { - if (String(opts.origin) === origin) return dispatch(opts, handler); + if (normalizeOrigin(String(opts.origin)) === normalizedOrigin) return dispatch(opts, handler); return dispatch({ ...opts, headers: withoutHeaders(opts.headers, drop) }, handler); }; } diff --git a/tests/token-queue/fetch-destination-policy.test.ts b/tests/token-queue/fetch-destination-policy.test.ts index 01663e0a..e1518e6f 100644 --- a/tests/token-queue/fetch-destination-policy.test.ts +++ b/tests/token-queue/fetch-destination-policy.test.ts @@ -14,6 +14,7 @@ import { isBlockedIpAddress, setLoopbackAllowedForTesting, } from '../../src/token-processor/util/fetch-destination-policy.js'; +import { stripHeadersOffOrigin } from '../../src/token-processor/util/fetch-header-policy.js'; import { fetchAllMetadataLocalesFromBaseUri, fetchMetadata, @@ -370,4 +371,26 @@ describe('Fetch destination policy', () => { DbJobInvalidReason.fetchDestinationBlocked ); }); + + test('keeps headers on same-origin redirects despite default port spelling differences', () => { + let dispatchedHeaders: Dispatcher.DispatchOptions['headers'] | undefined; + const intercept = stripHeadersOffOrigin('http://example.com', ['X-Api-Key'])( + ((opts: Dispatcher.DispatchOptions) => { + dispatchedHeaders = opts.headers; + return true; + }) as Dispatcher['dispatch'] + ); + + intercept( + { + origin: 'http://example.com:80', + path: '/', + method: 'GET', + headers: { 'X-Api-Key': 'gateway-secret' }, + }, + {} as Dispatcher.DispatchHandlers + ); + + assert.deepStrictEqual(dispatchedHeaders, { 'X-Api-Key': 'gateway-secret' }); + }); }); From 1abbb00e4ca5678f38089a9085abad86da53bb32 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:47:54 +0000 Subject: [PATCH 5/5] Clean up merged test comment Co-authored-by: rafa-stacks <253999660+rafa-stacks@users.noreply.github.com> --- tests/stacks-core/block-processor.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/stacks-core/block-processor.test.ts b/tests/stacks-core/block-processor.test.ts index 821ff99f..7637d809 100644 --- a/tests/stacks-core/block-processor.test.ts +++ b/tests/stacks-core/block-processor.test.ts @@ -311,7 +311,6 @@ describe('block processor', () => { }); // Re-orgs keep notification rows and only flip `canonical`. An orphaned 'frozen' event // must not stop token 1 from being refreshed, and token 2's only notification is orphaned - // must not stop token 1 from being refreshed, and token 2's only notification is orphaned // so it must not be treated as dynamic at all. await insertTestUpdateNotification(db, { token_id: 1,