diff --git a/docs/openapi/headers.yaml b/docs/openapi/headers.yaml index 06a7565..e41c1ac 100644 --- a/docs/openapi/headers.yaml +++ b/docs/openapi/headers.yaml @@ -8,3 +8,8 @@ location: description: The redirect location schema: type: string + +daContinuationToken: + description: Continuation token for fetching the next page of list results. + schema: + type: string diff --git a/docs/openapi/list-api.yaml b/docs/openapi/list-api.yaml index 2d682ee..c486f36 100644 --- a/docs/openapi/list-api.yaml +++ b/docs/openapi/list-api.yaml @@ -9,10 +9,11 @@ list: - $ref: "./parameters.yaml#/orgParam" - $ref: "./parameters.yaml#/repoParam" - $ref: "./parameters.yaml#/pathParam" + - $ref: "./parameters.yaml#/continuationTokenParam" responses: '200': $ref: "./responses.yaml#/list/200" '400': $ref: "./responses.yaml#/400" '404': - $ref: "./responses.yaml#/404" \ No newline at end of file + $ref: "./responses.yaml#/404" diff --git a/docs/openapi/parameters.yaml b/docs/openapi/parameters.yaml index 059a630..de0001f 100644 --- a/docs/openapi/parameters.yaml +++ b/docs/openapi/parameters.yaml @@ -26,6 +26,13 @@ pathParam: required: true schema: type: string +continuationTokenParam: + name: da-continuation-token + in: header + description: Continuation token from the previous list response header. + required: false + schema: + type: string guidParam: name: guid in: path diff --git a/docs/openapi/responses.yaml b/docs/openapi/responses.yaml index c7611e9..3811751 100644 --- a/docs/openapi/responses.yaml +++ b/docs/openapi/responses.yaml @@ -35,6 +35,9 @@ source: list: '200': description: The list of sources + headers: + da-continuation-token: + $ref: "./headers.yaml#/daContinuationToken" content: application/json: schema: diff --git a/src/storage/object/list.js b/src/storage/object/list.js index a00abab..72860f2 100644 --- a/src/storage/object/list.js +++ b/src/storage/object/list.js @@ -18,7 +18,7 @@ import getS3Config from '../utils/config.js'; import formatList from '../utils/list.js'; function buildInput({ - bucket, org, key, maxKeys, + bucket, org, key, maxKeys, continuationToken, }) { const input = { Bucket: bucket, @@ -26,6 +26,7 @@ function buildInput({ Delimiter: '/', }; if (maxKeys) input.MaxKeys = maxKeys; + if (continuationToken) input.ContinuationToken = continuationToken; return input; } @@ -33,16 +34,25 @@ export default async function listObjects(env, daCtx, maxKeys) { const config = getS3Config(env); const client = new S3Client(config); - const input = buildInput({ ...daCtx, maxKeys }); + const input = buildInput({ + ...daCtx, + maxKeys, + }); const command = new ListObjectsV2Command(input); try { const resp = await client.send(command); // console.log(resp); const body = formatList(resp); + const nextContinuationToken = resp.IsTruncated + && resp.NextContinuationToken + && resp.NextContinuationToken !== daCtx.continuationToken + ? resp.NextContinuationToken + : undefined; return { body: JSON.stringify(body), status: resp.$metadata.httpStatusCode, contentType: resp.ContentType, + continuationToken: nextContinuationToken, }; } catch (e) { return { body: '', status: 404 }; diff --git a/src/utils/daCtx.js b/src/utils/daCtx.js index 2964db7..2475dce 100644 --- a/src/utils/daCtx.js +++ b/src/utils/daCtx.js @@ -18,7 +18,8 @@ import { getAclCtx, getUsers } from './auth.js'; * @returns {DaCtx} The Dark Alley Context. */ export default async function getDaCtx(req, env) { - let { pathname } = new URL(req.url); + const url = new URL(req.url); + let { pathname } = url; // Remove proxied api route if (pathname.startsWith('/api')) pathname = pathname.replace('/api', ''); @@ -38,6 +39,7 @@ export default async function getDaCtx(req, env) { // Extract conditional headers const ifMatch = req.headers?.get('if-match') || null; const ifNoneMatch = req.headers?.get('if-none-match') || null; + const continuationToken = req.headers?.get('da-continuation-token') || null; // Set base details const daCtx = { @@ -53,6 +55,7 @@ export default async function getDaCtx(req, env) { ifMatch, ifNoneMatch, }, + continuationToken, }; // Sanitize the remaining path parts diff --git a/src/utils/daResp.js b/src/utils/daResp.js index 633972f..fde3964 100644 --- a/src/utils/daResp.js +++ b/src/utils/daResp.js @@ -27,12 +27,13 @@ export default function daResp({ contentLength, metadata, etag, + continuationToken, }, ctx = null) { const headers = new Headers(); headers.append('Access-Control-Allow-Origin', '*'); headers.append('Access-Control-Allow-Methods', 'HEAD, GET, PUT, POST, DELETE'); headers.append('Access-Control-Allow-Headers', '*'); - headers.append('Access-Control-Expose-Headers', 'X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, ETag'); + headers.append('Access-Control-Expose-Headers', 'X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, da-continuation-token, ETag'); headers.append('Content-Type', contentType); if (contentLength) { headers.append('Content-Length', contentLength); @@ -48,6 +49,9 @@ export default function daResp({ if (etag) { headers.append('ETag', etag); } + if (continuationToken) { + headers.append('da-continuation-token', continuationToken); + } if (ctx?.aclCtx && status < 500) { headers.append('X-da-actions', `/${ctx.key}=${[...ctx.aclCtx.actionSet]}`); diff --git a/test/index.test.js b/test/index.test.js index 5558063..529add4 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -90,6 +90,31 @@ describe('fetch', () => { const resp = await hnd.fetch({ method: 'GET', url: 'http://www.example.com/source/org/repo/file.html' }, {}); assert.strictEqual(resp.status, 500); }); + + it('should expose continuation token header for list responses', async () => { + const hnd = await esmock('../src/index.js', { + '../src/utils/daCtx.js': { + default: async () => ({ + authorized: true, + users: [{ email: 'test@example.com' }], + path: '/list/org/repo/path', + key: 'repo/path', + }), + }, + '../src/handlers/get.js': { + default: async () => ({ + status: 200, + body: '[]', + contentType: 'application/json', + continuationToken: 'next-token', + }), + }, + }); + + const resp = await hnd.fetch({ method: 'GET', url: 'http://www.example.com/list/org/repo/path' }, {}); + assert.strictEqual(resp.status, 200); + assert.strictEqual(resp.headers.get('da-continuation-token'), 'next-token'); + }); }); describe('invalid routes', () => { diff --git a/test/storage/object/list.test.js b/test/storage/object/list.test.js index 04cf3ca..993f2a8 100644 --- a/test/storage/object/list.test.js +++ b/test/storage/object/list.test.js @@ -56,4 +56,75 @@ describe('List Objects', () => { const data = JSON.parse(resp.body); assert.strictEqual(data.length, 2, 'Should only return 2 items'); }); + + it('passes continuation token and returns next token', async () => { + s3Mock.on(ListObjectsV2Command, { + Bucket: 'rt-bkt', + Prefix: 'acme/wknd/', + Delimiter: '/', + ContinuationToken: 'prev-token', + }).resolves({ + $metadata: { httpStatusCode: 200 }, + Contents: [Contents[0]], + IsTruncated: true, + NextContinuationToken: 'next-token', + }); + + const daCtx = { + bucket: 'rt-bkt', + org: 'acme', + key: 'wknd', + continuationToken: 'prev-token', + }; + const resp = await listObjects({}, daCtx); + const data = JSON.parse(resp.body); + assert.strictEqual(data.length, 1, 'Should only return 1 item'); + assert.strictEqual(resp.continuationToken, 'next-token'); + }); + + it('does not return continuation token on terminal page', async () => { + s3Mock.on(ListObjectsV2Command, { + Bucket: 'rt-bkt', + Prefix: 'acme/wknd/', + Delimiter: '/', + ContinuationToken: 'prev-token', + }).resolves({ + $metadata: { httpStatusCode: 200 }, + Contents: [Contents[1]], + IsTruncated: false, + NextContinuationToken: 'should-not-be-returned', + }); + + const daCtx = { + bucket: 'rt-bkt', + org: 'acme', + key: 'wknd', + continuationToken: 'prev-token', + }; + const resp = await listObjects({}, daCtx); + assert.strictEqual(resp.continuationToken, undefined); + }); + + it('does not return same continuation token again', async () => { + s3Mock.on(ListObjectsV2Command, { + Bucket: 'rt-bkt', + Prefix: 'acme/wknd/', + Delimiter: '/', + ContinuationToken: 'prev-token', + }).resolves({ + $metadata: { httpStatusCode: 200 }, + Contents: [Contents[2]], + IsTruncated: true, + NextContinuationToken: 'prev-token', + }); + + const daCtx = { + bucket: 'rt-bkt', + org: 'acme', + key: 'wknd', + continuationToken: 'prev-token', + }; + const resp = await listObjects({}, daCtx); + assert.strictEqual(resp.continuationToken, undefined); + }); }); diff --git a/test/utils/daCtx.test.js b/test/utils/daCtx.test.js index 4ecdd04..a29f600 100644 --- a/test/utils/daCtx.test.js +++ b/test/utils/daCtx.test.js @@ -217,4 +217,56 @@ describe('DA context', () => { assert.strictEqual(daCtx.conditionalHeaders.ifNoneMatch, null); }); }); + + describe('Continuation token', async () => { + it('should extract da-continuation-token header', async () => { + const req = { + url: 'http://localhost:8787/list/org/site/path', + headers: { + get: (name) => { + if (name === 'da-continuation-token') return 'header-token'; + return null; + }, + }, + }; + const daCtx = await getDaCtx(req, env); + assert.strictEqual(daCtx.continuationToken, 'header-token'); + }); + + it('should ignore continuation-token query param when header is present', async () => { + const req = { + url: 'http://localhost:8787/list/org/site/path?continuation-token=query-token', + headers: { + get: (name) => { + if (name === 'da-continuation-token') return 'header-token'; + return null; + }, + }, + }; + const daCtx = await getDaCtx(req, env); + assert.strictEqual(daCtx.continuationToken, 'header-token'); + }); + + it('should ignore continuation-token query param', async () => { + const req = { + url: 'http://localhost:8787/list/org/site/path?continuation-token=token123', + headers: { + get: () => null, + }, + }; + const daCtx = await getDaCtx(req, env); + assert.strictEqual(daCtx.continuationToken, null); + }); + + it('should default continuation token to null', async () => { + const req = { + url: 'http://localhost:8787/list/org/site/path', + headers: { + get: () => null, + }, + }; + const daCtx = await getDaCtx(req, env); + assert.strictEqual(daCtx.continuationToken, null); + }); + }); }); diff --git a/test/utils/daResp.test.js b/test/utils/daResp.test.js index 7878119..25134b8 100644 --- a/test/utils/daResp.test.js +++ b/test/utils/daResp.test.js @@ -28,7 +28,7 @@ describe('DA Resp', () => { assert.strictEqual('*', resp.headers.get('Access-Control-Allow-Origin')); assert.strictEqual('HEAD, GET, PUT, POST, DELETE', resp.headers.get('Access-Control-Allow-Methods')); assert.strictEqual('*', resp.headers.get('Access-Control-Allow-Headers')); - assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, ETag', resp.headers.get('Access-Control-Expose-Headers')); + assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, da-continuation-token, ETag', resp.headers.get('Access-Control-Expose-Headers')); assert.strictEqual('text/plain', resp.headers.get('Content-Type')); assert.strictEqual('777', resp.headers.get('Content-Length')); assert.strictEqual('/foo/bar.html=read,write', resp.headers.get('X-da-actions')); @@ -49,7 +49,7 @@ describe('DA Resp', () => { assert.strictEqual('*', resp.headers.get('Access-Control-Allow-Origin')); assert.strictEqual('HEAD, GET, PUT, POST, DELETE', resp.headers.get('Access-Control-Allow-Methods')); assert.strictEqual('*', resp.headers.get('Access-Control-Allow-Headers')); - assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, ETag', resp.headers.get('Access-Control-Expose-Headers')); + assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, da-continuation-token, ETag', resp.headers.get('Access-Control-Expose-Headers')); assert.strictEqual('application/json', resp.headers.get('Content-Type')); assert(!resp.headers.get('Content-Length')); assert.strictEqual('/foo/blah.html=read', resp.headers.get('X-da-actions')); @@ -76,7 +76,7 @@ describe('DA Resp', () => { const ctx = { key: 'foo/bar.html', aclCtx }; const resp = daResp({ status: 200, body: 'foobar' }, ctx); assert.strictEqual(200, resp.status); - assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, ETag', resp.headers.get('Access-Control-Expose-Headers')); + assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, da-continuation-token, ETag', resp.headers.get('Access-Control-Expose-Headers')); assert.strictEqual('/haha/hoho/**=read,write', resp.headers.get('X-da-child-actions')); }); });