Add pagination to list api

docs/openapi/headers.yaml

@@ -8,3 +8,8 @@ location:
   description: The redirect location
   schema:
     type: string
+
+daContinuationToken:
+  description: Continuation token for fetching the next page of list results.
+  schema:
+    type: string

docs/openapi/list-api.yaml

@@ -9,10 +9,11 @@ list:
       - $ref: "./parameters.yaml#/orgParam"
       - $ref: "./parameters.yaml#/repoParam"
       - $ref: "./parameters.yaml#/pathParam"
+      - $ref: "./parameters.yaml#/continuationTokenParam"
     responses:
       '200':
         $ref: "./responses.yaml#/list/200"
       '400':
         $ref: "./responses.yaml#/400"
       '404':
-        $ref: "./responses.yaml#/404"
+        $ref: "./responses.yaml#/404"

docs/openapi/parameters.yaml

@@ -26,6 +26,13 @@ pathParam:
   required: true
   schema:
     type: string
+continuationTokenParam:
+  name: continuation-token
+  in: query
+  description: Continuation token for paginated list results.
+  required: false
+  schema:
+    type: string
 guidParam:
   name: guid
   in: path

docs/openapi/responses.yaml

@@ -35,6 +35,9 @@ source:
 list:
   '200':
     description: The list of sources
+    headers:
+      da-continuation-token:
+        $ref: "./headers.yaml#/daContinuationToken"
     content:
       application/json:
         schema:

src/storage/object/list.js

@@ -18,22 +18,27 @@ import getS3Config from '../utils/config.js';
 import formatList from '../utils/list.js';
 
 function buildInput({
-  bucket, org, key, maxKeys,
+  bucket, org, key, maxKeys, continuationToken,
 }) {
   const input = {
     Bucket: bucket,
     Prefix: key ? `${org}/${key}/` : `${org}/`,
     Delimiter: '/',
   };
   if (maxKeys) input.MaxKeys = maxKeys;
+  if (continuationToken) input.ContinuationToken = continuationToken;
   return input;
 }
 
 export default async function listObjects(env, daCtx, maxKeys) {
   const config = getS3Config(env);
   const client = new S3Client(config);
 
-  const input = buildInput({ ...daCtx, maxKeys });
+  const input = buildInput({
+    ...daCtx,
+    maxKeys,
+    continuationToken: daCtx.continuationToken,
+  });
   const command = new ListObjectsV2Command(input);
   try {
     const resp = await client.send(command);
@@ -43,6 +48,7 @@ export default async function listObjects(env, daCtx, maxKeys) {
       body: JSON.stringify(body),
       status: resp.$metadata.httpStatusCode,
       contentType: resp.ContentType,
+      continuationToken: resp.NextContinuationToken,
     };
   } catch (e) {
     return { body: '', status: 404 };

src/utils/daCtx.js

@@ -18,7 +18,8 @@ import { getAclCtx, getUsers } from './auth.js';
  * @returns {DaCtx} The Dark Alley Context.
  */
 export default async function getDaCtx(req, env) {
-  let { pathname } = new URL(req.url);
+  const url = new URL(req.url);
+  let { pathname } = url;
   // Remove proxied api route
   if (pathname.startsWith('/api')) pathname = pathname.replace('/api', '');
 
@@ -38,6 +39,9 @@ export default async function getDaCtx(req, env) {
   // Extract conditional headers
   const ifMatch = req.headers?.get('if-match') || null;
   const ifNoneMatch = req.headers?.get('if-none-match') || null;
+  const continuationToken = url.searchParams.get('continuation-token')
+    || url.searchParams.get('continuationToken')
+    || null;
 
   // Set base details
   const daCtx = {
@@ -53,6 +57,7 @@ export default async function getDaCtx(req, env) {
       ifMatch,
       ifNoneMatch,
     },
+    continuationToken,
   };
 
   // Sanitize the remaining path parts

src/utils/daResp.js

@@ -27,12 +27,13 @@ export default function daResp({
   contentLength,
   metadata,
   etag,
+  continuationToken,
 }, ctx = null) {
   const headers = new Headers();
   headers.append('Access-Control-Allow-Origin', '*');
   headers.append('Access-Control-Allow-Methods', 'HEAD, GET, PUT, POST, DELETE');
   headers.append('Access-Control-Allow-Headers', '*');
-  headers.append('Access-Control-Expose-Headers', 'X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, ETag');
+  headers.append('Access-Control-Expose-Headers', 'X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, da-continuation-token, ETag');
   headers.append('Content-Type', contentType);
   if (contentLength) {
     headers.append('Content-Length', contentLength);
@@ -48,6 +49,9 @@ export default function daResp({
   if (etag) {
     headers.append('ETag', etag);
   }
+  if (continuationToken) {
+    headers.append('da-continuation-token', continuationToken);
+  }
 
   if (ctx?.aclCtx && status < 500) {
     headers.append('X-da-actions', `/${ctx.key}=${[...ctx.aclCtx.actionSet]}`);

test/index.test.js

@@ -90,6 +90,31 @@ describe('fetch', () => {
     const resp = await hnd.fetch({ method: 'GET', url: 'http://www.example.com/source/org/repo/file.html' }, {});
     assert.strictEqual(resp.status, 500);
   });
+
+  it('should expose continuation token header for list responses', async () => {
+    const hnd = await esmock('../src/index.js', {
+      '../src/utils/daCtx.js': {
+        default: async () => ({
+          authorized: true,
+          users: [{ email: 'test@example.com' }],
+          path: '/list/org/repo/path',
+          key: 'repo/path',
+        }),
+      },
+      '../src/handlers/get.js': {
+        default: async () => ({
+          status: 200,
+          body: '[]',
+          contentType: 'application/json',
+          continuationToken: 'next-token',
+        }),
+      },
+    });
+
+    const resp = await hnd.fetch({ method: 'GET', url: 'http://www.example.com/list/org/repo/path' }, {});
+    assert.strictEqual(resp.status, 200);
+    assert.strictEqual(resp.headers.get('da-continuation-token'), 'next-token');
+  });
 });
 
 describe('invalid routes', () => {

test/storage/object/list.test.js

@@ -56,4 +56,28 @@ describe('List Objects', () => {
     const data = JSON.parse(resp.body);
     assert.strictEqual(data.length, 2, 'Should only return 2 items');
   });
+
+  it('passes continuation token and returns next token', async () => {
+    s3Mock.on(ListObjectsV2Command, {
+      Bucket: 'rt-bkt',
+      Prefix: 'acme/wknd/',
+      Delimiter: '/',
+      ContinuationToken: 'prev-token',
+    }).resolves({
+      $metadata: { httpStatusCode: 200 },
+      Contents: [Contents[0]],
+      NextContinuationToken: 'next-token',
+    });
+
+    const daCtx = {
+      bucket: 'rt-bkt',
+      org: 'acme',
+      key: 'wknd',
+      continuationToken: 'prev-token',
+    };
+    const resp = await listObjects({}, daCtx);
+    const data = JSON.parse(resp.body);
+    assert.strictEqual(data.length, 1, 'Should only return 1 item');
+    assert.strictEqual(resp.continuationToken, 'next-token');
+  });
 });

test/utils/daCtx.test.js

@@ -217,4 +217,28 @@ describe('DA context', () => {
       assert.strictEqual(daCtx.conditionalHeaders.ifNoneMatch, null);
     });
   });
+
+  describe('Continuation token', async () => {
+    it('should extract continuation-token query param', async () => {
+      const req = {
+        url: 'http://localhost:8787/list/org/site/path?continuation-token=token123',
+        headers: {
+          get: () => null,
+        },
+      };
+      const daCtx = await getDaCtx(req, env);
+      assert.strictEqual(daCtx.continuationToken, 'token123');
+    });
+
+    it('should default continuation token to null', async () => {
+      const req = {
+        url: 'http://localhost:8787/list/org/site/path',
+        headers: {
+          get: () => null,
+        },
+      };
+      const daCtx = await getDaCtx(req, env);
+      assert.strictEqual(daCtx.continuationToken, null);
+    });
+  });
 });

test/utils/daResp.test.js

@@ -28,7 +28,7 @@ describe('DA Resp', () => {
     assert.strictEqual('*', resp.headers.get('Access-Control-Allow-Origin'));
     assert.strictEqual('HEAD, GET, PUT, POST, DELETE', resp.headers.get('Access-Control-Allow-Methods'));
     assert.strictEqual('*', resp.headers.get('Access-Control-Allow-Headers'));
-    assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, ETag', resp.headers.get('Access-Control-Expose-Headers'));
+    assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, da-continuation-token, ETag', resp.headers.get('Access-Control-Expose-Headers'));
     assert.strictEqual('text/plain', resp.headers.get('Content-Type'));
     assert.strictEqual('777', resp.headers.get('Content-Length'));
     assert.strictEqual('/foo/bar.html=read,write', resp.headers.get('X-da-actions'));
@@ -49,7 +49,7 @@ describe('DA Resp', () => {
     assert.strictEqual('*', resp.headers.get('Access-Control-Allow-Origin'));
     assert.strictEqual('HEAD, GET, PUT, POST, DELETE', resp.headers.get('Access-Control-Allow-Methods'));
     assert.strictEqual('*', resp.headers.get('Access-Control-Allow-Headers'));
-    assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, ETag', resp.headers.get('Access-Control-Expose-Headers'));
+    assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, da-continuation-token, ETag', resp.headers.get('Access-Control-Expose-Headers'));
     assert.strictEqual('application/json', resp.headers.get('Content-Type'));
     assert(!resp.headers.get('Content-Length'));
     assert.strictEqual('/foo/blah.html=read', resp.headers.get('X-da-actions'));
@@ -76,7 +76,7 @@ describe('DA Resp', () => {
     const ctx = { key: 'foo/bar.html', aclCtx };
     const resp = daResp({ status: 200, body: 'foobar' }, ctx);
     assert.strictEqual(200, resp.status);
-    assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, ETag', resp.headers.get('Access-Control-Expose-Headers'));
+    assert.strictEqual('X-da-actions, X-da-child-actions, X-da-acltrace, X-da-id, da-continuation-token, ETag', resp.headers.get('Access-Control-Expose-Headers'));
     assert.strictEqual('/haha/hoho/**=read,write', resp.headers.get('X-da-child-actions'));
   });
 });