fix(selectors): fetch all pages for paginated dropdown list routes (#4823)

* fix(selectors): fetch all pages for paginated dropdown list routes

Dropdown selectors fetched only the first page of paginated provider
APIs, silently hiding results past page one. Add bounded server-side
draining to the list routes across Microsoft Graph, Google, Notion,
Atlassian, Linear, AWS CloudWatch, and offset/token REST APIs, plus a
shared client-side drain cap in the selector hook. Response shapes,
stored values, and tool execution are unchanged; CloudWatch list tools
still honor a caller-supplied limit. Also fixes the Word file picker
that was searching for .xlsx files.

* fix(selectors): harden JSM and Monday pagination draining

- JSM service-desk/request-type drains advance `start` by the actual row
  count returned (not the fixed page size) and stop on an empty page, so a
  short non-final page can't skip items.
- Monday boards drain now checks `response.ok` per page, surfacing a
  mid-drain HTTP failure instead of treating it as an empty final page and
  returning a partial 200.

* docs(selectors): clarify JSM drain advances start by actual row count

The offset-advancement fix (advance `start` by the rows returned, not the
fixed page size) landed in 7b19788; update the TSDoc to match so it no
longer reads as advancing by `limit`.

* fix(selectors): drain fetchPage in direct fetchList callers

Making `fetchList` optional left three direct callers (outside the
useSelectorOptions hook) calling it unguarded, which broke the build's
type check. Route them through a shared `loadAllSelectorOptions` helper
that uses `fetchList` when present and otherwise drains `fetchPage`.
This also prevents a regression: `confluence.spaces` / `knowledge.documents`
now paginate via `fetchPage` only, and these callers (search/replace,
value resolution) would otherwise have silently returned no options.

* chore(selectors): rename MAX_PAGE_PAGES to MAX_NOTION_PAGES for readability

Author: waleedlatif1

apps/sim/app/api/auth/oauth/microsoft/files/route.ts

@@ -8,13 +8,57 @@ import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { getCredential, refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
 import { GRAPH_ID_PATTERN } from '@/tools/microsoft_excel/utils'
+import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
 
 export const dynamic = 'force-dynamic'
 
 const logger = createLogger('MicrosoftFilesAPI')
 
 /**
- * Get Excel files from Microsoft OneDrive
+ * Microsoft Graph paginates `search()` results via the `@odata.nextLink`
+ * absolute URL in the response body. Request the largest page (`$top` caps at
+ * 999) and drain following nextLink, bounded by a page cap.
+ * See https://learn.microsoft.com/en-us/graph/paging
+ */
+const MICROSOFT_FILES_PAGE_SIZE = 999
+const MAX_MICROSOFT_FILES_PAGES = 20
+
+interface MicrosoftGraphFile {
+  id: string
+  name?: string
+  mimeType?: string
+  webUrl?: string
+  size?: number
+  createdDateTime?: string
+  lastModifiedDateTime?: string
+  thumbnails?: Array<{ small?: { url?: string }; medium?: { url?: string } }>
+  createdBy?: { user?: { displayName?: string; email?: string } }
+}
+
+/**
+ * The shared `/api/auth/oauth/microsoft/files` route serves both the
+ * `microsoft.excel` and `microsoft.word` selectors. The two are distinguished
+ * by the `fileType` query parameter the selector forwards (defaulting to
+ * `excel` for backward compatibility), which drives both the search-query
+ * extension hint and the server-side result filter.
+ */
+const FILE_TYPE_CONFIG = {
+  excel: {
+    extension: '.xlsx',
+    mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  },
+  word: {
+    extension: '.docx',
+    mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  },
+} as const
+
+type MicrosoftFileType = keyof typeof FILE_TYPE_CONFIG
+
+/**
+ * Get Excel or Word files from Microsoft OneDrive / SharePoint. The
+ * `fileType` query parameter selects which Office document type to return
+ * (defaults to `excel`).
  */
 export const GET = withRouteHandler(async (request: NextRequest) => {
   const requestId = generateRequestId()
@@ -27,6 +71,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
       query: searchParams.get('query') ?? undefined,
       driveId: searchParams.get('driveId') ?? undefined,
       workflowId: searchParams.get('workflowId') ?? undefined,
+      fileType: searchParams.get('fileType') ?? undefined,
     })
 
     if (!parsedQuery.success) {
@@ -40,6 +85,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
     const { credentialId, driveId, workflowId } = parsedQuery.data
     const query = parsedQuery.data.query ?? ''
 
+    const fileType: MicrosoftFileType = parsedQuery.data.fileType ?? 'excel'
+    const { extension, mimeType: targetMimeType } = FILE_TYPE_CONFIG[fileType]
+
     const authz = await authorizeCredentialUse(request, {
       credentialId,
       workflowId,
@@ -72,19 +120,16 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
       return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 })
     }
 
-    // Build search query for Excel files
-    let searchQuery = '.xlsx'
-    if (query) {
-      searchQuery = `${query} .xlsx`
-    }
+    // Build search query for the requested Office document type
+    const searchQuery = query ? `${query} ${extension}` : extension
 
     // Build the query parameters for Microsoft Graph API
     const searchParams_new = new URLSearchParams()
     searchParams_new.append(
       '$select',
       'id,name,mimeType,webUrl,thumbnails,createdDateTime,lastModifiedDateTime,size,createdBy'
     )
-    searchParams_new.append('$top', '50')
+    searchParams_new.append('$top', String(MICROSOFT_FILES_PAGE_SIZE))
 
     // When driveId is provided (SharePoint), search within that specific drive.
     // Otherwise, search the user's personal OneDrive.
@@ -99,44 +144,57 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
     }
     const drivePath = driveId ? `drives/${driveId}` : 'me/drive'
 
-    const response = await fetch(
-      `https://graph.microsoft.com/v1.0/${drivePath}/root/search(q='${encodeURIComponent(searchQuery)}')?${searchParams_new.toString()}`,
-      {
+    const rawFiles: MicrosoftGraphFile[] = []
+    let nextUrl: string | undefined =
+      `https://graph.microsoft.com/v1.0/${drivePath}/root/search(q='${encodeURIComponent(searchQuery)}')?${searchParams_new.toString()}`
+
+    for (let page = 0; page < MAX_MICROSOFT_FILES_PAGES && nextUrl; page++) {
+      const response = await fetch(nextUrl, {
         headers: {
           Authorization: `Bearer ${accessToken}`,
         },
+      })
+
+      if (!response.ok) {
+        const errorData = await response
+          .json()
+          .catch(() => ({ error: { message: 'Unknown error' } }))
+        logger.error(`[${requestId}] Microsoft Graph API error`, {
+          status: response.status,
+          error: errorData.error?.message || 'Failed to fetch files from Microsoft OneDrive',
+        })
+        return NextResponse.json(
+          {
+            error: errorData.error?.message || 'Failed to fetch files from Microsoft OneDrive',
+          },
+          { status: response.status }
+        )
       }
-    )
 
-    if (!response.ok) {
-      const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } }))
-      logger.error(`[${requestId}] Microsoft Graph API error`, {
-        status: response.status,
-        error: errorData.error?.message || 'Failed to fetch Excel files from Microsoft OneDrive',
-      })
-      return NextResponse.json(
-        {
-          error: errorData.error?.message || 'Failed to fetch Excel files from Microsoft OneDrive',
-        },
-        { status: response.status }
-      )
-    }
+      const data = await response.json()
+      rawFiles.push(...((data.value as MicrosoftGraphFile[]) || []))
+
+      const nextLink = getGraphNextPageUrl(data)
+      nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
 
-    const data = await response.json()
-    let files = data.value || []
+      if (nextUrl && page === MAX_MICROSOFT_FILES_PAGES - 1) {
+        logger.warn(
+          `[${requestId}] Microsoft files search hit pagination cap; list may be incomplete`,
+          { fileType, pages: MAX_MICROSOFT_FILES_PAGES, collected: rawFiles.length }
+        )
+      }
+    }
 
-    // Transform Microsoft Graph response to match expected format and filter for Excel files
-    files = files
+    // Transform Microsoft Graph response and filter to the requested file type
+    const files = rawFiles
       .filter(
-        (file: any) =>
-          file.name?.toLowerCase().endsWith('.xlsx') ||
-          file.mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+        (file: MicrosoftGraphFile) =>
+          file.name?.toLowerCase().endsWith(extension) || file.mimeType === targetMimeType
       )
-      .map((file: any) => ({
+      .map((file: MicrosoftGraphFile) => ({
         id: file.id,
         name: file.name,
-        mimeType:
-          file.mimeType || 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+        mimeType: file.mimeType || targetMimeType,
         iconLink: file.thumbnails?.[0]?.small?.url,
         webViewLink: file.webUrl,
         thumbnailLink: file.thumbnails?.[0]?.medium?.url,
@@ -155,7 +213,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
 
     return NextResponse.json({ files }, { status: 200 })
   } catch (error) {
-    logger.error(`[${requestId}] Error fetching Excel files from Microsoft OneDrive`, error)
+    logger.error(`[${requestId}] Error fetching files from Microsoft OneDrive`, error)
     return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
   }
 })

apps/sim/app/api/tools/airtable/bases/route.ts

@@ -11,6 +11,71 @@ const logger = createLogger('AirtableBasesAPI')
 
 export const dynamic = 'force-dynamic'
 
+const AIRTABLE_MAX_BASES_PAGES = 50
+
+interface AirtableBase {
+  id: string
+  name: string
+}
+
+/**
+ * Lists all Airtable bases, following the `offset` continuation token the Meta
+ * API returns (an opaque string, passed back verbatim as `?offset=`) so the
+ * full set is returned. Bounded by `AIRTABLE_MAX_BASES_PAGES`; logs a warning
+ * rather than silently dropping bases when the cap is hit.
+ */
+async function fetchAllBases(accessToken: string): Promise<AirtableBase[]> {
+  const bases: AirtableBase[] = []
+  let offset: string | undefined
+
+  for (let page = 0; page < AIRTABLE_MAX_BASES_PAGES; page++) {
+    const url = new URL('https://api.airtable.com/v0/meta/bases')
+    if (offset) {
+      url.searchParams.set('offset', offset)
+    }
+
+    const response = await fetch(url.toString(), {
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        'Content-Type': 'application/json',
+      },
+    })
+
+    if (!response.ok) {
+      const errorData = await response.json().catch(() => ({}))
+      throw new AirtableFetchError(response.status, errorData)
+    }
+
+    const data = (await response.json()) as { bases?: AirtableBase[]; offset?: string }
+    if (Array.isArray(data.bases)) {
+      bases.push(...data.bases)
+    }
+
+    offset = data.offset || undefined
+    if (!offset) {
+      return bases
+    }
+
+    if (page === AIRTABLE_MAX_BASES_PAGES - 1) {
+      logger.warn('Airtable bases listing hit pagination cap; base list may be incomplete', {
+        pages: AIRTABLE_MAX_BASES_PAGES,
+      })
+    }
+  }
+
+  return bases
+}
+
+class AirtableFetchError extends Error {
+  constructor(
+    readonly status: number,
+    readonly details: unknown
+  ) {
+    super('Failed to fetch Airtable bases')
+    this.name = 'AirtableFetchError'
+  }
+}
+
 export const POST = withRouteHandler(async (request: NextRequest) => {
   const requestId = generateRequestId()
   try {
@@ -45,27 +110,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
       )
     }
 
-    const response = await fetch('https://api.airtable.com/v0/meta/bases', {
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        'Content-Type': 'application/json',
-      },
-    })
-
-    if (!response.ok) {
-      const errorData = await response.json().catch(() => ({}))
-      logger.error('Failed to fetch Airtable bases', {
-        status: response.status,
-        error: errorData,
-      })
-      return NextResponse.json(
-        { error: 'Failed to fetch Airtable bases', details: errorData },
-        { status: response.status }
-      )
+    let allBases: AirtableBase[]
+    try {
+      allBases = await fetchAllBases(accessToken)
+    } catch (error) {
+      if (error instanceof AirtableFetchError) {
+        logger.error('Failed to fetch Airtable bases', {
+          status: error.status,
+          error: error.details,
+        })
+        return NextResponse.json(
+          { error: 'Failed to fetch Airtable bases', details: error.details },
+          { status: error.status }
+        )
+      }
+      throw error
     }
 
-    const data = await response.json()
-    const bases = (data.bases || []).map((base: { id: string; name: string }) => ({
+    const bases = allBases.map((base) => ({
       id: base.id,
       name: base.name,
     }))