From a4d15896bccf6ad9880686d4356b8f2fd4c08e8b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 30 May 2026 20:00:55 -0700 Subject: [PATCH 1/5] fix(selectors): fetch all pages for paginated dropdown list routes Dropdown selectors fetched only the first page of paginated provider APIs, silently hiding results past page one. Add bounded server-side draining to the list routes across Microsoft Graph, Google, Notion, Atlassian, Linear, AWS CloudWatch, and offset/token REST APIs, plus a shared client-side drain cap in the selector hook. Response shapes, stored values, and tool execution are unchanged; CloudWatch list tools still honor a caller-supplied limit. Also fixes the Word file picker that was searching for .xlsx files. --- .../api/auth/oauth/microsoft/files/route.ts | 128 +++++++++++++----- .../sim/app/api/tools/airtable/bases/route.ts | 100 +++++++++++--- .../app/api/tools/asana/workspaces/route.ts | 110 ++++++++++++--- .../cloudwatch/describe-log-groups/route.ts | 66 +++++++-- apps/sim/app/api/tools/cloudwatch/utils.ts | 94 +++++++++---- apps/sim/app/api/tools/drive/files/route.ts | 79 +++++++---- .../tools/google_bigquery/datasets/route.ts | 77 +++++++---- .../api/tools/google_bigquery/tables/route.ts | 74 ++++++---- .../tools/google_calendar/calendars/route.ts | 74 ++++++---- .../tools/google_tasks/task-lists/route.ts | 61 ++++++--- apps/sim/app/api/tools/jira/projects/route.ts | 108 ++++++++++++--- .../tools/jsm/selector-requesttypes/route.ts | 91 +++++++++++-- .../tools/jsm/selector-servicedesks/route.ts | 90 ++++++++++-- .../app/api/tools/linear/projects/route.ts | 50 ++++++- apps/sim/app/api/tools/linear/teams/route.ts | 46 ++++++- .../tools/microsoft-teams/channels/route.ts | 79 +++++++---- .../api/tools/microsoft-teams/chats/route.ts | 91 +++++++++---- .../api/tools/microsoft-teams/teams/route.ts | 87 ++++++++---- .../api/tools/microsoft_excel/drives/route.ts | 54 ++++++-- .../tools/microsoft_planner/plans/route.ts | 55 +++++--- .../tools/microsoft_planner/tasks/route.ts | 51 +++++-- apps/sim/app/api/tools/monday/boards/route.ts | 86 ++++++++---- .../app/api/tools/notion/databases/route.ts | 81 +++++++---- apps/sim/app/api/tools/notion/pages/route.ts | 81 +++++++---- .../sim/app/api/tools/onedrive/files/route.ts | 69 +++++++--- .../app/api/tools/onedrive/folders/route.ts | 57 ++++++-- .../app/api/tools/outlook/folders/route.ts | 76 +++++++---- .../api/tools/pipedrive/pipelines/route.ts | 115 +++++++++++++--- .../app/api/tools/sharepoint/lists/route.ts | 54 ++++++-- .../app/api/tools/sharepoint/sites/route.ts | 54 ++++++-- apps/sim/app/api/tools/slack/users/route.ts | 71 +++++++--- .../app/api/tools/wealthbox/items/route.ts | 119 ++++++++++------ apps/sim/app/api/tools/webflow/items/route.ts | 109 +++++++++++---- apps/sim/app/api/tools/zoom/meetings/route.ts | 77 ++++++++--- .../providers/confluence/selectors.ts | 40 ++---- .../providers/knowledge/selectors.ts | 26 +++- .../providers/microsoft/selectors.ts | 2 + apps/sim/hooks/selectors/registry.test.ts | 4 +- apps/sim/hooks/selectors/types.ts | 7 +- .../sim/hooks/selectors/use-selector-query.ts | 45 +++++- .../lib/api/contracts/selectors/knowledge.ts | 1 + .../lib/api/contracts/selectors/microsoft.ts | 7 + apps/sim/lib/oauth/google-pagination.ts | 108 +++++++++++++++ 43 files changed, 2233 insertions(+), 721 deletions(-) create mode 100644 apps/sim/lib/oauth/google-pagination.ts diff --git a/apps/sim/app/api/auth/oauth/microsoft/files/route.ts b/apps/sim/app/api/auth/oauth/microsoft/files/route.ts index 35858ecd13..7dcd342d66 100644 --- a/apps/sim/app/api/auth/oauth/microsoft/files/route.ts +++ b/apps/sim/app/api/auth/oauth/microsoft/files/route.ts @@ -8,13 +8,57 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getCredential, refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { GRAPH_ID_PATTERN } from '@/tools/microsoft_excel/utils' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' const logger = createLogger('MicrosoftFilesAPI') /** - * Get Excel files from Microsoft OneDrive + * Microsoft Graph paginates `search()` results via the `@odata.nextLink` + * absolute URL in the response body. Request the largest page (`$top` caps at + * 999) and drain following nextLink, bounded by a page cap. + * See https://learn.microsoft.com/en-us/graph/paging + */ +const MICROSOFT_FILES_PAGE_SIZE = 999 +const MAX_MICROSOFT_FILES_PAGES = 20 + +interface MicrosoftGraphFile { + id: string + name?: string + mimeType?: string + webUrl?: string + size?: number + createdDateTime?: string + lastModifiedDateTime?: string + thumbnails?: Array<{ small?: { url?: string }; medium?: { url?: string } }> + createdBy?: { user?: { displayName?: string; email?: string } } +} + +/** + * The shared `/api/auth/oauth/microsoft/files` route serves both the + * `microsoft.excel` and `microsoft.word` selectors. The two are distinguished + * by the `fileType` query parameter the selector forwards (defaulting to + * `excel` for backward compatibility), which drives both the search-query + * extension hint and the server-side result filter. + */ +const FILE_TYPE_CONFIG = { + excel: { + extension: '.xlsx', + mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + }, + word: { + extension: '.docx', + mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + }, +} as const + +type MicrosoftFileType = keyof typeof FILE_TYPE_CONFIG + +/** + * Get Excel or Word files from Microsoft OneDrive / SharePoint. The + * `fileType` query parameter selects which Office document type to return + * (defaults to `excel`). */ export const GET = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() @@ -27,6 +71,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { query: searchParams.get('query') ?? undefined, driveId: searchParams.get('driveId') ?? undefined, workflowId: searchParams.get('workflowId') ?? undefined, + fileType: searchParams.get('fileType') ?? undefined, }) if (!parsedQuery.success) { @@ -40,6 +85,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const { credentialId, driveId, workflowId } = parsedQuery.data const query = parsedQuery.data.query ?? '' + const fileType: MicrosoftFileType = parsedQuery.data.fileType ?? 'excel' + const { extension, mimeType: targetMimeType } = FILE_TYPE_CONFIG[fileType] + const authz = await authorizeCredentialUse(request, { credentialId, workflowId, @@ -72,11 +120,8 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) } - // Build search query for Excel files - let searchQuery = '.xlsx' - if (query) { - searchQuery = `${query} .xlsx` - } + // Build search query for the requested Office document type + const searchQuery = query ? `${query} ${extension}` : extension // Build the query parameters for Microsoft Graph API const searchParams_new = new URLSearchParams() @@ -84,7 +129,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { '$select', 'id,name,mimeType,webUrl,thumbnails,createdDateTime,lastModifiedDateTime,size,createdBy' ) - searchParams_new.append('$top', '50') + searchParams_new.append('$top', String(MICROSOFT_FILES_PAGE_SIZE)) // When driveId is provided (SharePoint), search within that specific drive. // Otherwise, search the user's personal OneDrive. @@ -99,44 +144,57 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } const drivePath = driveId ? `drives/${driveId}` : 'me/drive' - const response = await fetch( - `https://graph.microsoft.com/v1.0/${drivePath}/root/search(q='${encodeURIComponent(searchQuery)}')?${searchParams_new.toString()}`, - { + const rawFiles: MicrosoftGraphFile[] = [] + let nextUrl: string | undefined = + `https://graph.microsoft.com/v1.0/${drivePath}/root/search(q='${encodeURIComponent(searchQuery)}')?${searchParams_new.toString()}` + + for (let page = 0; page < MAX_MICROSOFT_FILES_PAGES && nextUrl; page++) { + const response = await fetch(nextUrl, { headers: { Authorization: `Bearer ${accessToken}`, }, + }) + + if (!response.ok) { + const errorData = await response + .json() + .catch(() => ({ error: { message: 'Unknown error' } })) + logger.error(`[${requestId}] Microsoft Graph API error`, { + status: response.status, + error: errorData.error?.message || 'Failed to fetch files from Microsoft OneDrive', + }) + return NextResponse.json( + { + error: errorData.error?.message || 'Failed to fetch files from Microsoft OneDrive', + }, + { status: response.status } + ) } - ) - if (!response.ok) { - const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } })) - logger.error(`[${requestId}] Microsoft Graph API error`, { - status: response.status, - error: errorData.error?.message || 'Failed to fetch Excel files from Microsoft OneDrive', - }) - return NextResponse.json( - { - error: errorData.error?.message || 'Failed to fetch Excel files from Microsoft OneDrive', - }, - { status: response.status } - ) - } + const data = await response.json() + rawFiles.push(...((data.value as MicrosoftGraphFile[]) || [])) + + const nextLink = getGraphNextPageUrl(data) + nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined - const data = await response.json() - let files = data.value || [] + if (nextUrl && page === MAX_MICROSOFT_FILES_PAGES - 1) { + logger.warn( + `[${requestId}] Microsoft files search hit pagination cap; list may be incomplete`, + { fileType, pages: MAX_MICROSOFT_FILES_PAGES, collected: rawFiles.length } + ) + } + } - // Transform Microsoft Graph response to match expected format and filter for Excel files - files = files + // Transform Microsoft Graph response and filter to the requested file type + const files = rawFiles .filter( - (file: any) => - file.name?.toLowerCase().endsWith('.xlsx') || - file.mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + (file: MicrosoftGraphFile) => + file.name?.toLowerCase().endsWith(extension) || file.mimeType === targetMimeType ) - .map((file: any) => ({ + .map((file: MicrosoftGraphFile) => ({ id: file.id, name: file.name, - mimeType: - file.mimeType || 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + mimeType: file.mimeType || targetMimeType, iconLink: file.thumbnails?.[0]?.small?.url, webViewLink: file.webUrl, thumbnailLink: file.thumbnails?.[0]?.medium?.url, @@ -155,7 +213,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ files }, { status: 200 }) } catch (error) { - logger.error(`[${requestId}] Error fetching Excel files from Microsoft OneDrive`, error) + logger.error(`[${requestId}] Error fetching files from Microsoft OneDrive`, error) return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) } }) diff --git a/apps/sim/app/api/tools/airtable/bases/route.ts b/apps/sim/app/api/tools/airtable/bases/route.ts index b2545df0e8..20b3b3459d 100644 --- a/apps/sim/app/api/tools/airtable/bases/route.ts +++ b/apps/sim/app/api/tools/airtable/bases/route.ts @@ -11,6 +11,71 @@ const logger = createLogger('AirtableBasesAPI') export const dynamic = 'force-dynamic' +const AIRTABLE_MAX_BASES_PAGES = 50 + +interface AirtableBase { + id: string + name: string +} + +/** + * Lists all Airtable bases, following the `offset` continuation token the Meta + * API returns (an opaque string, passed back verbatim as `?offset=`) so the + * full set is returned. Bounded by `AIRTABLE_MAX_BASES_PAGES`; logs a warning + * rather than silently dropping bases when the cap is hit. + */ +async function fetchAllBases(accessToken: string): Promise { + const bases: AirtableBase[] = [] + let offset: string | undefined + + for (let page = 0; page < AIRTABLE_MAX_BASES_PAGES; page++) { + const url = new URL('https://api.airtable.com/v0/meta/bases') + if (offset) { + url.searchParams.set('offset', offset) + } + + const response = await fetch(url.toString(), { + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + throw new AirtableFetchError(response.status, errorData) + } + + const data = (await response.json()) as { bases?: AirtableBase[]; offset?: string } + if (Array.isArray(data.bases)) { + bases.push(...data.bases) + } + + offset = data.offset || undefined + if (!offset) { + return bases + } + + if (page === AIRTABLE_MAX_BASES_PAGES - 1) { + logger.warn('Airtable bases listing hit pagination cap; base list may be incomplete', { + pages: AIRTABLE_MAX_BASES_PAGES, + }) + } + } + + return bases +} + +class AirtableFetchError extends Error { + constructor( + readonly status: number, + readonly details: unknown + ) { + super('Failed to fetch Airtable bases') + this.name = 'AirtableFetchError' + } +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { @@ -45,27 +110,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://api.airtable.com/v0/meta/bases', { - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - }) - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error('Failed to fetch Airtable bases', { - status: response.status, - error: errorData, - }) - return NextResponse.json( - { error: 'Failed to fetch Airtable bases', details: errorData }, - { status: response.status } - ) + let allBases: AirtableBase[] + try { + allBases = await fetchAllBases(accessToken) + } catch (error) { + if (error instanceof AirtableFetchError) { + logger.error('Failed to fetch Airtable bases', { + status: error.status, + error: error.details, + }) + return NextResponse.json( + { error: 'Failed to fetch Airtable bases', details: error.details }, + { status: error.status } + ) + } + throw error } - const data = await response.json() - const bases = (data.bases || []).map((base: { id: string; name: string }) => ({ + const bases = allBases.map((base) => ({ id: base.id, name: base.name, })) diff --git a/apps/sim/app/api/tools/asana/workspaces/route.ts b/apps/sim/app/api/tools/asana/workspaces/route.ts index ee66783722..0f71376c6d 100644 --- a/apps/sim/app/api/tools/asana/workspaces/route.ts +++ b/apps/sim/app/api/tools/asana/workspaces/route.ts @@ -11,6 +11,81 @@ const logger = createLogger('AsanaWorkspacesAPI') export const dynamic = 'force-dynamic' +const ASANA_PAGE_LIMIT = 100 +const ASANA_MAX_WORKSPACES_PAGES = 50 + +interface AsanaWorkspace { + gid: string + name: string +} + +interface AsanaWorkspacesPage { + data?: AsanaWorkspace[] + next_page?: { + offset?: string + } | null +} + +/** + * Lists all Asana workspaces using `limit`/`offset` pagination, following + * `next_page.offset` (an opaque token, passed back verbatim as `?offset=`) + * until `next_page` is null so the full set is returned. Bounded by + * `ASANA_MAX_WORKSPACES_PAGES`; logs a warning rather than silently dropping + * workspaces when the cap is hit. + */ +async function fetchAllWorkspaces(accessToken: string): Promise { + const workspaces: AsanaWorkspace[] = [] + let offset: string | undefined + + for (let page = 0; page < ASANA_MAX_WORKSPACES_PAGES; page++) { + const url = new URL('https://app.asana.com/api/1.0/workspaces') + url.searchParams.set('limit', String(ASANA_PAGE_LIMIT)) + if (offset) { + url.searchParams.set('offset', offset) + } + + const response = await fetch(url.toString(), { + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + throw new AsanaFetchError(response.status, errorData) + } + + const data = (await response.json()) as AsanaWorkspacesPage + if (Array.isArray(data.data)) { + workspaces.push(...data.data) + } + + offset = data.next_page?.offset || undefined + if (!offset) { + return workspaces + } + + if (page === ASANA_MAX_WORKSPACES_PAGES - 1) { + logger.warn('Asana workspaces listing hit pagination cap; workspace list may be incomplete', { + pages: ASANA_MAX_WORKSPACES_PAGES, + }) + } + } + + return workspaces +} + +class AsanaFetchError extends Error { + constructor( + readonly status: number, + readonly details: unknown + ) { + super('Failed to fetch Asana workspaces') + this.name = 'AsanaFetchError' + } +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { @@ -42,27 +117,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://app.asana.com/api/1.0/workspaces', { - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error('Failed to fetch Asana workspaces', { - status: response.status, - error: errorData, - }) - return NextResponse.json( - { error: 'Failed to fetch Asana workspaces', details: errorData }, - { status: response.status } - ) + let allWorkspaces: AsanaWorkspace[] + try { + allWorkspaces = await fetchAllWorkspaces(accessToken) + } catch (error) { + if (error instanceof AsanaFetchError) { + logger.error('Failed to fetch Asana workspaces', { + status: error.status, + error: error.details, + }) + return NextResponse.json( + { error: 'Failed to fetch Asana workspaces', details: error.details }, + { status: error.status } + ) + } + throw error } - const data = await response.json() - const workspaces = (data.data || []).map((workspace: { gid: string; name: string }) => ({ + const workspaces = allWorkspaces.map((workspace) => ({ id: workspace.gid, name: workspace.name, })) diff --git a/apps/sim/app/api/tools/cloudwatch/describe-log-groups/route.ts b/apps/sim/app/api/tools/cloudwatch/describe-log-groups/route.ts index bb0bff4390..4a2aabea1c 100644 --- a/apps/sim/app/api/tools/cloudwatch/describe-log-groups/route.ts +++ b/apps/sim/app/api/tools/cloudwatch/describe-log-groups/route.ts @@ -10,6 +10,12 @@ import { createCloudWatchLogsClient } from '@/app/api/tools/cloudwatch/utils' const logger = createLogger('CloudWatchDescribeLogGroups') +/** AWS DescribeLogGroups caps `limit` at 50 items per page. */ +const LOG_GROUPS_PAGE_SIZE = 50 + +/** Upper bound on pages drained to avoid unbounded loops on very large accounts. */ +const MAX_LOG_GROUPS_PAGES = 20 + export const POST = withRouteHandler(async (request: NextRequest) => { try { const auth = await checkSessionOrInternalAuth(request) @@ -33,26 +39,58 @@ export const POST = withRouteHandler(async (request: NextRequest) => { }) try { - const command = new DescribeLogGroupsCommand({ - ...(validatedData.prefix && { logGroupNamePrefix: validatedData.prefix }), - ...(validatedData.limit !== undefined && { limit: validatedData.limit }), - }) + const totalLimit = validatedData.limit + const logGroups: { + logGroupName: string + arn: string + storedBytes: number + retentionInDays: number | undefined + creationTime: number | undefined + }[] = [] + let nextToken: string | undefined + + for (let page = 0; page < MAX_LOG_GROUPS_PAGES; page++) { + const pageLimit = + totalLimit !== undefined + ? Math.min(LOG_GROUPS_PAGE_SIZE, totalLimit - logGroups.length) + : LOG_GROUPS_PAGE_SIZE + + const command = new DescribeLogGroupsCommand({ + ...(validatedData.prefix && { logGroupNamePrefix: validatedData.prefix }), + limit: pageLimit, + ...(nextToken && { nextToken }), + }) + + const response = await client.send(command) + + for (const lg of response.logGroups ?? []) { + logGroups.push({ + logGroupName: lg.logGroupName ?? '', + arn: lg.arn ?? '', + storedBytes: lg.storedBytes ?? 0, + retentionInDays: lg.retentionInDays, + creationTime: lg.creationTime, + }) + } + + nextToken = response.nextToken + if (!nextToken) break + if (totalLimit !== undefined && logGroups.length >= totalLimit) break - const response = await client.send(command) + if (page === MAX_LOG_GROUPS_PAGES - 1) { + logger.warn( + `DescribeLogGroups hit pagination cap of ${MAX_LOG_GROUPS_PAGES} pages; log group list may be incomplete` + ) + } + } - const logGroups = (response.logGroups ?? []).map((lg) => ({ - logGroupName: lg.logGroupName ?? '', - arn: lg.arn ?? '', - storedBytes: lg.storedBytes ?? 0, - retentionInDays: lg.retentionInDays, - creationTime: lg.creationTime, - })) + const cappedLogGroups = totalLimit !== undefined ? logGroups.slice(0, totalLimit) : logGroups - logger.info(`Successfully described ${logGroups.length} log groups`) + logger.info(`Successfully described ${cappedLogGroups.length} log groups`) return NextResponse.json({ success: true, - output: { logGroups }, + output: { logGroups: cappedLogGroups }, }) } finally { client.destroy() diff --git a/apps/sim/app/api/tools/cloudwatch/utils.ts b/apps/sim/app/api/tools/cloudwatch/utils.ts index 966aa67b2b..2984229e75 100644 --- a/apps/sim/app/api/tools/cloudwatch/utils.ts +++ b/apps/sim/app/api/tools/cloudwatch/utils.ts @@ -5,6 +5,7 @@ import { GetQueryResultsCommand, type ResultField, } from '@aws-sdk/client-cloudwatch-logs' +import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' @@ -96,37 +97,82 @@ export async function pollQueryResults( } } +/** AWS DescribeLogStreams caps `limit` at 50 items per page. */ +const LOG_STREAMS_PAGE_SIZE = 50 + +/** Upper bound on pages drained to avoid unbounded loops on log groups with many streams. */ +const MAX_LOG_STREAMS_PAGES = 20 + +const logger = createLogger('CloudWatchUtils') + +interface DescribedLogStream { + logStreamName: string + lastEventTimestamp: number | undefined + firstEventTimestamp: number | undefined + creationTime: number | undefined + storedBytes: number +} + +/** + * Lists log streams for a log group, following `nextToken` so the complete set + * is returned rather than just the first page. Bounded by + * `MAX_LOG_STREAMS_PAGES`; logs a warning rather than silently dropping streams + * when the cap is hit. Ordering/prefix inputs are preserved across all pages. + * + * When `limit` is provided it is treated as a total result cap: draining stops + * once enough streams have been collected. When omitted, every page is drained. + */ export async function describeLogStreams( client: CloudWatchLogsClient, logGroupName: string, options?: { prefix?: string; limit?: number } -): Promise<{ - logStreams: { - logStreamName: string - lastEventTimestamp: number | undefined - firstEventTimestamp: number | undefined - creationTime: number | undefined - storedBytes: number - }[] -}> { +): Promise<{ logStreams: DescribedLogStream[] }> { const hasPrefix = Boolean(options?.prefix) - const command = new DescribeLogStreamsCommand({ - logGroupName, - ...(hasPrefix - ? { orderBy: 'LogStreamName', logStreamNamePrefix: options!.prefix } - : { orderBy: 'LastEventTime', descending: true }), - ...(options?.limit !== undefined && { limit: options.limit }), - }) + const totalLimit = options?.limit + const logStreams: DescribedLogStream[] = [] + let nextToken: string | undefined + + for (let page = 0; page < MAX_LOG_STREAMS_PAGES; page++) { + const pageLimit = + totalLimit !== undefined + ? Math.min(LOG_STREAMS_PAGE_SIZE, totalLimit - logStreams.length) + : LOG_STREAMS_PAGE_SIZE + + const command = new DescribeLogStreamsCommand({ + logGroupName, + ...(hasPrefix + ? { orderBy: 'LogStreamName', logStreamNamePrefix: options!.prefix } + : { orderBy: 'LastEventTime', descending: true }), + limit: pageLimit, + ...(nextToken && { nextToken }), + }) + + const response = await client.send(command) + + for (const ls of response.logStreams ?? []) { + logStreams.push({ + logStreamName: ls.logStreamName ?? '', + lastEventTimestamp: ls.lastEventTimestamp, + firstEventTimestamp: ls.firstEventTimestamp, + creationTime: ls.creationTime, + storedBytes: ls.storedBytes ?? 0, + }) + } + + nextToken = response.nextToken + if (!nextToken) break + if (totalLimit !== undefined && logStreams.length >= totalLimit) break + + if (page === MAX_LOG_STREAMS_PAGES - 1) { + logger.warn( + `DescribeLogStreams hit pagination cap of ${MAX_LOG_STREAMS_PAGES} pages; log stream list may be incomplete`, + { logGroupName } + ) + } + } - const response = await client.send(command) return { - logStreams: (response.logStreams ?? []).map((ls) => ({ - logStreamName: ls.logStreamName ?? '', - lastEventTimestamp: ls.lastEventTimestamp, - firstEventTimestamp: ls.firstEventTimestamp, - creationTime: ls.creationTime, - storedBytes: ls.storedBytes ?? 0, - })), + logStreams: totalLimit !== undefined ? logStreams.slice(0, totalLimit) : logStreams, } } diff --git a/apps/sim/app/api/tools/drive/files/route.ts b/apps/sim/app/api/tools/drive/files/route.ts index 773fd2ae97..4c38334b14 100644 --- a/apps/sim/app/api/tools/drive/files/route.ts +++ b/apps/sim/app/api/tools/drive/files/route.ts @@ -7,12 +7,21 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination' import { getScopesForService } from '@/lib/oauth/utils' import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' const logger = createLogger('GoogleDriveFilesAPI') +const MAX_DRIVE_FILE_PAGES = 20 +const DRIVE_FILE_PAGE_SIZE = 100 + +interface DriveFilesResponse { + files?: DriveFile[] + nextPageToken?: string +} + function escapeForDriveQuery(value: string): string { return value.replace(/\\/g, '\\\\').replace(/'/g, "\\'") } @@ -138,34 +147,56 @@ export const GET = withRouteHandler(async (request: NextRequest) => { if (query) { qParts.push(`name contains '${escapeForDriveQuery(query)}'`) } - const q = encodeURIComponent(qParts.join(' and ')) - - const response = await fetch( - `https://www.googleapis.com/drive/v3/files?q=${q}&corpora=allDrives&supportsAllDrives=true&includeItemsFromAllDrives=true&fields=files(id,name,mimeType,iconLink,webViewLink,thumbnailLink,createdTime,modifiedTime,size,owners,parents)`, - { - headers: { - Authorization: `Bearer ${accessToken}`, + const q = qParts.join(' and ') + + let files: DriveFile[] + try { + const drained = await drainGooglePagedList({ + buildUrl: (pageToken) => { + const url = new URL('https://www.googleapis.com/drive/v3/files') + url.searchParams.set('q', q) + url.searchParams.set('corpora', 'allDrives') + url.searchParams.set('supportsAllDrives', 'true') + url.searchParams.set('includeItemsFromAllDrives', 'true') + url.searchParams.set('pageSize', String(DRIVE_FILE_PAGE_SIZE)) + url.searchParams.set( + 'fields', + 'nextPageToken,files(id,name,mimeType,iconLink,webViewLink,thumbnailLink,createdTime,modifiedTime,size,owners,parents)' + ) + if (pageToken) url.searchParams.set('pageToken', pageToken) + return url.toString() }, - } - ) - - if (!response.ok) { - const error = await response.json().catch(() => ({ error: { message: 'Unknown error' } })) - logger.error(`[${requestId}] Google Drive API error`, { - status: response.status, - error: error.error?.message || 'Failed to fetch files from Google Drive', + fetch: (url) => + fetch(url, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }), + parseError: (response) => + response.json().catch(() => ({ error: { message: 'Unknown error' } })), + getItems: (body) => body.files, + getNextPageToken: (body) => body.nextPageToken, + maxPages: MAX_DRIVE_FILE_PAGES, + label: 'Google Drive files', }) - return NextResponse.json( - { - error: error.error?.message || 'Failed to fetch files from Google Drive', - }, - { status: response.status } - ) + files = drained.items + } catch (error) { + if (error instanceof GooglePageError) { + const errorBody = error.body as { error?: { message?: string } } + logger.error(`[${requestId}] Google Drive API error`, { + status: error.status, + error: errorBody?.error?.message || 'Failed to fetch files from Google Drive', + }) + return NextResponse.json( + { + error: errorBody?.error?.message || 'Failed to fetch files from Google Drive', + }, + { status: error.status } + ) + } + throw error } - const data = await response.json() - let files: DriveFile[] = data.files || [] - if (mimeType === 'application/vnd.google-apps.spreadsheet') { files = files.filter( (file: DriveFile) => file.mimeType === 'application/vnd.google-apps.spreadsheet' diff --git a/apps/sim/app/api/tools/google_bigquery/datasets/route.ts b/apps/sim/app/api/tools/google_bigquery/datasets/route.ts index a4c97d6850..695a371e17 100644 --- a/apps/sim/app/api/tools/google_bigquery/datasets/route.ts +++ b/apps/sim/app/api/tools/google_bigquery/datasets/route.ts @@ -5,6 +5,7 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination' import { getScopesForService } from '@/lib/oauth/utils' import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' @@ -12,6 +13,19 @@ const logger = createLogger('GoogleBigQueryDatasetsAPI') export const dynamic = 'force-dynamic' +const MAX_DATASET_PAGES = 20 +const DATASET_PAGE_SIZE = 200 + +interface BigQueryDataset { + datasetReference: { datasetId: string; projectId: string } + friendlyName?: string +} + +interface BigQueryDatasetsResponse { + datasets?: BigQueryDataset[] + nextPageToken?: string +} + /** * POST /api/tools/google_bigquery/datasets * @@ -71,41 +85,46 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch( - `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets?maxResults=200`, - { - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - } - ) + const { items } = await drainGooglePagedList({ + buildUrl: (pageToken) => { + const url = new URL( + `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets` + ) + url.searchParams.set('maxResults', String(DATASET_PAGE_SIZE)) + if (pageToken) url.searchParams.set('pageToken', pageToken) + return url.toString() + }, + fetch: (url) => + fetch(url, { + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + }), + parseError: (response) => response.json().catch(() => ({})), + getItems: (body) => body.datasets, + getNextPageToken: (body) => body.nextPageToken, + maxPages: MAX_DATASET_PAGES, + label: 'BigQuery datasets', + }) + + const datasets = items.map((ds) => ({ + datasetReference: ds.datasetReference, + friendlyName: ds.friendlyName, + })) - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) + return NextResponse.json({ datasets }) + } catch (error) { + if (error instanceof GooglePageError) { logger.error('Failed to fetch BigQuery datasets', { - status: response.status, - error: errorData, + status: error.status, + error: error.body, }) return NextResponse.json( - { error: 'Failed to fetch BigQuery datasets', details: errorData }, - { status: response.status } + { error: 'Failed to fetch BigQuery datasets', details: error.body }, + { status: error.status } ) } - - const data = await response.json() - const datasets = (data.datasets || []).map( - (ds: { - datasetReference: { datasetId: string; projectId: string } - friendlyName?: string - }) => ({ - datasetReference: ds.datasetReference, - friendlyName: ds.friendlyName, - }) - ) - - return NextResponse.json({ datasets }) - } catch (error) { if (error instanceof ServiceAccountTokenError) { return NextResponse.json({ error: error.message }, { status: 400 }) } diff --git a/apps/sim/app/api/tools/google_bigquery/tables/route.ts b/apps/sim/app/api/tools/google_bigquery/tables/route.ts index 2f6320a0fe..af01379059 100644 --- a/apps/sim/app/api/tools/google_bigquery/tables/route.ts +++ b/apps/sim/app/api/tools/google_bigquery/tables/route.ts @@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination' import { getScopesForService } from '@/lib/oauth/utils' import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' @@ -12,6 +13,19 @@ const logger = createLogger('GoogleBigQueryTablesAPI') export const dynamic = 'force-dynamic' +const MAX_TABLE_PAGES = 20 +const TABLE_PAGE_SIZE = 200 + +interface BigQueryTable { + tableReference: { tableId: string } + friendlyName?: string +} + +interface BigQueryTablesResponse { + tables?: BigQueryTable[] + nextPageToken?: string +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { @@ -68,38 +82,46 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch( - `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets/${encodeURIComponent(datasetId)}/tables?maxResults=200`, - { - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - } - ) + const { items } = await drainGooglePagedList({ + buildUrl: (pageToken) => { + const url = new URL( + `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets/${encodeURIComponent(datasetId)}/tables` + ) + url.searchParams.set('maxResults', String(TABLE_PAGE_SIZE)) + if (pageToken) url.searchParams.set('pageToken', pageToken) + return url.toString() + }, + fetch: (url) => + fetch(url, { + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + }), + parseError: (response) => response.json().catch(() => ({})), + getItems: (body) => body.tables, + getNextPageToken: (body) => body.nextPageToken, + maxPages: MAX_TABLE_PAGES, + label: 'BigQuery tables', + }) + + const tables = items.map((t) => ({ + tableReference: t.tableReference, + friendlyName: t.friendlyName, + })) - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) + return NextResponse.json({ tables }) + } catch (error) { + if (error instanceof GooglePageError) { logger.error('Failed to fetch BigQuery tables', { - status: response.status, - error: errorData, + status: error.status, + error: error.body, }) return NextResponse.json( - { error: 'Failed to fetch BigQuery tables', details: errorData }, - { status: response.status } + { error: 'Failed to fetch BigQuery tables', details: error.body }, + { status: error.status } ) } - - const data = await response.json() - const tables = (data.tables || []).map( - (t: { tableReference: { tableId: string }; friendlyName?: string }) => ({ - tableReference: t.tableReference, - friendlyName: t.friendlyName, - }) - ) - - return NextResponse.json({ tables }) - } catch (error) { if (error instanceof ServiceAccountTokenError) { return NextResponse.json({ error: error.message }, { status: 400 }) } diff --git a/apps/sim/app/api/tools/google_calendar/calendars/route.ts b/apps/sim/app/api/tools/google_calendar/calendars/route.ts index e1ac55b13e..752cc72b22 100644 --- a/apps/sim/app/api/tools/google_calendar/calendars/route.ts +++ b/apps/sim/app/api/tools/google_calendar/calendars/route.ts @@ -5,12 +5,16 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination' import { getScopesForService } from '@/lib/oauth/utils' import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' const logger = createLogger('GoogleCalendarAPI') +const MAX_CALENDAR_PAGES = 20 +const CALENDAR_PAGE_SIZE = 250 + interface CalendarListItem { id: string summary: string @@ -21,6 +25,11 @@ interface CalendarListItem { foregroundColor?: string } +interface CalendarListResponse { + items?: CalendarListItem[] + nextPageToken?: string +} + /** * Get calendars from Google Calendar */ @@ -64,35 +73,50 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } logger.info(`[${requestId}] Fetching calendars from Google Calendar API`) - const calendarResponse = await fetch( - 'https://www.googleapis.com/calendar/v3/users/me/calendarList', - { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - } - ) - if (!calendarResponse.ok) { - const errorData = await calendarResponse - .text() - .then((text) => JSON.parse(text)) - .catch(() => ({ error: { message: 'Unknown error' } })) - logger.error(`[${requestId}] Google Calendar API error`, { - status: calendarResponse.status, - error: errorData.error?.message || 'Failed to fetch calendars', + let calendars: CalendarListItem[] + try { + const drained = await drainGooglePagedList({ + buildUrl: (pageToken) => { + const url = new URL('https://www.googleapis.com/calendar/v3/users/me/calendarList') + url.searchParams.set('maxResults', String(CALENDAR_PAGE_SIZE)) + if (pageToken) url.searchParams.set('pageToken', pageToken) + return url.toString() + }, + fetch: (url) => + fetch(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + }), + parseError: (response) => + response + .text() + .then((text) => JSON.parse(text)) + .catch(() => ({ error: { message: 'Unknown error' } })), + getItems: (body) => body.items, + getNextPageToken: (body) => body.nextPageToken, + maxPages: MAX_CALENDAR_PAGES, + label: 'Google Calendar calendars', }) - return NextResponse.json( - { error: errorData.error?.message || 'Failed to fetch calendars' }, - { status: calendarResponse.status } - ) + calendars = drained.items + } catch (error) { + if (error instanceof GooglePageError) { + const errorData = error.body as { error?: { message?: string } } + logger.error(`[${requestId}] Google Calendar API error`, { + status: error.status, + error: errorData?.error?.message || 'Failed to fetch calendars', + }) + return NextResponse.json( + { error: errorData?.error?.message || 'Failed to fetch calendars' }, + { status: error.status } + ) + } + throw error } - const data = await calendarResponse.json() - const calendars: CalendarListItem[] = data.items || [] - calendars.sort((a, b) => { if (a.primary && !b.primary) return -1 if (!a.primary && b.primary) return 1 diff --git a/apps/sim/app/api/tools/google_tasks/task-lists/route.ts b/apps/sim/app/api/tools/google_tasks/task-lists/route.ts index c5ee97bb23..80c5a99f59 100644 --- a/apps/sim/app/api/tools/google_tasks/task-lists/route.ts +++ b/apps/sim/app/api/tools/google_tasks/task-lists/route.ts @@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination' import { getScopesForService } from '@/lib/oauth/utils' import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' @@ -12,6 +13,19 @@ const logger = createLogger('GoogleTasksTaskListsAPI') export const dynamic = 'force-dynamic' +const MAX_TASK_LIST_PAGES = 20 +const TASK_LIST_PAGE_SIZE = 1000 + +interface GoogleTaskList { + id: string + title: string +} + +interface GoogleTaskListsResponse { + items?: GoogleTaskList[] + nextPageToken?: string +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { @@ -56,33 +70,44 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://tasks.googleapis.com/tasks/v1/users/@me/lists', { - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', + const { items } = await drainGooglePagedList({ + buildUrl: (pageToken) => { + const url = new URL('https://tasks.googleapis.com/tasks/v1/users/@me/lists') + url.searchParams.set('maxResults', String(TASK_LIST_PAGE_SIZE)) + if (pageToken) url.searchParams.set('pageToken', pageToken) + return url.toString() }, + fetch: (url) => + fetch(url, { + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + }), + parseError: (response) => response.json().catch(() => ({})), + getItems: (body) => body.items, + getNextPageToken: (body) => body.nextPageToken, + maxPages: MAX_TASK_LIST_PAGES, + label: 'Google Tasks task lists', }) - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error('Failed to fetch Google Tasks task lists', { - status: response.status, - error: errorData, - }) - return NextResponse.json( - { error: 'Failed to fetch Google Tasks task lists', details: errorData }, - { status: response.status } - ) - } - - const data = await response.json() - const taskLists = (data.items || []).map((list: { id: string; title: string }) => ({ + const taskLists = items.map((list) => ({ id: list.id, title: list.title, })) return NextResponse.json({ taskLists }) } catch (error) { + if (error instanceof GooglePageError) { + logger.error('Failed to fetch Google Tasks task lists', { + status: error.status, + error: error.body, + }) + return NextResponse.json( + { error: 'Failed to fetch Google Tasks task lists', details: error.body }, + { status: error.status } + ) + } if (error instanceof ServiceAccountTokenError) { return NextResponse.json({ error: error.message }, { status: 400 }) } diff --git a/apps/sim/app/api/tools/jira/projects/route.ts b/apps/sim/app/api/tools/jira/projects/route.ts index 2c99f2e447..2ee1244cb2 100644 --- a/apps/sim/app/api/tools/jira/projects/route.ts +++ b/apps/sim/app/api/tools/jira/projects/route.ts @@ -14,6 +14,77 @@ export const dynamic = 'force-dynamic' const logger = createLogger('JiraProjectsAPI') +const JIRA_PROJECTS_PAGE_SIZE = 50 +const MAX_JIRA_PROJECTS_PAGES = 40 + +interface JiraProjectSearchPage { + values?: unknown[] + isLast?: boolean + maxResults?: number +} + +/** + * Drains the offset-paginated Jira `/project/search` endpoint, advancing + * `startAt` by the server-returned page size until `isLast === true` (or a short + * page is seen). Bounded by `MAX_JIRA_PROJECTS_PAGES`; emits a `logger.warn` and + * returns the partial set rather than looping unbounded when the cap is hit. + */ +async function fetchAllJiraProjects( + apiUrl: string, + baseParams: URLSearchParams, + accessToken: string +): Promise<{ values: unknown[]; lastResponse: Response }> { + const values: unknown[] = [] + let startAt = 0 + let lastResponse: Response + + for (let page = 0; page < MAX_JIRA_PROJECTS_PAGES; page++) { + const params = new URLSearchParams(baseParams) + params.set('startAt', String(startAt)) + params.set('maxResults', String(JIRA_PROJECTS_PAGE_SIZE)) + + const finalUrl = `${apiUrl}?${params.toString()}` + logger.info(`Fetching Jira projects from: ${finalUrl}`) + + const response = await fetch(finalUrl, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) + + logger.info(`Response status: ${response.status} ${response.statusText}`) + + if (!response.ok) { + return { values, lastResponse: response } + } + + const data = (await response.json()) as JiraProjectSearchPage + lastResponse = response + + const pageValues = data.values ?? [] + values.push(...pageValues) + + const pageSize = + data.maxResults && data.maxResults > 0 ? data.maxResults : JIRA_PROJECTS_PAGE_SIZE + if (data.isLast === true || pageValues.length < pageSize) { + return { values, lastResponse } + } + + startAt += pageValues.length + + if (page === MAX_JIRA_PROJECTS_PAGES - 1) { + logger.warn('Jira project search hit pagination cap; project list may be incomplete', { + pages: MAX_JIRA_PROJECTS_PAGES, + collected: values.length, + }) + } + } + + return { values, lastResponse: lastResponse! } +} + export const GET = withRouteHandler(async (request: NextRequest) => { try { const auth = await checkSessionOrInternalAuth(request) @@ -51,35 +122,28 @@ export const GET = withRouteHandler(async (request: NextRequest) => { queryParams.append('orderBy', 'name') queryParams.append('expand', 'description,lead,url,projectKeys') - const finalUrl = `${apiUrl}?${queryParams.toString()}` - logger.info(`Fetching Jira projects from: ${finalUrl}`) - - const response = await fetch(finalUrl, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - logger.info(`Response status: ${response.status} ${response.statusText}`) + const { values, lastResponse } = await fetchAllJiraProjects(apiUrl, queryParams, accessToken) - if (!response.ok) { - const errorText = await response.text() - logger.error('Jira API error:', { status: response.status, error: errorText }) + if (!lastResponse.ok) { + const errorText = await lastResponse.text() + logger.error('Jira API error:', { status: lastResponse.status, error: errorText }) return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } + { + error: parseAtlassianErrorMessage( + lastResponse.status, + lastResponse.statusText, + errorText + ), + }, + { status: lastResponse.status } ) } - const data = await response.json() - - logger.info(`Jira API Response Status: ${response.status}`) - logger.info(`Found projects: ${data.values?.length || 0}`) + logger.info(`Jira API Response Status: ${lastResponse.status}`) + logger.info(`Found projects: ${values.length}`) const projects = - data.values?.map((project: any) => ({ + values.map((project: any) => ({ id: project.id, key: project.key, name: project.name, diff --git a/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts b/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts index 01c1682e1e..0485e2686a 100644 --- a/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts +++ b/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts @@ -14,6 +14,69 @@ const logger = createLogger('JsmSelectorRequestTypesAPI') export const dynamic = 'force-dynamic' +const JSM_REQUEST_TYPES_PAGE_SIZE = 100 +const MAX_JSM_REQUEST_TYPES_PAGES = 50 + +interface JsmPagedResponse { + values?: T[] + isLastPage?: boolean + _links?: { next?: string } +} + +interface JsmRequestTypeValue { + id: string + name: string +} + +/** + * Drains the offset-paginated JSM `/servicedesk/{id}/requesttype` endpoint, + * advancing `start` by `limit` until `isLastPage === true` (or `_links.next` is + * absent). Bounded by `MAX_JSM_REQUEST_TYPES_PAGES`; emits a `logger.warn` and + * returns the partial set rather than looping unbounded when the cap is hit. + */ +async function fetchAllJsmRequestTypes( + requestTypeUrl: string, + accessToken: string +): Promise<{ values: JsmRequestTypeValue[]; lastResponse: Response }> { + const values: JsmRequestTypeValue[] = [] + let start = 0 + let lastResponse: Response + + for (let page = 0; page < MAX_JSM_REQUEST_TYPES_PAGES; page++) { + const url = `${requestTypeUrl}?start=${start}&limit=${JSM_REQUEST_TYPES_PAGE_SIZE}` + + const response = await fetch(url, { + method: 'GET', + headers: getJsmHeaders(accessToken), + }) + + if (!response.ok) { + return { values, lastResponse: response } + } + + const data = (await response.json()) as JsmPagedResponse + lastResponse = response + + const pageValues = data.values ?? [] + values.push(...pageValues) + + if (data.isLastPage === true || !data._links?.next) { + return { values, lastResponse } + } + + start += JSM_REQUEST_TYPES_PAGE_SIZE + + if (page === MAX_JSM_REQUEST_TYPES_PAGES - 1) { + logger.warn('JSM request type list hit pagination cap; list may be incomplete', { + pages: MAX_JSM_REQUEST_TYPES_PAGES, + collected: values.length, + }) + } + } + + return { values, lastResponse: lastResponse! } +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { @@ -72,28 +135,30 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const baseUrl = getJsmApiBaseUrl(cloudIdValidation.sanitized!) - const url = `${baseUrl}/servicedesk/${serviceDeskIdValidation.sanitized}/requesttype?limit=100` + const requestTypeUrl = `${baseUrl}/servicedesk/${serviceDeskIdValidation.sanitized}/requesttype` - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) + const { values, lastResponse } = await fetchAllJsmRequestTypes(requestTypeUrl, accessToken) - if (!response.ok) { - const errorText = await response.text() + if (!lastResponse.ok) { + const errorText = await lastResponse.text() logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, + status: lastResponse.status, + statusText: lastResponse.statusText, error: errorText, }) return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } + { + error: parseAtlassianErrorMessage( + lastResponse.status, + lastResponse.statusText, + errorText + ), + }, + { status: lastResponse.status } ) } - const data = await response.json() - const requestTypes = (data.values || []).map((rt: { id: string; name: string }) => ({ + const requestTypes = values.map((rt) => ({ id: rt.id, name: rt.name, })) diff --git a/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts b/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts index c1efdb0f93..38e91a6cdf 100644 --- a/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts +++ b/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts @@ -14,6 +14,69 @@ const logger = createLogger('JsmSelectorServiceDesksAPI') export const dynamic = 'force-dynamic' +const JSM_SERVICE_DESKS_PAGE_SIZE = 100 +const MAX_JSM_SERVICE_DESKS_PAGES = 50 + +interface JsmPagedResponse { + values?: T[] + isLastPage?: boolean + _links?: { next?: string } +} + +interface JsmServiceDeskValue { + id: string + projectName: string +} + +/** + * Drains the offset-paginated JSM `/servicedesk` endpoint, advancing `start` by + * `limit` until `isLastPage === true` (or `_links.next` is absent). Bounded by + * `MAX_JSM_SERVICE_DESKS_PAGES`; emits a `logger.warn` and returns the partial + * set rather than looping unbounded when the cap is hit. + */ +async function fetchAllJsmServiceDesks( + baseUrl: string, + accessToken: string +): Promise<{ values: JsmServiceDeskValue[]; lastResponse: Response }> { + const values: JsmServiceDeskValue[] = [] + let start = 0 + let lastResponse: Response + + for (let page = 0; page < MAX_JSM_SERVICE_DESKS_PAGES; page++) { + const url = `${baseUrl}/servicedesk?start=${start}&limit=${JSM_SERVICE_DESKS_PAGE_SIZE}` + + const response = await fetch(url, { + method: 'GET', + headers: getJsmHeaders(accessToken), + }) + + if (!response.ok) { + return { values, lastResponse: response } + } + + const data = (await response.json()) as JsmPagedResponse + lastResponse = response + + const pageValues = data.values ?? [] + values.push(...pageValues) + + if (data.isLastPage === true || !data._links?.next) { + return { values, lastResponse } + } + + start += JSM_SERVICE_DESKS_PAGE_SIZE + + if (page === MAX_JSM_SERVICE_DESKS_PAGES - 1) { + logger.warn('JSM service desk list hit pagination cap; list may be incomplete', { + pages: MAX_JSM_SERVICE_DESKS_PAGES, + collected: values.length, + }) + } + } + + return { values, lastResponse: lastResponse! } +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { @@ -63,28 +126,29 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const baseUrl = getJsmApiBaseUrl(cloudIdValidation.sanitized!) - const url = `${baseUrl}/servicedesk?limit=100` - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) + const { values, lastResponse } = await fetchAllJsmServiceDesks(baseUrl, accessToken) - if (!response.ok) { - const errorText = await response.text() + if (!lastResponse.ok) { + const errorText = await lastResponse.text() logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, + status: lastResponse.status, + statusText: lastResponse.statusText, error: errorText, }) return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } + { + error: parseAtlassianErrorMessage( + lastResponse.status, + lastResponse.statusText, + errorText + ), + }, + { status: lastResponse.status } ) } - const data = await response.json() - const serviceDesks = (data.values || []).map((sd: { id: string; projectName: string }) => ({ + const serviceDesks = values.map((sd) => ({ id: sd.id, name: sd.projectName, })) diff --git a/apps/sim/app/api/tools/linear/projects/route.ts b/apps/sim/app/api/tools/linear/projects/route.ts index e7ca9bab1a..9b0b500e0d 100644 --- a/apps/sim/app/api/tools/linear/projects/route.ts +++ b/apps/sim/app/api/tools/linear/projects/route.ts @@ -1,4 +1,4 @@ -import type { Project } from '@linear/sdk' +import type { Project, Team } from '@linear/sdk' import { LinearClient } from '@linear/sdk' import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' @@ -13,6 +13,50 @@ export const dynamic = 'force-dynamic' const logger = createLogger('LinearProjectsAPI') +/** Linear's maximum page size for a single connection request. */ +const LINEAR_PAGE_SIZE = 250 + +/** + * Upper bound on pages to drain from a single team's projects connection. At + * 250 projects/page this covers 2,500 projects per team; the cap guards + * against runaway loops on a broken `hasNextPage` rather than a realistic + * limit. + */ +const MAX_PROJECTS_PAGES = 10 + +/** + * Drains a single team's projects connection by following + * `pageInfo.endCursor` until `hasNextPage` is false. Bounded by + * `MAX_PROJECTS_PAGES`; logs a warning if the cap is hit so a truncated list + * is visible rather than silently dropped. + */ +async function fetchAllTeamProjects(team: Team): Promise { + const projects: Project[] = [] + let after: string | undefined + + for (let page = 0; page < MAX_PROJECTS_PAGES; page++) { + const result = await team.projects({ first: LINEAR_PAGE_SIZE, after }) + projects.push(...result.nodes) + + if (!result.pageInfo.hasNextPage) { + return projects + } + after = result.pageInfo.endCursor ?? undefined + if (!after) { + return projects + } + if (page === MAX_PROJECTS_PAGES - 1) { + logger.warn('Linear projects pagination hit cap; project list may be incomplete', { + teamId: team.id, + cap: MAX_PROJECTS_PAGES, + fetched: projects.length, + }) + } + } + + return projects +} + export const POST = withRouteHandler(async (request: NextRequest) => { try { const parsed = await parseRequest(linearProjectsSelectorContract, request, {}) @@ -59,8 +103,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const perTeam = await Promise.all( teamIds.map(async (id) => { const team = await linearClient.team(id) - const result = await team.projects() - return result.nodes.map((project: Project) => ({ + const teamProjects = await fetchAllTeamProjects(team) + return teamProjects.map((project: Project) => ({ id: project.id, name: project.name, })) diff --git a/apps/sim/app/api/tools/linear/teams/route.ts b/apps/sim/app/api/tools/linear/teams/route.ts index 89b02a6e24..f71adec6b0 100644 --- a/apps/sim/app/api/tools/linear/teams/route.ts +++ b/apps/sim/app/api/tools/linear/teams/route.ts @@ -13,6 +13,48 @@ export const dynamic = 'force-dynamic' const logger = createLogger('LinearTeamsAPI') +/** Linear's maximum page size for a single connection request. */ +const LINEAR_PAGE_SIZE = 250 + +/** + * Upper bound on pages to drain from the teams connection. At 250 teams/page + * this covers 2,500 teams; the cap guards against runaway loops on a broken + * `hasNextPage` rather than a realistic limit. + */ +const MAX_TEAMS_PAGES = 10 + +/** + * Drains the full Linear teams connection by following + * `pageInfo.endCursor` until `hasNextPage` is false. Bounded by + * `MAX_TEAMS_PAGES`; logs a warning if the cap is hit so a truncated list is + * visible rather than silently dropped. + */ +async function fetchAllTeams(linearClient: LinearClient): Promise { + const teams: Team[] = [] + let after: string | undefined + + for (let page = 0; page < MAX_TEAMS_PAGES; page++) { + const result = await linearClient.teams({ first: LINEAR_PAGE_SIZE, after }) + teams.push(...result.nodes) + + if (!result.pageInfo.hasNextPage) { + return teams + } + after = result.pageInfo.endCursor ?? undefined + if (!after) { + return teams + } + if (page === MAX_TEAMS_PAGES - 1) { + logger.warn('Linear teams pagination hit cap; team list may be incomplete', { + cap: MAX_TEAMS_PAGES, + fetched: teams.length, + }) + } + } + + return teams +} + export const POST = withRouteHandler(async (request: NextRequest) => { try { const requestId = generateRequestId() @@ -45,8 +87,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const linearClient = new LinearClient({ accessToken }) - const teamsResult = await linearClient.teams() - const teams = teamsResult.nodes.map((team: Team) => ({ + const allTeams = await fetchAllTeams(linearClient) + const teams = allTeams.map((team: Team) => ({ id: team.id, name: team.name, })) diff --git a/apps/sim/app/api/tools/microsoft-teams/channels/route.ts b/apps/sim/app/api/tools/microsoft-teams/channels/route.ts index 6570d5fdd4..c8bd6ddcb5 100644 --- a/apps/sim/app/api/tools/microsoft-teams/channels/route.ts +++ b/apps/sim/app/api/tools/microsoft-teams/channels/route.ts @@ -7,11 +7,25 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' const logger = createLogger('TeamsChannelsAPI') +/** + * Upper bound on Microsoft Graph pages drained when listing a team's channels. + * The `teams/{id}/channels` endpoint does not support `$top`, so paging is + * driven entirely by the server via `@odata.nextLink`. The cap prevents an + * unbounded loop; hitting it is logged as a warning. + */ +const MAX_CHANNELS_PAGES = 20 + +interface GraphChannel { + id: string + displayName?: string +} + export const POST = withRouteHandler(async (request: NextRequest) => { try { const parsed = await parseRequest(microsoftChannelsSelectorContract, request, {}) @@ -52,41 +66,60 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch( - `https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(teamId)}/channels`, - { + const channels: GraphChannel[] = [] + let nextPageUrl: string | undefined = + `https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(teamId)}/channels` + + for (let page = 0; page < MAX_CHANNELS_PAGES; page++) { + const response = await fetch(nextPageUrl, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json', }, + }) + + if (!response.ok) { + const errorData = await response.json() + logger.error('Microsoft Graph API error getting channels', { + status: response.status, + error: errorData, + endpoint: nextPageUrl, + }) + + if (response.status === 401) { + return NextResponse.json( + { + error: 'Authentication failed. Please reconnect your Microsoft Teams account.', + authRequired: true, + }, + { status: 401 } + ) + } + + throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`) } - ) - if (!response.ok) { - const errorData = await response.json() - logger.error('Microsoft Graph API error getting channels', { - status: response.status, - error: errorData, - endpoint: `https://graph.microsoft.com/v1.0/teams/${teamId}/channels`, - }) + const data = await response.json() + if (Array.isArray(data.value)) { + channels.push(...(data.value as GraphChannel[])) + } - if (response.status === 401) { - return NextResponse.json( - { - error: 'Authentication failed. Please reconnect your Microsoft Teams account.', - authRequired: true, - }, - { status: 401 } - ) + const rawNextLink = getGraphNextPageUrl(data) + if (!rawNextLink) { + nextPageUrl = undefined + break } + nextPageUrl = assertGraphNextPageUrl(rawNextLink) - throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`) + if (page === MAX_CHANNELS_PAGES - 1) { + logger.warn( + 'Hit Microsoft Graph channels pagination cap; channel list may be incomplete', + { maxPages: MAX_CHANNELS_PAGES, collected: channels.length } + ) + } } - const data = await response.json() - const channels = data.value - return NextResponse.json({ channels: channels, }) diff --git a/apps/sim/app/api/tools/microsoft-teams/chats/route.ts b/apps/sim/app/api/tools/microsoft-teams/chats/route.ts index 832c720014..d709bcd62e 100644 --- a/apps/sim/app/api/tools/microsoft-teams/chats/route.ts +++ b/apps/sim/app/api/tools/microsoft-teams/chats/route.ts @@ -7,11 +7,29 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' const logger = createLogger('TeamsChatsAPI') +/** + * Largest page size the `me/chats` Microsoft Graph endpoint permits via `$top`. + */ +const CHATS_PAGE_SIZE = 50 + +/** + * Upper bound on Microsoft Graph pages drained when listing the user's chats. + * Paging follows `@odata.nextLink`. The cap prevents an unbounded loop; hitting + * it is logged as a warning. + */ +const MAX_CHATS_PAGES = 20 + +interface GraphChat { + id: string + topic?: string +} + /** * Helper function to get chat members and create a meaningful name * @@ -153,39 +171,62 @@ export const POST = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: 'Could not retrieve access token' }, { status: 401 }) } - const response = await fetch('https://graph.microsoft.com/v1.0/me/chats', { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - }) + const rawChats: GraphChat[] = [] + let nextPageUrl: string | undefined = + `https://graph.microsoft.com/v1.0/me/chats?$top=${CHATS_PAGE_SIZE}` - if (!response.ok) { - const errorData = await response.json() - logger.error('Microsoft Graph API error getting chats', { - status: response.status, - error: errorData, - endpoint: 'https://graph.microsoft.com/v1.0/me/chats', + for (let page = 0; page < MAX_CHATS_PAGES; page++) { + const response = await fetch(nextPageUrl, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, }) - if (response.status === 401) { - return NextResponse.json( - { - error: 'Authentication failed. Please reconnect your Microsoft Teams account.', - authRequired: true, - }, - { status: 401 } - ) + if (!response.ok) { + const errorData = await response.json() + logger.error('Microsoft Graph API error getting chats', { + status: response.status, + error: errorData, + endpoint: nextPageUrl, + }) + + if (response.status === 401) { + return NextResponse.json( + { + error: 'Authentication failed. Please reconnect your Microsoft Teams account.', + authRequired: true, + }, + { status: 401 } + ) + } + + throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`) } - throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`) - } + const data = await response.json() + if (Array.isArray(data.value)) { + rawChats.push(...(data.value as GraphChat[])) + } - const data = await response.json() + const rawNextLink = getGraphNextPageUrl(data) + if (!rawNextLink) { + nextPageUrl = undefined + break + } + nextPageUrl = assertGraphNextPageUrl(rawNextLink) + + if (page === MAX_CHATS_PAGES - 1) { + logger.warn('Hit Microsoft Graph chats pagination cap; chat list may be incomplete', { + maxPages: MAX_CHATS_PAGES, + collected: rawChats.length, + }) + } + } const chats = await Promise.all( - data.value.map(async (chat: any) => ({ + rawChats.map(async (chat) => ({ id: chat.id, displayName: await getChatDisplayName(chat.id, accessToken, chat.topic), })) diff --git a/apps/sim/app/api/tools/microsoft-teams/teams/route.ts b/apps/sim/app/api/tools/microsoft-teams/teams/route.ts index 44c1d99706..990bfd282d 100644 --- a/apps/sim/app/api/tools/microsoft-teams/teams/route.ts +++ b/apps/sim/app/api/tools/microsoft-teams/teams/route.ts @@ -6,11 +6,25 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' const logger = createLogger('TeamsTeamsAPI') +/** + * Upper bound on Microsoft Graph pages drained when listing the user's joined + * teams. The `me/joinedTeams` endpoint does not support `$top`, so paging is + * driven entirely by the server via `@odata.nextLink`. The cap prevents an + * unbounded loop; hitting it is logged as a warning. + */ +const MAX_TEAMS_PAGES = 20 + +interface GraphTeam { + id: string + displayName?: string +} + export const POST = withRouteHandler(async (request: NextRequest) => { try { const parsed = await parseRequest(microsoftTeamsSelectorContract, request, {}) @@ -46,38 +60,59 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://graph.microsoft.com/v1.0/me/joinedTeams', { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - }) + const teams: GraphTeam[] = [] + let nextPageUrl: string | undefined = 'https://graph.microsoft.com/v1.0/me/joinedTeams' - if (!response.ok) { - const errorData = await response.json() - logger.error('Microsoft Graph API error getting teams', { - status: response.status, - error: errorData, - endpoint: 'https://graph.microsoft.com/v1.0/me/joinedTeams', + for (let page = 0; page < MAX_TEAMS_PAGES; page++) { + const response = await fetch(nextPageUrl, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, }) - // Check for auth errors specifically - if (response.status === 401) { - return NextResponse.json( - { - error: 'Authentication failed. Please reconnect your Microsoft Teams account.', - authRequired: true, - }, - { status: 401 } - ) + if (!response.ok) { + const errorData = await response.json() + logger.error('Microsoft Graph API error getting teams', { + status: response.status, + error: errorData, + endpoint: nextPageUrl, + }) + + // Check for auth errors specifically + if (response.status === 401) { + return NextResponse.json( + { + error: 'Authentication failed. Please reconnect your Microsoft Teams account.', + authRequired: true, + }, + { status: 401 } + ) + } + + throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`) } - throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`) - } + const data = await response.json() + if (Array.isArray(data.value)) { + teams.push(...(data.value as GraphTeam[])) + } + + const rawNextLink = getGraphNextPageUrl(data) + if (!rawNextLink) { + nextPageUrl = undefined + break + } + nextPageUrl = assertGraphNextPageUrl(rawNextLink) - const data = await response.json() - const teams = data.value + if (page === MAX_TEAMS_PAGES - 1) { + logger.warn('Hit Microsoft Graph teams pagination cap; team list may be incomplete', { + maxPages: MAX_TEAMS_PAGES, + collected: teams.length, + }) + } + } return NextResponse.json({ teams: teams, diff --git a/apps/sim/app/api/tools/microsoft_excel/drives/route.ts b/apps/sim/app/api/tools/microsoft_excel/drives/route.ts index 2e0d1d80e4..97d921a5ea 100644 --- a/apps/sim/app/api/tools/microsoft_excel/drives/route.ts +++ b/apps/sim/app/api/tools/microsoft_excel/drives/route.ts @@ -8,11 +8,19 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { extractGraphError, GRAPH_ID_PATTERN } from '@/tools/microsoft_excel/utils' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' const logger = createLogger('MicrosoftExcelDrivesAPI') +/** + * Upper bound on Microsoft Graph pages drained when listing site drives. + * Each page returns up to `$top=999` drives, so this caps the result set at + * roughly 10k drives while preventing an unbounded server-side loop. + */ +const MAX_DRIVES_PAGES = 10 + interface GraphDrive { id: string name: string @@ -88,25 +96,41 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } // List all drives for the site - const url = `https://graph.microsoft.com/v1.0/sites/${siteId}/drives?$select=id,name,driveType,webUrl` + let nextUrl: string | undefined = + `https://graph.microsoft.com/v1.0/sites/${siteId}/drives?$select=id,name,driveType,webUrl&$top=999` + + const rawDrives: GraphDrive[] = [] + for (let page = 0; page < MAX_DRIVES_PAGES && nextUrl; page++) { + const response = await fetch(nextUrl, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) - const response = await fetch(url, { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - }) + if (!response.ok) { + const errorMessage = await extractGraphError(response) + logger.error(`[${requestId}] Microsoft Graph API error fetching drives`, { + status: response.status, + error: errorMessage, + }) + return NextResponse.json({ error: errorMessage }, { status: response.status }) + } - if (!response.ok) { - const errorMessage = await extractGraphError(response) - logger.error(`[${requestId}] Microsoft Graph API error fetching drives`, { - status: response.status, - error: errorMessage, - }) - return NextResponse.json({ error: errorMessage }, { status: response.status }) + const data = await response.json() + if (Array.isArray(data.value)) { + rawDrives.push(...data.value) + } + + const nextLink = getGraphNextPageUrl(data) + nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined + if (nextUrl && page === MAX_DRIVES_PAGES - 1) { + logger.warn( + `[${requestId}] Site drives pagination hit ${MAX_DRIVES_PAGES}-page cap; result may be incomplete` + ) + } } - const data = await response.json() - const drives = (data.value || []).map((drive: GraphDrive) => ({ + const drives = rawDrives.map((drive: GraphDrive) => ({ id: drive.id, name: drive.name, driveType: drive.driveType, diff --git a/apps/sim/app/api/tools/microsoft_planner/plans/route.ts b/apps/sim/app/api/tools/microsoft_planner/plans/route.ts index a710f84525..604f7c85b3 100644 --- a/apps/sim/app/api/tools/microsoft_planner/plans/route.ts +++ b/apps/sim/app/api/tools/microsoft_planner/plans/route.ts @@ -6,11 +6,19 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' const logger = createLogger('MicrosoftPlannerPlansAPI') export const dynamic = 'force-dynamic' +/** + * Upper bound on Microsoft Graph pages drained when listing Planner plans. + * Planner uses server-side paging (`$top` is generally ignored), so this caps + * the `@odata.nextLink` follow loop to prevent an unbounded drain. + */ +const MAX_PLANS_PAGES = 20 + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() @@ -40,25 +48,40 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://graph.microsoft.com/v1.0/me/planner/plans', { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - }) + let nextUrl: string | undefined = 'https://graph.microsoft.com/v1.0/me/planner/plans' - if (!response.ok) { - const errorText = await response.text() - logger.error(`[${requestId}] Microsoft Graph API error:`, errorText) - return NextResponse.json( - { error: 'Failed to fetch plans from Microsoft Graph' }, - { status: response.status } - ) - } + const rawPlans: { id: string; title: string }[] = [] + for (let page = 0; page < MAX_PLANS_PAGES && nextUrl; page++) { + const response = await fetch(nextUrl, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) + + if (!response.ok) { + const errorText = await response.text() + logger.error(`[${requestId}] Microsoft Graph API error:`, errorText) + return NextResponse.json( + { error: 'Failed to fetch plans from Microsoft Graph' }, + { status: response.status } + ) + } - const data = await response.json() - const plans = data.value || [] + const data = await response.json() + if (Array.isArray(data.value)) { + rawPlans.push(...data.value) + } + + const nextLink = getGraphNextPageUrl(data) + nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined + if (nextUrl && page === MAX_PLANS_PAGES - 1) { + logger.warn( + `[${requestId}] Planner plans pagination hit ${MAX_PLANS_PAGES}-page cap; result may be incomplete` + ) + } + } - const filteredPlans = plans.map((plan: { id: string; title: string }) => ({ + const filteredPlans = rawPlans.map((plan: { id: string; title: string }) => ({ id: plan.id, title: plan.title, })) diff --git a/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts b/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts index 94bf43e832..b9b764089b 100644 --- a/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts +++ b/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts @@ -8,11 +8,19 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import type { PlannerTask } from '@/tools/microsoft_planner/types' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' const logger = createLogger('MicrosoftPlannerTasksAPI') export const dynamic = 'force-dynamic' +/** + * Upper bound on Microsoft Graph pages drained when listing a plan's tasks. + * Planner uses server-side paging (`$top` is generally ignored), so this caps + * the `@odata.nextLink` follow loop to prevent an unbounded drain. + */ +const MAX_TASKS_PAGES = 20 + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() @@ -48,28 +56,41 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch( - `https://graph.microsoft.com/v1.0/planner/plans/${planIdValidation.sanitized}/tasks`, - { + let nextUrl: string | undefined = + `https://graph.microsoft.com/v1.0/planner/plans/${planIdValidation.sanitized}/tasks` + + const rawTasks: PlannerTask[] = [] + for (let page = 0; page < MAX_TASKS_PAGES && nextUrl; page++) { + const response = await fetch(nextUrl, { headers: { Authorization: `Bearer ${accessToken}`, }, + }) + + if (!response.ok) { + const errorText = await response.text() + logger.error(`[${requestId}] Microsoft Graph API error:`, errorText) + return NextResponse.json( + { error: 'Failed to fetch tasks from Microsoft Graph' }, + { status: response.status } + ) } - ) - if (!response.ok) { - const errorText = await response.text() - logger.error(`[${requestId}] Microsoft Graph API error:`, errorText) - return NextResponse.json( - { error: 'Failed to fetch tasks from Microsoft Graph' }, - { status: response.status } - ) - } + const data = await response.json() + if (Array.isArray(data.value)) { + rawTasks.push(...data.value) + } - const data = await response.json() - const tasks = data.value || [] + const nextLink = getGraphNextPageUrl(data) + nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined + if (nextUrl && page === MAX_TASKS_PAGES - 1) { + logger.warn( + `[${requestId}] Planner tasks pagination hit ${MAX_TASKS_PAGES}-page cap; result may be incomplete` + ) + } + } - const filteredTasks = tasks.map((task: PlannerTask) => ({ + const filteredTasks = rawTasks.map((task: PlannerTask) => ({ id: task.id, title: task.title, planId: task.planId, diff --git a/apps/sim/app/api/tools/monday/boards/route.ts b/apps/sim/app/api/tools/monday/boards/route.ts index d20634b011..3c93b63a40 100644 --- a/apps/sim/app/api/tools/monday/boards/route.ts +++ b/apps/sim/app/api/tools/monday/boards/route.ts @@ -11,18 +11,29 @@ export const dynamic = 'force-dynamic' const logger = createLogger('MondayBoardsAPI') +/** + * Monday's GraphQL `boards(limit: N, page: P, state: active)` has no cursor: + * `page` starts at 1 and you stop once a page returns fewer than `limit` items + * (or an empty page). We request the largest page (`MONDAY_BOARDS_LIMIT`) and + * bound the drain with `MAX_MONDAY_PAGES`. + */ +const MONDAY_BOARDS_LIMIT = 100 +const MAX_MONDAY_PAGES = 50 + interface MondayGraphQLError { message?: string } +interface MondayBoard { + id: string + name: string +} + interface MondayBoardsResponse { errors?: MondayGraphQLError[] error_message?: string data?: { - boards?: Array<{ - id: string - name: string - }> + boards?: MondayBoard[] } } @@ -57,34 +68,55 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://api.monday.com/v2', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: accessToken, - 'API-Version': '2024-10', - }, - body: JSON.stringify({ - query: '{ boards(limit: 100, state: active) { id name } }', - }), - }) + const allBoards: MondayBoard[] = [] + let page = 1 - const data = (await response.json()) as MondayBoardsResponse + for (; page <= MAX_MONDAY_PAGES; page++) { + const response = await fetch('https://api.monday.com/v2', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: accessToken, + 'API-Version': '2024-10', + }, + body: JSON.stringify({ + query: `{ boards(limit: ${MONDAY_BOARDS_LIMIT}, page: ${page}, state: active) { id name } }`, + }), + }) - if (data.errors?.length) { - logger.error('Monday.com API error', { errors: data.errors }) - return NextResponse.json( - { error: data.errors[0].message || 'Monday.com API error' }, - { status: 500 } - ) - } + const data = (await response.json()) as MondayBoardsResponse + + if (data.errors?.length) { + logger.error('Monday.com API error', { errors: data.errors }) + return NextResponse.json( + { error: data.errors[0].message || 'Monday.com API error' }, + { status: 500 } + ) + } + + if (data.error_message) { + logger.error('Monday.com API error', { error_message: data.error_message }) + return NextResponse.json({ error: data.error_message }, { status: 500 }) + } + + const pageBoards = data.data?.boards || [] + allBoards.push(...pageBoards) + + if (pageBoards.length < MONDAY_BOARDS_LIMIT) { + break + } - if (data.error_message) { - logger.error('Monday.com API error', { error_message: data.error_message }) - return NextResponse.json({ error: data.error_message }, { status: 500 }) + if (page === MAX_MONDAY_PAGES) { + logger.warn( + 'Monday boards pagination hit MAX_MONDAY_PAGES cap; board list may be incomplete', + { + maxPages: MAX_MONDAY_PAGES, + } + ) + } } - const boards = (data.data?.boards || []).map((board) => ({ + const boards = allBoards.map((board) => ({ id: board.id, name: board.name, })) diff --git a/apps/sim/app/api/tools/notion/databases/route.ts b/apps/sim/app/api/tools/notion/databases/route.ts index 6ab772afa9..c3f844495d 100644 --- a/apps/sim/app/api/tools/notion/databases/route.ts +++ b/apps/sim/app/api/tools/notion/databases/route.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' import { notionDatabasesSelectorContract } from '@/lib/api/contracts/selectors' import { parseRequest } from '@/lib/api/server' @@ -12,6 +13,16 @@ const logger = createLogger('NotionDatabasesAPI') export const dynamic = 'force-dynamic' +const NOTION_PAGE_SIZE = 100 + +/** + * Notion's `POST /v1/search` returns at most `page_size` results per call and + * exposes `has_more`/`next_cursor` for pagination. This caps the number of + * pages drained so a tenant with a very large workspace cannot make this route + * loop unbounded. With `NOTION_PAGE_SIZE` of 100 this covers up to 2,000 items. + */ +const MAX_DATABASE_PAGES = 20 + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { @@ -43,33 +54,55 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://api.notion.com/v1/search', { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - 'Notion-Version': '2022-06-28', - }, - body: JSON.stringify({ - filter: { value: 'database', property: 'object' }, - page_size: 100, - }), - }) + const results: Record[] = [] + let startCursor: string | undefined - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error('Failed to fetch Notion databases', { - status: response.status, - error: errorData, + for (let page = 0; page < MAX_DATABASE_PAGES; page++) { + const response = await fetch('https://api.notion.com/v1/search', { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + 'Notion-Version': '2022-06-28', + }, + body: JSON.stringify({ + filter: { value: 'database', property: 'object' }, + page_size: NOTION_PAGE_SIZE, + ...(startCursor ? { start_cursor: startCursor } : {}), + }), }) - return NextResponse.json( - { error: 'Failed to fetch Notion databases', details: errorData }, - { status: response.status } - ) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + logger.error('Failed to fetch Notion databases', { + status: response.status, + error: errorData, + }) + return NextResponse.json( + { error: 'Failed to fetch Notion databases', details: errorData }, + { status: response.status } + ) + } + + const data = await response.json() + if (Array.isArray(data.results)) { + results.push(...(data.results as Record[])) + } + + if (!data.has_more || !data.next_cursor) { + break + } + startCursor = data.next_cursor as string + + if (page === MAX_DATABASE_PAGES - 1) { + logger.warn('Notion databases search hit pagination cap; results may be incomplete', { + maxPages: MAX_DATABASE_PAGES, + fetched: results.length, + }) + } } - const data = await response.json() - const databases = (data.results || []).map((db: Record) => ({ + const databases = results.map((db) => ({ id: db.id as string, name: extractTitleFromItem(db), })) @@ -78,7 +111,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { logger.error('Error processing Notion databases request:', error) return NextResponse.json( - { error: 'Failed to retrieve Notion databases', details: (error as Error).message }, + { error: 'Failed to retrieve Notion databases', details: getErrorMessage(error) }, { status: 500 } ) } diff --git a/apps/sim/app/api/tools/notion/pages/route.ts b/apps/sim/app/api/tools/notion/pages/route.ts index 419193fdc7..5da2205c30 100644 --- a/apps/sim/app/api/tools/notion/pages/route.ts +++ b/apps/sim/app/api/tools/notion/pages/route.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' import { notionPagesSelectorContract } from '@/lib/api/contracts/selectors' import { parseRequest } from '@/lib/api/server' @@ -12,6 +13,16 @@ const logger = createLogger('NotionPagesAPI') export const dynamic = 'force-dynamic' +const NOTION_PAGE_SIZE = 100 + +/** + * Notion's `POST /v1/search` returns at most `page_size` results per call and + * exposes `has_more`/`next_cursor` for pagination. This caps the number of + * pages drained so a tenant with a very large workspace cannot make this route + * loop unbounded. With `NOTION_PAGE_SIZE` of 100 this covers up to 2,000 items. + */ +const MAX_PAGE_PAGES = 20 + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { @@ -43,33 +54,55 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://api.notion.com/v1/search', { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - 'Notion-Version': '2022-06-28', - }, - body: JSON.stringify({ - filter: { value: 'page', property: 'object' }, - page_size: 100, - }), - }) + const results: Record[] = [] + let startCursor: string | undefined - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error('Failed to fetch Notion pages', { - status: response.status, - error: errorData, + for (let page = 0; page < MAX_PAGE_PAGES; page++) { + const response = await fetch('https://api.notion.com/v1/search', { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + 'Notion-Version': '2022-06-28', + }, + body: JSON.stringify({ + filter: { value: 'page', property: 'object' }, + page_size: NOTION_PAGE_SIZE, + ...(startCursor ? { start_cursor: startCursor } : {}), + }), }) - return NextResponse.json( - { error: 'Failed to fetch Notion pages', details: errorData }, - { status: response.status } - ) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + logger.error('Failed to fetch Notion pages', { + status: response.status, + error: errorData, + }) + return NextResponse.json( + { error: 'Failed to fetch Notion pages', details: errorData }, + { status: response.status } + ) + } + + const data = await response.json() + if (Array.isArray(data.results)) { + results.push(...(data.results as Record[])) + } + + if (!data.has_more || !data.next_cursor) { + break + } + startCursor = data.next_cursor as string + + if (page === MAX_PAGE_PAGES - 1) { + logger.warn('Notion pages search hit pagination cap; results may be incomplete', { + maxPages: MAX_PAGE_PAGES, + fetched: results.length, + }) + } } - const data = await response.json() - const pages = (data.results || []).map((page: Record) => ({ + const pages = results.map((page) => ({ id: page.id as string, name: extractTitleFromItem(page), })) @@ -78,7 +111,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { logger.error('Error processing Notion pages request:', error) return NextResponse.json( - { error: 'Failed to retrieve Notion pages', details: (error as Error).message }, + { error: 'Failed to retrieve Notion pages', details: getErrorMessage(error) }, { status: 500 } ) } diff --git a/apps/sim/app/api/tools/onedrive/files/route.ts b/apps/sim/app/api/tools/onedrive/files/route.ts index 4b3b727360..5bf2a580ac 100644 --- a/apps/sim/app/api/tools/onedrive/files/route.ts +++ b/apps/sim/app/api/tools/onedrive/files/route.ts @@ -8,11 +8,21 @@ import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' const logger = createLogger('OneDriveFilesAPI') +/** + * Microsoft Graph paginates drive item collections via the `@odata.nextLink` + * absolute URL in the response body. Request the largest page (`$top` caps at + * 999) and drain following nextLink, bounded by a page cap. + * See https://learn.microsoft.com/en-us/graph/paging + */ +const ONEDRIVE_FILES_PAGE_SIZE = 999 +const MAX_ONEDRIVE_FILES_PAGES = 20 + /** * Get files (not folders) from Microsoft OneDrive */ @@ -71,7 +81,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { '$select', 'id,name,file,webUrl,size,createdDateTime,lastModifiedDateTime,createdBy,thumbnails' ) - searchParams_new.append('$top', '50') + searchParams_new.append('$top', String(ONEDRIVE_FILES_PAGE_SIZE)) url = `https://graph.microsoft.com/v1.0/me/drive/root/search(q='${encodeURIComponent(query)}')?${searchParams_new.toString()}` } else { const searchParams_new = new URLSearchParams() @@ -79,34 +89,53 @@ export const GET = withRouteHandler(async (request: NextRequest) => { '$select', 'id,name,file,folder,webUrl,size,createdDateTime,lastModifiedDateTime,createdBy,thumbnails' ) - searchParams_new.append('$top', '50') + searchParams_new.append('$top', String(ONEDRIVE_FILES_PAGE_SIZE)) url = `https://graph.microsoft.com/v1.0/me/drive/root/children?${searchParams_new.toString()}` } logger.info(`[${requestId}] Fetching files from Microsoft Graph`, { url }) - const response = await fetch(url, { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - }) + const rawItems: MicrosoftGraphDriveItem[] = [] + let nextUrl: string | undefined = url - if (!response.ok) { - const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } })) - logger.error(`[${requestId}] Microsoft Graph API error`, { - status: response.status, - error: errorData.error?.message || 'Failed to fetch files from OneDrive', + for (let page = 0; page < MAX_ONEDRIVE_FILES_PAGES && nextUrl; page++) { + const response = await fetch(nextUrl, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, }) - return NextResponse.json( - { error: errorData.error?.message || 'Failed to fetch files from OneDrive' }, - { status: response.status } - ) + + if (!response.ok) { + const errorData = await response + .json() + .catch(() => ({ error: { message: 'Unknown error' } })) + logger.error(`[${requestId}] Microsoft Graph API error`, { + status: response.status, + error: errorData.error?.message || 'Failed to fetch files from OneDrive', + }) + return NextResponse.json( + { error: errorData.error?.message || 'Failed to fetch files from OneDrive' }, + { status: response.status } + ) + } + + const data = await response.json() + rawItems.push(...((data.value as MicrosoftGraphDriveItem[]) || [])) + + const nextLink = getGraphNextPageUrl(data) + nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined + + if (nextUrl && page === MAX_ONEDRIVE_FILES_PAGES - 1) { + logger.warn(`[${requestId}] OneDrive files hit pagination cap; list may be incomplete`, { + pages: MAX_ONEDRIVE_FILES_PAGES, + collected: rawItems.length, + }) + } } - const data = await response.json() - logger.info(`[${requestId}] Received ${data.value?.length || 0} items from Microsoft Graph`) + logger.info(`[${requestId}] Received ${rawItems.length} items from Microsoft Graph`) - const files = (data.value || []) + const files = rawItems .filter((item: MicrosoftGraphDriveItem) => !!item.file && !item.folder) .map((file: MicrosoftGraphDriveItem) => ({ id: file.id, @@ -129,7 +158,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { })) logger.info(`[${requestId}] Returning ${files.length} files`, { - totalItems: data.value?.length || 0, + totalItems: rawItems.length, }) return NextResponse.json({ files }, { status: 200 }) diff --git a/apps/sim/app/api/tools/onedrive/folders/route.ts b/apps/sim/app/api/tools/onedrive/folders/route.ts index 4c65c4190f..bcfd9273c2 100644 --- a/apps/sim/app/api/tools/onedrive/folders/route.ts +++ b/apps/sim/app/api/tools/onedrive/folders/route.ts @@ -8,11 +8,21 @@ import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' const logger = createLogger('OneDriveFoldersAPI') +/** + * Microsoft Graph paginates drive item collections via the `@odata.nextLink` + * absolute URL in the response body. Request the largest page (`$top` caps at + * 999) and drain following nextLink, bounded by a page cap. + * See https://learn.microsoft.com/en-us/graph/paging + */ +const ONEDRIVE_FOLDERS_PAGE_SIZE = 999 +const MAX_ONEDRIVE_FOLDERS_PAGES = 20 + /** * Get folders from Microsoft OneDrive */ @@ -60,28 +70,47 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) } - let url = `https://graph.microsoft.com/v1.0/me/drive/root/children?$filter=folder ne null&$select=id,name,folder,webUrl,createdDateTime,lastModifiedDateTime&$top=50` + let url = `https://graph.microsoft.com/v1.0/me/drive/root/children?$filter=folder ne null&$select=id,name,folder,webUrl,createdDateTime,lastModifiedDateTime&$top=${ONEDRIVE_FOLDERS_PAGE_SIZE}` if (query) { url += `&$search="${encodeURIComponent(query)}"` } - const response = await fetch(url, { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - }) + const rawItems: MicrosoftGraphDriveItem[] = [] + let nextUrl: string | undefined = url - if (!response.ok) { - const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } })) - return NextResponse.json( - { error: errorData.error?.message || 'Failed to fetch folders from OneDrive' }, - { status: response.status } - ) + for (let page = 0; page < MAX_ONEDRIVE_FOLDERS_PAGES && nextUrl; page++) { + const response = await fetch(nextUrl, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) + + if (!response.ok) { + const errorData = await response + .json() + .catch(() => ({ error: { message: 'Unknown error' } })) + return NextResponse.json( + { error: errorData.error?.message || 'Failed to fetch folders from OneDrive' }, + { status: response.status } + ) + } + + const data = await response.json() + rawItems.push(...((data.value as MicrosoftGraphDriveItem[]) || [])) + + const nextLink = getGraphNextPageUrl(data) + nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined + + if (nextUrl && page === MAX_ONEDRIVE_FOLDERS_PAGES - 1) { + logger.warn(`[${requestId}] OneDrive folders hit pagination cap; list may be incomplete`, { + pages: MAX_ONEDRIVE_FOLDERS_PAGES, + collected: rawItems.length, + }) + } } - const data = await response.json() - const folders = (data.value || []) + const folders = rawItems .filter((item: MicrosoftGraphDriveItem) => item.folder) .map((folder: MicrosoftGraphDriveItem) => ({ id: folder.id, diff --git a/apps/sim/app/api/tools/outlook/folders/route.ts b/apps/sim/app/api/tools/outlook/folders/route.ts index 2cd0addcd8..8ae9d3e9e2 100644 --- a/apps/sim/app/api/tools/outlook/folders/route.ts +++ b/apps/sim/app/api/tools/outlook/folders/route.ts @@ -8,11 +8,21 @@ import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' const logger = createLogger('OutlookFoldersAPI') +/** + * Microsoft Graph paginates `mailFolders` via the `@odata.nextLink` absolute + * URL in the response body (default page size is ~10). Bound the drain so a + * pathological account can't loop unbounded; `$top` is capped at 999 by Graph. + * See https://learn.microsoft.com/en-us/graph/paging + */ +const OUTLOOK_FOLDERS_PAGE_SIZE = 999 +const MAX_OUTLOOK_FOLDERS_PAGES = 20 + interface OutlookFolder { id: string displayName: string @@ -65,37 +75,53 @@ export const GET = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://graph.microsoft.com/v1.0/me/mailFolders', { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - }) + const folders: OutlookFolder[] = [] + let nextUrl: string | undefined = + `https://graph.microsoft.com/v1.0/me/mailFolders?$top=${OUTLOOK_FOLDERS_PAGE_SIZE}` - if (!response.ok) { - const errorData = await response.json() - logger.error('Microsoft Graph API error getting folders', { - status: response.status, - error: errorData, - endpoint: 'https://graph.microsoft.com/v1.0/me/mailFolders', + for (let page = 0; page < MAX_OUTLOOK_FOLDERS_PAGES && nextUrl; page++) { + const response = await fetch(nextUrl, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, }) - if (response.status === 401) { - return NextResponse.json( - { - error: 'Authentication failed. Please reconnect your Outlook account.', - authRequired: true, - }, - { status: 401 } - ) + if (!response.ok) { + const errorData = await response.json() + logger.error('Microsoft Graph API error getting folders', { + status: response.status, + error: errorData, + endpoint: nextUrl, + }) + + if (response.status === 401) { + return NextResponse.json( + { + error: 'Authentication failed. Please reconnect your Outlook account.', + authRequired: true, + }, + { status: 401 } + ) + } + + throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`) } - throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`) - } + const data = await response.json() + folders.push(...((data.value as OutlookFolder[]) || [])) - const data = await response.json() - const folders = data.value || [] + const nextLink = getGraphNextPageUrl(data) + nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined + + if (nextUrl && page === MAX_OUTLOOK_FOLDERS_PAGES - 1) { + logger.warn('Outlook mailFolders hit pagination cap; folder list may be incomplete', { + pages: MAX_OUTLOOK_FOLDERS_PAGES, + collected: folders.length, + }) + } + } const transformedFolders = folders.map((folder: OutlookFolder) => ({ id: folder.id, diff --git a/apps/sim/app/api/tools/pipedrive/pipelines/route.ts b/apps/sim/app/api/tools/pipedrive/pipelines/route.ts index 8e3900fe11..707a9ff9ee 100644 --- a/apps/sim/app/api/tools/pipedrive/pipelines/route.ts +++ b/apps/sim/app/api/tools/pipedrive/pipelines/route.ts @@ -11,6 +11,86 @@ const logger = createLogger('PipedrivePipelinesAPI') export const dynamic = 'force-dynamic' +const PIPEDRIVE_PAGE_LIMIT = 500 +const PIPEDRIVE_MAX_PIPELINES_PAGES = 50 + +interface PipedrivePipeline { + id: number + name: string +} + +interface PipedrivePipelinesPage { + data?: PipedrivePipeline[] + additional_data?: { + pagination?: { + more_items_in_collection?: boolean + next_start?: number + } + } +} + +/** + * Lists all Pipedrive pipelines using v1 offset pagination (`start`/`limit`), + * following `additional_data.pagination.next_start` while + * `more_items_in_collection` is true so the full set is returned. Bounded by + * `PIPEDRIVE_MAX_PIPELINES_PAGES`; logs a warning rather than silently dropping + * pipelines when the cap is hit. + */ +async function fetchAllPipelines(accessToken: string): Promise { + const pipelines: PipedrivePipeline[] = [] + let start = 0 + + for (let page = 0; page < PIPEDRIVE_MAX_PIPELINES_PAGES; page++) { + const url = new URL('https://api.pipedrive.com/v1/pipelines') + url.searchParams.set('start', String(start)) + url.searchParams.set('limit', String(PIPEDRIVE_PAGE_LIMIT)) + + const response = await fetch(url.toString(), { + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + throw new PipedriveFetchError(response.status, errorData) + } + + const data = (await response.json()) as PipedrivePipelinesPage + if (Array.isArray(data.data)) { + pipelines.push(...data.data) + } + + const pagination = data.additional_data?.pagination + if (!pagination?.more_items_in_collection || typeof pagination.next_start !== 'number') { + return pipelines + } + start = pagination.next_start + + if (page === PIPEDRIVE_MAX_PIPELINES_PAGES - 1) { + logger.warn( + 'Pipedrive pipelines listing hit pagination cap; pipeline list may be incomplete', + { + pages: PIPEDRIVE_MAX_PIPELINES_PAGES, + } + ) + } + } + + return pipelines +} + +class PipedriveFetchError extends Error { + constructor( + readonly status: number, + readonly details: unknown + ) { + super('Failed to fetch Pipedrive pipelines') + this.name = 'PipedriveFetchError' + } +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { @@ -42,27 +122,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://api.pipedrive.com/v1/pipelines', { - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - }) - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error('Failed to fetch Pipedrive pipelines', { - status: response.status, - error: errorData, - }) - return NextResponse.json( - { error: 'Failed to fetch Pipedrive pipelines', details: errorData }, - { status: response.status } - ) + let allPipelines: PipedrivePipeline[] + try { + allPipelines = await fetchAllPipelines(accessToken) + } catch (error) { + if (error instanceof PipedriveFetchError) { + logger.error('Failed to fetch Pipedrive pipelines', { + status: error.status, + error: error.details, + }) + return NextResponse.json( + { error: 'Failed to fetch Pipedrive pipelines', details: error.details }, + { status: error.status } + ) + } + throw error } - const data = await response.json() - const pipelines = (data.data || []).map((pipeline: { id: number; name: string }) => ({ + const pipelines = allPipelines.map((pipeline) => ({ id: String(pipeline.id), name: pipeline.name, })) diff --git a/apps/sim/app/api/tools/sharepoint/lists/route.ts b/apps/sim/app/api/tools/sharepoint/lists/route.ts index 109b410678..a3970a6f04 100644 --- a/apps/sim/app/api/tools/sharepoint/lists/route.ts +++ b/apps/sim/app/api/tools/sharepoint/lists/route.ts @@ -7,11 +7,19 @@ import { validateSharePointSiteId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' const logger = createLogger('SharePointListsAPI') +/** + * Upper bound on Microsoft Graph pages drained when listing SharePoint lists. + * Each page returns up to `$top=999` lists, so this caps the result set at + * roughly 10k lists while preventing an unbounded server-side loop. + */ +const MAX_LISTS_PAGES = 10 + interface SharePointList { id: string displayName: string @@ -60,24 +68,42 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const url = `https://graph.microsoft.com/v1.0/sites/${siteIdValidation.sanitized}/lists?$select=id,displayName,description,webUrl&$expand=list($select=hidden)&$top=100` + let nextUrl: string | undefined = + `https://graph.microsoft.com/v1.0/sites/${siteIdValidation.sanitized}/lists?$select=id,displayName,description,webUrl&$expand=list($select=hidden)&$top=999` - const response = await fetch(url, { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - }) + const rawLists: SharePointList[] = [] + for (let page = 0; page < MAX_LISTS_PAGES && nextUrl; page++) { + const response = await fetch(nextUrl, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) - if (!response.ok) { - const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } })) - return NextResponse.json( - { error: errorData.error?.message || 'Failed to fetch lists from SharePoint' }, - { status: response.status } - ) + if (!response.ok) { + const errorData = await response + .json() + .catch(() => ({ error: { message: 'Unknown error' } })) + return NextResponse.json( + { error: errorData.error?.message || 'Failed to fetch lists from SharePoint' }, + { status: response.status } + ) + } + + const data = await response.json() + if (Array.isArray(data.value)) { + rawLists.push(...data.value) + } + + const nextLink = getGraphNextPageUrl(data) + nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined + if (nextUrl && page === MAX_LISTS_PAGES - 1) { + logger.warn( + `[${requestId}] SharePoint lists pagination hit ${MAX_LISTS_PAGES}-page cap; result may be incomplete` + ) + } } - const data = await response.json() - const lists = (data.value || []) + const lists = rawLists .filter((list: SharePointList) => list.list?.hidden !== true) .map((list: SharePointList) => ({ id: list.id, diff --git a/apps/sim/app/api/tools/sharepoint/sites/route.ts b/apps/sim/app/api/tools/sharepoint/sites/route.ts index 4ca0c58cde..fc8db948c7 100644 --- a/apps/sim/app/api/tools/sharepoint/sites/route.ts +++ b/apps/sim/app/api/tools/sharepoint/sites/route.ts @@ -7,11 +7,19 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import type { SharepointSite } from '@/tools/sharepoint/types' +import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' const logger = createLogger('SharePointSitesAPI') +/** + * Upper bound on Microsoft Graph pages drained when listing SharePoint sites. + * Each page returns up to `$top=999` sites, so this caps the result set at + * roughly 10k sites while preventing an unbounded server-side loop. + */ +const MAX_SITES_PAGES = 10 + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() @@ -45,24 +53,42 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const searchQuery = query || '*' - const url = `https://graph.microsoft.com/v1.0/sites?search=${encodeURIComponent(searchQuery)}&$select=id,name,displayName,webUrl,createdDateTime,lastModifiedDateTime&$top=50` + let nextUrl: string | undefined = + `https://graph.microsoft.com/v1.0/sites?search=${encodeURIComponent(searchQuery)}&$select=id,name,displayName,webUrl,createdDateTime,lastModifiedDateTime&$top=999` - const response = await fetch(url, { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - }) + const rawSites: SharepointSite[] = [] + for (let page = 0; page < MAX_SITES_PAGES && nextUrl; page++) { + const response = await fetch(nextUrl, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) - if (!response.ok) { - const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } })) - return NextResponse.json( - { error: errorData.error?.message || 'Failed to fetch sites from SharePoint' }, - { status: response.status } - ) + if (!response.ok) { + const errorData = await response + .json() + .catch(() => ({ error: { message: 'Unknown error' } })) + return NextResponse.json( + { error: errorData.error?.message || 'Failed to fetch sites from SharePoint' }, + { status: response.status } + ) + } + + const data = await response.json() + if (Array.isArray(data.value)) { + rawSites.push(...data.value) + } + + const nextLink = getGraphNextPageUrl(data) + nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined + if (nextUrl && page === MAX_SITES_PAGES - 1) { + logger.warn( + `[${requestId}] SharePoint sites pagination hit ${MAX_SITES_PAGES}-page cap; result may be incomplete` + ) + } } - const data = await response.json() - const sites = (data.value || []).map((site: SharepointSite) => ({ + const sites = rawSites.map((site: SharepointSite) => ({ id: site.id, name: site.displayName || site.name, mimeType: 'application/vnd.microsoft.graph.site', diff --git a/apps/sim/app/api/tools/slack/users/route.ts b/apps/sim/app/api/tools/slack/users/route.ts index d9360d9b7c..cb1e69569f 100644 --- a/apps/sim/app/api/tools/slack/users/route.ts +++ b/apps/sim/app/api/tools/slack/users/route.ts @@ -12,6 +12,9 @@ export const dynamic = 'force-dynamic' const logger = createLogger('SlackUsersAPI') +const SLACK_PAGE_LIMIT = 200 +const SLACK_MAX_USER_PAGES = 10 + interface SlackUser { id: string name: string @@ -20,6 +23,11 @@ interface SlackUser { is_bot: boolean } +interface SlackUsersResult { + members: SlackUser[] + truncated: boolean +} + export const POST = withRouteHandler(async (request: NextRequest) => { try { const requestId = generateRequestId() @@ -86,6 +94,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const data = await fetchSlackUsers(accessToken) + if (data.truncated) { + logger.warn('users.list hit pagination cap; user list may be incomplete') + } const users = (data.members || []) .filter((user: SlackUser) => !user.deleted && !user.is_bot) @@ -134,27 +145,53 @@ async function fetchSlackUser(accessToken: string, userId: string) { return data } -async function fetchSlackUsers(accessToken: string) { - const url = new URL('https://slack.com/api/users.list') - url.searchParams.append('limit', '200') +/** + * Lists Slack workspace members, following `response_metadata.next_cursor` so + * the full set is returned. Bounded by `SLACK_MAX_USER_PAGES`; sets `truncated` + * rather than silently dropping members when the cap is hit. + */ +async function fetchSlackUsers(accessToken: string): Promise { + const members: SlackUser[] = [] + let cursor: string | undefined + let truncated = false + + for (let page = 0; page < SLACK_MAX_USER_PAGES; page++) { + const url = new URL('https://slack.com/api/users.list') + url.searchParams.append('limit', String(SLACK_PAGE_LIMIT)) + if (cursor) { + url.searchParams.append('cursor', cursor) + } - const response = await fetch(url.toString(), { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - }) + const response = await fetch(url.toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + }) - if (!response.ok) { - throw new Error(`Slack API error: ${response.status} ${response.statusText}`) - } + if (!response.ok) { + throw new Error(`Slack API error: ${response.status} ${response.statusText}`) + } - const data = await response.json() + const data = await response.json() - if (!data.ok) { - throw new Error(data.error || 'Failed to fetch users') + if (!data.ok) { + throw new Error(data.error || 'Failed to fetch users') + } + + if (Array.isArray(data.members)) { + members.push(...data.members) + } + + cursor = data.response_metadata?.next_cursor?.trim() || undefined + if (!cursor) { + return { members, truncated } + } + if (page === SLACK_MAX_USER_PAGES - 1) { + truncated = true + } } - return data + return { members, truncated } } diff --git a/apps/sim/app/api/tools/wealthbox/items/route.ts b/apps/sim/app/api/tools/wealthbox/items/route.ts index 00ce1aab98..a10c4672eb 100644 --- a/apps/sim/app/api/tools/wealthbox/items/route.ts +++ b/apps/sim/app/api/tools/wealthbox/items/route.ts @@ -12,6 +12,18 @@ export const dynamic = 'force-dynamic' const logger = createLogger('WealthboxItemsAPI') +/** + * Wealthbox `GET /v1/contacts` paginates with `?page=` / `?per_page=`, starting + * at page 1. Wealthbox documents no `per_page` maximum and its contacts response + * carries no pagination `meta`, so termination relies on the short-page check: + * we stop once a page returns fewer items than `WEALTHBOX_PAGE_SIZE` (the + * `meta.total_pages` / `meta.current_page` check is a defensive fallback for if + * Wealthbox ever adds that block). Bounded by `MAX_WEALTHBOX_PAGES` so a runaway + * response can't loop forever. + */ +const WEALTHBOX_PAGE_SIZE = 50 +const MAX_WEALTHBOX_PAGES = 50 + interface WealthboxItem { id: string name: string @@ -21,6 +33,15 @@ interface WealthboxItem { updatedAt: string } +interface WealthboxContactsPage { + contacts?: Array> + meta?: { + total_count?: number + total_pages?: number + current_page?: number + } +} + /** * Get items (notes, contacts, tasks) from Wealthbox */ @@ -69,63 +90,83 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } const endpoint = endpoints[type as keyof typeof endpoints] - const url = new URL(`https://api.crmworkspace.com/v1/${endpoint}`) - logger.info(`[${requestId}] Fetching ${type}s from Wealthbox`, { endpoint, - url: url.toString(), hasQuery: !!query.trim(), }) - const response = await fetch(url.toString(), { - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - }) + const allContacts: Array> = [] + let page = 1 - if (!response.ok) { - const errorText = await response.text() - logger.error( - `[${requestId}] Wealthbox API error: ${response.status} ${response.statusText}`, - { - error: errorText, - endpoint, - url: url.toString(), - } - ) - return NextResponse.json( - { error: `Failed to fetch ${type}s from Wealthbox` }, - { status: response.status } - ) - } + for (; page <= MAX_WEALTHBOX_PAGES; page++) { + const url = new URL(`https://api.crmworkspace.com/v1/${endpoint}`) + url.searchParams.set('per_page', String(WEALTHBOX_PAGE_SIZE)) + url.searchParams.set('page', String(page)) - const data = (await response.json()) as { contacts?: Array> } & Record< - string, - unknown - > + const response = await fetch(url.toString(), { + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + }) - logger.info(`[${requestId}] Wealthbox API raw response`, { - type, - status: response.status, - dataKeys: Object.keys(data || {}), - hasContacts: !!data.contacts, - dataStructure: typeof data === 'object' ? Object.keys(data) : 'not an object', - }) + if (!response.ok) { + const errorText = await response.text() + logger.error( + `[${requestId}] Wealthbox API error: ${response.status} ${response.statusText}`, + { + error: errorText, + endpoint, + url: url.toString(), + } + ) + return NextResponse.json( + { error: `Failed to fetch ${type}s from Wealthbox` }, + { status: response.status } + ) + } - let items: WealthboxItem[] = [] + const data = (await response.json()) as WealthboxContactsPage - if (type === 'contact') { const contacts = data.contacts || [] if (!Array.isArray(contacts)) { logger.warn(`[${requestId}] Contacts is not an array`, { contacts, dataType: typeof contacts, }) - return NextResponse.json({ items: [] }, { status: 200 }) + break + } + + allContacts.push(...contacts) + + const totalPages = data.meta?.total_pages + const currentPage = data.meta?.current_page ?? page + const reachedLastByMeta = + typeof totalPages === 'number' && totalPages > 0 && currentPage >= totalPages + const reachedLastByCount = contacts.length < WEALTHBOX_PAGE_SIZE + + if (reachedLastByMeta || reachedLastByCount) { + break } - items = contacts.map((item) => { + if (page === MAX_WEALTHBOX_PAGES) { + logger.warn( + `[${requestId}] Wealthbox pagination hit MAX_WEALTHBOX_PAGES cap; contact list may be incomplete`, + { endpoint, maxPages: MAX_WEALTHBOX_PAGES } + ) + } + } + + logger.info(`[${requestId}] Wealthbox API drained`, { + type, + pagesFetched: Math.min(page, MAX_WEALTHBOX_PAGES), + totalContacts: allContacts.length, + }) + + let items: WealthboxItem[] = [] + + if (type === 'contact') { + items = allContacts.map((item) => { const firstName = typeof item.first_name === 'string' ? item.first_name : '' const lastName = typeof item.last_name === 'string' ? item.last_name : '' return { diff --git a/apps/sim/app/api/tools/webflow/items/route.ts b/apps/sim/app/api/tools/webflow/items/route.ts index 1ed9884e0f..4feb0f4041 100644 --- a/apps/sim/app/api/tools/webflow/items/route.ts +++ b/apps/sim/app/api/tools/webflow/items/route.ts @@ -12,6 +12,9 @@ const logger = createLogger('WebflowItemsAPI') export const dynamic = 'force-dynamic' +const WEBFLOW_PAGE_LIMIT = 100 +const WEBFLOW_MAX_ITEMS_PAGES = 50 + interface WebflowItem { id: string fieldData?: { @@ -21,6 +24,74 @@ interface WebflowItem { } } +interface WebflowItemsPage { + items?: WebflowItem[] + pagination?: { + total?: number + limit?: number + offset?: number + } +} + +/** + * Lists all items in a Webflow collection using `offset`/`limit` pagination + * (limit capped at 100), advancing the numeric `offset` until the accumulated + * count reaches `pagination.total` so the full set is returned. Bounded by + * `WEBFLOW_MAX_ITEMS_PAGES`; logs a warning rather than silently dropping items + * when the cap is hit. + */ +async function fetchAllItems(accessToken: string, collectionId: string): Promise { + const items: WebflowItem[] = [] + let offset = 0 + + for (let page = 0; page < WEBFLOW_MAX_ITEMS_PAGES; page++) { + const url = new URL(`https://api.webflow.com/v2/collections/${collectionId}/items`) + url.searchParams.set('limit', String(WEBFLOW_PAGE_LIMIT)) + url.searchParams.set('offset', String(offset)) + + const response = await fetch(url.toString(), { + headers: { + Authorization: `Bearer ${accessToken}`, + accept: 'application/json', + }, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + throw new WebflowFetchError(response.status, errorData) + } + + const data = (await response.json()) as WebflowItemsPage + const pageItems = data.items || [] + items.push(...pageItems) + + const total = data.pagination?.total + offset += pageItems.length + if (pageItems.length === 0 || (typeof total === 'number' && items.length >= total)) { + return items + } + + if (page === WEBFLOW_MAX_ITEMS_PAGES - 1) { + logger.warn('Webflow items listing hit pagination cap; item list may be incomplete', { + collectionId, + pages: WEBFLOW_MAX_ITEMS_PAGES, + }) + } + } + + return items +} + +class WebflowFetchError extends Error { + constructor( + readonly status: number, + readonly details: unknown + ) { + super('Failed to fetch Webflow items') + this.name = 'WebflowFetchError' + } +} + export const POST = withRouteHandler(async (request: NextRequest) => { try { const requestId = generateRequestId() @@ -61,32 +132,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch( - `https://api.webflow.com/v2/collections/${collectionId}/items?limit=100`, - { - headers: { - Authorization: `Bearer ${accessToken}`, - accept: 'application/json', - }, + let items: WebflowItem[] + try { + items = await fetchAllItems(accessToken, collectionId) + } catch (error) { + if (error instanceof WebflowFetchError) { + logger.error('Failed to fetch Webflow items', { + status: error.status, + error: error.details, + collectionId, + }) + return NextResponse.json( + { error: 'Failed to fetch Webflow items', details: error.details }, + { status: error.status } + ) } - ) - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error('Failed to fetch Webflow items', { - status: response.status, - error: errorData, - collectionId, - }) - return NextResponse.json( - { error: 'Failed to fetch Webflow items', details: errorData }, - { status: response.status } - ) + throw error } - const data = (await response.json()) as { items?: WebflowItem[] } - const items = data.items || [] - let formattedItems = items.map((item) => { const fieldData = item.fieldData || {} const name = fieldData.name || fieldData.title || fieldData.slug || item.id diff --git a/apps/sim/app/api/tools/zoom/meetings/route.ts b/apps/sim/app/api/tools/zoom/meetings/route.ts index 36edf49878..53e78f408c 100644 --- a/apps/sim/app/api/tools/zoom/meetings/route.ts +++ b/apps/sim/app/api/tools/zoom/meetings/route.ts @@ -11,6 +11,24 @@ const logger = createLogger('ZoomMeetingsAPI') export const dynamic = 'force-dynamic' +/** + * Zoom `GET /v2/users/me/meetings` returns `next_page_token`, which is passed + * back as `?next_page_token=` until it comes back as an empty string. `page_size` + * max is 300. Bounded by `MAX_ZOOM_PAGES` so a runaway response can't loop forever. + */ +const ZOOM_PAGE_SIZE = 300 +const MAX_ZOOM_PAGES = 50 + +interface ZoomMeeting { + id: number + topic: string +} + +interface ZoomMeetingsPage { + meetings?: ZoomMeeting[] + next_page_token?: string +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { @@ -42,30 +60,57 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch( - 'https://api.zoom.us/v2/users/me/meetings?page_size=300&type=scheduled', - { + const allMeetings: ZoomMeeting[] = [] + let nextPageToken = '' + + for (let page = 0; page < MAX_ZOOM_PAGES; page++) { + const url = new URL('https://api.zoom.us/v2/users/me/meetings') + url.searchParams.set('page_size', String(ZOOM_PAGE_SIZE)) + url.searchParams.set('type', 'scheduled') + if (nextPageToken) { + url.searchParams.set('next_page_token', nextPageToken) + } + + const response = await fetch(url.toString(), { headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json', }, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + logger.error('Failed to fetch Zoom meetings', { + status: response.status, + error: errorData, + }) + return NextResponse.json( + { error: 'Failed to fetch Zoom meetings', details: errorData }, + { status: response.status } + ) } - ) - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error('Failed to fetch Zoom meetings', { - status: response.status, - error: errorData, - }) - return NextResponse.json( - { error: 'Failed to fetch Zoom meetings', details: errorData }, - { status: response.status } - ) + const data = (await response.json()) as ZoomMeetingsPage + if (Array.isArray(data.meetings)) { + allMeetings.push(...data.meetings) + } + + nextPageToken = data.next_page_token?.trim() || '' + if (!nextPageToken) { + break + } + + if (page === MAX_ZOOM_PAGES - 1) { + logger.warn( + 'Zoom meetings pagination hit MAX_ZOOM_PAGES cap; meeting list may be incomplete', + { + maxPages: MAX_ZOOM_PAGES, + } + ) + } } - const data = await response.json() - const meetings = (data.meetings || []).map((meeting: { id: number; topic: string }) => ({ + const meetings = allMeetings.map((meeting) => ({ id: String(meeting.id), name: meeting.topic, })) diff --git a/apps/sim/hooks/selectors/providers/confluence/selectors.ts b/apps/sim/hooks/selectors/providers/confluence/selectors.ts index 7ebf81640b..91cddd8cea 100644 --- a/apps/sim/hooks/selectors/providers/confluence/selectors.ts +++ b/apps/sim/hooks/selectors/providers/confluence/selectors.ts @@ -9,6 +9,13 @@ function formatConfluenceSpaceLabel(space: { name: string; key: string; status?: return space.status === 'archived' ? `${base} — archived` : base } +function toSpaceOption(space: { name: string; key: string; status?: string }): { + id: string + label: string +} { + return { id: space.key, label: formatConfluenceSpaceLabel(space) } +} + export const confluenceSelectors = { 'confluence.spaces': { key: 'confluence.spaces', @@ -21,28 +28,10 @@ export const confluenceSelectors = { context.domain ?? 'none', ], enabled: ({ context }) => Boolean(context.oauthCredential && context.domain), - fetchList: async ({ context, signal }: SelectorQueryArgs) => { - const credentialId = ensureCredential(context, 'confluence.spaces') - const domain = ensureDomain(context, 'confluence.spaces') - const collected: { id: string; label: string }[] = [] - let cursor: string | undefined - do { - const data = await requestJson(selectorContracts.confluenceSpacesSelectorContract, { - body: { - credential: credentialId, - workflowId: context.workflowId, - domain, - cursor, - }, - signal, - }) - for (const space of data.spaces || []) { - collected.push({ id: space.key, label: formatConfluenceSpaceLabel(space) }) - } - cursor = data.nextCursor - } while (cursor) - return collected - }, + /** + * Drives pagination through {@link useSelectorOptions}, which drains every + * page via this callback. No `fetchList` — the paged path supersedes it. + */ fetchPage: async ({ context, cursor, signal }) => { const credentialId = ensureCredential(context, 'confluence.spaces') const domain = ensureDomain(context, 'confluence.spaces') @@ -56,10 +45,7 @@ export const confluenceSelectors = { signal, }) return { - items: (data.spaces || []).map((space) => ({ - id: space.key, - label: formatConfluenceSpaceLabel(space), - })), + items: (data.spaces || []).map(toSpaceOption), nextCursor: data.nextCursor, } }, @@ -83,7 +69,7 @@ export const confluenceSelectors = { }) const space = (data.spaces || []).find((s) => s.key === detailId) ?? null if (!space) return null - return { id: space.key, label: formatConfluenceSpaceLabel(space) } + return toSpaceOption(space) }, }, 'confluence.pages': { diff --git a/apps/sim/hooks/selectors/providers/knowledge/selectors.ts b/apps/sim/hooks/selectors/providers/knowledge/selectors.ts index 01a76f44a1..c379d96c91 100644 --- a/apps/sim/hooks/selectors/providers/knowledge/selectors.ts +++ b/apps/sim/hooks/selectors/providers/knowledge/selectors.ts @@ -3,6 +3,8 @@ import * as selectorContracts from '@/lib/api/contracts/selectors' import { ensureKnowledgeBase, SELECTOR_STALE } from '@/hooks/selectors/providers/shared' import type { SelectorDefinition, SelectorKey, SelectorQueryArgs } from '@/hooks/selectors/types' +const KNOWLEDGE_DOCUMENTS_PAGE_LIMIT = 100 + export const knowledgeSelectors = { 'knowledge.documents': { key: 'knowledge.documents', @@ -18,20 +20,32 @@ export const knowledgeSelectors = { search ?? '', ], enabled: ({ context }) => Boolean(context.knowledgeBaseId), - fetchList: async ({ context, search, signal }: SelectorQueryArgs) => { + /** + * Drives pagination through {@link useSelectorOptions}, which drains every + * page via this callback. The `pagination.hasMore` flag from the route + * decides when to stop; `nextCursor` encodes the next `offset`. + */ + fetchPage: async ({ context, search, cursor, signal }) => { const knowledgeBaseId = ensureKnowledgeBase(context) + const offset = cursor ? Number(cursor) : 0 const result = await requestJson(selectorContracts.listKnowledgeSelectorDocumentsContract, { params: { id: knowledgeBaseId }, query: { - limit: 100, + limit: KNOWLEDGE_DOCUMENTS_PAGE_LIMIT, + offset, search, }, signal, }) - return result.data.documents.map((doc) => ({ - id: doc.id, - label: doc.filename, - })) + const { pagination } = result.data + const nextOffset = pagination.offset + pagination.limit + return { + items: result.data.documents.map((doc) => ({ + id: doc.id, + label: doc.filename, + })), + nextCursor: pagination.hasMore ? String(nextOffset) : undefined, + } }, fetchById: async ({ context, detailId, signal }: SelectorQueryArgs) => { if (!detailId) return null diff --git a/apps/sim/hooks/selectors/providers/microsoft/selectors.ts b/apps/sim/hooks/selectors/providers/microsoft/selectors.ts index e9b1dad6d2..f18a854cc4 100644 --- a/apps/sim/hooks/selectors/providers/microsoft/selectors.ts +++ b/apps/sim/hooks/selectors/providers/microsoft/selectors.ts @@ -320,6 +320,7 @@ export const microsoftSelectors = { query: search, driveId: context.driveId, workflowId: context.workflowId, + fileType: 'excel', }, signal, }) @@ -347,6 +348,7 @@ export const microsoftSelectors = { credentialId, query: search, workflowId: context.workflowId, + fileType: 'word', }, signal, }) diff --git a/apps/sim/hooks/selectors/registry.test.ts b/apps/sim/hooks/selectors/registry.test.ts index 8c287ea549..08f58b0572 100644 --- a/apps/sim/hooks/selectors/registry.test.ts +++ b/apps/sim/hooks/selectors/registry.test.ts @@ -61,7 +61,7 @@ describe('sim.workflows selector', () => { it('reads workflow options from the scoped workflow cache', async () => { const definition = getSelectorDefinition('sim.workflows') - const options = await definition.fetchList({ + const options = await definition.fetchList!({ key: 'sim.workflows', context: { workspaceId: 'ws-1', excludeWorkflowId: 'wf-2' }, }) @@ -112,7 +112,7 @@ describe('sim.workflows selector', () => { }) const definition = getSelectorDefinition('sim.workflows') - const options = await definition.fetchList({ + const options = await definition.fetchList!({ key: 'sim.workflows', context: { workspaceId: 'ws-1' }, }) diff --git a/apps/sim/hooks/selectors/types.ts b/apps/sim/hooks/selectors/types.ts index ccf814fdc1..48af9cfda8 100644 --- a/apps/sim/hooks/selectors/types.ts +++ b/apps/sim/hooks/selectors/types.ts @@ -114,7 +114,12 @@ export interface SelectorDefinition { key: SelectorKey contracts?: readonly AnyApiRouteContract[] getQueryKey: (args: SelectorQueryArgs) => QueryKey - fetchList: (args: SelectorQueryArgs) => Promise + /** + * Loads the full option list in a single call. Required unless `fetchPage` is + * defined, in which case the hook drives pagination through `fetchPage` and + * `fetchList` is never invoked — provide one or the other, not both. + */ + fetchList?: (args: SelectorQueryArgs) => Promise /** * Optional. When defined, the selector hook fetches one page at a time and * auto-drains remaining pages so the dropdown populates progressively. diff --git a/apps/sim/hooks/selectors/use-selector-query.ts b/apps/sim/hooks/selectors/use-selector-query.ts index 8eb4755834..0bf7979d4c 100644 --- a/apps/sim/hooks/selectors/use-selector-query.ts +++ b/apps/sim/hooks/selectors/use-selector-query.ts @@ -1,4 +1,5 @@ import { useEffect, useMemo } from 'react' +import { createLogger } from '@sim/logger' import { useInfiniteQuery, useQuery } from '@tanstack/react-query' import { extractEnvVarName, isEnvVarReference, isReference } from '@/executor/constants' import { usePersonalEnvironment } from '@/hooks/queries/environment' @@ -30,11 +31,26 @@ export interface SelectorOptionsResult { * for non-paginated selectors. */ hasMore: boolean + /** + * True when the paginated drain stopped at {@link MAX_AUTO_DRAIN_PAGES} with + * pages still remaining, so the option list is a partial view. Always false + * for non-paginated selectors. + */ + truncated: boolean error: Error | null } +const logger = createLogger('SelectorQuery') + const EMPTY_PAGE: SelectorPage = { items: [], nextCursor: undefined } +/** + * Safety bound on the background auto-drain. Real dropdowns settle in a handful + * of pages; this only trips for pathological result sets and prevents an + * unbounded request loop when a provider keeps handing back cursors. + */ +const MAX_AUTO_DRAIN_PAGES = 50 + export function useSelectorOptions( key: SelectorKey, args: SelectorHookArgs @@ -50,7 +66,8 @@ export function useSelectorOptions( const flatQuery = useQuery({ queryKey: definition.getQueryKey(queryArgs), - queryFn: ({ signal }) => definition.fetchList({ ...queryArgs, signal }), + queryFn: ({ signal }) => + definition.fetchList?.({ ...queryArgs, signal }) ?? Promise.resolve([]), enabled: !supportsPagination && isEnabled, staleTime: definition.staleTime ?? 30_000, }) @@ -72,13 +89,33 @@ export function useSelectorOptions( }) const { hasNextPage, isFetchingNextPage, fetchNextPage, isError } = pagedQuery + const pageCount = pagedQuery.data?.pages.length ?? 0 + const reachedDrainCap = pageCount >= MAX_AUTO_DRAIN_PAGES useEffect(() => { if (!supportsPagination) return if (isError) return + if (reachedDrainCap) { + if (hasNextPage) { + logger.warn('Selector hit auto-drain cap; option list is truncated', { + key, + pages: pageCount, + }) + } + return + } if (hasNextPage && !isFetchingNextPage) { void fetchNextPage() } - }, [supportsPagination, hasNextPage, isFetchingNextPage, isError, fetchNextPage]) + }, [ + supportsPagination, + hasNextPage, + isFetchingNextPage, + isError, + fetchNextPage, + reachedDrainCap, + pageCount, + key, + ]) const pagedOptions = useMemo(() => { if (!supportsPagination) return undefined @@ -92,7 +129,8 @@ export function useSelectorOptions( isLoading: pagedQuery.isLoading, isFetching: pagedQuery.isFetching, isFetchingMore: pagedQuery.isFetchingNextPage, - hasMore: pagedQuery.hasNextPage ?? false, + hasMore: (pagedQuery.hasNextPage ?? false) && !reachedDrainCap, + truncated: reachedDrainCap && (pagedQuery.hasNextPage ?? false), error: (pagedQuery.error as Error | null) ?? null, } } @@ -103,6 +141,7 @@ export function useSelectorOptions( isFetching: flatQuery.isFetching, isFetchingMore: false, hasMore: false, + truncated: false, error: (flatQuery.error as Error | null) ?? null, } } diff --git a/apps/sim/lib/api/contracts/selectors/knowledge.ts b/apps/sim/lib/api/contracts/selectors/knowledge.ts index 6971a301db..1234c868f6 100644 --- a/apps/sim/lib/api/contracts/selectors/knowledge.ts +++ b/apps/sim/lib/api/contracts/selectors/knowledge.ts @@ -11,6 +11,7 @@ const knowledgeDocumentParamsSchema = knowledgeDocumentsParamsSchema.extend({ const knowledgeDocumentsQuerySchema = z.object({ limit: z.coerce.number().int().min(1).max(100).optional(), + offset: z.coerce.number().int().min(0).optional(), search: optionalString, }) diff --git a/apps/sim/lib/api/contracts/selectors/microsoft.ts b/apps/sim/lib/api/contracts/selectors/microsoft.ts index 513a48df98..0930489692 100644 --- a/apps/sim/lib/api/contracts/selectors/microsoft.ts +++ b/apps/sim/lib/api/contracts/selectors/microsoft.ts @@ -38,10 +38,17 @@ export const microsoftExcelDrivesBodySchema = credentialWorkflowBodySchema.exten driveId: optionalString, }) +/** + * The `/api/auth/oauth/microsoft/files` route is shared by the + * `microsoft.excel` and `microsoft.word` selectors. `fileType` lets the route + * search for and filter to the correct Office document type; it is optional and + * defaults to `excel` on the server for backward compatibility. + */ export const microsoftFilesQuerySchema = credentialIdQuerySchema.extend({ query: optionalString, driveId: optionalString, workflowId: optionalString, + fileType: z.enum(['excel', 'word']).optional(), }) export const microsoftFileQuerySchema = credentialIdQuerySchema.extend({ diff --git a/apps/sim/lib/oauth/google-pagination.ts b/apps/sim/lib/oauth/google-pagination.ts new file mode 100644 index 0000000000..c907288496 --- /dev/null +++ b/apps/sim/lib/oauth/google-pagination.ts @@ -0,0 +1,108 @@ +import { createLogger } from '@sim/logger' + +const logger = createLogger('GooglePagination') + +/** + * Thrown by {@link drainGooglePagedList} when a page request returns a non-OK + * HTTP status. Carries the status and parsed error body so callers can shape + * their existing error responses unchanged. + */ +export class GooglePageError extends Error { + readonly status: number + readonly body: unknown + + constructor(status: number, body: unknown) { + super(`Google API error: ${status}`) + this.name = 'GooglePageError' + this.status = status + this.body = body + } +} + +/** + * Result of draining a token-paginated Google REST list endpoint. + */ +export interface GoogleDrainResult { + /** All items accumulated across every fetched page. */ + items: T[] + /** True when the page cap was reached before the API stopped returning a token. */ + truncated: boolean +} + +/** + * Options for {@link drainGooglePagedList}. + */ +export interface DrainGooglePagedListOptions { + /** + * Builds the request URL for a given page. `pageToken` is `undefined` for the + * first page, then the value of the previous response's `nextPageToken`. + */ + buildUrl: (pageToken: string | undefined) => string + /** Performs the HTTP request for a built URL. */ + fetch: (url: string) => Promise + /** Parses an error body from a non-OK response (used to build {@link GooglePageError}). */ + parseError: (response: Response) => Promise + /** Extracts the array of items from a single page's JSON body. */ + getItems: (body: R) => T[] | undefined + /** Reads the continuation token from a single page's JSON body. */ + getNextPageToken: (body: R) => string | undefined + /** Maximum number of pages to fetch before stopping and flagging `truncated`. */ + maxPages: number + /** Label used in the cap-reached warning log. */ + label: string +} + +/** + * Drains a token-paginated Google REST list endpoint, following each response's + * `nextPageToken` until it is absent or the `maxPages` cap is hit. + * + * Mirrors the bounded-loop pattern used by the Slack channels selector: the loop + * is bounded and emits a `logger.warn` (and sets `truncated`) when the cap is + * reached rather than silently dropping items. A non-OK page response throws a + * {@link GooglePageError} carrying the status and parsed body so callers preserve + * their existing error-response shapes. + */ +export async function drainGooglePagedList( + options: DrainGooglePagedListOptions +): Promise> { + const { + buildUrl, + fetch: fetchPage, + parseError, + getItems, + getNextPageToken, + maxPages, + label, + } = options + + const items: T[] = [] + let pageToken: string | undefined + let truncated = false + + for (let page = 0; page < maxPages; page++) { + const response = await fetchPage(buildUrl(pageToken)) + + if (!response.ok) { + throw new GooglePageError(response.status, await parseError(response)) + } + + const body = (await response.json()) as R + + const pageItems = getItems(body) + if (pageItems?.length) { + items.push(...pageItems) + } + + pageToken = getNextPageToken(body)?.trim() || undefined + if (!pageToken) { + return { items, truncated } + } + + if (page === maxPages - 1) { + truncated = true + logger.warn(`${label}: hit pagination cap of ${maxPages} pages; results may be incomplete`) + } + } + + return { items, truncated } +} From 7b19788a8a1cd2061bcb44b10241ea4b16653eb3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 30 May 2026 20:05:36 -0700 Subject: [PATCH 2/5] fix(selectors): harden JSM and Monday pagination draining - JSM service-desk/request-type drains advance `start` by the actual row count returned (not the fixed page size) and stop on an empty page, so a short non-final page can't skip items. - Monday boards drain now checks `response.ok` per page, surfacing a mid-drain HTTP failure instead of treating it as an empty final page and returning a partial 200. --- .../api/tools/jsm/selector-requesttypes/route.ts | 4 ++-- .../api/tools/jsm/selector-servicedesks/route.ts | 4 ++-- apps/sim/app/api/tools/monday/boards/route.ts | 13 +++++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts b/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts index 0485e2686a..be8ba4dfaf 100644 --- a/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts +++ b/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts @@ -60,11 +60,11 @@ async function fetchAllJsmRequestTypes( const pageValues = data.values ?? [] values.push(...pageValues) - if (data.isLastPage === true || !data._links?.next) { + if (data.isLastPage === true || !data._links?.next || pageValues.length === 0) { return { values, lastResponse } } - start += JSM_REQUEST_TYPES_PAGE_SIZE + start += pageValues.length if (page === MAX_JSM_REQUEST_TYPES_PAGES - 1) { logger.warn('JSM request type list hit pagination cap; list may be incomplete', { diff --git a/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts b/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts index 38e91a6cdf..8235de2781 100644 --- a/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts +++ b/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts @@ -60,11 +60,11 @@ async function fetchAllJsmServiceDesks( const pageValues = data.values ?? [] values.push(...pageValues) - if (data.isLastPage === true || !data._links?.next) { + if (data.isLastPage === true || !data._links?.next || pageValues.length === 0) { return { values, lastResponse } } - start += JSM_SERVICE_DESKS_PAGE_SIZE + start += pageValues.length if (page === MAX_JSM_SERVICE_DESKS_PAGES - 1) { logger.warn('JSM service desk list hit pagination cap; list may be incomplete', { diff --git a/apps/sim/app/api/tools/monday/boards/route.ts b/apps/sim/app/api/tools/monday/boards/route.ts index 3c93b63a40..e5d3dc5fad 100644 --- a/apps/sim/app/api/tools/monday/boards/route.ts +++ b/apps/sim/app/api/tools/monday/boards/route.ts @@ -84,6 +84,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => { }), }) + if (!response.ok) { + const details = await response.text().catch(() => '') + logger.error('Monday.com API HTTP error', { + status: response.status, + statusText: response.statusText, + details, + }) + return NextResponse.json( + { error: `Monday.com API error: ${response.status} ${response.statusText}` }, + { status: 500 } + ) + } + const data = (await response.json()) as MondayBoardsResponse if (data.errors?.length) { From dc41d23be7dc0adca2e2ecc58148a056f3ef21bc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 30 May 2026 20:15:06 -0700 Subject: [PATCH 3/5] docs(selectors): clarify JSM drain advances start by actual row count The offset-advancement fix (advance `start` by the rows returned, not the fixed page size) landed in 7b19788a8; update the TSDoc to match so it no longer reads as advancing by `limit`. --- .../sim/app/api/tools/jsm/selector-requesttypes/route.ts | 9 ++++++--- .../sim/app/api/tools/jsm/selector-servicedesks/route.ts | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts b/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts index be8ba4dfaf..b23b4b7c7a 100644 --- a/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts +++ b/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts @@ -30,9 +30,12 @@ interface JsmRequestTypeValue { /** * Drains the offset-paginated JSM `/servicedesk/{id}/requesttype` endpoint, - * advancing `start` by `limit` until `isLastPage === true` (or `_links.next` is - * absent). Bounded by `MAX_JSM_REQUEST_TYPES_PAGES`; emits a `logger.warn` and - * returns the partial set rather than looping unbounded when the cap is hit. + * advancing `start` by the number of rows actually returned until + * `isLastPage === true` (or `_links.next` is absent, or a page comes back + * empty). Advancing by the real row count — not the requested `limit` — + * prevents skipping items if the server returns a short non-final page. Bounded + * by `MAX_JSM_REQUEST_TYPES_PAGES`; emits a `logger.warn` and returns the + * partial set rather than looping unbounded when the cap is hit. */ async function fetchAllJsmRequestTypes( requestTypeUrl: string, diff --git a/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts b/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts index 8235de2781..786483630d 100644 --- a/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts +++ b/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts @@ -30,9 +30,12 @@ interface JsmServiceDeskValue { /** * Drains the offset-paginated JSM `/servicedesk` endpoint, advancing `start` by - * `limit` until `isLastPage === true` (or `_links.next` is absent). Bounded by - * `MAX_JSM_SERVICE_DESKS_PAGES`; emits a `logger.warn` and returns the partial - * set rather than looping unbounded when the cap is hit. + * the number of rows actually returned until `isLastPage === true` (or + * `_links.next` is absent, or a page comes back empty). Advancing by the real + * row count — not the requested `limit` — prevents skipping items if the server + * returns a short non-final page. Bounded by `MAX_JSM_SERVICE_DESKS_PAGES`; + * emits a `logger.warn` and returns the partial set rather than looping + * unbounded when the cap is hit. */ async function fetchAllJsmServiceDesks( baseUrl: string, From abdaf2c85b9fee9f43a3d28e783498aed62c4553 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 30 May 2026 20:28:55 -0700 Subject: [PATCH 4/5] fix(selectors): drain fetchPage in direct fetchList callers Making `fetchList` optional left three direct callers (outside the useSelectorOptions hook) calling it unguarded, which broke the build's type check. Route them through a shared `loadAllSelectorOptions` helper that uses `fetchList` when present and otherwise drains `fetchPage`. This also prevents a regression: `confluence.spaces` / `knowledge.documents` now paginate via `fetchPage` only, and these callers (search/replace, value resolution) would otherwise have silently returned no options. --- .../hooks/queries/workflow-search-replace.ts | 10 +++-- apps/sim/hooks/selectors/registry.ts | 39 ++++++++++++++++++- .../workflows/comparison/resolve-values.ts | 4 +- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/apps/sim/hooks/queries/workflow-search-replace.ts b/apps/sim/hooks/queries/workflow-search-replace.ts index 0dd264e6e4..02e3bf1d50 100644 --- a/apps/sim/hooks/queries/workflow-search-replace.ts +++ b/apps/sim/hooks/queries/workflow-search-replace.ts @@ -33,7 +33,7 @@ import { fetchOAuthCredentialDetail, fetchOAuthCredentials, } from '@/hooks/queries/oauth/oauth-credentials' -import { getSelectorDefinition } from '@/hooks/selectors/registry' +import { getSelectorDefinition, loadAllSelectorOptions } from '@/hooks/selectors/registry' import type { SelectorKey, SelectorOption } from '@/hooks/selectors/types' export interface WorkflowSearchResolvedResource { @@ -374,7 +374,11 @@ export function useWorkflowSearchSelectorDetails(matches: WorkflowSearchMatch[]) return definition.fetchById({ ...queryArgs, signal }) } - const options = await definition.fetchList({ key: selectorKey, context, signal }) + const options = await loadAllSelectorOptions(definition, { + key: selectorKey, + context, + signal, + }) return options.find((option) => option.id === match.rawValue) ?? null }, enabled: Boolean(selectorKey && match.rawValue && baseEnabled), @@ -620,7 +624,7 @@ export function useWorkflowSearchSelectorReplacementOptions(matches: WorkflowSea return { queryKey: workflowSearchReplaceKeys.selectorReplacementOptions(selectorKey, contextKey), queryFn: ({ signal }: { signal: AbortSignal }) => - definition.fetchList({ ...queryArgs, signal }), + loadAllSelectorOptions(definition, { ...queryArgs, signal }), enabled: Boolean(selectorKey && baseEnabled), staleTime: definition.staleTime ?? 60 * 1000, select: (options: SelectorOption[]): WorkflowSearchReplacementOption[] => diff --git a/apps/sim/hooks/selectors/registry.ts b/apps/sim/hooks/selectors/registry.ts index 26c9ac9494..e3af0f2b17 100644 --- a/apps/sim/hooks/selectors/registry.ts +++ b/apps/sim/hooks/selectors/registry.ts @@ -21,7 +21,12 @@ import { trelloSelectors } from '@/hooks/selectors/providers/trello/selectors' import { wealthboxSelectors } from '@/hooks/selectors/providers/wealthbox/selectors' import { webflowSelectors } from '@/hooks/selectors/providers/webflow/selectors' import { zoomSelectors } from '@/hooks/selectors/providers/zoom/selectors' -import type { SelectorDefinition, SelectorKey, SelectorOption } from '@/hooks/selectors/types' +import type { + SelectorDefinition, + SelectorKey, + SelectorOption, + SelectorQueryArgs, +} from '@/hooks/selectors/types' export const selectorRegistry = { ...airtableSelectors, @@ -57,6 +62,38 @@ export function getSelectorDefinition(key: SelectorKey): SelectorDefinition { return definition } +const MAX_LOAD_ALL_PAGES = 50 + +/** + * Loads the complete option list for a selector outside the React Query hook — + * for callers (search/replace, value resolution) that need every option in one + * call. Uses `fetchList` when defined, otherwise drains `fetchPage` (bounded by + * {@link MAX_LOAD_ALL_PAGES}). Returns an empty array for a selector that + * provides neither. + */ +export async function loadAllSelectorOptions( + definition: SelectorDefinition, + args: SelectorQueryArgs +): Promise { + if (definition.fetchList) { + return definition.fetchList(args) + } + + if (definition.fetchPage) { + const items: SelectorOption[] = [] + let cursor: string | undefined + for (let page = 0; page < MAX_LOAD_ALL_PAGES; page++) { + const { items: pageItems, nextCursor } = await definition.fetchPage({ ...args, cursor }) + items.push(...pageItems) + cursor = nextCursor + if (!cursor) break + } + return items + } + + return [] +} + export function mergeOption(options: SelectorOption[], option?: SelectorOption | null) { if (!option) return options if (options.some((item) => item.id === option.id)) { diff --git a/apps/sim/lib/workflows/comparison/resolve-values.ts b/apps/sim/lib/workflows/comparison/resolve-values.ts index 651a03ad50..bc9d6c7a75 100644 --- a/apps/sim/lib/workflows/comparison/resolve-values.ts +++ b/apps/sim/lib/workflows/comparison/resolve-values.ts @@ -6,7 +6,7 @@ import { SELECTOR_TYPES_HYDRATION_REQUIRED, type SubBlockConfig } from '@/blocks import { CREDENTIAL_SET, isUuid } from '@/executor/constants' import { fetchCredentialSetById } from '@/hooks/queries/credential-sets' import { fetchOAuthCredentialDetail } from '@/hooks/queries/oauth/oauth-credentials' -import { getSelectorDefinition } from '@/hooks/selectors/registry' +import { getSelectorDefinition, loadAllSelectorOptions } from '@/hooks/selectors/registry' import { resolveSelectorForSubBlock } from '@/hooks/selectors/resolution' import type { SelectorContext, SelectorKey } from '@/hooks/selectors/types' import type { WorkflowState } from '@/stores/workflows/workflow/types' @@ -107,7 +107,7 @@ async function resolveSelectorValue( } } - const options = await definition.fetchList({ + const options = await loadAllSelectorOptions(definition, { key: selectorKey, context: selectorContext, }) From 76d87bd0276d71dac71cc6d2edf9630038e116ff Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 30 May 2026 20:47:44 -0700 Subject: [PATCH 5/5] chore(selectors): rename MAX_PAGE_PAGES to MAX_NOTION_PAGES for readability --- apps/sim/app/api/tools/notion/pages/route.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/sim/app/api/tools/notion/pages/route.ts b/apps/sim/app/api/tools/notion/pages/route.ts index 5da2205c30..e48eadf8a4 100644 --- a/apps/sim/app/api/tools/notion/pages/route.ts +++ b/apps/sim/app/api/tools/notion/pages/route.ts @@ -21,7 +21,7 @@ const NOTION_PAGE_SIZE = 100 * pages drained so a tenant with a very large workspace cannot make this route * loop unbounded. With `NOTION_PAGE_SIZE` of 100 this covers up to 2,000 items. */ -const MAX_PAGE_PAGES = 20 +const MAX_NOTION_PAGES = 20 export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() @@ -57,7 +57,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const results: Record[] = [] let startCursor: string | undefined - for (let page = 0; page < MAX_PAGE_PAGES; page++) { + for (let page = 0; page < MAX_NOTION_PAGES; page++) { const response = await fetch('https://api.notion.com/v1/search', { method: 'POST', headers: { @@ -94,9 +94,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } startCursor = data.next_cursor as string - if (page === MAX_PAGE_PAGES - 1) { + if (page === MAX_NOTION_PAGES - 1) { logger.warn('Notion pages search hit pagination cap; results may be incomplete', { - maxPages: MAX_PAGE_PAGES, + maxPages: MAX_NOTION_PAGES, fetched: results.length, }) }