From ac3a9d9e582c8a4ebe3e5b34dee02bce3fcf2a58 Mon Sep 17 00:00:00 2001 From: Naomi Carrigan Date: Wed, 17 Dec 2025 18:54:50 -0800 Subject: [PATCH] fix: paginaton needs to be more generic Not every API is the same. --- src/gitea/uploadToAllRepos.ts | 16 +++++++++ src/utils/paginatedFetch.ts | 61 ++++++++++++++++++++++++++++++++--- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/src/gitea/uploadToAllRepos.ts b/src/gitea/uploadToAllRepos.ts index 47eeb4e..74a0dc6 100644 --- a/src/gitea/uploadToAllRepos.ts +++ b/src/gitea/uploadToAllRepos.ts @@ -47,10 +47,17 @@ if (uploadPath === "") { const orgs = [ "nhcarrigan", "nhcarrigan-private", "nhcarrigan-games" ]; +let totalReposProcessed = 0; +let totalReposSucceeded = 0; +let totalReposFailed = 0; + for (const org of orgs) { + console.log(`\n=== Fetching repositories for org: ${org} ===`); const repos = await paginatedFetch>(`${giteaUrl}/api/v1/orgs/${org}/repos`, 100, { headers: { authorization: `Bearer ${giteaToken}` } }); + console.log(`Found ${repos.length.toString()} repositories in ${org}`); for (const repo of repos) { + totalReposProcessed = totalReposProcessed + 1; console.log(`Checking if file exists in ${org}/${repo.name}`); const fileResponse = await fetch(`${giteaUrl}/api/v1/repos/${org}/${repo.name}/contents/${uploadPath}`, { headers: { @@ -77,10 +84,12 @@ for (const org of orgs) { method: "PUT", }); if (!response.ok) { + totalReposFailed = totalReposFailed + 1; console.error(`Failed to update ${fileName} in ${org}/${repo.name}: ${response.statusText}`); console.error(await response.text()); continue; } + totalReposSucceeded = totalReposSucceeded + 1; console.log(`Updated ${fileName} in ${org}/${repo.name}`); continue; } @@ -99,10 +108,17 @@ for (const org of orgs) { method: "POST", }); if (!response.ok) { + totalReposFailed = totalReposFailed + 1; console.error(`Failed to upload ${fileName} to ${org}/${repo.name}: ${response.statusText}`); console.error(await response.text()); continue; } + totalReposSucceeded = totalReposSucceeded + 1; console.log(`Uploaded ${fileName} to ${org}/${repo.name}`); } } + +console.log(`\n=== Summary ===`); +console.log(`Total repositories processed: ${totalReposProcessed.toString()}`); +console.log(`Successfully uploaded/updated: ${totalReposSucceeded.toString()}`); +console.log(`Failed: ${totalReposFailed.toString()}`); diff --git a/src/utils/paginatedFetch.ts b/src/utils/paginatedFetch.ts index 7a2d586..844e53b 100644 --- a/src/utils/paginatedFetch.ts +++ b/src/utils/paginatedFetch.ts @@ -7,12 +7,13 @@ /** * Fetches a paginated resource from a URL. Automatically handles pagination, * and returns the complete data as an array. - * @type {Array>} T - The type of data returned from the API endpoint. This should be an array of objects. + * @type {Array>} - The type of data returned from the API endpoint. This should be an array of objects. * @param url - The URL to fetch. * @param limit - The number of items to fetch per page. * @param options - The standard fetch options object. * @returns The complete data as type T. */ +// eslint-disable-next-line max-lines-per-function, max-statements -- We're doing some complex logic here. export const paginatedFetch = async >>( url: string, limit: number, @@ -22,15 +23,67 @@ export const paginatedFetch = async >>( let offset = 0; // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- This is a workaround to avoid type errors. const data: T = [] as unknown as T; - let request = await fetch(`${url}?limit=${limit.toString()}&page=${page.toString()}&offset=${offset.toString()}`, options); + + // First page + const firstUrl = `${url}?limit=${limit.toString()}&page=${page.toString()}&offset=${offset.toString()}`; + console.log(`Fetching page ${page.toString()} (offset ${offset.toString()}, limit ${limit.toString()})...`); + let request = await fetch(firstUrl, options); + + if (!request.ok) { + throw new Error(`Failed to fetch ${firstUrl}: ${request.status.toString()} ${request.statusText}`); + } + let response: T = await request.json(); + + // Check if response is actually an array + if (!Array.isArray(response)) { + console.error( + "API response is not an array:", + typeof response, + Object.keys(response), + ); + const errorMessage + = `Expected array response but got ${typeof response}. ` + + `Response keys: ${Object.keys(response).join(", ")}`; + throw new Error(errorMessage); + } + + console.log(`Page ${page.toString()}: Received ${response.length.toString()} items`); data.push(...response); - while (response.length >= limit) { + + /** + * Continue paginating while we get items back. + * Keep fetching until we get an empty array (0 items), which means we've reached the end. + */ + while (response.length > 0) { page = page + 1; offset = offset + limit; - request = await fetch(`${url}?limit=${limit.toString()}&page=${page.toString()}&offset=${offset.toString()}`, options); + const pageUrl = `${url}?limit=${limit.toString()}&page=${page.toString()}&offset=${offset.toString()}`; + console.log(`Fetching page ${page.toString()} (offset ${offset.toString()}, limit ${limit.toString()})...`); + request = await fetch(pageUrl, options); + + if (!request.ok) { + console.error(`Failed to fetch page ${page.toString()}: ${request.status.toString()} ${request.statusText}`); + break; + } + response = await request.json(); + + if (!Array.isArray(response)) { + console.error(`Page ${page.toString()} response is not an array:`, typeof response); + break; + } + + console.log(`Page ${page.toString()}: Received ${response.length.toString()} items`); + + // If we get an empty array, we've reached the end + if (response.length === 0) { + break; + } + data.push(...response); } + + console.log(`Total items fetched: ${data.length.toString()}`); return data; };