fix: paginaton needs to be more generic

Not every API is the same.

src/gitea/uploadToAllRepos.ts

@@ -47,10 +47,17 @@ if (uploadPath === "") {
 
 const orgs = [ "nhcarrigan", "nhcarrigan-private", "nhcarrigan-games" ];
 
+let totalReposProcessed = 0;
+let totalReposSucceeded = 0;
+let totalReposFailed = 0;
+
 for (const org of orgs) {
+  console.log(`\n=== Fetching repositories for org: ${org} ===`);
   const repos = await paginatedFetch<Array<Repository>>(`${giteaUrl}/api/v1/orgs/${org}/repos`, 100, { headers: { authorization: `Bearer ${giteaToken}` } });
+  console.log(`Found ${repos.length.toString()} repositories in ${org}`);
 
   for (const repo of repos) {
+    totalReposProcessed = totalReposProcessed + 1;
     console.log(`Checking if file exists in ${org}/${repo.name}`);
     const fileResponse = await fetch(`${giteaUrl}/api/v1/repos/${org}/${repo.name}/contents/${uploadPath}`, {
       headers: {
@@ -77,10 +84,12 @@ for (const org of orgs) {
         method: "PUT",
       });
       if (!response.ok) {
+        totalReposFailed = totalReposFailed + 1;
         console.error(`Failed to update ${fileName} in ${org}/${repo.name}: ${response.statusText}`);
         console.error(await response.text());
         continue;
       }
+      totalReposSucceeded = totalReposSucceeded + 1;
       console.log(`Updated ${fileName} in ${org}/${repo.name}`);
       continue;
     }
@@ -99,10 +108,17 @@ for (const org of orgs) {
       method: "POST",
     });
     if (!response.ok) {
+      totalReposFailed = totalReposFailed + 1;
       console.error(`Failed to upload ${fileName} to ${org}/${repo.name}: ${response.statusText}`);
       console.error(await response.text());
       continue;
     }
+    totalReposSucceeded = totalReposSucceeded + 1;
     console.log(`Uploaded ${fileName} to ${org}/${repo.name}`);
   }
 }
+
+console.log(`\n=== Summary ===`);
+console.log(`Total repositories processed: ${totalReposProcessed.toString()}`);
+console.log(`Successfully uploaded/updated: ${totalReposSucceeded.toString()}`);
+console.log(`Failed: ${totalReposFailed.toString()}`);

src/utils/paginatedFetch.ts

@@ -7,12 +7,13 @@
 /**
  * Fetches a paginated resource from a URL. Automatically handles pagination,
  * and returns the complete data as an array.
- * @type {Array<Record<string, unknown>>} T - The type of data returned from the API endpoint. This should be an array of objects.
+ * @type {Array<Record<string, unknown>>} - The type of data returned from the API endpoint. This should be an array of objects.
  * @param url - The URL to fetch.
  * @param limit - The number of items to fetch per page.
  * @param options - The standard fetch options object.
  * @returns The complete data as type T.
  */
+// eslint-disable-next-line max-lines-per-function, max-statements -- We're doing some complex logic here.
 export const paginatedFetch = async <T extends Array<Record<string, unknown>>>(
   url: string,
   limit: number,
@@ -22,15 +23,67 @@ export const paginatedFetch = async <T extends Array<Record<string, unknown>>>(
   let offset = 0;
   // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- This is a workaround to avoid type errors.
   const data: T = [] as unknown as T;
-  let request = await fetch(`${url}?limit=${limit.toString()}&page=${page.toString()}&offset=${offset.toString()}`, options);
+
+  // First page
+  const firstUrl = `${url}?limit=${limit.toString()}&page=${page.toString()}&offset=${offset.toString()}`;
+  console.log(`Fetching page ${page.toString()} (offset ${offset.toString()}, limit ${limit.toString()})...`);
+  let request = await fetch(firstUrl, options);
+
+  if (!request.ok) {
+    throw new Error(`Failed to fetch ${firstUrl}: ${request.status.toString()} ${request.statusText}`);
+  }
+
   let response: T = await request.json();
+
+  // Check if response is actually an array
+  if (!Array.isArray(response)) {
+    console.error(
+      "API response is not an array:",
+      typeof response,
+      Object.keys(response),
+    );
+    const errorMessage
+      = `Expected array response but got ${typeof response}. `
+      + `Response keys: ${Object.keys(response).join(", ")}`;
+    throw new Error(errorMessage);
+  }
+
+  console.log(`Page ${page.toString()}: Received ${response.length.toString()} items`);
   data.push(...response);
-  while (response.length >= limit) {
+
+  /**
+   * Continue paginating while we get items back.
+   * Keep fetching until we get an empty array (0 items), which means we've reached the end.
+   */
+  while (response.length > 0) {
     page = page + 1;
     offset = offset + limit;
-    request = await fetch(`${url}?limit=${limit.toString()}&page=${page.toString()}&offset=${offset.toString()}`, options);
+    const pageUrl = `${url}?limit=${limit.toString()}&page=${page.toString()}&offset=${offset.toString()}`;
+    console.log(`Fetching page ${page.toString()} (offset ${offset.toString()}, limit ${limit.toString()})...`);
+    request = await fetch(pageUrl, options);
+
+    if (!request.ok) {
+      console.error(`Failed to fetch page ${page.toString()}: ${request.status.toString()} ${request.statusText}`);
+      break;
+    }
+
     response = await request.json();
+
+    if (!Array.isArray(response)) {
+      console.error(`Page ${page.toString()} response is not an array:`, typeof response);
+      break;
+    }
+
+    console.log(`Page ${page.toString()}: Received ${response.length.toString()} items`);
+
+    // If we get an empty array, we've reached the end
+    if (response.length === 0) {
+      break;
+    }
+
     data.push(...response);
   }
+
+  console.log(`Total items fetched: ${data.length.toString()}`);
   return data;
 };