/** * @copyright NHCarrigan * @license Naomi's Public License * @author Naomi Carrigan */ /** * Fetches a paginated resource from a URL. Automatically handles pagination, * and returns the complete data as an array. * @type {Array>} - The type of data returned from the API endpoint. This should be an array of objects. * @param url - The URL to fetch. * @param limit - The number of items to fetch per page. * @param options - The standard fetch options object. * @returns The complete data as type T. */ // eslint-disable-next-line max-lines-per-function, max-statements -- We're doing some complex logic here. export const paginatedFetch = async >>( url: string, limit: number, options: RequestInit = {}, ): Promise => { let page = 1; let offset = 0; // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- This is a workaround to avoid type errors. const data: T = [] as unknown as T; // First page const firstUrl = `${url}?limit=${limit.toString()}&page=${page.toString()}&offset=${offset.toString()}`; console.log(`Fetching page ${page.toString()} (offset ${offset.toString()}, limit ${limit.toString()})...`); let request = await fetch(firstUrl, options); if (!request.ok) { throw new Error(`Failed to fetch ${firstUrl}: ${request.status.toString()} ${request.statusText}`); } let response: T = await request.json(); // Check if response is actually an array if (!Array.isArray(response)) { console.error( "API response is not an array:", typeof response, Object.keys(response), ); const errorMessage = `Expected array response but got ${typeof response}. ` + `Response keys: ${Object.keys(response).join(", ")}`; throw new Error(errorMessage); } console.log(`Page ${page.toString()}: Received ${response.length.toString()} items`); data.push(...response); /** * Continue paginating while we get items back. * Keep fetching until we get an empty array (0 items), which means we've reached the end. */ while (response.length > 0) { page = page + 1; offset = offset + limit; const pageUrl = `${url}?limit=${limit.toString()}&page=${page.toString()}&offset=${offset.toString()}`; console.log(`Fetching page ${page.toString()} (offset ${offset.toString()}, limit ${limit.toString()})...`); request = await fetch(pageUrl, options); if (!request.ok) { console.error(`Failed to fetch page ${page.toString()}: ${request.status.toString()} ${request.statusText}`); break; } response = await request.json(); if (!Array.isArray(response)) { console.error(`Page ${page.toString()} response is not an array:`, typeof response); break; } console.log(`Page ${page.toString()}: Received ${response.length.toString()} items`); // If we get an empty array, we've reached the end if (response.length === 0) { break; } data.push(...response); } console.log(`Total items fetched: ${data.length.toString()}`); return data; };