From 8ea523de3cd8c6d9d7bceace1f252a017b9053a4 Mon Sep 17 00:00:00 2001 From: Jared Schoeny Date: Tue, 7 Jul 2026 15:46:04 -0600 Subject: [PATCH] Add array chunk helper for discover data fetching --- src/app/discover/actions.ts | 109 +++++++++++++++++++++--------------- src/utils/array.ts | 26 +++++++++ 2 files changed, 89 insertions(+), 46 deletions(-) create mode 100644 src/utils/array.ts diff --git a/src/app/discover/actions.ts b/src/app/discover/actions.ts index 34e2c5f..3e11c24 100644 --- a/src/app/discover/actions.ts +++ b/src/app/discover/actions.ts @@ -4,11 +4,13 @@ import { unstable_cache as cache } from "next/cache"; import { createServiceClient } from "@/utils/supabase/server"; import { getCachedTagsWithUsage, buildTagFilterGroups } from "@/data/tags"; import { sortOrderedTags, OrderedTag, getCoverUrls } from "@/utils/format"; +import { fetchInChunks } from "@/utils/array"; import { HackCardAttributes } from "@/components/HackCard"; import type { DiscoverSortOption } from "@/types/discover"; const TRENDING_WINDOW_DAYS = 3; const TIME_TO_LIVE = 600; // 10 minutes +const CHUNK_SIZE = 150; export interface DiscoverDataResult { hacks: HackCardAttributes[]; @@ -66,11 +68,14 @@ export async function getDiscoverData(sort: DiscoverSortOption): Promise r.slug); // Fetch covers - const { data: coverRows, error: coversError } = await supabase - .from("hack_covers") - .select("hack_slug,url,position") - .in("hack_slug", slugs) - .order("position", { ascending: true }); + const { data: coverRows, error: coversError } = await fetchInChunks(slugs, CHUNK_SIZE, async (chunk) => { + const { data, error } = await supabase + .from("hack_covers") + .select("hack_slug,url,position") + .in("hack_slug", chunk) + .order("position", { ascending: true }); + return { data, error }; + }); if (coversError) throw coversError; const coversBySlug = new Map(); @@ -92,43 +97,47 @@ export async function getDiscoverData(sort: DiscoverSortOption): Promise(); - const BATCH_SIZE = 1000; - let offset = 0; - let hasMore = true; + // Fetch tags - chunk slugs to avoid URI limits, + // paginate rows per chunk to avoid 1000 row limit per query + const ROW_BATCH_SIZE = 1000; + const { data: tagRows, error: tagsError } = await fetchInChunks(slugs, CHUNK_SIZE, async (slugChunk) => { + const rows: any[] = []; + let offset = 0; + let hasMore = true; - while (hasMore) { - const { data: tagRows, error: tagsError } = await supabase - .from("hack_tags") - .select("hack_slug,order,tags(name,category)") - .in("hack_slug", slugs) - .range(offset, offset + BATCH_SIZE - 1) - .order("hack_slug", { ascending: true }); + while (hasMore) { + const { data, error } = await supabase + .from("hack_tags") + .select("hack_slug,order,tags(name,category)") + .in("hack_slug", slugChunk) + .range(offset, offset + ROW_BATCH_SIZE - 1) + .order("hack_slug", { ascending: true }); - if (tagsError) throw tagsError; + if (error) return { data: null, error }; - if (!tagRows || tagRows.length === 0) { - hasMore = false; - } else { - tagRows.forEach((r: any) => { - if (!r.tags?.name) return; - const arr = tagsBySlug.get(r.hack_slug) || []; - arr.push({ - name: r.tags.name, - order: r.order, - }); - tagsBySlug.set(r.hack_slug, arr); - }); - - // If we got fewer rows than the batch size, we've reached the end - if (tagRows.length < BATCH_SIZE) { + if (!data || data.length === 0) { hasMore = false; } else { - offset += BATCH_SIZE; + rows.push(...data); + hasMore = data.length === ROW_BATCH_SIZE; + if (hasMore) offset += ROW_BATCH_SIZE; } } - } + + return { data: rows, error: null }; + }); + if (tagsError) throw tagsError; + + const tagsBySlug = new Map(); + (tagRows || []).forEach((r: any) => { + if (!r.tags?.name) return; + const arr = tagsBySlug.get(r.hack_slug) || []; + arr.push({ + name: r.tags.name, + order: r.order, + }); + tagsBySlug.set(r.hack_slug, arr); + }); // Fetch patches for version mapping const patchIds = Array.from( @@ -142,10 +151,13 @@ export async function getDiscoverData(sort: DiscoverSortOption): Promise(); const publishedAtByPatchId = new Map(); if (patchIds.length > 0) { - const { data: patchRows, error: patchesError } = await supabase + const { data: patchRows, error: patchesError } = await fetchInChunks(patchIds, CHUNK_SIZE, async (chunk) => { + const { data, error } = await supabase .from("patches") .select("id,version,published_at") - .in("id", patchIds); + .in("id", chunk); + return { data, error }; + }); if (patchesError) throw patchesError; (patchRows || []).forEach((p: any) => { @@ -160,10 +172,13 @@ export async function getDiscoverData(sort: DiscoverSortOption): Promise | null = null; if (sort === "trending") { // Get all patches for all hacks, grouped by slug - const { data: allPatches, error: allPatchesError } = await supabase - .from("patches") - .select("id,parent_hack") - .in("parent_hack", slugs); + const { data: allPatches, error: allPatchesError } = await fetchInChunks(slugs, CHUNK_SIZE, async (chunk) => { + const { data, error } = await supabase + .from("patches") + .select("id,parent_hack") + .in("parent_hack", chunk); + return { data, error }; + }); if (allPatchesError) throw allPatchesError; // Group patch IDs by parent_hack (slug) @@ -233,12 +248,14 @@ export async function getDiscoverData(sort: DiscoverSortOption): Promise r.created_by))]; - const { data: profiles, error: profilesError } = await supabase - .from("profiles") - .select("id,username") - .in("id", creatorIds); + const { data: profiles, error: profilesError } = await fetchInChunks(creatorIds, CHUNK_SIZE, async (chunk) => { + const { data, error } = await supabase + .from("profiles") + .select("id,username") + .in("id", chunk); + return { data, error }; + }); if (profilesError) throw profilesError; const usernameById = new Map(); diff --git a/src/utils/array.ts b/src/utils/array.ts new file mode 100644 index 0000000..a611fec --- /dev/null +++ b/src/utils/array.ts @@ -0,0 +1,26 @@ +export function chunk(array: T[], size: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < array.length; i += size) { + chunks.push(array.slice(i, i + size)); + } + return chunks; +} + +type FetchInChunksFn = ( + chunk: TItem[], +) => Promise<{ data: TResult[] | null; error: unknown }>; +export async function fetchInChunks( + array: TItem[], + size: number, + fn: FetchInChunksFn, +): Promise<{ data: TResult[]; error: unknown | null }> { + const chunks = chunk(array, size); + const results = await Promise.all(chunks.map(fn)); + + const data: TResult[] = []; + for (const result of results) { + if (result.error) return { data: [], error: result.error }; + if (result.data) data.push(...result.data); + } + return { data, error: null }; +}