From 03f333d011f20d22327be2b04caaec945514a0b4 Mon Sep 17 00:00:00 2001 From: ld3zian <42660576+ld3z@users.noreply.github.com> Date: Mon, 31 Mar 2025 20:32:44 -0700 Subject: [PATCH 1/3] 10 at a time rule --- src/lib/db.ts | 68 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/src/lib/db.ts b/src/lib/db.ts index 4094f66..c6106c1 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -654,66 +654,80 @@ export async function getFeedMapping( /** * Pre-warm the chapter cache for better performance - * This function checks cache status and uses the getChaptersForSlugs function from api.ts - * to fetch and cache data in the background + * This function checks cache status and uses the getChaptersForSlugs function from api.ts + * to fetch and cache data in the background, processing only a limited batch at a time. */ export async function warmChapterCache( slugs: string[], lang: string ): Promise { + const BATCH_SIZE = 10; // Define the maximum number of slugs to process per run + try { - console.log(`Warming chapter cache for ${slugs.length} comics in ${lang}`); + console.log(`[warmChapterCache] Received request to warm cache for ${slugs.length} comics in ${lang}`); const { client } = await getRedisClient(); // This already handles failover - + // Validate slugs - const validSlugs = slugs.filter(slug => + const validSlugs = slugs.filter(slug => slug && typeof slug === 'string' && slug.match(/^[a-z0-9-]+$/)); - + if (validSlugs.length !== slugs.length) { - console.warn(`Filtered out ${slugs.length - validSlugs.length} invalid slugs`); + console.warn(`[warmChapterCache] Filtered out ${slugs.length - validSlugs.length} invalid slugs`); } - + if (validSlugs.length === 0) { + console.log("[warmChapterCache] No valid slugs provided."); + return; + } + // Check which slugs are already in cache const pipeline = client.pipeline(); const cacheKeys = validSlugs.map(slug => `chapters:${slug}:${lang}`); - + cacheKeys.forEach(key => { pipeline.exists(key); }); - + const results = await pipeline.exec(); - const slugsToFetch: string[] = []; - + let slugsMissingFromCache: string[] = []; // Renamed for clarity + results?.forEach(([err, exists], index) => { if (err) { - console.error(`Error checking cache for ${cacheKeys[index]}:`, err); - slugsToFetch.push(validSlugs[index]); + console.error(`[warmChapterCache] Error checking cache for ${cacheKeys[index]}:`, err); + // Optionally decide if you still want to try fetching on error + // slugsMissingFromCache.push(validSlugs[index]); } else if (!exists) { - slugsToFetch.push(validSlugs[index]); + slugsMissingFromCache.push(validSlugs[index]); } }); - - if (slugsToFetch.length === 0) { - console.log('All chapters already in cache, no warming needed'); + + if (slugsMissingFromCache.length === 0) { + console.log('[warmChapterCache] All requested chapters already in cache, no warming needed.'); return; } - - console.log(`Cache miss for ${slugsToFetch.length} comics - starting background fetch`); - + + // --- Batching Logic --- + const slugsToProcessNow = slugsMissingFromCache.slice(0, BATCH_SIZE); + const remainingSlugs = slugsMissingFromCache.length - slugsToProcessNow.length; + + console.log(`[warmChapterCache] Cache miss for ${slugsMissingFromCache.length} comics. Processing batch of ${slugsToProcessNow.length}. Remaining: ${remainingSlugs}`); + // --- End Batching Logic --- + + // To avoid circular dependencies, we'll import getChaptersForSlugs dynamically const { getChaptersForSlugs } = await import('./api'); - - // Start fetching in the background without waiting for the result - getChaptersForSlugs(slugsToFetch, lang) + + // Start fetching in the background *only for the current batch* without waiting for the result + getChaptersForSlugs(slugsToProcessNow, lang) // Use the batched list .then(chapters => { - console.log(`Successfully warmed cache with ${chapters.length} chapters`); + // Note: chapters.length might not equal slugsToProcessNow.length if some fetches failed internally + console.log(`[warmChapterCache] Background cache warming initiated for ${slugsToProcessNow.length} slugs. Successfully fetched data for ${chapters.length} chapters.`); }) .catch(error => { - console.error('Error warming chapter cache:', error); + console.error('[warmChapterCache] Error during background chapter cache warming:', error); }); } catch (error) { - console.error('Error in warmChapterCache:', error); + console.error('[warmChapterCache] Error in warmChapterCache:', error); } } From ef16dc81d633861c5eaf86ad3db2ce50d2d1ca30 Mon Sep 17 00:00:00 2001 From: ld3zian <42660576+ld3z@users.noreply.github.com> Date: Mon, 31 Mar 2025 20:47:18 -0700 Subject: [PATCH 2/3] yo batch --- src/lib/db.ts | 51 +++++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/src/lib/db.ts b/src/lib/db.ts index c6106c1..caee75c 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -1,5 +1,5 @@ import Redis from "ioredis"; -import type { FeedMapping } from "./types"; +import type { FeedMapping, ChapterDetail } from "./types"; import { ReJSON } from 'redis-modules-sdk'; import * as zlib from 'zlib'; import { promisify } from 'util'; @@ -653,18 +653,18 @@ export async function getFeedMapping( } /** - * Pre-warm the chapter cache for better performance - * This function checks cache status and uses the getChaptersForSlugs function from api.ts - * to fetch and cache data in the background, processing only a limited batch at a time. + * Fetches and caches a batch of chapters if they are missing from the cache. + * Returns the chapter data for the processed batch. + * Processes a maximum of BATCH_SIZE comics per call. */ export async function warmChapterCache( slugs: string[], lang: string -): Promise { - const BATCH_SIZE = 10; // Define the maximum number of slugs to process per run +): Promise { + const BATCH_SIZE = 5; // Define the maximum number of slugs to process per run try { - console.log(`[warmChapterCache] Received request to warm cache for ${slugs.length} comics in ${lang}`); + console.log(`[warmChapterCache] Received request to fetch/warm cache for ${slugs.length} comics in ${lang}`); const { client } = await getRedisClient(); // This already handles failover // Validate slugs @@ -676,7 +676,7 @@ export async function warmChapterCache( } if (validSlugs.length === 0) { console.log("[warmChapterCache] No valid slugs provided."); - return; + return []; // Return empty array if no valid slugs } // Check which slugs are already in cache @@ -688,12 +688,12 @@ export async function warmChapterCache( }); const results = await pipeline.exec(); - let slugsMissingFromCache: string[] = []; // Renamed for clarity + let slugsMissingFromCache: string[] = []; results?.forEach(([err, exists], index) => { if (err) { console.error(`[warmChapterCache] Error checking cache for ${cacheKeys[index]}:`, err); - // Optionally decide if you still want to try fetching on error + // Decide if you want to attempt fetching even if cache check failed // slugsMissingFromCache.push(validSlugs[index]); } else if (!exists) { slugsMissingFromCache.push(validSlugs[index]); @@ -701,8 +701,11 @@ export async function warmChapterCache( }); if (slugsMissingFromCache.length === 0) { - console.log('[warmChapterCache] All requested chapters already in cache, no warming needed.'); - return; + console.log('[warmChapterCache] All requested chapters already in cache. No fetching needed for this request.'); + // If all are cached, you might want to fetch them all now, or return empty + // Depending on the calling context's expectation. Let's return empty for now, + // assuming the caller will fetch cached data separately if needed. + return []; } // --- Batching Logic --- @@ -716,18 +719,22 @@ export async function warmChapterCache( // To avoid circular dependencies, we'll import getChaptersForSlugs dynamically const { getChaptersForSlugs } = await import('./api'); - // Start fetching in the background *only for the current batch* without waiting for the result - getChaptersForSlugs(slugsToProcessNow, lang) // Use the batched list - .then(chapters => { - // Note: chapters.length might not equal slugsToProcessNow.length if some fetches failed internally - console.log(`[warmChapterCache] Background cache warming initiated for ${slugsToProcessNow.length} slugs. Successfully fetched data for ${chapters.length} chapters.`); - }) - .catch(error => { - console.error('[warmChapterCache] Error during background chapter cache warming:', error); - }); + // --- Fetch the batch SYNCHRONOUSLY and return the result --- + console.log(`[warmChapterCache] Calling getChaptersForSlugs for batch: [${slugsToProcessNow.join(', ')}]`); + try { + const chaptersForBatch: ChapterDetail[] = await getChaptersForSlugs(slugsToProcessNow, lang); // Await the result + console.log(`[warmChapterCache] Successfully fetched ${chaptersForBatch.length} chapters for the batch.`); + // The getChaptersForSlugs function is assumed to handle caching internally + return chaptersForBatch; // Return the fetched data for this batch + } catch (fetchError) { + console.error(`[warmChapterCache] Error fetching chapters for batch [${slugsToProcessNow.join(', ')}]:`, fetchError); + return []; // Return empty array on fetch error for this batch + } + // --- End Fetch and Return --- } catch (error) { - console.error('[warmChapterCache] Error in warmChapterCache:', error); + console.error('[warmChapterCache] CRITICAL Error in warmChapterCache:', error); + return []; // Return empty array on critical error } } From 4cebcc40b18fd76ee7bb647825bb80e16ee37358 Mon Sep 17 00:00:00 2001 From: ld3zian <42660576+ld3z@users.noreply.github.com> Date: Mon, 31 Mar 2025 20:53:28 -0700 Subject: [PATCH 3/3] bump deps --- package-lock.json | 50 +++++++++++++++++++++++------------------------ package.json | 6 +++--- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 109fad9..b740aa8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,12 +9,12 @@ "version": "1.0.0", "dependencies": { "@astrojs/rss": "^4.0.11", - "@astrojs/tailwind": "^6.0.1", + "@astrojs/tailwind": "^6.0.2", "@astrojs/vercel": "^8.1.3", "@floating-ui/dom": "^1.6.13", - "@types/node": "^22.13.11", + "@types/node": "^22.13.14", "@vercel/analytics": "^1.5.0", - "astro": "^5.5.4", + "astro": "^5.5.6", "feed": "^4.2.2", "ioredis": "^5.6.0", "p-throttle": "^7.0.0", @@ -92,9 +92,9 @@ } }, "node_modules/@astrojs/tailwind": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@astrojs/tailwind/-/tailwind-6.0.1.tgz", - "integrity": "sha512-jYrqG6dXwRnS3xUcRIoeYGFB2L2SISuTNaqMMFp0otW55HQ4fhiz941VaE9EVbs49tF2+1aAZCT3xgt2l/j5tQ==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@astrojs/tailwind/-/tailwind-6.0.2.tgz", + "integrity": "sha512-j3mhLNeugZq6A8dMNXVarUa8K6X9AW+QHU9u3lKNrPLMHhOQ0S7VeWhHwEeJFpEK1BTKEUY1U78VQv2gN6hNGg==", "dependencies": { "autoprefixer": "^10.4.21", "postcss": "^8.5.3", @@ -1459,9 +1459,9 @@ } }, "node_modules/@types/node": { - "version": "22.13.11", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.11.tgz", - "integrity": "sha512-iEUCUJoU0i3VnrCmgoWCXttklWcvoCIx4jzcP22fioIVSdTmjgoEvmAO/QPw6TcS9k5FrNgn4w7q5lGOd1CT5g==", + "version": "22.13.14", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.14.tgz", + "integrity": "sha512-Zs/Ollc1SJ8nKUAgc7ivOEdIBM8JAKgrqqUYi2J997JuKO7/tpQC+WCetQ1sypiKCQWHdvdg9wBNpUPEWZae7w==", "dependencies": { "undici-types": "~6.20.0" } @@ -1739,9 +1739,9 @@ } }, "node_modules/astro": { - "version": "5.5.4", - "resolved": "https://registry.npmjs.org/astro/-/astro-5.5.4.tgz", - "integrity": "sha512-rc9Mj6dIKM9ylTSQ0/S1l5f1iSDNs0PEjRMnvVpZ0W+wfBCu99JEuXy+fiWDNT6gf60bC4dKQehZxZGdSJB9kg==", + "version": "5.5.6", + "resolved": "https://registry.npmjs.org/astro/-/astro-5.5.6.tgz", + "integrity": "sha512-SjU6zAlRe/gQrZ558Qhm211Yl8XOZBfjMpZig3XtQpG6zlyHJkElbWQIWXAVPzMg17cFVdv/qZPVHfcP2YOGGw==", "dependencies": { "@astrojs/compiler": "^2.11.0", "@astrojs/internal-helpers": "0.6.1", @@ -1779,12 +1779,12 @@ "neotraverse": "^0.6.18", "p-limit": "^6.2.0", "p-queue": "^8.1.0", - "package-manager-detector": "^1.0.0", + "package-manager-detector": "^1.1.0", "picomatch": "^4.0.2", "prompts": "^2.4.2", "rehype": "^13.0.2", "semver": "^7.7.1", - "shiki": "^3.0.0", + "shiki": "^3.2.1", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.12", "tsconfck": "^3.1.5", @@ -1792,13 +1792,13 @@ "unist-util-visit": "^5.0.0", "unstorage": "^1.15.0", "vfile": "^6.0.3", - "vite": "^6.2.1", + "vite": "^6.2.4", "vitefu": "^1.0.6", "xxhash-wasm": "^1.1.0", "yargs-parser": "^21.1.1", "yocto-spinner": "^0.2.1", "zod": "^3.24.2", - "zod-to-json-schema": "^3.24.3", + "zod-to-json-schema": "^3.24.5", "zod-to-ts": "^1.2.0" }, "bin": { @@ -4403,9 +4403,9 @@ "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==" }, "node_modules/package-manager-detector": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.0.0.tgz", - "integrity": "sha512-7elnH+9zMsRo7aS72w6MeRugTpdRvInmEB4Kmm9BVvPw/SLG8gXUGQ+4wF0Mys0RSWPz0B9nuBbDe8vFeA2sfg==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.1.0.tgz", + "integrity": "sha512-Y8f9qUlBzW8qauJjd/eu6jlpJZsuPJm2ZAV0cDVd420o4EdpH5RPdoCv+60/TdJflGatr4sDfpAL6ArWZbM5tA==" }, "node_modules/parse-latin": { "version": "7.0.0", @@ -5822,9 +5822,9 @@ } }, "node_modules/vite": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.1.tgz", - "integrity": "sha512-n2GnqDb6XPhlt9B8olZPrgMD/es/Nd1RdChF6CBD/fHW6pUyUTt2sQW2fPRX5GiD9XEa6+8A6A4f2vT6pSsE7Q==", + "version": "6.2.4", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.4.tgz", + "integrity": "sha512-veHMSew8CcRzhL5o8ONjy8gkfmFJAd5Ac16oxBUjlwgX3Gq2Wqr+qNC3TjPIpy7TPV/KporLga5GT9HqdrCizw==", "dependencies": { "esbuild": "^0.25.0", "postcss": "^8.5.3", @@ -6139,9 +6139,9 @@ } }, "node_modules/zod-to-json-schema": { - "version": "3.24.3", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.3.tgz", - "integrity": "sha512-HIAfWdYIt1sssHfYZFCXp4rU1w2r8hVVXYIlmoa0r0gABLs5di3RCqPU5DDROogVz1pAdYBaz7HK5n9pSUNs3A==", + "version": "3.24.5", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz", + "integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==", "peerDependencies": { "zod": "^3.24.1" } diff --git a/package.json b/package.json index 1350193..a96f78f 100644 --- a/package.json +++ b/package.json @@ -12,12 +12,12 @@ }, "dependencies": { "@astrojs/rss": "^4.0.11", - "@astrojs/tailwind": "^6.0.1", + "@astrojs/tailwind": "^6.0.2", "@astrojs/vercel": "^8.1.3", "@floating-ui/dom": "^1.6.13", - "@types/node": "^22.13.11", + "@types/node": "^22.13.14", "@vercel/analytics": "^1.5.0", - "astro": "^5.5.4", + "astro": "^5.5.6", "feed": "^4.2.2", "ioredis": "^5.6.0", "p-throttle": "^7.0.0",