ldez/new-manga
1
0
Fork 0
mirror of https://github.com/ld3z/new-manga.git synced 2026-11-09 20:33:44 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

10 at a time rule

Browse source
This commit is contained in:
ld3zian 2025-03-31 20:32:44 -07:00
parent 3e98aeccba
commit 03f333d011
1 changed files with 41 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

68
src/lib/db.ts
View file

@ -654,66 +654,80 @@ export async function getFeedMapping(
/** /**
* Pre-warm the chapter cache for better performance * Pre-warm the chapter cache for better performance
* This function checks cache status and uses the getChaptersForSlugs function from api.ts * This function checks cache status and uses the getChaptersForSlugs function from api.ts
* to fetch and cache data in the background * to fetch and cache data in the background, processing only a limited batch at a time.
*/ */
export async function warmChapterCache( export async function warmChapterCache(
slugs: string[], slugs: string[],
lang: string lang: string
): Promise<void> { ): Promise<void> {
const BATCH_SIZE = 10; // Define the maximum number of slugs to process per run
try { try {
console.log(`Warming chapter cache for ${slugs.length} comics in ${lang}`); console.log(`[warmChapterCache] Received request to warm cache for ${slugs.length} comics in ${lang}`);
const { client } = await getRedisClient(); // This already handles failover const { client } = await getRedisClient(); // This already handles failover
// Validate slugs // Validate slugs
const validSlugs = slugs.filter(slug => const validSlugs = slugs.filter(slug =>
slug && typeof slug === 'string' && slug.match(/^[a-z0-9-]+$/)); slug && typeof slug === 'string' && slug.match(/^[a-z0-9-]+$/));
if (validSlugs.length !== slugs.length) { if (validSlugs.length !== slugs.length) {
console.warn(`Filtered out ${slugs.length - validSlugs.length} invalid slugs`); console.warn(`[warmChapterCache] Filtered out ${slugs.length - validSlugs.length} invalid slugs`);
} }
if (validSlugs.length === 0) {
console.log("[warmChapterCache] No valid slugs provided.");
return;
}
// Check which slugs are already in cache // Check which slugs are already in cache
const pipeline = client.pipeline(); const pipeline = client.pipeline();
const cacheKeys = validSlugs.map(slug => `chapters:${slug}:${lang}`); const cacheKeys = validSlugs.map(slug => `chapters:${slug}:${lang}`);
cacheKeys.forEach(key => { cacheKeys.forEach(key => {
pipeline.exists(key); pipeline.exists(key);
}); });
const results = await pipeline.exec(); const results = await pipeline.exec();
const slugsToFetch: string[] = []; let slugsMissingFromCache: string[] = []; // Renamed for clarity
results?.forEach(([err, exists], index) => { results?.forEach(([err, exists], index) => {
if (err) { if (err) {
console.error(`Error checking cache for ${cacheKeys[index]}:`, err); console.error(`[warmChapterCache] Error checking cache for ${cacheKeys[index]}:`, err);
slugsToFetch.push(validSlugs[index]); // Optionally decide if you still want to try fetching on error
// slugsMissingFromCache.push(validSlugs[index]);
} else if (!exists) { } else if (!exists) {
slugsToFetch.push(validSlugs[index]); slugsMissingFromCache.push(validSlugs[index]);
} }
}); });
if (slugsToFetch.length === 0) { if (slugsMissingFromCache.length === 0) {
console.log('All chapters already in cache, no warming needed'); console.log('[warmChapterCache] All requested chapters already in cache, no warming needed.');
return; return;
} }
console.log(`Cache miss for ${slugsToFetch.length} comics - starting background fetch`); // --- Batching Logic ---
const slugsToProcessNow = slugsMissingFromCache.slice(0, BATCH_SIZE);
const remainingSlugs = slugsMissingFromCache.length - slugsToProcessNow.length;
console.log(`[warmChapterCache] Cache miss for ${slugsMissingFromCache.length} comics. Processing batch of ${slugsToProcessNow.length}. Remaining: ${remainingSlugs}`);
// --- End Batching Logic ---
// To avoid circular dependencies, we'll import getChaptersForSlugs dynamically // To avoid circular dependencies, we'll import getChaptersForSlugs dynamically
const { getChaptersForSlugs } = await import('./api'); const { getChaptersForSlugs } = await import('./api');
// Start fetching in the background without waiting for the result // Start fetching in the background *only for the current batch* without waiting for the result
getChaptersForSlugs(slugsToFetch, lang) getChaptersForSlugs(slugsToProcessNow, lang) // Use the batched list
.then(chapters => { .then(chapters => {
console.log(`Successfully warmed cache with ${chapters.length} chapters`); // Note: chapters.length might not equal slugsToProcessNow.length if some fetches failed internally
console.log(`[warmChapterCache] Background cache warming initiated for ${slugsToProcessNow.length} slugs. Successfully fetched data for ${chapters.length} chapters.`);
}) })
.catch(error => { .catch(error => {
console.error('Error warming chapter cache:', error); console.error('[warmChapterCache] Error during background chapter cache warming:', error);
}); });
} catch (error) { } catch (error) {
console.error('Error in warmChapterCache:', error); console.error('[warmChapterCache] Error in warmChapterCache:', error);
} }
} }