ldez/new-manga
1
0
Fork 0
mirror of https://github.com/ld3z/new-manga.git synced 2026-11-09 20:23:45 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

yo batch

Browse source
This commit is contained in:
ld3zian 2025-03-31 20:47:18 -07:00
parent 03f333d011
commit ef16dc81d6
1 changed files with 29 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

51
src/lib/db.ts
View file

@ -1,5 +1,5 @@
import Redis from "ioredis"; import Redis from "ioredis";
import type { FeedMapping } from "./types"; import type { FeedMapping, ChapterDetail } from "./types";
import { ReJSON } from 'redis-modules-sdk'; import { ReJSON } from 'redis-modules-sdk';
import * as zlib from 'zlib'; import * as zlib from 'zlib';
import { promisify } from 'util'; import { promisify } from 'util';
@ -653,18 +653,18 @@ export async function getFeedMapping(
} }
/** /**
* Pre-warm the chapter cache for better performance * Fetches and caches a batch of chapters if they are missing from the cache.
* This function checks cache status and uses the getChaptersForSlugs function from api.ts * Returns the chapter data for the processed batch.
* to fetch and cache data in the background, processing only a limited batch at a time. * Processes a maximum of BATCH_SIZE comics per call.
*/ */
export async function warmChapterCache( export async function warmChapterCache(
slugs: string[], slugs: string[],
lang: string lang: string
): Promise<void> { ): Promise<ChapterDetail[]> {
const BATCH_SIZE = 10; // Define the maximum number of slugs to process per run const BATCH_SIZE = 5; // Define the maximum number of slugs to process per run
try { try {
console.log(`[warmChapterCache] Received request to warm cache for ${slugs.length} comics in ${lang}`); console.log(`[warmChapterCache] Received request to fetch/warm cache for ${slugs.length} comics in ${lang}`);
const { client } = await getRedisClient(); // This already handles failover const { client } = await getRedisClient(); // This already handles failover
// Validate slugs // Validate slugs
@ -676,7 +676,7 @@ export async function warmChapterCache(
} }
if (validSlugs.length === 0) { if (validSlugs.length === 0) {
console.log("[warmChapterCache] No valid slugs provided."); console.log("[warmChapterCache] No valid slugs provided.");
return; return []; // Return empty array if no valid slugs
} }
// Check which slugs are already in cache // Check which slugs are already in cache
@ -688,12 +688,12 @@ export async function warmChapterCache(
}); });
const results = await pipeline.exec(); const results = await pipeline.exec();
let slugsMissingFromCache: string[] = []; // Renamed for clarity let slugsMissingFromCache: string[] = [];
results?.forEach(([err, exists], index) => { results?.forEach(([err, exists], index) => {
if (err) { if (err) {
console.error(`[warmChapterCache] Error checking cache for ${cacheKeys[index]}:`, err); console.error(`[warmChapterCache] Error checking cache for ${cacheKeys[index]}:`, err);
// Optionally decide if you still want to try fetching on error // Decide if you want to attempt fetching even if cache check failed
// slugsMissingFromCache.push(validSlugs[index]); // slugsMissingFromCache.push(validSlugs[index]);
} else if (!exists) { } else if (!exists) {
slugsMissingFromCache.push(validSlugs[index]); slugsMissingFromCache.push(validSlugs[index]);
@ -701,8 +701,11 @@ export async function warmChapterCache(
}); });
if (slugsMissingFromCache.length === 0) { if (slugsMissingFromCache.length === 0) {
console.log('[warmChapterCache] All requested chapters already in cache, no warming needed.'); console.log('[warmChapterCache] All requested chapters already in cache. No fetching needed for this request.');
return; // If all are cached, you might want to fetch them all now, or return empty
// Depending on the calling context's expectation. Let's return empty for now,
// assuming the caller will fetch cached data separately if needed.
return [];
} }
// --- Batching Logic --- // --- Batching Logic ---
@ -716,18 +719,22 @@ export async function warmChapterCache(
// To avoid circular dependencies, we'll import getChaptersForSlugs dynamically // To avoid circular dependencies, we'll import getChaptersForSlugs dynamically
const { getChaptersForSlugs } = await import('./api'); const { getChaptersForSlugs } = await import('./api');
// Start fetching in the background *only for the current batch* without waiting for the result // --- Fetch the batch SYNCHRONOUSLY and return the result ---
getChaptersForSlugs(slugsToProcessNow, lang) // Use the batched list console.log(`[warmChapterCache] Calling getChaptersForSlugs for batch: [${slugsToProcessNow.join(', ')}]`);
.then(chapters => { try {
// Note: chapters.length might not equal slugsToProcessNow.length if some fetches failed internally const chaptersForBatch: ChapterDetail[] = await getChaptersForSlugs(slugsToProcessNow, lang); // Await the result
console.log(`[warmChapterCache] Background cache warming initiated for ${slugsToProcessNow.length} slugs. Successfully fetched data for ${chapters.length} chapters.`); console.log(`[warmChapterCache] Successfully fetched ${chaptersForBatch.length} chapters for the batch.`);
}) // The getChaptersForSlugs function is assumed to handle caching internally
.catch(error => { return chaptersForBatch; // Return the fetched data for this batch
console.error('[warmChapterCache] Error during background chapter cache warming:', error); } catch (fetchError) {
}); console.error(`[warmChapterCache] Error fetching chapters for batch [${slugsToProcessNow.join(', ')}]:`, fetchError);
return []; // Return empty array on fetch error for this batch
}
// --- End Fetch and Return ---
} catch (error) { } catch (error) {
console.error('[warmChapterCache] Error in warmChapterCache:', error); console.error('[warmChapterCache] CRITICAL Error in warmChapterCache:', error);
return []; // Return empty array on critical error
} }
} }