mirror of
https://github.com/ld3z/new-manga.git
synced 2026-11-09 20:53:45 -07:00
Compare commits
No commits in common. "4cebcc40b18fd76ee7bb647825bb80e16ee37358" and "3e98aeccba0b5ed275bd282ba627a80f6a2ca5f7" have entirely different histories.
4cebcc40b1
...
3e98aeccba
3 changed files with 63 additions and 84 deletions
50
package-lock.json
generated
50
package-lock.json
generated
|
|
@ -9,12 +9,12 @@
|
|||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@astrojs/rss": "^4.0.11",
|
||||
"@astrojs/tailwind": "^6.0.2",
|
||||
"@astrojs/tailwind": "^6.0.1",
|
||||
"@astrojs/vercel": "^8.1.3",
|
||||
"@floating-ui/dom": "^1.6.13",
|
||||
"@types/node": "^22.13.14",
|
||||
"@types/node": "^22.13.11",
|
||||
"@vercel/analytics": "^1.5.0",
|
||||
"astro": "^5.5.6",
|
||||
"astro": "^5.5.4",
|
||||
"feed": "^4.2.2",
|
||||
"ioredis": "^5.6.0",
|
||||
"p-throttle": "^7.0.0",
|
||||
|
|
@ -92,9 +92,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@astrojs/tailwind": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@astrojs/tailwind/-/tailwind-6.0.2.tgz",
|
||||
"integrity": "sha512-j3mhLNeugZq6A8dMNXVarUa8K6X9AW+QHU9u3lKNrPLMHhOQ0S7VeWhHwEeJFpEK1BTKEUY1U78VQv2gN6hNGg==",
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@astrojs/tailwind/-/tailwind-6.0.1.tgz",
|
||||
"integrity": "sha512-jYrqG6dXwRnS3xUcRIoeYGFB2L2SISuTNaqMMFp0otW55HQ4fhiz941VaE9EVbs49tF2+1aAZCT3xgt2l/j5tQ==",
|
||||
"dependencies": {
|
||||
"autoprefixer": "^10.4.21",
|
||||
"postcss": "^8.5.3",
|
||||
|
|
@ -1459,9 +1459,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "22.13.14",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.14.tgz",
|
||||
"integrity": "sha512-Zs/Ollc1SJ8nKUAgc7ivOEdIBM8JAKgrqqUYi2J997JuKO7/tpQC+WCetQ1sypiKCQWHdvdg9wBNpUPEWZae7w==",
|
||||
"version": "22.13.11",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.11.tgz",
|
||||
"integrity": "sha512-iEUCUJoU0i3VnrCmgoWCXttklWcvoCIx4jzcP22fioIVSdTmjgoEvmAO/QPw6TcS9k5FrNgn4w7q5lGOd1CT5g==",
|
||||
"dependencies": {
|
||||
"undici-types": "~6.20.0"
|
||||
}
|
||||
|
|
@ -1739,9 +1739,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/astro": {
|
||||
"version": "5.5.6",
|
||||
"resolved": "https://registry.npmjs.org/astro/-/astro-5.5.6.tgz",
|
||||
"integrity": "sha512-SjU6zAlRe/gQrZ558Qhm211Yl8XOZBfjMpZig3XtQpG6zlyHJkElbWQIWXAVPzMg17cFVdv/qZPVHfcP2YOGGw==",
|
||||
"version": "5.5.4",
|
||||
"resolved": "https://registry.npmjs.org/astro/-/astro-5.5.4.tgz",
|
||||
"integrity": "sha512-rc9Mj6dIKM9ylTSQ0/S1l5f1iSDNs0PEjRMnvVpZ0W+wfBCu99JEuXy+fiWDNT6gf60bC4dKQehZxZGdSJB9kg==",
|
||||
"dependencies": {
|
||||
"@astrojs/compiler": "^2.11.0",
|
||||
"@astrojs/internal-helpers": "0.6.1",
|
||||
|
|
@ -1779,12 +1779,12 @@
|
|||
"neotraverse": "^0.6.18",
|
||||
"p-limit": "^6.2.0",
|
||||
"p-queue": "^8.1.0",
|
||||
"package-manager-detector": "^1.1.0",
|
||||
"package-manager-detector": "^1.0.0",
|
||||
"picomatch": "^4.0.2",
|
||||
"prompts": "^2.4.2",
|
||||
"rehype": "^13.0.2",
|
||||
"semver": "^7.7.1",
|
||||
"shiki": "^3.2.1",
|
||||
"shiki": "^3.0.0",
|
||||
"tinyexec": "^0.3.2",
|
||||
"tinyglobby": "^0.2.12",
|
||||
"tsconfck": "^3.1.5",
|
||||
|
|
@ -1792,13 +1792,13 @@
|
|||
"unist-util-visit": "^5.0.0",
|
||||
"unstorage": "^1.15.0",
|
||||
"vfile": "^6.0.3",
|
||||
"vite": "^6.2.4",
|
||||
"vite": "^6.2.1",
|
||||
"vitefu": "^1.0.6",
|
||||
"xxhash-wasm": "^1.1.0",
|
||||
"yargs-parser": "^21.1.1",
|
||||
"yocto-spinner": "^0.2.1",
|
||||
"zod": "^3.24.2",
|
||||
"zod-to-json-schema": "^3.24.5",
|
||||
"zod-to-json-schema": "^3.24.3",
|
||||
"zod-to-ts": "^1.2.0"
|
||||
},
|
||||
"bin": {
|
||||
|
|
@ -4403,9 +4403,9 @@
|
|||
"integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw=="
|
||||
},
|
||||
"node_modules/package-manager-detector": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.1.0.tgz",
|
||||
"integrity": "sha512-Y8f9qUlBzW8qauJjd/eu6jlpJZsuPJm2ZAV0cDVd420o4EdpH5RPdoCv+60/TdJflGatr4sDfpAL6ArWZbM5tA=="
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.0.0.tgz",
|
||||
"integrity": "sha512-7elnH+9zMsRo7aS72w6MeRugTpdRvInmEB4Kmm9BVvPw/SLG8gXUGQ+4wF0Mys0RSWPz0B9nuBbDe8vFeA2sfg=="
|
||||
},
|
||||
"node_modules/parse-latin": {
|
||||
"version": "7.0.0",
|
||||
|
|
@ -5822,9 +5822,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/vite": {
|
||||
"version": "6.2.4",
|
||||
"resolved": "https://registry.npmjs.org/vite/-/vite-6.2.4.tgz",
|
||||
"integrity": "sha512-veHMSew8CcRzhL5o8ONjy8gkfmFJAd5Ac16oxBUjlwgX3Gq2Wqr+qNC3TjPIpy7TPV/KporLga5GT9HqdrCizw==",
|
||||
"version": "6.2.1",
|
||||
"resolved": "https://registry.npmjs.org/vite/-/vite-6.2.1.tgz",
|
||||
"integrity": "sha512-n2GnqDb6XPhlt9B8olZPrgMD/es/Nd1RdChF6CBD/fHW6pUyUTt2sQW2fPRX5GiD9XEa6+8A6A4f2vT6pSsE7Q==",
|
||||
"dependencies": {
|
||||
"esbuild": "^0.25.0",
|
||||
"postcss": "^8.5.3",
|
||||
|
|
@ -6139,9 +6139,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/zod-to-json-schema": {
|
||||
"version": "3.24.5",
|
||||
"resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz",
|
||||
"integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==",
|
||||
"version": "3.24.3",
|
||||
"resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.3.tgz",
|
||||
"integrity": "sha512-HIAfWdYIt1sssHfYZFCXp4rU1w2r8hVVXYIlmoa0r0gABLs5di3RCqPU5DDROogVz1pAdYBaz7HK5n9pSUNs3A==",
|
||||
"peerDependencies": {
|
||||
"zod": "^3.24.1"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,12 +12,12 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@astrojs/rss": "^4.0.11",
|
||||
"@astrojs/tailwind": "^6.0.2",
|
||||
"@astrojs/tailwind": "^6.0.1",
|
||||
"@astrojs/vercel": "^8.1.3",
|
||||
"@floating-ui/dom": "^1.6.13",
|
||||
"@types/node": "^22.13.14",
|
||||
"@types/node": "^22.13.11",
|
||||
"@vercel/analytics": "^1.5.0",
|
||||
"astro": "^5.5.6",
|
||||
"astro": "^5.5.4",
|
||||
"feed": "^4.2.2",
|
||||
"ioredis": "^5.6.0",
|
||||
"p-throttle": "^7.0.0",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import Redis from "ioredis";
|
||||
import type { FeedMapping, ChapterDetail } from "./types";
|
||||
import type { FeedMapping } from "./types";
|
||||
import { ReJSON } from 'redis-modules-sdk';
|
||||
import * as zlib from 'zlib';
|
||||
import { promisify } from 'util';
|
||||
|
|
@ -653,18 +653,16 @@ export async function getFeedMapping(
|
|||
}
|
||||
|
||||
/**
|
||||
* Fetches and caches a batch of chapters if they are missing from the cache.
|
||||
* Returns the chapter data for the processed batch.
|
||||
* Processes a maximum of BATCH_SIZE comics per call.
|
||||
* Pre-warm the chapter cache for better performance
|
||||
* This function checks cache status and uses the getChaptersForSlugs function from api.ts
|
||||
* to fetch and cache data in the background
|
||||
*/
|
||||
export async function warmChapterCache(
|
||||
slugs: string[],
|
||||
lang: string
|
||||
): Promise<ChapterDetail[]> {
|
||||
const BATCH_SIZE = 5; // Define the maximum number of slugs to process per run
|
||||
|
||||
): Promise<void> {
|
||||
try {
|
||||
console.log(`[warmChapterCache] Received request to fetch/warm cache for ${slugs.length} comics in ${lang}`);
|
||||
console.log(`Warming chapter cache for ${slugs.length} comics in ${lang}`);
|
||||
const { client } = await getRedisClient(); // This already handles failover
|
||||
|
||||
// Validate slugs
|
||||
|
|
@ -672,11 +670,7 @@ export async function warmChapterCache(
|
|||
slug && typeof slug === 'string' && slug.match(/^[a-z0-9-]+$/));
|
||||
|
||||
if (validSlugs.length !== slugs.length) {
|
||||
console.warn(`[warmChapterCache] Filtered out ${slugs.length - validSlugs.length} invalid slugs`);
|
||||
}
|
||||
if (validSlugs.length === 0) {
|
||||
console.log("[warmChapterCache] No valid slugs provided.");
|
||||
return []; // Return empty array if no valid slugs
|
||||
console.warn(`Filtered out ${slugs.length - validSlugs.length} invalid slugs`);
|
||||
}
|
||||
|
||||
// Check which slugs are already in cache
|
||||
|
|
@ -688,53 +682,38 @@ export async function warmChapterCache(
|
|||
});
|
||||
|
||||
const results = await pipeline.exec();
|
||||
let slugsMissingFromCache: string[] = [];
|
||||
const slugsToFetch: string[] = [];
|
||||
|
||||
results?.forEach(([err, exists], index) => {
|
||||
if (err) {
|
||||
console.error(`[warmChapterCache] Error checking cache for ${cacheKeys[index]}:`, err);
|
||||
// Decide if you want to attempt fetching even if cache check failed
|
||||
// slugsMissingFromCache.push(validSlugs[index]);
|
||||
console.error(`Error checking cache for ${cacheKeys[index]}:`, err);
|
||||
slugsToFetch.push(validSlugs[index]);
|
||||
} else if (!exists) {
|
||||
slugsMissingFromCache.push(validSlugs[index]);
|
||||
slugsToFetch.push(validSlugs[index]);
|
||||
}
|
||||
});
|
||||
|
||||
if (slugsMissingFromCache.length === 0) {
|
||||
console.log('[warmChapterCache] All requested chapters already in cache. No fetching needed for this request.');
|
||||
// If all are cached, you might want to fetch them all now, or return empty
|
||||
// Depending on the calling context's expectation. Let's return empty for now,
|
||||
// assuming the caller will fetch cached data separately if needed.
|
||||
return [];
|
||||
if (slugsToFetch.length === 0) {
|
||||
console.log('All chapters already in cache, no warming needed');
|
||||
return;
|
||||
}
|
||||
|
||||
// --- Batching Logic ---
|
||||
const slugsToProcessNow = slugsMissingFromCache.slice(0, BATCH_SIZE);
|
||||
const remainingSlugs = slugsMissingFromCache.length - slugsToProcessNow.length;
|
||||
|
||||
console.log(`[warmChapterCache] Cache miss for ${slugsMissingFromCache.length} comics. Processing batch of ${slugsToProcessNow.length}. Remaining: ${remainingSlugs}`);
|
||||
// --- End Batching Logic ---
|
||||
|
||||
console.log(`Cache miss for ${slugsToFetch.length} comics - starting background fetch`);
|
||||
|
||||
// To avoid circular dependencies, we'll import getChaptersForSlugs dynamically
|
||||
const { getChaptersForSlugs } = await import('./api');
|
||||
|
||||
// --- Fetch the batch SYNCHRONOUSLY and return the result ---
|
||||
console.log(`[warmChapterCache] Calling getChaptersForSlugs for batch: [${slugsToProcessNow.join(', ')}]`);
|
||||
try {
|
||||
const chaptersForBatch: ChapterDetail[] = await getChaptersForSlugs(slugsToProcessNow, lang); // Await the result
|
||||
console.log(`[warmChapterCache] Successfully fetched ${chaptersForBatch.length} chapters for the batch.`);
|
||||
// The getChaptersForSlugs function is assumed to handle caching internally
|
||||
return chaptersForBatch; // Return the fetched data for this batch
|
||||
} catch (fetchError) {
|
||||
console.error(`[warmChapterCache] Error fetching chapters for batch [${slugsToProcessNow.join(', ')}]:`, fetchError);
|
||||
return []; // Return empty array on fetch error for this batch
|
||||
}
|
||||
// --- End Fetch and Return ---
|
||||
// Start fetching in the background without waiting for the result
|
||||
getChaptersForSlugs(slugsToFetch, lang)
|
||||
.then(chapters => {
|
||||
console.log(`Successfully warmed cache with ${chapters.length} chapters`);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('Error warming chapter cache:', error);
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('[warmChapterCache] CRITICAL Error in warmChapterCache:', error);
|
||||
return []; // Return empty array on critical error
|
||||
console.error('Error in warmChapterCache:', error);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue