ldez/new-manga
1
0
Fork 0
mirror of https://github.com/ld3z/new-manga.git synced 2026-11-09 20:53:45 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Compare commits

base: ldez:4cebcc40b18fd76ee7bb647825bb80e16ee37358
Branches Tags
ldez:main
..
compare: ldez:3e98aeccba0b5ed275bd282ba627a80f6a2ca5f7
Branches Tags
ldez:main

No commits in common. "4cebcc40b18fd76ee7bb647825bb80e16ee37358" and "3e98aeccba0b5ed275bd282ba627a80f6a2ca5f7" have entirely different histories.
4cebcc40b1 ... 3e98aeccba

3 changed files with 63 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

50
package-lock.json generated
View file

@ -9,12 +9,12 @@
"version": "1.0.0", "version": "1.0.0",
"dependencies": { "dependencies": {
"@astrojs/rss": "^4.0.11", "@astrojs/rss": "^4.0.11",
"@astrojs/tailwind": "^6.0.2", "@astrojs/tailwind": "^6.0.1",
"@astrojs/vercel": "^8.1.3", "@astrojs/vercel": "^8.1.3",
"@floating-ui/dom": "^1.6.13", "@floating-ui/dom": "^1.6.13",
"@types/node": "^22.13.14", "@types/node": "^22.13.11",
"@vercel/analytics": "^1.5.0", "@vercel/analytics": "^1.5.0",
"astro": "^5.5.6", "astro": "^5.5.4",
"feed": "^4.2.2", "feed": "^4.2.2",
"ioredis": "^5.6.0", "ioredis": "^5.6.0",
"p-throttle": "^7.0.0", "p-throttle": "^7.0.0",
@ -92,9 +92,9 @@
} }
}, },
"node_modules/@astrojs/tailwind": { "node_modules/@astrojs/tailwind": {
"version": "6.0.2", "version": "6.0.1",
"resolved": "https://registry.npmjs.org/@astrojs/tailwind/-/tailwind-6.0.2.tgz", "resolved": "https://registry.npmjs.org/@astrojs/tailwind/-/tailwind-6.0.1.tgz",
"integrity": "sha512-j3mhLNeugZq6A8dMNXVarUa8K6X9AW+QHU9u3lKNrPLMHhOQ0S7VeWhHwEeJFpEK1BTKEUY1U78VQv2gN6hNGg==", "integrity": "sha512-jYrqG6dXwRnS3xUcRIoeYGFB2L2SISuTNaqMMFp0otW55HQ4fhiz941VaE9EVbs49tF2+1aAZCT3xgt2l/j5tQ==",
"dependencies": { "dependencies": {
"autoprefixer": "^10.4.21", "autoprefixer": "^10.4.21",
"postcss": "^8.5.3", "postcss": "^8.5.3",
@ -1459,9 +1459,9 @@
} }
}, },
"node_modules/@types/node": { "node_modules/@types/node": {
"version": "22.13.14", "version": "22.13.11",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.14.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.11.tgz",
"integrity": "sha512-Zs/Ollc1SJ8nKUAgc7ivOEdIBM8JAKgrqqUYi2J997JuKO7/tpQC+WCetQ1sypiKCQWHdvdg9wBNpUPEWZae7w==", "integrity": "sha512-iEUCUJoU0i3VnrCmgoWCXttklWcvoCIx4jzcP22fioIVSdTmjgoEvmAO/QPw6TcS9k5FrNgn4w7q5lGOd1CT5g==",
"dependencies": { "dependencies": {
"undici-types": "~6.20.0" "undici-types": "~6.20.0"
} }
@ -1739,9 +1739,9 @@
} }
}, },
"node_modules/astro": { "node_modules/astro": {
"version": "5.5.6", "version": "5.5.4",
"resolved": "https://registry.npmjs.org/astro/-/astro-5.5.6.tgz", "resolved": "https://registry.npmjs.org/astro/-/astro-5.5.4.tgz",
"integrity": "sha512-SjU6zAlRe/gQrZ558Qhm211Yl8XOZBfjMpZig3XtQpG6zlyHJkElbWQIWXAVPzMg17cFVdv/qZPVHfcP2YOGGw==", "integrity": "sha512-rc9Mj6dIKM9ylTSQ0/S1l5f1iSDNs0PEjRMnvVpZ0W+wfBCu99JEuXy+fiWDNT6gf60bC4dKQehZxZGdSJB9kg==",
"dependencies": { "dependencies": {
"@astrojs/compiler": "^2.11.0", "@astrojs/compiler": "^2.11.0",
"@astrojs/internal-helpers": "0.6.1", "@astrojs/internal-helpers": "0.6.1",
@ -1779,12 +1779,12 @@
"neotraverse": "^0.6.18", "neotraverse": "^0.6.18",
"p-limit": "^6.2.0", "p-limit": "^6.2.0",
"p-queue": "^8.1.0", "p-queue": "^8.1.0",
"package-manager-detector": "^1.1.0", "package-manager-detector": "^1.0.0",
"picomatch": "^4.0.2", "picomatch": "^4.0.2",
"prompts": "^2.4.2", "prompts": "^2.4.2",
"rehype": "^13.0.2", "rehype": "^13.0.2",
"semver": "^7.7.1", "semver": "^7.7.1",
"shiki": "^3.2.1", "shiki": "^3.0.0",
"tinyexec": "^0.3.2", "tinyexec": "^0.3.2",
"tinyglobby": "^0.2.12", "tinyglobby": "^0.2.12",
"tsconfck": "^3.1.5", "tsconfck": "^3.1.5",
@ -1792,13 +1792,13 @@
"unist-util-visit": "^5.0.0", "unist-util-visit": "^5.0.0",
"unstorage": "^1.15.0", "unstorage": "^1.15.0",
"vfile": "^6.0.3", "vfile": "^6.0.3",
"vite": "^6.2.4", "vite": "^6.2.1",
"vitefu": "^1.0.6", "vitefu": "^1.0.6",
"xxhash-wasm": "^1.1.0", "xxhash-wasm": "^1.1.0",
"yargs-parser": "^21.1.1", "yargs-parser": "^21.1.1",
"yocto-spinner": "^0.2.1", "yocto-spinner": "^0.2.1",
"zod": "^3.24.2", "zod": "^3.24.2",
"zod-to-json-schema": "^3.24.5", "zod-to-json-schema": "^3.24.3",
"zod-to-ts": "^1.2.0" "zod-to-ts": "^1.2.0"
}, },
"bin": { "bin": {
@ -4403,9 +4403,9 @@
"integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==" "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw=="
}, },
"node_modules/package-manager-detector": { "node_modules/package-manager-detector": {
"version": "1.1.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.1.0.tgz", "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.0.0.tgz",
"integrity": "sha512-Y8f9qUlBzW8qauJjd/eu6jlpJZsuPJm2ZAV0cDVd420o4EdpH5RPdoCv+60/TdJflGatr4sDfpAL6ArWZbM5tA==" "integrity": "sha512-7elnH+9zMsRo7aS72w6MeRugTpdRvInmEB4Kmm9BVvPw/SLG8gXUGQ+4wF0Mys0RSWPz0B9nuBbDe8vFeA2sfg=="
}, },
"node_modules/parse-latin": { "node_modules/parse-latin": {
"version": "7.0.0", "version": "7.0.0",
@ -5822,9 +5822,9 @@
} }
}, },
"node_modules/vite": { "node_modules/vite": {
"version": "6.2.4", "version": "6.2.1",
"resolved": "https://registry.npmjs.org/vite/-/vite-6.2.4.tgz", "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.1.tgz",
"integrity": "sha512-veHMSew8CcRzhL5o8ONjy8gkfmFJAd5Ac16oxBUjlwgX3Gq2Wqr+qNC3TjPIpy7TPV/KporLga5GT9HqdrCizw==", "integrity": "sha512-n2GnqDb6XPhlt9B8olZPrgMD/es/Nd1RdChF6CBD/fHW6pUyUTt2sQW2fPRX5GiD9XEa6+8A6A4f2vT6pSsE7Q==",
"dependencies": { "dependencies": {
"esbuild": "^0.25.0", "esbuild": "^0.25.0",
"postcss": "^8.5.3", "postcss": "^8.5.3",
@ -6139,9 +6139,9 @@
} }
}, },
"node_modules/zod-to-json-schema": { "node_modules/zod-to-json-schema": {
"version": "3.24.5", "version": "3.24.3",
"resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz", "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.3.tgz",
"integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==", "integrity": "sha512-HIAfWdYIt1sssHfYZFCXp4rU1w2r8hVVXYIlmoa0r0gABLs5di3RCqPU5DDROogVz1pAdYBaz7HK5n9pSUNs3A==",
"peerDependencies": { "peerDependencies": {
"zod": "^3.24.1" "zod": "^3.24.1"
} }

6
package.json
View file

@ -12,12 +12,12 @@
}, },
"dependencies": { "dependencies": {
"@astrojs/rss": "^4.0.11", "@astrojs/rss": "^4.0.11",
"@astrojs/tailwind": "^6.0.2", "@astrojs/tailwind": "^6.0.1",
"@astrojs/vercel": "^8.1.3", "@astrojs/vercel": "^8.1.3",
"@floating-ui/dom": "^1.6.13", "@floating-ui/dom": "^1.6.13",
"@types/node": "^22.13.14", "@types/node": "^22.13.11",
"@vercel/analytics": "^1.5.0", "@vercel/analytics": "^1.5.0",
"astro": "^5.5.6", "astro": "^5.5.4",
"feed": "^4.2.2", "feed": "^4.2.2",
"ioredis": "^5.6.0", "ioredis": "^5.6.0",
"p-throttle": "^7.0.0", "p-throttle": "^7.0.0",

69
src/lib/db.ts
View file

@ -1,5 +1,5 @@
import Redis from "ioredis"; import Redis from "ioredis";
import type { FeedMapping, ChapterDetail } from "./types"; import type { FeedMapping } from "./types";
import { ReJSON } from 'redis-modules-sdk'; import { ReJSON } from 'redis-modules-sdk';
import * as zlib from 'zlib'; import * as zlib from 'zlib';
import { promisify } from 'util'; import { promisify } from 'util';
@ -653,18 +653,16 @@ export async function getFeedMapping(
} }
/** /**
* Fetches and caches a batch of chapters if they are missing from the cache. * Pre-warm the chapter cache for better performance
* Returns the chapter data for the processed batch. * This function checks cache status and uses the getChaptersForSlugs function from api.ts
* Processes a maximum of BATCH_SIZE comics per call. * to fetch and cache data in the background
*/ */
export async function warmChapterCache( export async function warmChapterCache(
slugs: string[], slugs: string[],
lang: string lang: string
): Promise<ChapterDetail[]> { ): Promise<void> {
const BATCH_SIZE = 5; // Define the maximum number of slugs to process per run
try { try {
console.log(`[warmChapterCache] Received request to fetch/warm cache for ${slugs.length} comics in ${lang}`); console.log(`Warming chapter cache for ${slugs.length} comics in ${lang}`);
const { client } = await getRedisClient(); // This already handles failover const { client } = await getRedisClient(); // This already handles failover
// Validate slugs // Validate slugs
@ -672,11 +670,7 @@ export async function warmChapterCache(
slug && typeof slug === 'string' && slug.match(/^[a-z0-9-]+$/)); slug && typeof slug === 'string' && slug.match(/^[a-z0-9-]+$/));
if (validSlugs.length !== slugs.length) { if (validSlugs.length !== slugs.length) {
console.warn(`[warmChapterCache] Filtered out ${slugs.length - validSlugs.length} invalid slugs`); console.warn(`Filtered out ${slugs.length - validSlugs.length} invalid slugs`);
}
if (validSlugs.length === 0) {
console.log("[warmChapterCache] No valid slugs provided.");
return []; // Return empty array if no valid slugs
} }
// Check which slugs are already in cache // Check which slugs are already in cache
@ -688,53 +682,38 @@ export async function warmChapterCache(
}); });
const results = await pipeline.exec(); const results = await pipeline.exec();
let slugsMissingFromCache: string[] = []; const slugsToFetch: string[] = [];
results?.forEach(([err, exists], index) => { results?.forEach(([err, exists], index) => {
if (err) { if (err) {
console.error(`[warmChapterCache] Error checking cache for ${cacheKeys[index]}:`, err); console.error(`Error checking cache for ${cacheKeys[index]}:`, err);
// Decide if you want to attempt fetching even if cache check failed slugsToFetch.push(validSlugs[index]);
// slugsMissingFromCache.push(validSlugs[index]);
} else if (!exists) { } else if (!exists) {
slugsMissingFromCache.push(validSlugs[index]); slugsToFetch.push(validSlugs[index]);
} }
}); });
if (slugsMissingFromCache.length === 0) { if (slugsToFetch.length === 0) {
console.log('[warmChapterCache] All requested chapters already in cache. No fetching needed for this request.'); console.log('All chapters already in cache, no warming needed');
// If all are cached, you might want to fetch them all now, or return empty return;
// Depending on the calling context's expectation. Let's return empty for now,
// assuming the caller will fetch cached data separately if needed.
return [];
} }
// --- Batching Logic --- console.log(`Cache miss for ${slugsToFetch.length} comics - starting background fetch`);
const slugsToProcessNow = slugsMissingFromCache.slice(0, BATCH_SIZE);
const remainingSlugs = slugsMissingFromCache.length - slugsToProcessNow.length;
console.log(`[warmChapterCache] Cache miss for ${slugsMissingFromCache.length} comics. Processing batch of ${slugsToProcessNow.length}. Remaining: ${remainingSlugs}`);
// --- End Batching Logic ---
// To avoid circular dependencies, we'll import getChaptersForSlugs dynamically // To avoid circular dependencies, we'll import getChaptersForSlugs dynamically
const { getChaptersForSlugs } = await import('./api'); const { getChaptersForSlugs } = await import('./api');
// --- Fetch the batch SYNCHRONOUSLY and return the result --- // Start fetching in the background without waiting for the result
console.log(`[warmChapterCache] Calling getChaptersForSlugs for batch: [${slugsToProcessNow.join(', ')}]`); getChaptersForSlugs(slugsToFetch, lang)
try { .then(chapters => {
const chaptersForBatch: ChapterDetail[] = await getChaptersForSlugs(slugsToProcessNow, lang); // Await the result console.log(`Successfully warmed cache with ${chapters.length} chapters`);
console.log(`[warmChapterCache] Successfully fetched ${chaptersForBatch.length} chapters for the batch.`); })
// The getChaptersForSlugs function is assumed to handle caching internally .catch(error => {
return chaptersForBatch; // Return the fetched data for this batch console.error('Error warming chapter cache:', error);
} catch (fetchError) { });
console.error(`[warmChapterCache] Error fetching chapters for batch [${slugsToProcessNow.join(', ')}]:`, fetchError);
return []; // Return empty array on fetch error for this batch
}
// --- End Fetch and Return ---
} catch (error) { } catch (error) {
console.error('[warmChapterCache] CRITICAL Error in warmChapterCache:', error); console.error('Error in warmChapterCache:', error);
return []; // Return empty array on critical error
} }
} }