Compare commits

..

No commits in common. "01734e8ce5ef9fa8bec9a21301cf5b2480f824ad" and "b8499d90c62e3cd36f945c9bbe598c56b85c2743" have entirely different histories.

7 changed files with 212 additions and 314 deletions

View file

@ -9,9 +9,4 @@ export default defineConfig({
remotePatterns: [{ protocol: "https" }], remotePatterns: [{ protocol: "https" }],
}, },
integrations: [tailwind()], integrations: [tailwind()],
vite: {
define: {
'process.env.REDIS_URL': JSON.stringify(process.env.REDIS_URL)
}
}
}); });

56
package-lock.json generated
View file

@ -12,10 +12,9 @@
"@astrojs/tailwind": "^6.0.0", "@astrojs/tailwind": "^6.0.0",
"@astrojs/vercel": "^8.0.5", "@astrojs/vercel": "^8.0.5",
"@floating-ui/dom": "^1.6.13", "@floating-ui/dom": "^1.6.13",
"@types/node": "^22.13.1",
"@vercel/analytics": "^1.4.1", "@vercel/analytics": "^1.4.1",
"@vercel/kv": "^3.0.0", "@vercel/kv": "^3.0.0",
"astro": "^5.2.5", "astro": "^5.2.3",
"feed": "^4.2.2", "feed": "^4.2.2",
"ioredis": "^5.4.2", "ioredis": "^5.4.2",
"tailwindcss": "^3.4.1" "tailwindcss": "^3.4.1"
@ -1463,14 +1462,6 @@
"@types/unist": "*" "@types/unist": "*"
} }
}, },
"node_modules/@types/node": {
"version": "22.13.1",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.1.tgz",
"integrity": "sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==",
"dependencies": {
"undici-types": "~6.20.0"
}
},
"node_modules/@types/unist": { "node_modules/@types/unist": {
"version": "3.0.3", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz",
@ -1753,12 +1744,12 @@
} }
}, },
"node_modules/astro": { "node_modules/astro": {
"version": "5.2.5", "version": "5.2.3",
"resolved": "https://registry.npmjs.org/astro/-/astro-5.2.5.tgz", "resolved": "https://registry.npmjs.org/astro/-/astro-5.2.3.tgz",
"integrity": "sha512-AYXyYkc+c5xbKTm48FyQA91y81nXyNPAaoyafR0LUugE4lAwuvIUcXDBfMzmbuP1lGRvsE33G2oypv6gbGaPFg==", "integrity": "sha512-04AceE/gVihtO3V28Ss0+tPPgLHGoulXrm1E7fUVPYyQu7y6w8+oXfYyzh1lpy+uEYiebjH7AFlJoz73anBmZA==",
"dependencies": { "dependencies": {
"@astrojs/compiler": "^2.10.3", "@astrojs/compiler": "^2.10.3",
"@astrojs/internal-helpers": "0.5.1", "@astrojs/internal-helpers": "0.5.0",
"@astrojs/markdown-remark": "6.1.0", "@astrojs/markdown-remark": "6.1.0",
"@astrojs/telemetry": "3.2.0", "@astrojs/telemetry": "3.2.0",
"@oslojs/encoding": "^1.1.0", "@oslojs/encoding": "^1.1.0",
@ -1796,10 +1787,10 @@
"neotraverse": "^0.6.18", "neotraverse": "^0.6.18",
"p-limit": "^6.2.0", "p-limit": "^6.2.0",
"p-queue": "^8.1.0", "p-queue": "^8.1.0",
"preferred-pm": "^4.1.1", "preferred-pm": "^4.1.0",
"prompts": "^2.4.2", "prompts": "^2.4.2",
"rehype": "^13.0.2", "rehype": "^13.0.2",
"semver": "^7.7.1", "semver": "^7.7.0",
"shiki": "^1.29.2", "shiki": "^1.29.2",
"tinyexec": "^0.3.2", "tinyexec": "^0.3.2",
"tsconfck": "^3.1.4", "tsconfck": "^3.1.4",
@ -1809,7 +1800,7 @@
"vfile": "^6.0.3", "vfile": "^6.0.3",
"vite": "^6.0.11", "vite": "^6.0.11",
"vitefu": "^1.0.5", "vitefu": "^1.0.5",
"which-pm": "^3.0.1", "which-pm": "^3.0.0",
"xxhash-wasm": "^1.1.0", "xxhash-wasm": "^1.1.0",
"yargs-parser": "^21.1.1", "yargs-parser": "^21.1.1",
"yocto-spinner": "^0.2.0", "yocto-spinner": "^0.2.0",
@ -1834,9 +1825,9 @@
} }
}, },
"node_modules/astro/node_modules/@astrojs/internal-helpers": { "node_modules/astro/node_modules/@astrojs/internal-helpers": {
"version": "0.5.1", "version": "0.5.0",
"resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.5.1.tgz", "resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.5.0.tgz",
"integrity": "sha512-M7rAge1n2+aOSxNvKUFa0u/KFn0W+sZy7EW91KOSERotm2Ti8qs+1K0xx3zbOxtAVrmJb5/J98eohVvvEqtNkw==" "integrity": "sha512-CgB5ZaZO1PFG+rbjF3HnA7G6gIBjJ070xb7bUjeu5Gqqufma+t6fpuRWMXnK2iEO3zVyX7e/xplPlqtFKy/lvw=="
}, },
"node_modules/astro/node_modules/@vercel/kv": { "node_modules/astro/node_modules/@vercel/kv": {
"version": "1.0.1", "version": "1.0.1",
@ -4798,13 +4789,13 @@
"integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==" "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
}, },
"node_modules/preferred-pm": { "node_modules/preferred-pm": {
"version": "4.1.1", "version": "4.1.0",
"resolved": "https://registry.npmjs.org/preferred-pm/-/preferred-pm-4.1.1.tgz", "resolved": "https://registry.npmjs.org/preferred-pm/-/preferred-pm-4.1.0.tgz",
"integrity": "sha512-rU+ZAv1Ur9jAUZtGPebQVQPzdGhNzaEiQ7VL9+cjsAWPHFYOccNXPNiev1CCDSOg/2j7UujM7ojNhpkuILEVNQ==", "integrity": "sha512-cNKMVcSvE3hZBRukdRsvPtlUaM2fYLFbIvBz620XaDA5SjHsCRsAjNE2baI90Hh7QvJIjCHJYnpPtyqJ1/kOjQ==",
"dependencies": { "dependencies": {
"find-up-simple": "^1.0.0", "find-up-simple": "^1.0.0",
"find-yarn-workspace-root2": "1.2.16", "find-yarn-workspace-root2": "1.2.16",
"which-pm": "^3.0.1" "which-pm": "^3.0.0"
}, },
"engines": { "engines": {
"node": ">=18.12" "node": ">=18.12"
@ -5297,9 +5288,9 @@
"integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==" "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg=="
}, },
"node_modules/semver": { "node_modules/semver": {
"version": "7.7.1", "version": "7.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.0.tgz",
"integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", "integrity": "sha512-DrfFnPzblFmNrIZzg5RzHegbiRWg7KMR7btwi2yjHwx06zsUbO5g613sVwEV7FTwmzJu+Io0lJe2GJ3LxqpvBQ==",
"bin": { "bin": {
"semver": "bin/semver.js" "semver": "bin/semver.js"
}, },
@ -5820,11 +5811,6 @@
"resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz", "resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz",
"integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==" "integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q=="
}, },
"node_modules/undici-types": {
"version": "6.20.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
},
"node_modules/unenv": { "node_modules/unenv": {
"version": "1.10.0", "version": "1.10.0",
"resolved": "https://registry.npmjs.org/unenv/-/unenv-1.10.0.tgz", "resolved": "https://registry.npmjs.org/unenv/-/unenv-1.10.0.tgz",
@ -6176,9 +6162,9 @@
} }
}, },
"node_modules/which-pm": { "node_modules/which-pm": {
"version": "3.0.1", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/which-pm/-/which-pm-3.0.1.tgz", "resolved": "https://registry.npmjs.org/which-pm/-/which-pm-3.0.0.tgz",
"integrity": "sha512-v2JrMq0waAI4ju1xU5x3blsxBBMgdgZve580iYMN5frDaLGjbA24fok7wKCsya8KLVO19Ju4XDc5+zTZCJkQfg==", "integrity": "sha512-ysVYmw6+ZBhx3+ZkcPwRuJi38ZOTLJJ33PSHaitLxSKUMsh0LkKd0nC69zZCwt5D+AYUcMK2hhw4yWny20vSGg==",
"dependencies": { "dependencies": {
"load-yaml-file": "^0.2.0" "load-yaml-file": "^0.2.0"
}, },

View file

@ -15,12 +15,11 @@
"@astrojs/tailwind": "^6.0.0", "@astrojs/tailwind": "^6.0.0",
"@astrojs/vercel": "^8.0.5", "@astrojs/vercel": "^8.0.5",
"@floating-ui/dom": "^1.6.13", "@floating-ui/dom": "^1.6.13",
"@types/node": "^22.13.1",
"@vercel/analytics": "^1.4.1", "@vercel/analytics": "^1.4.1",
"@vercel/kv": "^3.0.0", "@vercel/kv": "^3.0.0",
"astro": "^5.2.5", "astro": "^5.2.3",
"feed": "^4.2.2", "feed": "^4.2.2",
"ioredis": "^5.4.2", "ioredis": "^5.4.2",
"tailwindcss": "^3.4.1" "tailwindcss": "^3.4.1"
} }
} }

View file

@ -209,31 +209,39 @@ export async function getChaptersByHid(
const now = new Date(); const now = new Date();
const chapters: ChapterDetail[] = []; const chapters = data.chapters
const rawChapters = data.chapters || []; .filter(chapter => {
// Basic validation
const isValid = chapter &&
chapter.chap &&
chapter.hid;
for (const chap of rawChapters) { if (!isValid) return false;
try {
if (!chap.id || !chap.chap || !chap.updated_at) { // Check publish time
console.warn(`Skipping chapter with missing fields in ${hid}:`, chap); if (chapter.publish_at) {
continue; const publishTime = new Date(chapter.publish_at);
} if (publishTime > now) {
console.log(`Chapter ${chapter.chap} will be published at ${publishTime}`);
chapters.push({ return false;
id: chap.id,
chap: chap.chap.toString(),
title: chap.title || `Chapter ${chap.chap}`,
updated_at: chap.updated_at,
md_comics: {
title: comicTitle,
slug: slug,
md_covers: covers
} }
}); }
} catch (e) {
console.error(`Error processing chapter ${chap.id} for ${hid}:`, e); // Use the earliest available timestamp
} const hasValidTimestamp = chapter.publish_at || chapter.created_at || chapter.updated_at;
} return Boolean(hasValidTimestamp);
})
.map(chapter => ({
id: chapter.hid,
chap: chapter.chap,
title: chapter.title || `Chapter ${chapter.chap}`,
updated_at: chapter.publish_at || chapter.created_at || chapter.updated_at,
md_comics: {
title: comicTitle,
slug: slug,
md_covers: covers
}
}));
console.log(`Successfully processed ${chapters.length} chapters for ${comicTitle}`); console.log(`Successfully processed ${chapters.length} chapters for ${comicTitle}`);
return chapters; return chapters;
@ -243,140 +251,88 @@ export async function getChaptersByHid(
} }
} }
// Add deduplication function
function deduplicateChapters(chapters: ChapterDetail[]): ChapterDetail[] {
const seen = new Set<string>();
return chapters.filter(chapter => {
const key = `${chapter.md_comics.slug}-${chapter.chap}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
});
}
export async function getChaptersForSlugs( export async function getChaptersForSlugs(
slugs: string[], slugs: string[],
lang: string = 'en' lang: string = 'en'
): Promise<ChapterDetail[]> { ): Promise<ChapterDetail[]> {
console.log(`Fetching chapters for ${slugs.length} comics`); console.log(`Fetching chapters for slugs:`, slugs, `in language: ${lang}`);
const chapters: ChapterDetail[] = [];
const redis = await getRedisClient(); const redis = await getRedisClient();
const CACHE_TTL = 60 * 60;
const uniqueSlugs = [...new Set(slugs)]; // Cache results for 1 hour (3600 seconds)
const pipeline = redis.pipeline(); const CACHE_TTL = 60 * 60;
// Add cache key validation for (const slug of slugs) {
uniqueSlugs.forEach(slug => { console.log(`Processing slug: ${slug}`);
if (!slug.match(/^[a-z0-9-]+$/)) {
console.error(`Invalid slug format: ${slug}`);
}
const cacheKey = `chapters:${slug}:${lang}`; const cacheKey = `chapters:${slug}:${lang}`;
pipeline.get(cacheKey);
});
const [cacheResults, slugsToFetch] = await Promise.all([
pipeline.exec(),
validateSlugs(uniqueSlugs) // Add this new function
]);
const cachedChapters: ChapterDetail[] = [];
cacheResults.forEach(([err, result], index) => {
if (!err && result) {
try {
const parsed = JSON.parse(result);
if (Array.isArray(parsed)) {
cachedChapters.push(...parsed);
}
} catch (e) {
console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e);
}
}
});
// Add progress tracking
const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug));
console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`);
console.log(`Fetching ${slugsToFetch.length} comics from API`);
const fetchPromises = slugsToFetch.map(async (slug) => {
const timer = Date.now();
try { try {
const comicDetail = await getComicBySlug(slug); // Try to get cached chapters
const cachedChapters = await redis.get(cacheKey);
if (!comicDetail?.comic?.hid) { if (cachedChapters) {
console.error(`❌ No HID found for ${slug}`); console.log(`Using cached chapters for ${slug}`);
return []; chapters.push(...JSON.parse(cachedChapters));
continue;
} }
} catch (error) {
const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, { console.error('Redis cache read error:', error);
}
const comicDetail = await getComicBySlug(slug);
if (!comicDetail) {
console.error(`Failed to get comic details for slug: ${slug}`);
continue;
}
if (!comicDetail.comic?.hid) {
console.error(`No HID found for slug ${slug}`, comicDetail);
continue;
}
console.log(`Found HID ${comicDetail.comic.hid} for slug ${slug}`);
const comicChapters = await getChaptersByHid(
comicDetail.comic.hid,
slug,
{
limit: 5, limit: 5,
lang: lang lang: lang
});
console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`);
return chapters;
} catch (error) {
console.error(`🚨 Error processing ${slug}:`, error);
return [];
}
});
const liveChapters = (await Promise.all(fetchPromises)).flat();
// Cache new results in batch
const cachePipeline = redis.pipeline();
const chaptersBySlug: Record<string, ChapterDetail[]> = {};
liveChapters.forEach(chapter => {
const slug = chapter.md_comics.slug;
if (!chaptersBySlug[slug]) {
chaptersBySlug[slug] = [];
}
chaptersBySlug[slug].push(chapter);
});
Object.entries(chaptersBySlug).forEach(([slug, chaps]) => {
const cacheKey = `chapters:${slug}:${lang}`;
cachePipeline.set(
cacheKey,
JSON.stringify(chaps),
'EX',
CACHE_TTL
);
});
await cachePipeline.exec();
// Combine and sort all chapters
const allChapters = [...cachedChapters, ...liveChapters];
const deduplicatedChapters = deduplicateChapters(allChapters);
return deduplicatedChapters.sort((a, b) =>
parseFloat(b.chap) - parseFloat(a.chap) ||
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime()
);
}
// Add export to the validation function
export async function validateSlugs(slugs: string[]): Promise<string[]> {
const validSlugs: string[] = [];
const invalidSlugs: string[] = [];
await Promise.all(slugs.map(async (slug) => {
try {
const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1);
if (response.ok) {
validSlugs.push(slug);
} else {
invalidSlugs.push(slug);
} }
} catch { );
invalidSlugs.push(slug); console.log(`Found ${comicChapters.length} chapters for ${slug}`);
if (comicChapters.length > 0) {
chapters.push(...comicChapters);
// Cache the results
try {
await redis.set(
cacheKey,
JSON.stringify(comicChapters),
'EX',
CACHE_TTL
);
console.log(`Cached chapters for ${slug}`);
} catch (error) {
console.error('Redis cache write error:', error);
}
} }
}));
if (invalidSlugs.length > 0) {
console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`);
} }
return validSlugs; // Sort by chapter number (descending) first, then by date
const sortedChapters = chapters.sort((a, b) => {
// Convert chapter numbers to floats for proper numerical sorting
const chapA = parseFloat(a.chap);
const chapB = parseFloat(b.chap);
if (chapB !== chapA) {
return chapB - chapA; // Sort by chapter number first
}
// If chapters are the same, sort by date
return new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime();
});
console.log(`Total chapters found: ${sortedChapters.length}`);
return sortedChapters;
} }

View file

@ -1,115 +1,104 @@
import Redis from 'ioredis'; import Redis from 'ioredis';
import type { FeedMapping } from './types'; import type { FeedMapping } from './types';
import { getChaptersForSlugs } from './api';
const REDIS_URL = import.meta.env.REDIS_URL || process.env.REDIS_URL; const REDIS_URL = import.meta.env.REDIS_URL || process.env.REDIS_URL;
let redisClient: Redis | null = null; let redis: Redis | null = null;
let redisRetryCount = 0;
const MAX_RETRIES = 3;
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
const MAX_RETRIES = 5;
const BASE_DELAY_MS = 100;
// Improved connection handling with exponential backoff try {
async function createRedisClient(): Promise<Redis> { console.log('Environment check:', {
if (!REDIS_URL) throw new Error('REDIS_URL environment variable is not set'); hasEnvVar: !!REDIS_URL,
envVarStart: REDIS_URL?.substring(0, 20) + '...',
nodeEnv: process.env.NODE_ENV
});
if (!REDIS_URL) {
throw new Error('REDIS_URL environment variable is not set');
}
console.log('Attempting to connect to Redis...');
return new Redis(REDIS_URL, { redis = new Redis(REDIS_URL, {
retryStrategy: (times) => { retryStrategy(times) {
if (times >= MAX_RETRIES) return null; const delay = Math.min(times * 50, 2000);
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000); console.log(`Retry attempt ${times} with delay ${delay}ms`);
console.log(`Redis retry attempt ${times}, next try in ${delay}ms`); return delay;
return delay + Math.random() * 200; // Add jitter
}, },
maxRetriesPerRequest: null, // Allow retries for queued commands maxRetriesPerRequest: 2,
enableOfflineQueue: true, // Enable command queuing
enableReadyCheck: true, enableReadyCheck: true,
reconnectOnError: (err) => { reconnectOnError: (err) => {
return err.message.includes('READONLY'); const targetError = 'READONLY';
} if (err.message.includes(targetError)) {
}); return true;
}
// Singleton pattern with connection pooling
export async function getRedisClient(): Promise<Redis> {
if (redisClient?.status === 'ready') {
// Validate connection with a quick ping
try {
await redisClient.ping();
return redisClient;
} catch {
// Fall through to reconnect
}
}
console.log('Initializing new Redis connection...');
redisClient = await createRedisClient();
// Add event listeners for better monitoring
redisClient
.on('ready', () => console.log('Redis connection ready'))
.on('error', (err) => console.error('Redis error:', err))
.on('reconnecting', () => console.log('Redis reconnecting'))
.on('end', () => console.log('Redis connection closed'));
return redisClient;
}
// Batch operations for better performance
export async function storeFeedMappings(mappings: Array<{ feedId: string, slugs: string[], lang: string }>): Promise<void> {
const redis = await getRedisClient();
const pipeline = redis.pipeline();
mappings.forEach(({ feedId, slugs, lang }) => {
const mapping = {
slugs,
lang,
created_at: new Date().toISOString()
};
pipeline.set(feedId, JSON.stringify(mapping), 'EX', DEFAULT_CACHE_TTL);
});
await pipeline.exec();
}
// Update warmChapterCache to prevent empty cache entries
export async function warmChapterCache(slugs: string[], lang: string): Promise<void> {
const redis = await getRedisClient();
const needsCache = await checkCacheStatus(slugs, lang);
if (needsCache.length > 0) {
console.log(`Pre-warming cache for ${needsCache.length} comics`);
const pipeline = redis.pipeline();
// Replace empty array with actual chapter fetching
await Promise.all(needsCache.map(async slug => {
try {
const chapters = await getChaptersForSlugs([slug], lang);
const cacheKey = `chapters:${slug}:${lang}`;
pipeline.set(cacheKey, JSON.stringify(chapters), 'EX', 60 * 60);
} catch (error) {
console.error(`Failed to warm cache for ${slug}:`, error);
} }
})); return false;
}
await pipeline.exec(); });
redis.on('error', (error) => {
console.error('Redis connection error:', error.message);
});
redis.on('connect', () => {
console.log('Successfully connected to Redis');
});
redis.on('ready', () => {
console.log('Redis client is ready');
});
redis.on('reconnecting', () => {
console.log('Redis client is reconnecting');
});
} catch (error) {
console.error('Failed to initialize Redis:', error);
if (error instanceof Error) {
console.error('Error details:', error.message);
console.error('Stack trace:', error.stack);
} }
} }
// Add batch cache checking export async function getRedisClient(): Promise<Redis> {
export async function checkCacheStatus(slugs: string[], lang: string): Promise<string[]> { if (redis?.status === 'ready') return redis;
const redis = await getRedisClient();
const pipeline = redis.pipeline();
slugs.forEach(slug => { if (redisRetryCount >= MAX_RETRIES) {
const cacheKey = `chapters:${slug}:${lang}`; throw new Error('Failed to connect to Redis after multiple attempts');
pipeline.exists(cacheKey); }
});
try {
const results = await pipeline.exec(); redis = new Redis(REDIS_URL, {
return slugs.filter((_, index) => results![index][1] === 0); retryStrategy(times) {
const delay = Math.min(times * 50, 2000);
console.log(`Retry attempt ${times} with delay ${delay}ms`);
return delay;
},
maxRetriesPerRequest: 2,
enableReadyCheck: true,
reconnectOnError: (err) => {
const targetError = 'READONLY';
if (err.message.includes(targetError)) {
return true;
}
return false;
}
});
redisRetryCount = 0;
return redis;
} catch (error) {
redisRetryCount++;
throw error;
}
} }
export async function storeFeedMapping(feedId: string, slugs: string[], lang: string): Promise<void> { export async function storeFeedMapping(feedId: string, slugs: string[], lang: string): Promise<void> {
const redis = await getRedisClient(); if (!redis) {
console.error('Redis client not initialized');
throw new Error('Storage system unavailable');
}
const mapping = { const mapping = {
slugs, slugs,
lang, lang,
@ -130,8 +119,11 @@ export async function storeFeedMapping(feedId: string, slugs: string[], lang: st
} }
export async function getFeedMapping(feedId: string): Promise<FeedMapping | null> { export async function getFeedMapping(feedId: string): Promise<FeedMapping | null> {
const redis = await getRedisClient(); if (!redis) {
console.error('Redis client not initialized');
throw new Error('Storage system unavailable');
}
try { try {
const mapping = await redis.get(feedId); const mapping = await redis.get(feedId);
if (!mapping) return null; if (!mapping) return null;
@ -149,15 +141,4 @@ export async function getFeedMapping(feedId: string): Promise<FeedMapping | null
} }
} }
// No need for manual cleanup as we're using TTL (expire) // No need for manual cleanup as we're using TTL (expire)
// Warm connection on startup
(async () => {
try {
const client = await getRedisClient();
await client.ping();
console.log('Redis pre-connected successfully');
} catch (error) {
console.error('Redis pre-connection failed:', error);
}
})();

View file

@ -1,17 +0,0 @@
import type { APIRoute } from "astro";
import { validateSlugs } from "../../lib/api";
export const POST: APIRoute = async ({ request }) => {
try {
const { slugs } = await request.json();
const validSlugs = await validateSlugs(slugs);
return new Response(JSON.stringify({ validSlugs }), {
status: 200,
headers: { 'Content-Type': 'application/json' }
});
} catch (error) {
console.error('Validation error:', error);
return new Response("Validation failed", { status: 500 });
}
};

View file

@ -1,6 +1,6 @@
import { Feed } from 'feed'; import { Feed } from 'feed';
import { getChaptersForSlugs, isValidLanguage } from "../../lib/api"; import { getChaptersForSlugs, isValidLanguage } from "../../lib/api";
import { getFeedMapping, warmChapterCache } from "../../lib/db"; import { getFeedMapping } from "../../lib/db";
import type { APIRoute } from "astro"; import type { APIRoute } from "astro";
export const GET: APIRoute = async ({ params, request }) => { export const GET: APIRoute = async ({ params, request }) => {
@ -22,8 +22,6 @@ export const GET: APIRoute = async ({ params, request }) => {
return new Response("Invalid language", { status: 400 }); return new Response("Invalid language", { status: 400 });
} }
await warmChapterCache(slugs, lang);
const chapters = await getChaptersForSlugs(slugs, lang); const chapters = await getChaptersForSlugs(slugs, lang);
if (chapters.length === 0) { if (chapters.length === 0) {