diff --git a/src/lib/api.ts b/src/lib/api.ts index 95c42c4..1692838 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -209,39 +209,31 @@ export async function getChaptersByHid( const now = new Date(); - const chapters = data.chapters - .filter(chapter => { - // Basic validation - const isValid = chapter && - chapter.chap && - chapter.hid; + const chapters: ChapterDetail[] = []; + const rawChapters = data.chapters || []; - if (!isValid) return false; - - // Check publish time - if (chapter.publish_at) { - const publishTime = new Date(chapter.publish_at); - if (publishTime > now) { - console.log(`Chapter ${chapter.chap} will be published at ${publishTime}`); - return false; + for (const chap of rawChapters) { + try { + if (!chap.id || !chap.chap || !chap.updated_at) { + console.warn(`Skipping chapter with missing fields in ${hid}:`, chap); + continue; + } + + chapters.push({ + id: chap.id, + chap: chap.chap.toString(), + title: chap.title || `Chapter ${chap.chap}`, + updated_at: chap.updated_at, + md_comics: { + title: comicTitle, + slug: slug, + md_covers: covers } - } - - // Use the earliest available timestamp - const hasValidTimestamp = chapter.publish_at || chapter.created_at || chapter.updated_at; - return Boolean(hasValidTimestamp); - }) - .map(chapter => ({ - id: chapter.hid, - chap: chapter.chap, - title: chapter.title || `Chapter ${chapter.chap}`, - updated_at: chapter.publish_at || chapter.created_at || chapter.updated_at, - md_comics: { - title: comicTitle, - slug: slug, - md_covers: covers - } - })); + }); + } catch (e) { + console.error(`Error processing chapter ${chap.id} for ${hid}:`, e); + } + } console.log(`Successfully processed ${chapters.length} chapters for ${comicTitle}`); return chapters; @@ -251,88 +243,139 @@ export async function getChaptersByHid( } } +// Add deduplication function +function deduplicateChapters(chapters: ChapterDetail[]): ChapterDetail[] { + const seen = new Set(); + return chapters.filter(chapter => { + const key = `${chapter.md_comics.slug}-${chapter.chap}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + export async function getChaptersForSlugs( slugs: string[], lang: string = 'en' ): Promise { - console.log(`Fetching chapters for slugs:`, slugs, `in language: ${lang}`); - const chapters: ChapterDetail[] = []; + console.log(`Fetching chapters for ${slugs.length} comics`); const redis = await getRedisClient(); - - // Cache results for 1 hour (3600 seconds) const CACHE_TTL = 60 * 60; + + const uniqueSlugs = [...new Set(slugs)]; + const pipeline = redis.pipeline(); - for (const slug of slugs) { - console.log(`Processing slug: ${slug}`); + // Add cache key validation + uniqueSlugs.forEach(slug => { + if (!slug.match(/^[a-z0-9-]+$/)) { + console.error(`Invalid slug format: ${slug}`); + } const cacheKey = `chapters:${slug}:${lang}`; - - try { - // Try to get cached chapters - const cachedChapters = await redis.get(cacheKey); - if (cachedChapters) { - console.log(`Using cached chapters for ${slug}`); - chapters.push(...JSON.parse(cachedChapters)); - continue; - } - } catch (error) { - console.error('Redis cache read error:', error); - } + pipeline.get(cacheKey); + }); - const comicDetail = await getComicBySlug(slug); - - if (!comicDetail) { - console.error(`Failed to get comic details for slug: ${slug}`); - continue; + const [cacheResults, slugsToFetch] = await Promise.all([ + pipeline.exec(), + validateSlugs(uniqueSlugs) // Add this new function + ]); + + const cachedChapters: ChapterDetail[] = []; + cacheResults.forEach(([err, result], index) => { + if (!err && result) { + try { + const parsed = JSON.parse(result); + if (Array.isArray(parsed)) { + cachedChapters.push(...parsed); + } + } catch (e) { + console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e); + } } - - if (!comicDetail.comic?.hid) { - console.error(`No HID found for slug ${slug}`, comicDetail); - continue; - } - - console.log(`Found HID ${comicDetail.comic.hid} for slug ${slug}`); - const comicChapters = await getChaptersByHid( - comicDetail.comic.hid, - slug, - { + }); + + // Add progress tracking + console.log(`Cache hits: ${cachedChapters.length} chapters from ${slugs.length - slugsToFetch.length} comics`); + console.log(`Fetching ${slugsToFetch.length} comics from API`); + + const fetchPromises = slugsToFetch.map(async (slug) => { + const timer = Date.now(); + try { + const comicDetail = await getComicBySlug(slug); + + if (!comicDetail?.comic?.hid) { + console.error(`❌ No HID found for ${slug}`); + return []; + } + + const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, { limit: 5, lang: lang - } - ); - console.log(`Found ${comicChapters.length} chapters for ${slug}`); - - if (comicChapters.length > 0) { - chapters.push(...comicChapters); + }); - // Cache the results - try { - await redis.set( - cacheKey, - JSON.stringify(comicChapters), - 'EX', - CACHE_TTL - ); - console.log(`Cached chapters for ${slug}`); - } catch (error) { - console.error('Redis cache write error:', error); - } + console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`); + return chapters; + } catch (error) { + console.error(`🚨 Error processing ${slug}:`, error); + return []; } + }); + + const liveChapters = (await Promise.all(fetchPromises)).flat(); + + // Cache new results in batch + const cachePipeline = redis.pipeline(); + const chaptersBySlug: Record = {}; + + liveChapters.forEach(chapter => { + const slug = chapter.md_comics.slug; + if (!chaptersBySlug[slug]) { + chaptersBySlug[slug] = []; + } + chaptersBySlug[slug].push(chapter); + }); + + Object.entries(chaptersBySlug).forEach(([slug, chaps]) => { + const cacheKey = `chapters:${slug}:${lang}`; + cachePipeline.set( + cacheKey, + JSON.stringify(chaps), + 'EX', + CACHE_TTL + ); + }); + + await cachePipeline.exec(); + + // Combine and sort all chapters + const allChapters = [...cachedChapters, ...liveChapters]; + const deduplicatedChapters = deduplicateChapters(allChapters); + return deduplicatedChapters.sort((a, b) => + parseFloat(b.chap) - parseFloat(a.chap) || + new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime() + ); +} + +// Add this new validation function +async function validateSlugs(slugs: string[]): Promise { + const validSlugs: string[] = []; + const invalidSlugs: string[] = []; + + await Promise.all(slugs.map(async (slug) => { + try { + const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1); + if (response.ok) { + validSlugs.push(slug); + } else { + invalidSlugs.push(slug); + } + } catch { + invalidSlugs.push(slug); + } + })); + + if (invalidSlugs.length > 0) { + console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`); } - // Sort by chapter number (descending) first, then by date - const sortedChapters = chapters.sort((a, b) => { - // Convert chapter numbers to floats for proper numerical sorting - const chapA = parseFloat(a.chap); - const chapB = parseFloat(b.chap); - - if (chapB !== chapA) { - return chapB - chapA; // Sort by chapter number first - } - - // If chapters are the same, sort by date - return new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(); - }); - - console.log(`Total chapters found: ${sortedChapters.length}`); - return sortedChapters; + return validSlugs; } \ No newline at end of file diff --git a/src/lib/db.ts b/src/lib/db.ts index 90f6642..a25297a 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -2,103 +2,113 @@ import Redis from 'ioredis'; import type { FeedMapping } from './types'; const REDIS_URL = import.meta.env.REDIS_URL || process.env.REDIS_URL; -let redis: Redis | null = null; -let redisRetryCount = 0; -const MAX_RETRIES = 3; +let redisClient: Redis | null = null; const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days +const MAX_RETRIES = 5; +const BASE_DELAY_MS = 100; -try { - console.log('Environment check:', { - hasEnvVar: !!REDIS_URL, - envVarStart: REDIS_URL?.substring(0, 20) + '...', - nodeEnv: process.env.NODE_ENV - }); - - if (!REDIS_URL) { - throw new Error('REDIS_URL environment variable is not set'); - } - - console.log('Attempting to connect to Redis...'); +// Improved connection handling with exponential backoff +async function createRedisClient(): Promise { + if (!REDIS_URL) throw new Error('REDIS_URL environment variable is not set'); - redis = new Redis(REDIS_URL, { - retryStrategy(times) { - const delay = Math.min(times * 50, 2000); - console.log(`Retry attempt ${times} with delay ${delay}ms`); - return delay; + return new Redis(REDIS_URL, { + retryStrategy: (times) => { + if (times >= MAX_RETRIES) return null; + const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000); + console.log(`Redis retry attempt ${times}, next try in ${delay}ms`); + return delay + Math.random() * 200; // Add jitter }, - maxRetriesPerRequest: 2, + maxRetriesPerRequest: 1, + enableOfflineQueue: false, enableReadyCheck: true, reconnectOnError: (err) => { - const targetError = 'READONLY'; - if (err.message.includes(targetError)) { - return true; - } - return false; + return err.message.includes('READONLY'); } }); +} - redis.on('error', (error) => { - console.error('Redis connection error:', error.message); +// Singleton pattern with connection pooling +export async function getRedisClient(): Promise { + if (redisClient?.status === 'ready') { + // Validate connection with a quick ping + try { + await redisClient.ping(); + return redisClient; + } catch { + // Fall through to reconnect + } + } + + console.log('Initializing new Redis connection...'); + redisClient = await createRedisClient(); + + // Add event listeners for better monitoring + redisClient + .on('ready', () => console.log('Redis connection ready')) + .on('error', (err) => console.error('Redis error:', err)) + .on('reconnecting', () => console.log('Redis reconnecting')) + .on('end', () => console.log('Redis connection closed')); + + return redisClient; +} + +// Batch operations for better performance +export async function storeFeedMappings(mappings: Array<{ feedId: string, slugs: string[], lang: string }>): Promise { + const redis = await getRedisClient(); + const pipeline = redis.pipeline(); + + mappings.forEach(({ feedId, slugs, lang }) => { + const mapping = { + slugs, + lang, + created_at: new Date().toISOString() + }; + pipeline.set(feedId, JSON.stringify(mapping), 'EX', DEFAULT_CACHE_TTL); }); - redis.on('connect', () => { - console.log('Successfully connected to Redis'); - }); + await pipeline.exec(); +} - redis.on('ready', () => { - console.log('Redis client is ready'); - }); - - redis.on('reconnecting', () => { - console.log('Redis client is reconnecting'); - }); - -} catch (error) { - console.error('Failed to initialize Redis:', error); - if (error instanceof Error) { - console.error('Error details:', error.message); - console.error('Stack trace:', error.stack); +// Update warmChapterCache to prevent empty cache entries +export async function warmChapterCache(slugs: string[], lang: string): Promise { + const redis = await getRedisClient(); + const needsCache = await checkCacheStatus(slugs, lang); + + if (needsCache.length > 0) { + console.log(`Pre-warming cache for ${needsCache.length} comics`); + const pipeline = redis.pipeline(); + + // Replace empty array with actual chapter fetching + await Promise.all(needsCache.map(async slug => { + try { + const chapters = await getChaptersForSlugs([slug], lang); + const cacheKey = `chapters:${slug}:${lang}`; + pipeline.set(cacheKey, JSON.stringify(chapters), 'EX', 60 * 60); + } catch (error) { + console.error(`Failed to warm cache for ${slug}:`, error); + } + })); + + await pipeline.exec(); } } -export async function getRedisClient(): Promise { - if (redis?.status === 'ready') return redis; +// Add batch cache checking +export async function checkCacheStatus(slugs: string[], lang: string): Promise { + const redis = await getRedisClient(); + const pipeline = redis.pipeline(); - if (redisRetryCount >= MAX_RETRIES) { - throw new Error('Failed to connect to Redis after multiple attempts'); - } - - try { - redis = new Redis(REDIS_URL, { - retryStrategy(times) { - const delay = Math.min(times * 50, 2000); - console.log(`Retry attempt ${times} with delay ${delay}ms`); - return delay; - }, - maxRetriesPerRequest: 2, - enableReadyCheck: true, - reconnectOnError: (err) => { - const targetError = 'READONLY'; - if (err.message.includes(targetError)) { - return true; - } - return false; - } - }); - redisRetryCount = 0; - return redis; - } catch (error) { - redisRetryCount++; - throw error; - } + slugs.forEach(slug => { + const cacheKey = `chapters:${slug}:${lang}`; + pipeline.exists(cacheKey); + }); + + const results = await pipeline.exec(); + return slugs.filter((_, index) => results![index][1] === 0); } export async function storeFeedMapping(feedId: string, slugs: string[], lang: string): Promise { - if (!redis) { - console.error('Redis client not initialized'); - throw new Error('Storage system unavailable'); - } - + const redis = await getRedisClient(); const mapping = { slugs, lang, @@ -119,11 +129,8 @@ export async function storeFeedMapping(feedId: string, slugs: string[], lang: st } export async function getFeedMapping(feedId: string): Promise { - if (!redis) { - console.error('Redis client not initialized'); - throw new Error('Storage system unavailable'); - } - + const redis = await getRedisClient(); + try { const mapping = await redis.get(feedId); if (!mapping) return null; diff --git a/src/pages/api/validate-slugs.ts b/src/pages/api/validate-slugs.ts new file mode 100644 index 0000000..a12fc42 --- /dev/null +++ b/src/pages/api/validate-slugs.ts @@ -0,0 +1,17 @@ +import type { APIRoute } from "astro"; +import { validateSlugs } from "../../lib/api"; + +export const POST: APIRoute = async ({ request }) => { + try { + const { slugs } = await request.json(); + const validSlugs = await validateSlugs(slugs); + + return new Response(JSON.stringify({ validSlugs }), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }); + } catch (error) { + console.error('Validation error:', error); + return new Response("Validation failed", { status: 500 }); + } +}; \ No newline at end of file diff --git a/src/pages/rss/[feedId].xml.ts b/src/pages/rss/[feedId].xml.ts index 9429878..b2d16dd 100644 --- a/src/pages/rss/[feedId].xml.ts +++ b/src/pages/rss/[feedId].xml.ts @@ -1,6 +1,6 @@ import { Feed } from 'feed'; import { getChaptersForSlugs, isValidLanguage } from "../../lib/api"; -import { getFeedMapping } from "../../lib/db"; +import { getFeedMapping, warmChapterCache } from "../../lib/db"; import type { APIRoute } from "astro"; export const GET: APIRoute = async ({ params, request }) => { @@ -22,6 +22,8 @@ export const GET: APIRoute = async ({ params, request }) => { return new Response("Invalid language", { status: 400 }); } + await warmChapterCache(slugs, lang); + const chapters = await getChaptersForSlugs(slugs, lang); if (chapters.length === 0) {