add better feed handling

src/lib/api.ts

@@ -209,39 +209,31 @@ export async function getChaptersByHid(
     
     const now = new Date();
     
-    const chapters = data.chapters
+    const chapters: ChapterDetail[] = [];
-      .filter(chapter => {
+    const rawChapters = data.chapters || [];
-        // Basic validation
-        const isValid = chapter && 
-               chapter.chap &&
-               chapter.hid;
 
-        if (!isValid) return false;
+    for (const chap of rawChapters) {
-
+      try {
-        // Check publish time
+        if (!chap.id || !chap.chap || !chap.updated_at) {
-        if (chapter.publish_at) {
+          console.warn(`Skipping chapter with missing fields in ${hid}:`, chap);
-          const publishTime = new Date(chapter.publish_at);
+          continue;
-          if (publishTime > now) {
-            console.log(`Chapter ${chapter.chap} will be published at ${publishTime}`);
-            return false;
-          }
         }
         
-        // Use the earliest available timestamp
+        chapters.push({
-        const hasValidTimestamp = chapter.publish_at || chapter.created_at || chapter.updated_at;
+          id: chap.id,
-        return Boolean(hasValidTimestamp);
+          chap: chap.chap.toString(),
-      })
+          title: chap.title || `Chapter ${chap.chap}`,
-      .map(chapter => ({
+          updated_at: chap.updated_at,
-        id: chapter.hid,
-        chap: chapter.chap,
-        title: chapter.title || `Chapter ${chapter.chap}`,
-        updated_at: chapter.publish_at || chapter.created_at || chapter.updated_at,
           md_comics: {
             title: comicTitle,
             slug: slug,
             md_covers: covers
           }
-      }));
+        });
+      } catch (e) {
+        console.error(`Error processing chapter ${chap.id} for ${hid}:`, e);
+      }
+    }
 
     console.log(`Successfully processed ${chapters.length} chapters for ${comicTitle}`);
     return chapters;
@@ -251,88 +243,139 @@ export async function getChaptersByHid(
   }
 }
 
+// Add deduplication function
+function deduplicateChapters(chapters: ChapterDetail[]): ChapterDetail[] {
+  const seen = new Set<string>();
+  return chapters.filter(chapter => {
+    const key = `${chapter.md_comics.slug}-${chapter.chap}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
 export async function getChaptersForSlugs(
   slugs: string[], 
   lang: string = 'en'
 ): Promise<ChapterDetail[]> {
-  console.log(`Fetching chapters for slugs:`, slugs, `in language: ${lang}`);
+  console.log(`Fetching chapters for ${slugs.length} comics`);
-  const chapters: ChapterDetail[] = [];
   const redis = await getRedisClient();
-  
-  // Cache results for 1 hour (3600 seconds)
   const CACHE_TTL = 60 * 60;
   
-  for (const slug of slugs) {
+  const uniqueSlugs = [...new Set(slugs)];
-    console.log(`Processing slug: ${slug}`);
+  const pipeline = redis.pipeline();
+
+  // Add cache key validation
+  uniqueSlugs.forEach(slug => {
+    if (!slug.match(/^[a-z0-9-]+$/)) {
+      console.error(`Invalid slug format: ${slug}`);
+    }
     const cacheKey = `chapters:${slug}:${lang}`;
+    pipeline.get(cacheKey);
+  });
 
+  const [cacheResults, slugsToFetch] = await Promise.all([
+    pipeline.exec(),
+    validateSlugs(uniqueSlugs) // Add this new function
+  ]);
+
+  const cachedChapters: ChapterDetail[] = [];
+  cacheResults.forEach(([err, result], index) => {
+    if (!err && result) {
       try {
-      // Try to get cached chapters
+        const parsed = JSON.parse(result);
-      const cachedChapters = await redis.get(cacheKey);
+        if (Array.isArray(parsed)) {
-      if (cachedChapters) {
+          cachedChapters.push(...parsed);
-        console.log(`Using cached chapters for ${slug}`);
-        chapters.push(...JSON.parse(cachedChapters));
-        continue;
         }
-    } catch (error) {
+      } catch (e) {
-      console.error('Redis cache read error:', error);
+        console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e);
       }
+    }
+  });
 
+  // Add progress tracking
+  console.log(`Cache hits: ${cachedChapters.length} chapters from ${slugs.length - slugsToFetch.length} comics`);
+  console.log(`Fetching ${slugsToFetch.length} comics from API`);
+
+  const fetchPromises = slugsToFetch.map(async (slug) => {
+    const timer = Date.now();
+    try {
       const comicDetail = await getComicBySlug(slug);
       
-    if (!comicDetail) {
+      if (!comicDetail?.comic?.hid) {
-      console.error(`Failed to get comic details for slug: ${slug}`);
+        console.error(`❌ No HID found for ${slug}`);
-      continue;
+        return [];
       }
       
-    if (!comicDetail.comic?.hid) {
+      const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, {
-      console.error(`No HID found for slug ${slug}`, comicDetail);
-      continue;
-    }
-    
-    console.log(`Found HID ${comicDetail.comic.hid} for slug ${slug}`);
-    const comicChapters = await getChaptersByHid(
-      comicDetail.comic.hid,
-      slug,
-      {
         limit: 5,
         lang: lang
+      });
+      
+      console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`);
+      return chapters;
+    } catch (error) {
+      console.error(`🚨 Error processing ${slug}:`, error);
+      return [];
     }
-    );
+  });
-    console.log(`Found ${comicChapters.length} chapters for ${slug}`);
 
-    if (comicChapters.length > 0) {
+  const liveChapters = (await Promise.all(fetchPromises)).flat();
-      chapters.push(...comicChapters);
   
-      // Cache the results
+  // Cache new results in batch
-      try {
+  const cachePipeline = redis.pipeline();
-        await redis.set(
+  const chaptersBySlug: Record<string, ChapterDetail[]> = {};
+
+  liveChapters.forEach(chapter => {
+    const slug = chapter.md_comics.slug;
+    if (!chaptersBySlug[slug]) {
+      chaptersBySlug[slug] = [];
+    }
+    chaptersBySlug[slug].push(chapter);
+  });
+
+  Object.entries(chaptersBySlug).forEach(([slug, chaps]) => {
+    const cacheKey = `chapters:${slug}:${lang}`;
+    cachePipeline.set(
       cacheKey, 
-          JSON.stringify(comicChapters),
+      JSON.stringify(chaps), 
       'EX', 
       CACHE_TTL
     );
-        console.log(`Cached chapters for ${slug}`);
-      } catch (error) {
-        console.error('Redis cache write error:', error);
-      }
-    }
-  }
-  
-  // Sort by chapter number (descending) first, then by date
-  const sortedChapters = chapters.sort((a, b) => {
-    // Convert chapter numbers to floats for proper numerical sorting
-    const chapA = parseFloat(a.chap);
-    const chapB = parseFloat(b.chap);
-    
-    if (chapB !== chapA) {
-      return chapB - chapA; // Sort by chapter number first
-    }
-    
-    // If chapters are the same, sort by date
-    return new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime();
   });
 
-  console.log(`Total chapters found: ${sortedChapters.length}`);
+  await cachePipeline.exec();
-  return sortedChapters;
+
+  // Combine and sort all chapters
+  const allChapters = [...cachedChapters, ...liveChapters];
+  const deduplicatedChapters = deduplicateChapters(allChapters);
+  return deduplicatedChapters.sort((a, b) => 
+    parseFloat(b.chap) - parseFloat(a.chap) ||
+    new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime()
+  );
+}
+
+// Add this new validation function
+async function validateSlugs(slugs: string[]): Promise<string[]> {
+  const validSlugs: string[] = [];
+  const invalidSlugs: string[] = [];
+  
+  await Promise.all(slugs.map(async (slug) => {
+    try {
+      const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1);
+      if (response.ok) {
+        validSlugs.push(slug);
+      } else {
+        invalidSlugs.push(slug);
+      }
+    } catch {
+      invalidSlugs.push(slug);
+    }
+  }));
+  
+  if (invalidSlugs.length > 0) {
+    console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`);
+  }
+  
+  return validSlugs;
 }

src/lib/db.ts

@@ -2,103 +2,113 @@ import Redis from 'ioredis';
 import type { FeedMapping } from './types';
 
 const REDIS_URL = import.meta.env.REDIS_URL || process.env.REDIS_URL;
-let redis: Redis | null = null;
+let redisClient: Redis | null = null;
-let redisRetryCount = 0;
-const MAX_RETRIES = 3;
 const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
+const MAX_RETRIES = 5;
+const BASE_DELAY_MS = 100;
 
-try {
+// Improved connection handling with exponential backoff
-  console.log('Environment check:', {
+async function createRedisClient(): Promise<Redis> {
-    hasEnvVar: !!REDIS_URL,
+  if (!REDIS_URL) throw new Error('REDIS_URL environment variable is not set');
-    envVarStart: REDIS_URL?.substring(0, 20) + '...',
-    nodeEnv: process.env.NODE_ENV
-  });
   
-  if (!REDIS_URL) {
+  return new Redis(REDIS_URL, {
-    throw new Error('REDIS_URL environment variable is not set');
+    retryStrategy: (times) => {
-  }
+      if (times >= MAX_RETRIES) return null;
-
+      const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000);
-  console.log('Attempting to connect to Redis...');
+      console.log(`Redis retry attempt ${times}, next try in ${delay}ms`);
-  
+      return delay + Math.random() * 200; // Add jitter
-  redis = new Redis(REDIS_URL, {
-    retryStrategy(times) {
-      const delay = Math.min(times * 50, 2000);
-      console.log(`Retry attempt ${times} with delay ${delay}ms`);
-      return delay;
     },
-    maxRetriesPerRequest: 2,
+    maxRetriesPerRequest: 1,
+    enableOfflineQueue: false,
     enableReadyCheck: true,
     reconnectOnError: (err) => {
-      const targetError = 'READONLY';
+      return err.message.includes('READONLY');
-      if (err.message.includes(targetError)) {
-        return true;
-      }
-      return false;
     }
   });
+}
 
-  redis.on('error', (error) => {
+// Singleton pattern with connection pooling
-    console.error('Redis connection error:', error.message);
+export async function getRedisClient(): Promise<Redis> {
+  if (redisClient?.status === 'ready') {
+    // Validate connection with a quick ping
+    try {
+      await redisClient.ping();
+      return redisClient;
+    } catch {
+      // Fall through to reconnect
+    }
+  }
+
+  console.log('Initializing new Redis connection...');
+  redisClient = await createRedisClient();
+  
+  // Add event listeners for better monitoring
+  redisClient
+    .on('ready', () => console.log('Redis connection ready'))
+    .on('error', (err) => console.error('Redis error:', err))
+    .on('reconnecting', () => console.log('Redis reconnecting'))
+    .on('end', () => console.log('Redis connection closed'));
+
+  return redisClient;
+}
+
+// Batch operations for better performance
+export async function storeFeedMappings(mappings: Array<{ feedId: string, slugs: string[], lang: string }>): Promise<void> {
+  const redis = await getRedisClient();
+  const pipeline = redis.pipeline();
+  
+  mappings.forEach(({ feedId, slugs, lang }) => {
+    const mapping = {
+      slugs,
+      lang,
+      created_at: new Date().toISOString()
+    };
+    pipeline.set(feedId, JSON.stringify(mapping), 'EX', DEFAULT_CACHE_TTL);
   });
 
-  redis.on('connect', () => {
+  await pipeline.exec();
-    console.log('Successfully connected to Redis');
+}
-  });
 
-  redis.on('ready', () => {
+// Update warmChapterCache to prevent empty cache entries
-    console.log('Redis client is ready');
+export async function warmChapterCache(slugs: string[], lang: string): Promise<void> {
-  });
+  const redis = await getRedisClient();
+  const needsCache = await checkCacheStatus(slugs, lang);
   
-  redis.on('reconnecting', () => {
+  if (needsCache.length > 0) {
-    console.log('Redis client is reconnecting');
+    console.log(`Pre-warming cache for ${needsCache.length} comics`);
-  });
+    const pipeline = redis.pipeline();
     
-} catch (error) {
+    // Replace empty array with actual chapter fetching
-  console.error('Failed to initialize Redis:', error);
+    await Promise.all(needsCache.map(async slug => {
-  if (error instanceof Error) {
+      try {
-    console.error('Error details:', error.message);
+        const chapters = await getChaptersForSlugs([slug], lang);
-    console.error('Stack trace:', error.stack);
+        const cacheKey = `chapters:${slug}:${lang}`;
+        pipeline.set(cacheKey, JSON.stringify(chapters), 'EX', 60 * 60);
+      } catch (error) {
+        console.error(`Failed to warm cache for ${slug}:`, error);
+      }
+    }));
+    
+    await pipeline.exec();
   }
 }
 
-export async function getRedisClient(): Promise<Redis> {
+// Add batch cache checking
-  if (redis?.status === 'ready') return redis;
+export async function checkCacheStatus(slugs: string[], lang: string): Promise<string[]> {
+  const redis = await getRedisClient();
+  const pipeline = redis.pipeline();
   
-  if (redisRetryCount >= MAX_RETRIES) {
+  slugs.forEach(slug => {
-    throw new Error('Failed to connect to Redis after multiple attempts');
+    const cacheKey = `chapters:${slug}:${lang}`;
-  }
+    pipeline.exists(cacheKey);
-  
-  try {
-    redis = new Redis(REDIS_URL, {
-      retryStrategy(times) {
-        const delay = Math.min(times * 50, 2000);
-        console.log(`Retry attempt ${times} with delay ${delay}ms`);
-        return delay;
-      },
-      maxRetriesPerRequest: 2,
-      enableReadyCheck: true,
-      reconnectOnError: (err) => {
-        const targetError = 'READONLY';
-        if (err.message.includes(targetError)) {
-          return true;
-        }
-        return false;
-      }
   });
-    redisRetryCount = 0;
+
-    return redis;
+  const results = await pipeline.exec();
-  } catch (error) {
+  return slugs.filter((_, index) => results![index][1] === 0);
-    redisRetryCount++;
-    throw error;
-  }
 }
 
 export async function storeFeedMapping(feedId: string, slugs: string[], lang: string): Promise<void> {
-  if (!redis) {
+  const redis = await getRedisClient();
-    console.error('Redis client not initialized');
-    throw new Error('Storage system unavailable');
-  }
-
   const mapping = {
     slugs,
     lang,
@@ -119,10 +129,7 @@ export async function storeFeedMapping(feedId: string, slugs: string[], lang: st
 }
 
 export async function getFeedMapping(feedId: string): Promise<FeedMapping | null> {
-  if (!redis) {
+  const redis = await getRedisClient();
-    console.error('Redis client not initialized');
-    throw new Error('Storage system unavailable');
-  }
   
   try {
     const mapping = await redis.get(feedId);

src/pages/api/validate-slugs.ts

@@ -0,0 +1,17 @@
+import type { APIRoute } from "astro";
+import { validateSlugs } from "../../lib/api";
+
+export const POST: APIRoute = async ({ request }) => {
+  try {
+    const { slugs } = await request.json();
+    const validSlugs = await validateSlugs(slugs);
+    
+    return new Response(JSON.stringify({ validSlugs }), {
+      status: 200,
+      headers: { 'Content-Type': 'application/json' }
+    });
+  } catch (error) {
+    console.error('Validation error:', error);
+    return new Response("Validation failed", { status: 500 });
+  }
+}; 

src/pages/rss/[feedId].xml.ts

@@ -1,6 +1,6 @@
 import { Feed } from 'feed';
 import { getChaptersForSlugs, isValidLanguage } from "../../lib/api";
-import { getFeedMapping } from "../../lib/db";
+import { getFeedMapping, warmChapterCache } from "../../lib/db";
 import type { APIRoute } from "astro";
 
 export const GET: APIRoute = async ({ params, request }) => {
@@ -22,6 +22,8 @@ export const GET: APIRoute = async ({ params, request }) => {
     return new Response("Invalid language", { status: 400 });
   }
 
+  await warmChapterCache(slugs, lang);
+
   const chapters = await getChaptersForSlugs(slugs, lang);
 
   if (chapters.length === 0) {