this will be slower

package-lock.json

@@ -17,6 +17,8 @@
         "astro": "^5.5.4",
         "feed": "^4.2.2",
         "ioredis": "^5.6.0",
+        "p-throttle": "^7.0.0",
+        "redis-modules-sdk": "^1.4.0",
         "tailwindcss": "^3.4.1"
       }
     },
@@ -4373,6 +4375,17 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/p-throttle": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-7.0.0.tgz",
+      "integrity": "sha512-aio0v+S0QVkH1O+9x4dHtD4dgCExACcL+3EtNaGqC01GBudS9ijMuUsmN8OVScyV4OOp0jqdLShZFuSlbL/AsA==",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/p-timeout": {
       "version": "6.1.4",
       "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz",
@@ -4737,6 +4750,37 @@
         "node": ">=4"
       }
     },
+    "node_modules/redis-modules-sdk": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/redis-modules-sdk/-/redis-modules-sdk-1.4.0.tgz",
+      "integrity": "sha512-HYCr3ThtyoMrR9MUGmJblPU5aua0G4sanIZHK5wz/QwhVACe3BBgIZpHeymidBaukiuC0uzKla9xJ8exRffjnA==",
+      "dependencies": {
+        "ioredis": "5.2.3"
+      }
+    },
+    "node_modules/redis-modules-sdk/node_modules/ioredis": {
+      "version": "5.2.3",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz",
+      "integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==",
+      "dependencies": {
+        "@ioredis/commands": "^1.1.1",
+        "cluster-key-slot": "^1.1.0",
+        "debug": "^4.3.4",
+        "denque": "^2.0.1",
+        "lodash.defaults": "^4.2.0",
+        "lodash.isarguments": "^3.1.0",
+        "redis-errors": "^1.2.0",
+        "redis-parser": "^3.0.0",
+        "standard-as-callback": "^2.1.0"
+      },
+      "engines": {
+        "node": ">=12.22.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/ioredis"
+      }
+    },
     "node_modules/redis-parser": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",

package.json

@@ -20,6 +20,8 @@
     "astro": "^5.5.4",
     "feed": "^4.2.2",
     "ioredis": "^5.6.0",
+    "p-throttle": "^7.0.0",
+    "redis-modules-sdk": "^1.4.0",
     "tailwindcss": "^3.4.1"
   }
 }

src/lib/api.ts

@@ -1,5 +1,6 @@
 import type { Comic, Genre, ComicType, ChapterDetail } from './types';
 import { getRedisClient, CHAPTER_CACHE_TTL } from './db';
+import pThrottle from 'p-throttle';
 
 const languageMap = {
   "en": "English",
@@ -26,6 +27,20 @@ let genreCache: Genre[] = [];
 const RETRY_COUNT = 3;
 const RETRY_DELAY = 1000;
 
+// Rate limiting configuration - adjust these values if needed
+const API_REQUESTS_PER_SECOND = 5;  // Maximum requests per second
+const API_CONCURRENT_REQUESTS = 3;  // Maximum concurrent requests
+const DELAY_AFTER_429 = 2000;       // Wait 2 seconds after a rate limit hit
+
+// Create throttled fetch function to avoid rate limits
+const throttledFetch = pThrottle({
+  limit: API_REQUESTS_PER_SECOND,
+  interval: 1000,
+  concurrency: API_CONCURRENT_REQUESTS
+} as any)((url: string, options: RequestInit = {}, retries = RETRY_COUNT) => {
+  return fetchWithRetry(url, options, retries);
+});
+
 async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise<Response> {
   try {
     const response = await fetch(url, {
@@ -37,6 +52,19 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries =
       },
     });
 
+    // Special handling for rate limit errors
+    if (response.status === 429) {
+      console.warn(`Rate limited on ${url}. Waiting ${DELAY_AFTER_429}ms before retry.`);
+      
+      // Wait for the specified delay
+      await new Promise(resolve => setTimeout(resolve, DELAY_AFTER_429));
+      
+      // Retry with one less retry count but only if we have retries left
+      if (retries > 0) {
+        return fetchWithRetry(url, options, retries - 1);
+      }
+    }
+
     if (!response.ok) {
       throw new Error(`HTTP error! status: ${response.status}`);
     }
@@ -44,7 +72,10 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries =
     return response;
   } catch (error) {
     if (retries > 0) {
-      await new Promise(resolve => setTimeout(resolve, RETRY_DELAY));
+      // For non-429 errors, use exponential backoff
+      const delay = Math.min(RETRY_DELAY * Math.pow(2, RETRY_COUNT - retries), 10000);
+      console.log(`Retrying after ${delay}ms. Retries left: ${retries-1}`);
+      await new Promise(resolve => setTimeout(resolve, delay));
       return fetchWithRetry(url, options, retries - 1);
     }
     throw error;
@@ -55,7 +86,8 @@ export async function fetchGenres(): Promise<Genre[]> {
   if (genreCache.length > 0) return genreCache;
 
   try {
-    const response = await fetchWithRetry('https://api.comick.fun/genre');
+    // No tachiyomi param for genre endpoint
+    const response = await throttledFetch('https://api.comick.fun/genre');
     genreCache = await response.json();
     return genreCache;
   } catch (error) {
@@ -87,12 +119,12 @@ export async function fetchComics(
   comicType: ComicType = 'all',
   page: number = 1
 ): Promise<Comic[]> {
-  const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}`;
+  const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}&tachiyomi=true`;
   
   try {
     console.log('Fetching comics from:', apiUrl);
     
-    const response = await fetchWithRetry(apiUrl);
+    const response = await throttledFetch(apiUrl);
     const data = await response.json();
 
     if (!Array.isArray(data)) {
@@ -146,7 +178,7 @@ interface ChapterParams {
 
 export async function getComicBySlug(slug: string): Promise<ComicDetail | null> {
   try {
-    const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`);
+    const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`);
     const data = await response.json();
     
     if (!data?.comic?.hid) {
@@ -170,18 +202,19 @@ export async function getChaptersByHid(
     const queryParams = new URLSearchParams({
       limit: params.limit?.toString() || '5',
       lang: params.lang || 'en',
-      ordering: '-created_at'
+      ordering: '-created_at',
+      tachiyomi: 'true' // Add tachiyomi parameter
     });
     
     // Fetch comic details including cover
-    const comicResponse = await fetchWithRetry(
+    const comicResponse = await throttledFetch(
-      `https://api.comick.fun/comic/${hid}`
+      `https://api.comick.fun/comic/${hid}?tachiyomi=true`
     );
     const comicData = await comicResponse.json();
     const comicTitle = comicData?.comic?.title || 'Unknown Comic';
     const covers = comicData?.comic?.md_covers || [];
     
-    const response = await fetchWithRetry(
+    const response = await throttledFetch(
       `https://api.comick.fun/comic/${hid}/chapters?${queryParams}`
     );
     const data = await response.json();
@@ -364,9 +397,14 @@ export async function validateSlugs(slugs: string[]): Promise<string[]> {
   const validSlugs: string[] = [];
   const invalidSlugs: string[] = [];
   
-  await Promise.all(slugs.map(async (slug) => {
+  // Process in small batches to avoid rate limiting
+  const batchSize = 3;
+  for (let i = 0; i < slugs.length; i += batchSize) {
+    const batch = slugs.slice(i, i + batchSize);
+    
+    await Promise.all(batch.map(async (slug) => {
       try {
-      const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1);
+        const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`, {}, 1);
         if (response.ok) {
           validSlugs.push(slug);
         } else {
@@ -377,6 +415,12 @@ export async function validateSlugs(slugs: string[]): Promise<string[]> {
       }
     }));
     
+    // Add a small delay between batches
+    if (i + batchSize < slugs.length) {
+      await new Promise(resolve => setTimeout(resolve, 300));
+    }
+  }
+  
   if (invalidSlugs.length > 0) {
     console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`);
   }

src/lib/db.ts

@@ -1,5 +1,8 @@
 import Redis from "ioredis";
 import type { FeedMapping } from "./types";
+import { ReJSON } from 'redis-modules-sdk';
+import * as zlib from 'zlib';
+import { promisify } from 'util';
 
 // Configuration constants
 const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
@@ -55,6 +58,18 @@ const operationBuffer: RedisOperation[] = [];
 // Redis instance naming for logging
 const redisInstanceNames = new Map<string, string>();
 
+// Add zlib for compression
+const gzip = promisify(zlib.gzip);
+const gunzip = promisify(zlib.gunzip);
+
+// Compression threshold (in bytes)
+const COMPRESSION_THRESHOLD = 1024; // 1KB
+
+// Helper function to determine if compression should be used
+function shouldCompress(data: string): boolean {
+  return Buffer.byteLength(data, 'utf8') > COMPRESSION_THRESHOLD;
+}
+
 /**
  * Initialize Redis instance names for better logging
  */
@@ -75,6 +90,14 @@ initializeRedisNames();
  */
 const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown";
 
+// Add at the top of your file, after the imports
+declare module "ioredis" {
+  interface Redis {
+    json_set(key: string, path: string, json: any): Promise<string>;
+    json_get(key: string, ...paths: string[]): Promise<string>;
+  }
+}
+
 /**
  * Create a new Redis client with optimized settings
  */
@@ -455,15 +478,37 @@ export async function storeFeedMapping(
   const stringValue = JSON.stringify(mapping);
   
   try {
-    // First write to primary/active client
+    // Get Redis client
-    const redis = await getRedisClient(); // This already handles failover
+    const redis = await getRedisClient();
     const activeUrl = redis.options.host || '';
     
-    await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
+    // Check if compression should be used
-    console.log(`Successfully stored mapping for feed ${feedId}`);
+    if (shouldCompress(stringValue)) {
+      console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`);
+      const compressed = await gzip(stringValue);
+      const base64Compressed = compressed.toString('base64');
       
-    // Immediately replicate to other instances
+      // Store with compression flag
-    await replicateOperation(activeUrl, feedId, stringValue, DEFAULT_CACHE_TTL);
+      await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
+      console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`);
+      
+      // Replicate to other instances
+      const replicas = await getAvailableReplicas(activeUrl);
+      for (const replica of replicas) {
+        await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
+      }
+    } else {
+      // Use standard set operations without compression
+      await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
+      
+      // Replicate to other instances
+      const replicas = await getAvailableReplicas(activeUrl);
+      for (const replica of replicas) {
+        await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
+      }
+    }
+    
+    console.log(`Successfully stored mapping for feed ${feedId}`);
   } catch (error) {
     console.error(`Error storing feed mapping for ${feedId}:`, error);
     throw error;
@@ -477,29 +522,75 @@ export async function getFeedMapping(
   feedId: string
 ): Promise<FeedMapping | null> {
   try {
-    const redis = await getRedisClient(); // This already handles failover
+    // Try primary first
-    const mapping = await redis.get(feedId);
+    const redis = await getRedisClient();
     
-    if (!mapping) return null;
+    // Check for compressed version first
-    
+    const compressedKey = `${feedId}:compressed`;
-    // Reset expiration on access
+    const compressedData = await redis.get(compressedKey);
-    const activeUrl = redis.options.host || '';
-    await redis.expire(feedId, DEFAULT_CACHE_TTL);
-    
-    // Also reset TTL on other replicas
-    replicateOperation(activeUrl, feedId, mapping, DEFAULT_CACHE_TTL)
-      .catch(err => console.error(`Failed to reset TTL on replicas for ${feedId}:`, err));
     
+    if (compressedData) {
+      console.log(`Found compressed data for ${feedId}`);
       try {
-      const data = JSON.parse(mapping);
+        // Decompress and parse
+        const buffer = Buffer.from(compressedData, 'base64');
+        const decompressed = await gunzip(buffer);
+        const data = JSON.parse(decompressed.toString());
+        
+        // Reset expiration
+        await redis.expire(compressedKey, DEFAULT_CACHE_TTL);
+        
         return {
           slugs: data.slugs,
           lang: data.lang
         };
-    } catch (e) {
+      } catch (error) {
-      console.error(`Error parsing feed mapping for ${feedId}:`, e);
+        console.error(`Error decompressing data for ${feedId}:`, error);
-      return null;
+        // Fall through to try uncompressed version
       }
+    }
+    
+    // If no compressed version, try regular key
+    let jsonString = await redis.get(feedId);
+    
+    // If not found in primary but we have replicas, check them
+    if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) {
+      console.log(`Key ${feedId} not found in primary, checking replicas...`);
+      
+      // Try each replica
+      for (const [url, replicaClient] of replicaClients.entries()) {
+        if (replicaClient.status === "ready") {
+          try {
+            jsonString = await replicaClient.get(feedId);
+            if (jsonString) {
+              console.log(`Found key ${feedId} in replica ${getRedisName(url)}`);
+              
+              // Sync this key back to primary for future requests
+              if (primaryClient && primaryClient.status === "ready") {
+                primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL)
+                  .catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err));
+              }
+              
+              break;
+            }
+          } catch (error) {
+            console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error);
+          }
+        }
+      }
+    }
+    
+    if (!jsonString) return null;
+    
+    // Reset expiration on access
+    await redis.expire(feedId, DEFAULT_CACHE_TTL);
+    
+    // Parse the result
+    const data = JSON.parse(jsonString as string);
+    return {
+      slugs: data.slugs,
+      lang: data.lang
+    };
   } catch (error) {
     console.error(`Error retrieving feed mapping for ${feedId}:`, error);
     return null;
@@ -571,6 +662,143 @@ export async function warmChapterCache(
   }
 }
 
+// Add a new function for handling large feed mappings with compression
+export async function storeLargeFeedMapping(
+  feedId: string,
+  slugs: string[],
+  lang: string
+): Promise<void> {
+  const mapping = { 
+    slugs, 
+    lang, 
+    created_at: new Date().toISOString() 
+  };
+  
+  const jsonString = JSON.stringify(mapping);
+  
+  try {
+    const redis = await getRedisClient();
+    const activeUrl = redis.options.host || '';
+    
+    // Use compression for large mappings
+    if (shouldCompress(jsonString)) {
+      const compressed = await gzip(jsonString);
+      const base64Compressed = compressed.toString('base64');
+      
+      // Store with compression flag
+      await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
+      
+      // Replicate to other instances
+      await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL);
+      
+      console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`);
+    } else {
+      // Use normal storage for small mappings
+      await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL);
+      await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL);
+      console.log(`Successfully stored mapping for feed ${feedId}`);
+    }
+  } catch (error) {
+    console.error(`Error storing feed mapping for ${feedId}:`, error);
+    throw error;
+  }
+}
+
+// Batch get multiple feed mappings at once
+export async function getBatchFeedMappings(
+  feedIds: string[]
+): Promise<Record<string, FeedMapping | null>> {
+  if (!feedIds.length) return {};
+  
+  try {
+    const redis = await getRedisClient();
+    const pipeline = redis.pipeline();
+    
+    // Get all mappings in a single round trip
+    feedIds.forEach(id => {
+      pipeline.get(id);
+      // Also check for compressed version
+      pipeline.exists(`${id}:compressed`);
+    });
+    
+    const results = await pipeline.exec();
+    if (!results) return {};
+    
+    // Process results
+    const mappings: Record<string, FeedMapping | null> = {};
+    const ttlUpdates = redis.pipeline();
+    
+    for (let i = 0; i < feedIds.length; i++) {
+      const id = feedIds[i];
+      const mappingResult = results[i * 2];
+      const compressedResult = results[i * 2 + 1];
+      
+      if (mappingResult[0]) {
+        console.error(`Error retrieving mapping for ${id}:`, mappingResult[0]);
+        mappings[id] = null;
+        continue;
+      }
+      
+      if (compressedResult[0]) {
+        console.error(`Error checking compressed mapping for ${id}:`, compressedResult[0]);
+        mappings[id] = null;
+        continue;
+      }
+      
+      // Check if we have a compressed version
+      if (compressedResult[1]) {
+        // Fetch and decompress
+        const compressedData = await redis.get(`${id}:compressed`);
+        if (compressedData) {
+          try {
+            const buffer = Buffer.from(compressedData, 'base64');
+            const decompressed = await gunzip(buffer);
+            const data = JSON.parse(decompressed.toString() as string);
+            
+            mappings[id] = {
+              slugs: data.slugs,
+              lang: data.lang
+            };
+            
+            // Reset TTL
+            ttlUpdates.expire(`${id}:compressed`, DEFAULT_CACHE_TTL);
+          } catch (e) {
+            console.error(`Error decompressing data for ${id}:`, e);
+            mappings[id] = null;
+          }
+        } else {
+          mappings[id] = null;
+        }
+      } else if (mappingResult[1]) {
+        // Process regular mapping
+        try {
+          const data = JSON.parse(mappingResult[1] as string);
+          mappings[id] = {
+            slugs: data.slugs,
+            lang: data.lang
+          };
+          
+          // Reset TTL
+          ttlUpdates.expire(id, DEFAULT_CACHE_TTL);
+        } catch (e) {
+          console.error(`Error parsing feed mapping for ${id}:`, e);
+          mappings[id] = null;
+        }
+      } else {
+        mappings[id] = null;
+      }
+    }
+    
+    // Execute all TTL updates
+    await ttlUpdates.exec();
+    
+    return mappings;
+  } catch (error) {
+    console.error(`Error retrieving batch feed mappings:`, error);
+    return {};
+  }
+}
+
 // Initialize connections eagerly to establish connections as soon as possible
 if (!import.meta.env.SSR) {
   initializeAllClients()