diff --git a/package-lock.json b/package-lock.json index 01d33cf..109fad9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,8 @@ "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", + "p-throttle": "^7.0.0", + "redis-modules-sdk": "^1.4.0", "tailwindcss": "^3.4.1" } }, @@ -4373,6 +4375,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-throttle": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-7.0.0.tgz", + "integrity": "sha512-aio0v+S0QVkH1O+9x4dHtD4dgCExACcL+3EtNaGqC01GBudS9ijMuUsmN8OVScyV4OOp0jqdLShZFuSlbL/AsA==", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-timeout": { "version": "6.1.4", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz", @@ -4737,6 +4750,37 @@ "node": ">=4" } }, + "node_modules/redis-modules-sdk": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/redis-modules-sdk/-/redis-modules-sdk-1.4.0.tgz", + "integrity": "sha512-HYCr3ThtyoMrR9MUGmJblPU5aua0G4sanIZHK5wz/QwhVACe3BBgIZpHeymidBaukiuC0uzKla9xJ8exRffjnA==", + "dependencies": { + "ioredis": "5.2.3" + } + }, + "node_modules/redis-modules-sdk/node_modules/ioredis": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz", + "integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.0.1", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/redis-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", diff --git a/package.json b/package.json index 15ad66e..1350193 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,8 @@ "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", + "p-throttle": "^7.0.0", + "redis-modules-sdk": "^1.4.0", "tailwindcss": "^3.4.1" } } diff --git a/src/lib/api.ts b/src/lib/api.ts index c22762a..3854e74 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -1,5 +1,6 @@ import type { Comic, Genre, ComicType, ChapterDetail } from './types'; import { getRedisClient, CHAPTER_CACHE_TTL } from './db'; +import pThrottle from 'p-throttle'; const languageMap = { "en": "English", @@ -26,6 +27,20 @@ let genreCache: Genre[] = []; const RETRY_COUNT = 3; const RETRY_DELAY = 1000; +// Rate limiting configuration - adjust these values if needed +const API_REQUESTS_PER_SECOND = 5; // Maximum requests per second +const API_CONCURRENT_REQUESTS = 3; // Maximum concurrent requests +const DELAY_AFTER_429 = 2000; // Wait 2 seconds after a rate limit hit + +// Create throttled fetch function to avoid rate limits +const throttledFetch = pThrottle({ + limit: API_REQUESTS_PER_SECOND, + interval: 1000, + concurrency: API_CONCURRENT_REQUESTS +} as any)((url: string, options: RequestInit = {}, retries = RETRY_COUNT) => { + return fetchWithRetry(url, options, retries); +}); + async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise { try { const response = await fetch(url, { @@ -37,6 +52,19 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries = }, }); + // Special handling for rate limit errors + if (response.status === 429) { + console.warn(`Rate limited on ${url}. Waiting ${DELAY_AFTER_429}ms before retry.`); + + // Wait for the specified delay + await new Promise(resolve => setTimeout(resolve, DELAY_AFTER_429)); + + // Retry with one less retry count but only if we have retries left + if (retries > 0) { + return fetchWithRetry(url, options, retries - 1); + } + } + if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } @@ -44,7 +72,10 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries = return response; } catch (error) { if (retries > 0) { - await new Promise(resolve => setTimeout(resolve, RETRY_DELAY)); + // For non-429 errors, use exponential backoff + const delay = Math.min(RETRY_DELAY * Math.pow(2, RETRY_COUNT - retries), 10000); + console.log(`Retrying after ${delay}ms. Retries left: ${retries-1}`); + await new Promise(resolve => setTimeout(resolve, delay)); return fetchWithRetry(url, options, retries - 1); } throw error; @@ -55,7 +86,8 @@ export async function fetchGenres(): Promise { if (genreCache.length > 0) return genreCache; try { - const response = await fetchWithRetry('https://api.comick.fun/genre'); + // No tachiyomi param for genre endpoint + const response = await throttledFetch('https://api.comick.fun/genre'); genreCache = await response.json(); return genreCache; } catch (error) { @@ -87,12 +119,12 @@ export async function fetchComics( comicType: ComicType = 'all', page: number = 1 ): Promise { - const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}`; + const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}&tachiyomi=true`; try { console.log('Fetching comics from:', apiUrl); - const response = await fetchWithRetry(apiUrl); + const response = await throttledFetch(apiUrl); const data = await response.json(); if (!Array.isArray(data)) { @@ -146,7 +178,7 @@ interface ChapterParams { export async function getComicBySlug(slug: string): Promise { try { - const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`); + const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`); const data = await response.json(); if (!data?.comic?.hid) { @@ -170,18 +202,19 @@ export async function getChaptersByHid( const queryParams = new URLSearchParams({ limit: params.limit?.toString() || '5', lang: params.lang || 'en', - ordering: '-created_at' + ordering: '-created_at', + tachiyomi: 'true' // Add tachiyomi parameter }); // Fetch comic details including cover - const comicResponse = await fetchWithRetry( - `https://api.comick.fun/comic/${hid}` + const comicResponse = await throttledFetch( + `https://api.comick.fun/comic/${hid}?tachiyomi=true` ); const comicData = await comicResponse.json(); const comicTitle = comicData?.comic?.title || 'Unknown Comic'; const covers = comicData?.comic?.md_covers || []; - const response = await fetchWithRetry( + const response = await throttledFetch( `https://api.comick.fun/comic/${hid}/chapters?${queryParams}` ); const data = await response.json(); @@ -364,18 +397,29 @@ export async function validateSlugs(slugs: string[]): Promise { const validSlugs: string[] = []; const invalidSlugs: string[] = []; - await Promise.all(slugs.map(async (slug) => { - try { - const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1); - if (response.ok) { - validSlugs.push(slug); - } else { + // Process in small batches to avoid rate limiting + const batchSize = 3; + for (let i = 0; i < slugs.length; i += batchSize) { + const batch = slugs.slice(i, i + batchSize); + + await Promise.all(batch.map(async (slug) => { + try { + const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`, {}, 1); + if (response.ok) { + validSlugs.push(slug); + } else { + invalidSlugs.push(slug); + } + } catch { invalidSlugs.push(slug); } - } catch { - invalidSlugs.push(slug); + })); + + // Add a small delay between batches + if (i + batchSize < slugs.length) { + await new Promise(resolve => setTimeout(resolve, 300)); } - })); + } if (invalidSlugs.length > 0) { console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`); diff --git a/src/lib/db.ts b/src/lib/db.ts index 32cea62..7eabf97 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -1,5 +1,8 @@ import Redis from "ioredis"; import type { FeedMapping } from "./types"; +import { ReJSON } from 'redis-modules-sdk'; +import * as zlib from 'zlib'; +import { promisify } from 'util'; // Configuration constants const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days @@ -55,6 +58,18 @@ const operationBuffer: RedisOperation[] = []; // Redis instance naming for logging const redisInstanceNames = new Map(); +// Add zlib for compression +const gzip = promisify(zlib.gzip); +const gunzip = promisify(zlib.gunzip); + +// Compression threshold (in bytes) +const COMPRESSION_THRESHOLD = 1024; // 1KB + +// Helper function to determine if compression should be used +function shouldCompress(data: string): boolean { + return Buffer.byteLength(data, 'utf8') > COMPRESSION_THRESHOLD; +} + /** * Initialize Redis instance names for better logging */ @@ -75,6 +90,14 @@ initializeRedisNames(); */ const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown"; +// Add at the top of your file, after the imports +declare module "ioredis" { + interface Redis { + json_set(key: string, path: string, json: any): Promise; + json_get(key: string, ...paths: string[]): Promise; + } +} + /** * Create a new Redis client with optimized settings */ @@ -455,15 +478,37 @@ export async function storeFeedMapping( const stringValue = JSON.stringify(mapping); try { - // First write to primary/active client - const redis = await getRedisClient(); // This already handles failover + // Get Redis client + const redis = await getRedisClient(); const activeUrl = redis.options.host || ''; - await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); - console.log(`Successfully stored mapping for feed ${feedId}`); + // Check if compression should be used + if (shouldCompress(stringValue)) { + console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`); + const compressed = await gzip(stringValue); + const base64Compressed = compressed.toString('base64'); + + // Store with compression flag + await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`); + + // Replicate to other instances + const replicas = await getAvailableReplicas(activeUrl); + for (const replica of replicas) { + await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + } + } else { + // Use standard set operations without compression + await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); + + // Replicate to other instances + const replicas = await getAvailableReplicas(activeUrl); + for (const replica of replicas) { + await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); + } + } - // Immediately replicate to other instances - await replicateOperation(activeUrl, feedId, stringValue, DEFAULT_CACHE_TTL); + console.log(`Successfully stored mapping for feed ${feedId}`); } catch (error) { console.error(`Error storing feed mapping for ${feedId}:`, error); throw error; @@ -477,29 +522,75 @@ export async function getFeedMapping( feedId: string ): Promise { try { - const redis = await getRedisClient(); // This already handles failover - const mapping = await redis.get(feedId); + // Try primary first + const redis = await getRedisClient(); - if (!mapping) return null; + // Check for compressed version first + const compressedKey = `${feedId}:compressed`; + const compressedData = await redis.get(compressedKey); + + if (compressedData) { + console.log(`Found compressed data for ${feedId}`); + try { + // Decompress and parse + const buffer = Buffer.from(compressedData, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + // Reset expiration + await redis.expire(compressedKey, DEFAULT_CACHE_TTL); + + return { + slugs: data.slugs, + lang: data.lang + }; + } catch (error) { + console.error(`Error decompressing data for ${feedId}:`, error); + // Fall through to try uncompressed version + } + } + + // If no compressed version, try regular key + let jsonString = await redis.get(feedId); + + // If not found in primary but we have replicas, check them + if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) { + console.log(`Key ${feedId} not found in primary, checking replicas...`); + + // Try each replica + for (const [url, replicaClient] of replicaClients.entries()) { + if (replicaClient.status === "ready") { + try { + jsonString = await replicaClient.get(feedId); + if (jsonString) { + console.log(`Found key ${feedId} in replica ${getRedisName(url)}`); + + // Sync this key back to primary for future requests + if (primaryClient && primaryClient.status === "ready") { + primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL) + .catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err)); + } + + break; + } + } catch (error) { + console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error); + } + } + } + } + + if (!jsonString) return null; // Reset expiration on access - const activeUrl = redis.options.host || ''; await redis.expire(feedId, DEFAULT_CACHE_TTL); - // Also reset TTL on other replicas - replicateOperation(activeUrl, feedId, mapping, DEFAULT_CACHE_TTL) - .catch(err => console.error(`Failed to reset TTL on replicas for ${feedId}:`, err)); - - try { - const data = JSON.parse(mapping); - return { - slugs: data.slugs, - lang: data.lang - }; - } catch (e) { - console.error(`Error parsing feed mapping for ${feedId}:`, e); - return null; - } + // Parse the result + const data = JSON.parse(jsonString as string); + return { + slugs: data.slugs, + lang: data.lang + }; } catch (error) { console.error(`Error retrieving feed mapping for ${feedId}:`, error); return null; @@ -571,6 +662,143 @@ export async function warmChapterCache( } } +// Add a new function for handling large feed mappings with compression +export async function storeLargeFeedMapping( + feedId: string, + slugs: string[], + lang: string +): Promise { + const mapping = { + slugs, + lang, + created_at: new Date().toISOString() + }; + + const jsonString = JSON.stringify(mapping); + + try { + const redis = await getRedisClient(); + const activeUrl = redis.options.host || ''; + + // Use compression for large mappings + if (shouldCompress(jsonString)) { + const compressed = await gzip(jsonString); + const base64Compressed = compressed.toString('base64'); + + // Store with compression flag + await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + + // Replicate to other instances + await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL); + + console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`); + } else { + // Use normal storage for small mappings + await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL); + await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL); + console.log(`Successfully stored mapping for feed ${feedId}`); + } + } catch (error) { + console.error(`Error storing feed mapping for ${feedId}:`, error); + throw error; + } +} + +// Batch get multiple feed mappings at once +export async function getBatchFeedMappings( + feedIds: string[] +): Promise> { + if (!feedIds.length) return {}; + + try { + const redis = await getRedisClient(); + const pipeline = redis.pipeline(); + + // Get all mappings in a single round trip + feedIds.forEach(id => { + pipeline.get(id); + // Also check for compressed version + pipeline.exists(`${id}:compressed`); + }); + + const results = await pipeline.exec(); + if (!results) return {}; + + // Process results + const mappings: Record = {}; + const ttlUpdates = redis.pipeline(); + + for (let i = 0; i < feedIds.length; i++) { + const id = feedIds[i]; + const mappingResult = results[i * 2]; + const compressedResult = results[i * 2 + 1]; + + if (mappingResult[0]) { + console.error(`Error retrieving mapping for ${id}:`, mappingResult[0]); + mappings[id] = null; + continue; + } + + if (compressedResult[0]) { + console.error(`Error checking compressed mapping for ${id}:`, compressedResult[0]); + mappings[id] = null; + continue; + } + + // Check if we have a compressed version + if (compressedResult[1]) { + // Fetch and decompress + const compressedData = await redis.get(`${id}:compressed`); + if (compressedData) { + try { + const buffer = Buffer.from(compressedData, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString() as string); + + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL + ttlUpdates.expire(`${id}:compressed`, DEFAULT_CACHE_TTL); + } catch (e) { + console.error(`Error decompressing data for ${id}:`, e); + mappings[id] = null; + } + } else { + mappings[id] = null; + } + } else if (mappingResult[1]) { + // Process regular mapping + try { + const data = JSON.parse(mappingResult[1] as string); + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL + ttlUpdates.expire(id, DEFAULT_CACHE_TTL); + } catch (e) { + console.error(`Error parsing feed mapping for ${id}:`, e); + mappings[id] = null; + } + } else { + mappings[id] = null; + } + } + + // Execute all TTL updates + await ttlUpdates.exec(); + + return mappings; + } catch (error) { + console.error(`Error retrieving batch feed mappings:`, error); + return {}; + } +} + // Initialize connections eagerly to establish connections as soon as possible if (!import.meta.env.SSR) { initializeAllClients()