diff --git a/package-lock.json b/package-lock.json index 01d33cf..109fad9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,8 @@ "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", + "p-throttle": "^7.0.0", + "redis-modules-sdk": "^1.4.0", "tailwindcss": "^3.4.1" } }, @@ -4373,6 +4375,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-throttle": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-7.0.0.tgz", + "integrity": "sha512-aio0v+S0QVkH1O+9x4dHtD4dgCExACcL+3EtNaGqC01GBudS9ijMuUsmN8OVScyV4OOp0jqdLShZFuSlbL/AsA==", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-timeout": { "version": "6.1.4", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz", @@ -4737,6 +4750,37 @@ "node": ">=4" } }, + "node_modules/redis-modules-sdk": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/redis-modules-sdk/-/redis-modules-sdk-1.4.0.tgz", + "integrity": "sha512-HYCr3ThtyoMrR9MUGmJblPU5aua0G4sanIZHK5wz/QwhVACe3BBgIZpHeymidBaukiuC0uzKla9xJ8exRffjnA==", + "dependencies": { + "ioredis": "5.2.3" + } + }, + "node_modules/redis-modules-sdk/node_modules/ioredis": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz", + "integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.0.1", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/redis-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", diff --git a/package.json b/package.json index 15ad66e..1350193 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,8 @@ "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", + "p-throttle": "^7.0.0", + "redis-modules-sdk": "^1.4.0", "tailwindcss": "^3.4.1" } } diff --git a/src/components/SlugInput.astro b/src/components/SlugInput.astro index 43079a1..06ba25c 100644 --- a/src/components/SlugInput.astro +++ b/src/components/SlugInput.astro @@ -1,25 +1,17 @@ --- -import { getFeedMapping } from '../lib/db'; +import { getFeedMapping } from "../lib/db"; interface Props { currentSlugs?: string[]; } const { currentSlugs = [] } = Astro.props; -import { availableLanguages, languageMap } from '../lib/api'; +import { availableLanguages, languageMap } from "../lib/api"; -const languages = [ - { value: 'en', label: 'English' }, - { value: 'ja', label: 'Japanese' }, - { value: 'ko', label: 'Korean' }, - { value: 'zh', label: 'Chinese' }, - { value: 'fr', label: 'French' }, - { value: 'es', label: 'Spanish' }, - { value: 'it', label: 'Italian' }, - { value: 'de', label: 'German' }, - { value: 'tr', label: 'Turkish' }, - { value: 'pl', label: 'Polish' }, -]; +const languages = Object.entries(languageMap).map(([value, label]) => ({ + value, + label, +})); ---
@@ -37,9 +29,11 @@ const languages = [ id="lang-select" class="px-3 sm:px-4 py-2 bg-primary rounded-lg border border-hover/30 focus:outline-none focus:border-accent/50 transition-colors text-sm sm:text-base" > - {languages.map(lang => ( - - ))} + { + languages.map((lang) => ( + + )) + }
@@ -54,8 +48,16 @@ const languages = [ id="import-feed" class="px-4 py-2 bg-secondary hover:bg-hover transition-colors rounded-lg text-sm sm:text-base flex items-center gap-2" > - - + + Import Feed @@ -73,27 +75,46 @@ const languages = [ class="px-3 py-1.5 text-sm text-red-500 hover:text-red-600 hover:bg-red-500/10 rounded-lg transition-colors flex items-center gap-2" style="display: none;" > - - + + Clear All
- {currentSlugs.map(slug => ( -
- {slug} - -
- ))} + {slug} + +
+ )) + }
@@ -123,20 +144,26 @@ const languages = [
\ No newline at end of file + diff --git a/src/lib/api.ts b/src/lib/api.ts index c22762a..c0329e2 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -1,10 +1,12 @@ import type { Comic, Genre, ComicType, ChapterDetail } from './types'; import { getRedisClient, CHAPTER_CACHE_TTL } from './db'; +import pThrottle from 'p-throttle'; const languageMap = { "en": "English", "fr": "French", "es": "Spanish", + "es-419": "Spanish (Latin America)", "it": "Italian", "pl": "Polish", "tr": "Turkish", @@ -13,7 +15,10 @@ const languageMap = { "sv": "Swedish", "ar": "Arabic", "de": "German", - "ko": "Korean" + "ko": "Korean", + "pt": "Portuguese", + "pt-br": "Portuguese (Brazil)", + "pt-pt": "Portuguese (Portugal)" } as const; const languages = new Set(Object.keys(languageMap)); @@ -26,6 +31,20 @@ let genreCache: Genre[] = []; const RETRY_COUNT = 3; const RETRY_DELAY = 1000; +// Rate limiting configuration - adjust these values if needed +const API_REQUESTS_PER_SECOND = 10; // Maximum requests per second +const API_CONCURRENT_REQUESTS = 6; // Maximum concurrent requests +const DELAY_AFTER_429 = 1000; // Wait 1 second after a rate limit hit + +// Create throttled fetch function to avoid rate limits +const throttledFetch = pThrottle({ + limit: API_REQUESTS_PER_SECOND, + interval: 1000, + concurrency: API_CONCURRENT_REQUESTS +} as any)((url: string, options: RequestInit = {}, retries = RETRY_COUNT) => { + return fetchWithRetry(url, options, retries); +}); + async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise { try { const response = await fetch(url, { @@ -37,6 +56,19 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries = }, }); + // Special handling for rate limit errors + if (response.status === 429) { + console.warn(`Rate limited on ${url}. Waiting ${DELAY_AFTER_429}ms before retry.`); + + // Wait for the specified delay + await new Promise(resolve => setTimeout(resolve, DELAY_AFTER_429)); + + // Retry with one less retry count but only if we have retries left + if (retries > 0) { + return fetchWithRetry(url, options, retries - 1); + } + } + if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } @@ -44,7 +76,10 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries = return response; } catch (error) { if (retries > 0) { - await new Promise(resolve => setTimeout(resolve, RETRY_DELAY)); + // For non-429 errors, use exponential backoff + const delay = Math.min(RETRY_DELAY * Math.pow(2, RETRY_COUNT - retries), 10000); + console.log(`Retrying after ${delay}ms. Retries left: ${retries-1}`); + await new Promise(resolve => setTimeout(resolve, delay)); return fetchWithRetry(url, options, retries - 1); } throw error; @@ -55,7 +90,8 @@ export async function fetchGenres(): Promise { if (genreCache.length > 0) return genreCache; try { - const response = await fetchWithRetry('https://api.comick.fun/genre'); + // No tachiyomi param for genre endpoint + const response = await throttledFetch('https://api.comick.fun/genre'); genreCache = await response.json(); return genreCache; } catch (error) { @@ -92,7 +128,7 @@ export async function fetchComics( try { console.log('Fetching comics from:', apiUrl); - const response = await fetchWithRetry(apiUrl); + const response = await throttledFetch(apiUrl); const data = await response.json(); if (!Array.isArray(data)) { @@ -146,7 +182,7 @@ interface ChapterParams { export async function getComicBySlug(slug: string): Promise { try { - const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`); + const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`); const data = await response.json(); if (!data?.comic?.hid) { @@ -174,14 +210,14 @@ export async function getChaptersByHid( }); // Fetch comic details including cover - const comicResponse = await fetchWithRetry( + const comicResponse = await throttledFetch( `https://api.comick.fun/comic/${hid}` ); const comicData = await comicResponse.json(); const comicTitle = comicData?.comic?.title || 'Unknown Comic'; const covers = comicData?.comic?.md_covers || []; - const response = await fetchWithRetry( + const response = await throttledFetch( `https://api.comick.fun/comic/${hid}/chapters?${queryParams}` ); const data = await response.json(); @@ -260,22 +296,25 @@ export async function getChaptersForSlugs( const uniqueSlugs = [...new Set(slugs)]; const pipeline = redis.pipeline(); - // Add cache key validation + // Check cache for all slugs uniqueSlugs.forEach(slug => { if (!slug.match(/^[a-z0-9-]+$/)) { console.error(`Invalid slug format: ${slug}`); + return; // Skip invalid slugs immediately } const cacheKey = `chapters:${slug}:${lang}`; pipeline.get(cacheKey); }); - const [cacheResults, slugsToFetch] = await Promise.all([ - pipeline.exec(), - validateSlugs(uniqueSlugs) // Add this new function - ]); - + const cacheResults = await pipeline.exec(); + + // Process cache results const cachedChapters: ChapterDetail[] = []; + const slugsToFetch: string[] = []; + + // Separate cached and uncached slugs in one pass cacheResults.forEach(([err, result], index) => { + const slug = uniqueSlugs[index]; if (!err && result) { try { const parsed = JSON.parse(result as string); @@ -283,102 +322,107 @@ export async function getChaptersForSlugs( cachedChapters.push(...parsed); } } catch (e) { - console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e); + console.error(`Cache parse error for ${slug}:`, e); + slugsToFetch.push(slug); } + } else { + slugsToFetch.push(slug); } }); - // Add progress tracking + // Skip validation if we have recent cache data const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug)); console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`); - console.log(`Fetching ${slugsToFetch.length} comics from API`); - - const fetchPromises = slugsToFetch.map(async (slug) => { - const timer = Date.now(); - try { - const comicDetail = await getComicBySlug(slug); - - if (!comicDetail?.comic?.hid) { - console.error(`❌ No HID found for ${slug}`); + + // Only fetch if we have slugs that need fetching + let liveChapters: ChapterDetail[] = []; + if (slugsToFetch.length > 0) { + console.log(`Fetching ${slugsToFetch.length} comics from API`); + + // Use Promise.all for concurrent fetching with higher concurrency + const fetchPromises = slugsToFetch.map(async (slug) => { + try { + const comicDetail = await getComicBySlug(slug); + + if (!comicDetail?.comic?.hid) { + return []; + } + + return getChaptersByHid(comicDetail.comic.hid, slug, { + limit: 5, + lang: lang + }); + } catch (error) { + console.error(`Error processing ${slug}:`, error); return []; } - - const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, { - limit: 5, - lang: lang + }); + + liveChapters = (await Promise.all(fetchPromises)).flat(); + + // Cache new results in batch if we have any + if (liveChapters.length > 0) { + const cachePipeline = redis.pipeline(); + const chaptersBySlug: Record = {}; + + liveChapters.forEach(chapter => { + const slug = chapter.md_comics.slug; + if (!chaptersBySlug[slug]) { + chaptersBySlug[slug] = []; + } + chaptersBySlug[slug].push(chapter); }); - - console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`); - return chapters; - } catch (error) { - console.error(`🚨 Error processing ${slug}:`, error); - return []; + + Object.entries(chaptersBySlug).forEach(([slug, chaps]) => { + const cacheKey = `chapters:${slug}:${lang}`; + cachePipeline.set( + cacheKey, + JSON.stringify(chaps), + 'EX', + CHAPTER_CACHE_TTL + ); + }); + + await cachePipeline.exec(); } - }); + } - const liveChapters = (await Promise.all(fetchPromises)).flat(); - - // Cache new results in batch - const cachePipeline = redis.pipeline(); - const chaptersBySlug: Record = {}; - - liveChapters.forEach(chapter => { - const slug = chapter.md_comics.slug; - if (!chaptersBySlug[slug]) { - chaptersBySlug[slug] = []; - } - chaptersBySlug[slug].push(chapter); - }); - - Object.entries(chaptersBySlug).forEach(([slug, chaps]) => { - const cacheKey = `chapters:${slug}:${lang}`; - cachePipeline.set( - cacheKey, - JSON.stringify(chaps), - 'EX', - CHAPTER_CACHE_TTL - ); - }); - - await cachePipeline.exec(); - - // Combine and sort all chapters - const allChapters = [...cachedChapters, ...liveChapters]; - const deduplicatedChapters = deduplicateChapters(allChapters); - return deduplicatedChapters.sort((a, b) => { - // Ensure the created_at fields are valid dates - const dateA = new Date(a.created_at).getTime(); - const dateB = new Date(b.created_at).getTime(); - - // First try to sort by chapter number - const chapDiff = parseFloat(b.chap) - parseFloat(a.chap); - if (!isNaN(chapDiff)) return chapDiff; - - // Fall back to date if chapter numbers are equal or invalid - return dateB - dateA; - }); + // Combine and deduplicate + return deduplicateChapters([...cachedChapters, ...liveChapters]); } -// Add export to the validation function +// Replace validateSlugs with a more efficient implementation export async function validateSlugs(slugs: string[]): Promise { + // Skip validation in production for cached items + if (process.env.NODE_ENV === 'production') { + return slugs; + } + const validSlugs: string[] = []; const invalidSlugs: string[] = []; - await Promise.all(slugs.map(async (slug) => { - try { - const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1); - if (response.ok) { - validSlugs.push(slug); - } else { + // Increase batch size for faster processing + const batchSize = 5; + for (let i = 0; i < slugs.length; i += batchSize) { + const batch = slugs.slice(i, i + batchSize); + + await Promise.all(batch.map(async (slug) => { + try { + // Reduced retries (1 instead of 3) and removed tachiyomi parameter + const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`, {}, 1); + if (response.ok) { + validSlugs.push(slug); + } else { + invalidSlugs.push(slug); + } + } catch { invalidSlugs.push(slug); } - } catch { - invalidSlugs.push(slug); - } - })); + })); + } if (invalidSlugs.length > 0) { - console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`); + console.warn(`Skipping invalid slugs: ${invalidSlugs.length} items`); } return validSlugs; diff --git a/src/lib/db.ts b/src/lib/db.ts index 32cea62..f1fe845 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -1,5 +1,8 @@ import Redis from "ioredis"; import type { FeedMapping } from "./types"; +import { ReJSON } from 'redis-modules-sdk'; +import * as zlib from 'zlib'; +import { promisify } from 'util'; // Configuration constants const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days @@ -55,6 +58,18 @@ const operationBuffer: RedisOperation[] = []; // Redis instance naming for logging const redisInstanceNames = new Map(); +// Add zlib for compression +const gzip = promisify(zlib.gzip); +const gunzip = promisify(zlib.gunzip); + +// Compression threshold (in bytes) +const COMPRESSION_THRESHOLD = 1024; // 1KB + +// Helper function to determine if compression should be used +function shouldCompress(data: string): boolean { + return Buffer.byteLength(data, 'utf8') > COMPRESSION_THRESHOLD; +} + /** * Initialize Redis instance names for better logging */ @@ -75,6 +90,14 @@ initializeRedisNames(); */ const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown"; +// Add at the top of your file, after the imports +declare module "ioredis" { + interface Redis { + json_set(key: string, path: string, json: any): Promise; + json_get(key: string, ...paths: string[]): Promise; + } +} + /** * Create a new Redis client with optimized settings */ @@ -83,24 +106,19 @@ async function createRedisClient(url: string): Promise { throw new Error("Redis URL not provided"); } - console.log(`Creating Redis client for ${getRedisName(url)}: ${url.slice(0, 20)}...`); const redisName = getRedisName(url); const client = new Redis(url, { retryStrategy: (times) => { if (times >= MAX_RETRIES) return null; - const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 1000); // Cap at 1s - console.log( - `${redisName} retry ${times}/${MAX_RETRIES}, delay ${delay}ms` - ); - return delay; + return Math.min(BASE_DELAY_MS * Math.pow(1.5, times), 1000); }, - maxRetriesPerRequest: null, + maxRetriesPerRequest: 2, enableOfflineQueue: true, enableReadyCheck: true, - connectTimeout: IS_VERCEL ? 5000 : 10000, // 5s for Vercel, 10s local - keepAlive: IS_VERCEL ? 2000 : 10000, // 2s ping for Vercel - commandTimeout: 5000, // Align with Vercel's timeout + connectTimeout: 3000, + keepAlive: 5000, + commandTimeout: 3000, reconnectOnError: (err) => ["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) => err.message.includes(e) @@ -455,15 +473,37 @@ export async function storeFeedMapping( const stringValue = JSON.stringify(mapping); try { - // First write to primary/active client - const redis = await getRedisClient(); // This already handles failover + // Get Redis client + const redis = await getRedisClient(); const activeUrl = redis.options.host || ''; - await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); - console.log(`Successfully stored mapping for feed ${feedId}`); + // Check if compression should be used + if (shouldCompress(stringValue)) { + console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`); + const compressed = await gzip(stringValue); + const base64Compressed = compressed.toString('base64'); + + // Store with compression flag + await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`); + + // Replicate to other instances + const replicas = await getAvailableReplicas(activeUrl); + for (const replica of replicas) { + await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + } + } else { + // Use standard set operations without compression + await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); + + // Replicate to other instances + const replicas = await getAvailableReplicas(activeUrl); + for (const replica of replicas) { + await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); + } + } - // Immediately replicate to other instances - await replicateOperation(activeUrl, feedId, stringValue, DEFAULT_CACHE_TTL); + console.log(`Successfully stored mapping for feed ${feedId}`); } catch (error) { console.error(`Error storing feed mapping for ${feedId}:`, error); throw error; @@ -477,29 +517,75 @@ export async function getFeedMapping( feedId: string ): Promise { try { - const redis = await getRedisClient(); // This already handles failover - const mapping = await redis.get(feedId); + // Try primary first + const redis = await getRedisClient(); - if (!mapping) return null; + // Check for compressed version first + const compressedKey = `${feedId}:compressed`; + const compressedData = await redis.get(compressedKey); + + if (compressedData) { + console.log(`Found compressed data for ${feedId}`); + try { + // Decompress and parse + const buffer = Buffer.from(compressedData, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + // Reset expiration + await redis.expire(compressedKey, DEFAULT_CACHE_TTL); + + return { + slugs: data.slugs, + lang: data.lang + }; + } catch (error) { + console.error(`Error decompressing data for ${feedId}:`, error); + // Fall through to try uncompressed version + } + } + + // If no compressed version, try regular key + let jsonString = await redis.get(feedId); + + // If not found in primary but we have replicas, check them + if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) { + console.log(`Key ${feedId} not found in primary, checking replicas...`); + + // Try each replica + for (const [url, replicaClient] of replicaClients.entries()) { + if (replicaClient.status === "ready") { + try { + jsonString = await replicaClient.get(feedId); + if (jsonString) { + console.log(`Found key ${feedId} in replica ${getRedisName(url)}`); + + // Sync this key back to primary for future requests + if (primaryClient && primaryClient.status === "ready") { + primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL) + .catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err)); + } + + break; + } + } catch (error) { + console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error); + } + } + } + } + + if (!jsonString) return null; // Reset expiration on access - const activeUrl = redis.options.host || ''; await redis.expire(feedId, DEFAULT_CACHE_TTL); - // Also reset TTL on other replicas - replicateOperation(activeUrl, feedId, mapping, DEFAULT_CACHE_TTL) - .catch(err => console.error(`Failed to reset TTL on replicas for ${feedId}:`, err)); - - try { - const data = JSON.parse(mapping); - return { - slugs: data.slugs, - lang: data.lang - }; - } catch (e) { - console.error(`Error parsing feed mapping for ${feedId}:`, e); - return null; - } + // Parse the result + const data = JSON.parse(jsonString as string); + return { + slugs: data.slugs, + lang: data.lang + }; } catch (error) { console.error(`Error retrieving feed mapping for ${feedId}:`, error); return null; @@ -571,6 +657,161 @@ export async function warmChapterCache( } } +// Add a new function for handling large feed mappings with compression +export async function storeLargeFeedMapping( + feedId: string, + slugs: string[], + lang: string +): Promise { + const mapping = { + slugs, + lang, + created_at: new Date().toISOString() + }; + + const jsonString = JSON.stringify(mapping); + + try { + const redis = await getRedisClient(); + const activeUrl = redis.options.host || ''; + + // Use compression for large mappings + if (shouldCompress(jsonString)) { + const compressed = await gzip(jsonString); + const base64Compressed = compressed.toString('base64'); + + // Store with compression flag + await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + + // Replicate to other instances + await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL); + + console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`); + } else { + // Use normal storage for small mappings + await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL); + await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL); + console.log(`Successfully stored mapping for feed ${feedId}`); + } + } catch (error) { + console.error(`Error storing feed mapping for ${feedId}:`, error); + throw error; + } +} + +// Batch get multiple feed mappings at once +export async function getBatchFeedMappings( + feedIds: string[] +): Promise> { + if (!feedIds.length) return {}; + + try { + const redis = await getRedisClient(); + const pipeline = redis.pipeline(); + + // Check regular keys first + feedIds.forEach(id => { + pipeline.get(id); + }); + + const results = await pipeline.exec(); + if (!results) return {}; + + // Process results + const mappings: Record = {}; + const ttlUpdates = redis.pipeline(); + const compressedChecks = redis.pipeline(); + const compressedIds: string[] = []; + + for (let i = 0; i < feedIds.length; i++) { + const id = feedIds[i]; + const result = results[i]; + + if (result[0]) { + mappings[id] = null; + continue; + } + + if (result[1]) { + // Process regular mapping + try { + const data = JSON.parse(result[1] as string); + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL + ttlUpdates.expire(id, DEFAULT_CACHE_TTL); + } catch (e) { + mappings[id] = null; + } + } else { + // Check for compressed version + compressedChecks.exists(`${id}:compressed`); + compressedIds.push(id); + } + } + + // Execute TTL updates in background + ttlUpdates.exec().catch(err => console.error('TTL update error:', err)); + + // Check compressed versions if needed + if (compressedIds.length > 0) { + const compressedResults = await compressedChecks.exec(); + + if (compressedResults) { + const compressedFetches = redis.pipeline(); + const compressedToFetch: string[] = []; + + compressedResults.forEach((result, index) => { + if (!result[0] && result[1]) { + const id = compressedIds[index]; + compressedFetches.get(`${id}:compressed`); + compressedToFetch.push(id); + } + }); + + if (compressedToFetch.length > 0) { + const fetchResults = await compressedFetches.exec(); + + if (fetchResults) { + for (let i = 0; i < compressedToFetch.length; i++) { + const id = compressedToFetch[i]; + const [err, compressedData] = fetchResults[i]; + + if (!err && compressedData) { + try { + const buffer = Buffer.from(compressedData as string, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL in background + redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {}); + } catch (e) { + mappings[id] = null; + } + } else { + mappings[id] = null; + } + } + } + } + } + } + + return mappings; + } catch (error) { + console.error(`Error retrieving batch feed mappings:`, error); + return {}; + } +} + // Initialize connections eagerly to establish connections as soon as possible if (!import.meta.env.SSR) { initializeAllClients()