diff --git a/package-lock.json b/package-lock.json index 109fad9..01d33cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,8 +17,6 @@ "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", - "p-throttle": "^7.0.0", - "redis-modules-sdk": "^1.4.0", "tailwindcss": "^3.4.1" } }, @@ -4375,17 +4373,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-throttle": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-7.0.0.tgz", - "integrity": "sha512-aio0v+S0QVkH1O+9x4dHtD4dgCExACcL+3EtNaGqC01GBudS9ijMuUsmN8OVScyV4OOp0jqdLShZFuSlbL/AsA==", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/p-timeout": { "version": "6.1.4", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz", @@ -4750,37 +4737,6 @@ "node": ">=4" } }, - "node_modules/redis-modules-sdk": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/redis-modules-sdk/-/redis-modules-sdk-1.4.0.tgz", - "integrity": "sha512-HYCr3ThtyoMrR9MUGmJblPU5aua0G4sanIZHK5wz/QwhVACe3BBgIZpHeymidBaukiuC0uzKla9xJ8exRffjnA==", - "dependencies": { - "ioredis": "5.2.3" - } - }, - "node_modules/redis-modules-sdk/node_modules/ioredis": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz", - "integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==", - "dependencies": { - "@ioredis/commands": "^1.1.1", - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.4", - "denque": "^2.0.1", - "lodash.defaults": "^4.2.0", - "lodash.isarguments": "^3.1.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" - }, - "engines": { - "node": ">=12.22.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/ioredis" - } - }, "node_modules/redis-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", diff --git a/package.json b/package.json index 1350193..15ad66e 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,6 @@ "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", - "p-throttle": "^7.0.0", - "redis-modules-sdk": "^1.4.0", "tailwindcss": "^3.4.1" } } diff --git a/src/components/SlugInput.astro b/src/components/SlugInput.astro index 06ba25c..43079a1 100644 --- a/src/components/SlugInput.astro +++ b/src/components/SlugInput.astro @@ -1,17 +1,25 @@ --- -import { getFeedMapping } from "../lib/db"; +import { getFeedMapping } from '../lib/db'; interface Props { currentSlugs?: string[]; } const { currentSlugs = [] } = Astro.props; -import { availableLanguages, languageMap } from "../lib/api"; +import { availableLanguages, languageMap } from '../lib/api'; -const languages = Object.entries(languageMap).map(([value, label]) => ({ - value, - label, -})); +const languages = [ + { value: 'en', label: 'English' }, + { value: 'ja', label: 'Japanese' }, + { value: 'ko', label: 'Korean' }, + { value: 'zh', label: 'Chinese' }, + { value: 'fr', label: 'French' }, + { value: 'es', label: 'Spanish' }, + { value: 'it', label: 'Italian' }, + { value: 'de', label: 'German' }, + { value: 'tr', label: 'Turkish' }, + { value: 'pl', label: 'Polish' }, +]; ---
@@ -29,11 +37,9 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({ id="lang-select" class="px-3 sm:px-4 py-2 bg-primary rounded-lg border border-hover/30 focus:outline-none focus:border-accent/50 transition-colors text-sm sm:text-base" > - { - languages.map((lang) => ( - - )) - } + {languages.map(lang => ( + + ))}
@@ -48,16 +54,8 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({ id="import-feed" class="px-4 py-2 bg-secondary hover:bg-hover transition-colors rounded-lg text-sm sm:text-base flex items-center gap-2" > - - + + Import Feed @@ -75,46 +73,27 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({ class="px-3 py-1.5 text-sm text-red-500 hover:text-red-600 hover:bg-red-500/10 rounded-lg transition-colors flex items-center gap-2" style="display: none;" > - - + + Clear All
- { - currentSlugs.map((slug) => ( -
( +
+ {slug} + -
- )) - } + + + + +
+ ))}
@@ -144,26 +123,20 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
+ \ No newline at end of file diff --git a/src/lib/api.ts b/src/lib/api.ts index c0329e2..c22762a 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -1,12 +1,10 @@ import type { Comic, Genre, ComicType, ChapterDetail } from './types'; import { getRedisClient, CHAPTER_CACHE_TTL } from './db'; -import pThrottle from 'p-throttle'; const languageMap = { "en": "English", "fr": "French", "es": "Spanish", - "es-419": "Spanish (Latin America)", "it": "Italian", "pl": "Polish", "tr": "Turkish", @@ -15,10 +13,7 @@ const languageMap = { "sv": "Swedish", "ar": "Arabic", "de": "German", - "ko": "Korean", - "pt": "Portuguese", - "pt-br": "Portuguese (Brazil)", - "pt-pt": "Portuguese (Portugal)" + "ko": "Korean" } as const; const languages = new Set(Object.keys(languageMap)); @@ -31,20 +26,6 @@ let genreCache: Genre[] = []; const RETRY_COUNT = 3; const RETRY_DELAY = 1000; -// Rate limiting configuration - adjust these values if needed -const API_REQUESTS_PER_SECOND = 10; // Maximum requests per second -const API_CONCURRENT_REQUESTS = 6; // Maximum concurrent requests -const DELAY_AFTER_429 = 1000; // Wait 1 second after a rate limit hit - -// Create throttled fetch function to avoid rate limits -const throttledFetch = pThrottle({ - limit: API_REQUESTS_PER_SECOND, - interval: 1000, - concurrency: API_CONCURRENT_REQUESTS -} as any)((url: string, options: RequestInit = {}, retries = RETRY_COUNT) => { - return fetchWithRetry(url, options, retries); -}); - async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise { try { const response = await fetch(url, { @@ -56,19 +37,6 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries = }, }); - // Special handling for rate limit errors - if (response.status === 429) { - console.warn(`Rate limited on ${url}. Waiting ${DELAY_AFTER_429}ms before retry.`); - - // Wait for the specified delay - await new Promise(resolve => setTimeout(resolve, DELAY_AFTER_429)); - - // Retry with one less retry count but only if we have retries left - if (retries > 0) { - return fetchWithRetry(url, options, retries - 1); - } - } - if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } @@ -76,10 +44,7 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries = return response; } catch (error) { if (retries > 0) { - // For non-429 errors, use exponential backoff - const delay = Math.min(RETRY_DELAY * Math.pow(2, RETRY_COUNT - retries), 10000); - console.log(`Retrying after ${delay}ms. Retries left: ${retries-1}`); - await new Promise(resolve => setTimeout(resolve, delay)); + await new Promise(resolve => setTimeout(resolve, RETRY_DELAY)); return fetchWithRetry(url, options, retries - 1); } throw error; @@ -90,8 +55,7 @@ export async function fetchGenres(): Promise { if (genreCache.length > 0) return genreCache; try { - // No tachiyomi param for genre endpoint - const response = await throttledFetch('https://api.comick.fun/genre'); + const response = await fetchWithRetry('https://api.comick.fun/genre'); genreCache = await response.json(); return genreCache; } catch (error) { @@ -128,7 +92,7 @@ export async function fetchComics( try { console.log('Fetching comics from:', apiUrl); - const response = await throttledFetch(apiUrl); + const response = await fetchWithRetry(apiUrl); const data = await response.json(); if (!Array.isArray(data)) { @@ -182,7 +146,7 @@ interface ChapterParams { export async function getComicBySlug(slug: string): Promise { try { - const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`); + const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`); const data = await response.json(); if (!data?.comic?.hid) { @@ -210,14 +174,14 @@ export async function getChaptersByHid( }); // Fetch comic details including cover - const comicResponse = await throttledFetch( + const comicResponse = await fetchWithRetry( `https://api.comick.fun/comic/${hid}` ); const comicData = await comicResponse.json(); const comicTitle = comicData?.comic?.title || 'Unknown Comic'; const covers = comicData?.comic?.md_covers || []; - const response = await throttledFetch( + const response = await fetchWithRetry( `https://api.comick.fun/comic/${hid}/chapters?${queryParams}` ); const data = await response.json(); @@ -296,25 +260,22 @@ export async function getChaptersForSlugs( const uniqueSlugs = [...new Set(slugs)]; const pipeline = redis.pipeline(); - // Check cache for all slugs + // Add cache key validation uniqueSlugs.forEach(slug => { if (!slug.match(/^[a-z0-9-]+$/)) { console.error(`Invalid slug format: ${slug}`); - return; // Skip invalid slugs immediately } const cacheKey = `chapters:${slug}:${lang}`; pipeline.get(cacheKey); }); - const cacheResults = await pipeline.exec(); - - // Process cache results + const [cacheResults, slugsToFetch] = await Promise.all([ + pipeline.exec(), + validateSlugs(uniqueSlugs) // Add this new function + ]); + const cachedChapters: ChapterDetail[] = []; - const slugsToFetch: string[] = []; - - // Separate cached and uncached slugs in one pass cacheResults.forEach(([err, result], index) => { - const slug = uniqueSlugs[index]; if (!err && result) { try { const parsed = JSON.parse(result as string); @@ -322,107 +283,102 @@ export async function getChaptersForSlugs( cachedChapters.push(...parsed); } } catch (e) { - console.error(`Cache parse error for ${slug}:`, e); - slugsToFetch.push(slug); + console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e); } - } else { - slugsToFetch.push(slug); } }); - // Skip validation if we have recent cache data + // Add progress tracking const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug)); console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`); - - // Only fetch if we have slugs that need fetching - let liveChapters: ChapterDetail[] = []; - if (slugsToFetch.length > 0) { - console.log(`Fetching ${slugsToFetch.length} comics from API`); - - // Use Promise.all for concurrent fetching with higher concurrency - const fetchPromises = slugsToFetch.map(async (slug) => { - try { - const comicDetail = await getComicBySlug(slug); - - if (!comicDetail?.comic?.hid) { - return []; - } - - return getChaptersByHid(comicDetail.comic.hid, slug, { - limit: 5, - lang: lang - }); - } catch (error) { - console.error(`Error processing ${slug}:`, error); + console.log(`Fetching ${slugsToFetch.length} comics from API`); + + const fetchPromises = slugsToFetch.map(async (slug) => { + const timer = Date.now(); + try { + const comicDetail = await getComicBySlug(slug); + + if (!comicDetail?.comic?.hid) { + console.error(`❌ No HID found for ${slug}`); return []; } - }); - - liveChapters = (await Promise.all(fetchPromises)).flat(); - - // Cache new results in batch if we have any - if (liveChapters.length > 0) { - const cachePipeline = redis.pipeline(); - const chaptersBySlug: Record = {}; - - liveChapters.forEach(chapter => { - const slug = chapter.md_comics.slug; - if (!chaptersBySlug[slug]) { - chaptersBySlug[slug] = []; - } - chaptersBySlug[slug].push(chapter); + + const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, { + limit: 5, + lang: lang }); - - Object.entries(chaptersBySlug).forEach(([slug, chaps]) => { - const cacheKey = `chapters:${slug}:${lang}`; - cachePipeline.set( - cacheKey, - JSON.stringify(chaps), - 'EX', - CHAPTER_CACHE_TTL - ); - }); - - await cachePipeline.exec(); + + console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`); + return chapters; + } catch (error) { + console.error(`🚨 Error processing ${slug}:`, error); + return []; } - } + }); - // Combine and deduplicate - return deduplicateChapters([...cachedChapters, ...liveChapters]); + const liveChapters = (await Promise.all(fetchPromises)).flat(); + + // Cache new results in batch + const cachePipeline = redis.pipeline(); + const chaptersBySlug: Record = {}; + + liveChapters.forEach(chapter => { + const slug = chapter.md_comics.slug; + if (!chaptersBySlug[slug]) { + chaptersBySlug[slug] = []; + } + chaptersBySlug[slug].push(chapter); + }); + + Object.entries(chaptersBySlug).forEach(([slug, chaps]) => { + const cacheKey = `chapters:${slug}:${lang}`; + cachePipeline.set( + cacheKey, + JSON.stringify(chaps), + 'EX', + CHAPTER_CACHE_TTL + ); + }); + + await cachePipeline.exec(); + + // Combine and sort all chapters + const allChapters = [...cachedChapters, ...liveChapters]; + const deduplicatedChapters = deduplicateChapters(allChapters); + return deduplicatedChapters.sort((a, b) => { + // Ensure the created_at fields are valid dates + const dateA = new Date(a.created_at).getTime(); + const dateB = new Date(b.created_at).getTime(); + + // First try to sort by chapter number + const chapDiff = parseFloat(b.chap) - parseFloat(a.chap); + if (!isNaN(chapDiff)) return chapDiff; + + // Fall back to date if chapter numbers are equal or invalid + return dateB - dateA; + }); } -// Replace validateSlugs with a more efficient implementation +// Add export to the validation function export async function validateSlugs(slugs: string[]): Promise { - // Skip validation in production for cached items - if (process.env.NODE_ENV === 'production') { - return slugs; - } - const validSlugs: string[] = []; const invalidSlugs: string[] = []; - // Increase batch size for faster processing - const batchSize = 5; - for (let i = 0; i < slugs.length; i += batchSize) { - const batch = slugs.slice(i, i + batchSize); - - await Promise.all(batch.map(async (slug) => { - try { - // Reduced retries (1 instead of 3) and removed tachiyomi parameter - const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`, {}, 1); - if (response.ok) { - validSlugs.push(slug); - } else { - invalidSlugs.push(slug); - } - } catch { + await Promise.all(slugs.map(async (slug) => { + try { + const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1); + if (response.ok) { + validSlugs.push(slug); + } else { invalidSlugs.push(slug); } - })); - } + } catch { + invalidSlugs.push(slug); + } + })); if (invalidSlugs.length > 0) { - console.warn(`Skipping invalid slugs: ${invalidSlugs.length} items`); + console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`); } return validSlugs; diff --git a/src/lib/db.ts b/src/lib/db.ts index f1fe845..32cea62 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -1,8 +1,5 @@ import Redis from "ioredis"; import type { FeedMapping } from "./types"; -import { ReJSON } from 'redis-modules-sdk'; -import * as zlib from 'zlib'; -import { promisify } from 'util'; // Configuration constants const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days @@ -58,18 +55,6 @@ const operationBuffer: RedisOperation[] = []; // Redis instance naming for logging const redisInstanceNames = new Map(); -// Add zlib for compression -const gzip = promisify(zlib.gzip); -const gunzip = promisify(zlib.gunzip); - -// Compression threshold (in bytes) -const COMPRESSION_THRESHOLD = 1024; // 1KB - -// Helper function to determine if compression should be used -function shouldCompress(data: string): boolean { - return Buffer.byteLength(data, 'utf8') > COMPRESSION_THRESHOLD; -} - /** * Initialize Redis instance names for better logging */ @@ -90,14 +75,6 @@ initializeRedisNames(); */ const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown"; -// Add at the top of your file, after the imports -declare module "ioredis" { - interface Redis { - json_set(key: string, path: string, json: any): Promise; - json_get(key: string, ...paths: string[]): Promise; - } -} - /** * Create a new Redis client with optimized settings */ @@ -106,19 +83,24 @@ async function createRedisClient(url: string): Promise { throw new Error("Redis URL not provided"); } + console.log(`Creating Redis client for ${getRedisName(url)}: ${url.slice(0, 20)}...`); const redisName = getRedisName(url); const client = new Redis(url, { retryStrategy: (times) => { if (times >= MAX_RETRIES) return null; - return Math.min(BASE_DELAY_MS * Math.pow(1.5, times), 1000); + const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 1000); // Cap at 1s + console.log( + `${redisName} retry ${times}/${MAX_RETRIES}, delay ${delay}ms` + ); + return delay; }, - maxRetriesPerRequest: 2, + maxRetriesPerRequest: null, enableOfflineQueue: true, enableReadyCheck: true, - connectTimeout: 3000, - keepAlive: 5000, - commandTimeout: 3000, + connectTimeout: IS_VERCEL ? 5000 : 10000, // 5s for Vercel, 10s local + keepAlive: IS_VERCEL ? 2000 : 10000, // 2s ping for Vercel + commandTimeout: 5000, // Align with Vercel's timeout reconnectOnError: (err) => ["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) => err.message.includes(e) @@ -473,37 +455,15 @@ export async function storeFeedMapping( const stringValue = JSON.stringify(mapping); try { - // Get Redis client - const redis = await getRedisClient(); + // First write to primary/active client + const redis = await getRedisClient(); // This already handles failover const activeUrl = redis.options.host || ''; - // Check if compression should be used - if (shouldCompress(stringValue)) { - console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`); - const compressed = await gzip(stringValue); - const base64Compressed = compressed.toString('base64'); - - // Store with compression flag - await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); - console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`); - - // Replicate to other instances - const replicas = await getAvailableReplicas(activeUrl); - for (const replica of replicas) { - await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); - } - } else { - // Use standard set operations without compression - await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); - - // Replicate to other instances - const replicas = await getAvailableReplicas(activeUrl); - for (const replica of replicas) { - await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); - } - } - + await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); console.log(`Successfully stored mapping for feed ${feedId}`); + + // Immediately replicate to other instances + await replicateOperation(activeUrl, feedId, stringValue, DEFAULT_CACHE_TTL); } catch (error) { console.error(`Error storing feed mapping for ${feedId}:`, error); throw error; @@ -517,75 +477,29 @@ export async function getFeedMapping( feedId: string ): Promise { try { - // Try primary first - const redis = await getRedisClient(); + const redis = await getRedisClient(); // This already handles failover + const mapping = await redis.get(feedId); - // Check for compressed version first - const compressedKey = `${feedId}:compressed`; - const compressedData = await redis.get(compressedKey); - - if (compressedData) { - console.log(`Found compressed data for ${feedId}`); - try { - // Decompress and parse - const buffer = Buffer.from(compressedData, 'base64'); - const decompressed = await gunzip(buffer); - const data = JSON.parse(decompressed.toString()); - - // Reset expiration - await redis.expire(compressedKey, DEFAULT_CACHE_TTL); - - return { - slugs: data.slugs, - lang: data.lang - }; - } catch (error) { - console.error(`Error decompressing data for ${feedId}:`, error); - // Fall through to try uncompressed version - } - } - - // If no compressed version, try regular key - let jsonString = await redis.get(feedId); - - // If not found in primary but we have replicas, check them - if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) { - console.log(`Key ${feedId} not found in primary, checking replicas...`); - - // Try each replica - for (const [url, replicaClient] of replicaClients.entries()) { - if (replicaClient.status === "ready") { - try { - jsonString = await replicaClient.get(feedId); - if (jsonString) { - console.log(`Found key ${feedId} in replica ${getRedisName(url)}`); - - // Sync this key back to primary for future requests - if (primaryClient && primaryClient.status === "ready") { - primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL) - .catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err)); - } - - break; - } - } catch (error) { - console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error); - } - } - } - } - - if (!jsonString) return null; + if (!mapping) return null; // Reset expiration on access + const activeUrl = redis.options.host || ''; await redis.expire(feedId, DEFAULT_CACHE_TTL); - // Parse the result - const data = JSON.parse(jsonString as string); - return { - slugs: data.slugs, - lang: data.lang - }; + // Also reset TTL on other replicas + replicateOperation(activeUrl, feedId, mapping, DEFAULT_CACHE_TTL) + .catch(err => console.error(`Failed to reset TTL on replicas for ${feedId}:`, err)); + + try { + const data = JSON.parse(mapping); + return { + slugs: data.slugs, + lang: data.lang + }; + } catch (e) { + console.error(`Error parsing feed mapping for ${feedId}:`, e); + return null; + } } catch (error) { console.error(`Error retrieving feed mapping for ${feedId}:`, error); return null; @@ -657,161 +571,6 @@ export async function warmChapterCache( } } -// Add a new function for handling large feed mappings with compression -export async function storeLargeFeedMapping( - feedId: string, - slugs: string[], - lang: string -): Promise { - const mapping = { - slugs, - lang, - created_at: new Date().toISOString() - }; - - const jsonString = JSON.stringify(mapping); - - try { - const redis = await getRedisClient(); - const activeUrl = redis.options.host || ''; - - // Use compression for large mappings - if (shouldCompress(jsonString)) { - const compressed = await gzip(jsonString); - const base64Compressed = compressed.toString('base64'); - - // Store with compression flag - await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); - - // Replicate to other instances - await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL); - - console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`); - } else { - // Use normal storage for small mappings - await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL); - await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL); - console.log(`Successfully stored mapping for feed ${feedId}`); - } - } catch (error) { - console.error(`Error storing feed mapping for ${feedId}:`, error); - throw error; - } -} - -// Batch get multiple feed mappings at once -export async function getBatchFeedMappings( - feedIds: string[] -): Promise> { - if (!feedIds.length) return {}; - - try { - const redis = await getRedisClient(); - const pipeline = redis.pipeline(); - - // Check regular keys first - feedIds.forEach(id => { - pipeline.get(id); - }); - - const results = await pipeline.exec(); - if (!results) return {}; - - // Process results - const mappings: Record = {}; - const ttlUpdates = redis.pipeline(); - const compressedChecks = redis.pipeline(); - const compressedIds: string[] = []; - - for (let i = 0; i < feedIds.length; i++) { - const id = feedIds[i]; - const result = results[i]; - - if (result[0]) { - mappings[id] = null; - continue; - } - - if (result[1]) { - // Process regular mapping - try { - const data = JSON.parse(result[1] as string); - mappings[id] = { - slugs: data.slugs, - lang: data.lang - }; - - // Reset TTL - ttlUpdates.expire(id, DEFAULT_CACHE_TTL); - } catch (e) { - mappings[id] = null; - } - } else { - // Check for compressed version - compressedChecks.exists(`${id}:compressed`); - compressedIds.push(id); - } - } - - // Execute TTL updates in background - ttlUpdates.exec().catch(err => console.error('TTL update error:', err)); - - // Check compressed versions if needed - if (compressedIds.length > 0) { - const compressedResults = await compressedChecks.exec(); - - if (compressedResults) { - const compressedFetches = redis.pipeline(); - const compressedToFetch: string[] = []; - - compressedResults.forEach((result, index) => { - if (!result[0] && result[1]) { - const id = compressedIds[index]; - compressedFetches.get(`${id}:compressed`); - compressedToFetch.push(id); - } - }); - - if (compressedToFetch.length > 0) { - const fetchResults = await compressedFetches.exec(); - - if (fetchResults) { - for (let i = 0; i < compressedToFetch.length; i++) { - const id = compressedToFetch[i]; - const [err, compressedData] = fetchResults[i]; - - if (!err && compressedData) { - try { - const buffer = Buffer.from(compressedData as string, 'base64'); - const decompressed = await gunzip(buffer); - const data = JSON.parse(decompressed.toString()); - - mappings[id] = { - slugs: data.slugs, - lang: data.lang - }; - - // Reset TTL in background - redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {}); - } catch (e) { - mappings[id] = null; - } - } else { - mappings[id] = null; - } - } - } - } - } - } - - return mappings; - } catch (error) { - console.error(`Error retrieving batch feed mappings:`, error); - return {}; - } -} - // Initialize connections eagerly to establish connections as soon as possible if (!import.meta.env.SSR) { initializeAllClients()