From eaeaeae7d29cc94384eef229860da4f67ff96a01 Mon Sep 17 00:00:00 2001 From: ld3zian <42660576+ld3z@users.noreply.github.com> Date: Sat, 22 Mar 2025 13:25:51 -0700 Subject: [PATCH] maybe its fixed? --- src/lib/api.ts | 170 ++++++++++++++++++++++++------------------------- src/lib/db.ts | 119 +++++++++++++++++++--------------- 2 files changed, 149 insertions(+), 140 deletions(-) diff --git a/src/lib/api.ts b/src/lib/api.ts index 3854e74..b0a4a49 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -28,9 +28,9 @@ const RETRY_COUNT = 3; const RETRY_DELAY = 1000; // Rate limiting configuration - adjust these values if needed -const API_REQUESTS_PER_SECOND = 5; // Maximum requests per second -const API_CONCURRENT_REQUESTS = 3; // Maximum concurrent requests -const DELAY_AFTER_429 = 2000; // Wait 2 seconds after a rate limit hit +const API_REQUESTS_PER_SECOND = 10; // Maximum requests per second +const API_CONCURRENT_REQUESTS = 6; // Maximum concurrent requests +const DELAY_AFTER_429 = 1000; // Wait 1 second after a rate limit hit // Create throttled fetch function to avoid rate limits const throttledFetch = pThrottle({ @@ -119,7 +119,7 @@ export async function fetchComics( comicType: ComicType = 'all', page: number = 1 ): Promise { - const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}&tachiyomi=true`; + const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}`; try { console.log('Fetching comics from:', apiUrl); @@ -178,7 +178,7 @@ interface ChapterParams { export async function getComicBySlug(slug: string): Promise { try { - const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`); + const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`); const data = await response.json(); if (!data?.comic?.hid) { @@ -202,13 +202,12 @@ export async function getChaptersByHid( const queryParams = new URLSearchParams({ limit: params.limit?.toString() || '5', lang: params.lang || 'en', - ordering: '-created_at', - tachiyomi: 'true' // Add tachiyomi parameter + ordering: '-created_at' }); // Fetch comic details including cover const comicResponse = await throttledFetch( - `https://api.comick.fun/comic/${hid}?tachiyomi=true` + `https://api.comick.fun/comic/${hid}` ); const comicData = await comicResponse.json(); const comicTitle = comicData?.comic?.title || 'Unknown Comic'; @@ -293,22 +292,25 @@ export async function getChaptersForSlugs( const uniqueSlugs = [...new Set(slugs)]; const pipeline = redis.pipeline(); - // Add cache key validation + // Check cache for all slugs uniqueSlugs.forEach(slug => { if (!slug.match(/^[a-z0-9-]+$/)) { console.error(`Invalid slug format: ${slug}`); + return; // Skip invalid slugs immediately } const cacheKey = `chapters:${slug}:${lang}`; pipeline.get(cacheKey); }); - const [cacheResults, slugsToFetch] = await Promise.all([ - pipeline.exec(), - validateSlugs(uniqueSlugs) // Add this new function - ]); - + const cacheResults = await pipeline.exec(); + + // Process cache results const cachedChapters: ChapterDetail[] = []; + const slugsToFetch: string[] = []; + + // Separate cached and uncached slugs in one pass cacheResults.forEach(([err, result], index) => { + const slug = uniqueSlugs[index]; if (!err && result) { try { const parsed = JSON.parse(result as string); @@ -316,95 +318,94 @@ export async function getChaptersForSlugs( cachedChapters.push(...parsed); } } catch (e) { - console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e); + console.error(`Cache parse error for ${slug}:`, e); + slugsToFetch.push(slug); } + } else { + slugsToFetch.push(slug); } }); - // Add progress tracking + // Skip validation if we have recent cache data const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug)); console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`); - console.log(`Fetching ${slugsToFetch.length} comics from API`); - - const fetchPromises = slugsToFetch.map(async (slug) => { - const timer = Date.now(); - try { - const comicDetail = await getComicBySlug(slug); - - if (!comicDetail?.comic?.hid) { - console.error(`❌ No HID found for ${slug}`); + + // Only fetch if we have slugs that need fetching + let liveChapters: ChapterDetail[] = []; + if (slugsToFetch.length > 0) { + console.log(`Fetching ${slugsToFetch.length} comics from API`); + + // Use Promise.all for concurrent fetching with higher concurrency + const fetchPromises = slugsToFetch.map(async (slug) => { + try { + const comicDetail = await getComicBySlug(slug); + + if (!comicDetail?.comic?.hid) { + return []; + } + + return getChaptersByHid(comicDetail.comic.hid, slug, { + limit: 5, + lang: lang + }); + } catch (error) { + console.error(`Error processing ${slug}:`, error); return []; } - - const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, { - limit: 5, - lang: lang + }); + + liveChapters = (await Promise.all(fetchPromises)).flat(); + + // Cache new results in batch if we have any + if (liveChapters.length > 0) { + const cachePipeline = redis.pipeline(); + const chaptersBySlug: Record = {}; + + liveChapters.forEach(chapter => { + const slug = chapter.md_comics.slug; + if (!chaptersBySlug[slug]) { + chaptersBySlug[slug] = []; + } + chaptersBySlug[slug].push(chapter); }); - - console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`); - return chapters; - } catch (error) { - console.error(`🚨 Error processing ${slug}:`, error); - return []; + + Object.entries(chaptersBySlug).forEach(([slug, chaps]) => { + const cacheKey = `chapters:${slug}:${lang}`; + cachePipeline.set( + cacheKey, + JSON.stringify(chaps), + 'EX', + CHAPTER_CACHE_TTL + ); + }); + + await cachePipeline.exec(); } - }); + } - const liveChapters = (await Promise.all(fetchPromises)).flat(); - - // Cache new results in batch - const cachePipeline = redis.pipeline(); - const chaptersBySlug: Record = {}; - - liveChapters.forEach(chapter => { - const slug = chapter.md_comics.slug; - if (!chaptersBySlug[slug]) { - chaptersBySlug[slug] = []; - } - chaptersBySlug[slug].push(chapter); - }); - - Object.entries(chaptersBySlug).forEach(([slug, chaps]) => { - const cacheKey = `chapters:${slug}:${lang}`; - cachePipeline.set( - cacheKey, - JSON.stringify(chaps), - 'EX', - CHAPTER_CACHE_TTL - ); - }); - - await cachePipeline.exec(); - - // Combine and sort all chapters - const allChapters = [...cachedChapters, ...liveChapters]; - const deduplicatedChapters = deduplicateChapters(allChapters); - return deduplicatedChapters.sort((a, b) => { - // Ensure the created_at fields are valid dates - const dateA = new Date(a.created_at).getTime(); - const dateB = new Date(b.created_at).getTime(); - - // First try to sort by chapter number - const chapDiff = parseFloat(b.chap) - parseFloat(a.chap); - if (!isNaN(chapDiff)) return chapDiff; - - // Fall back to date if chapter numbers are equal or invalid - return dateB - dateA; - }); + // Combine and deduplicate + return deduplicateChapters([...cachedChapters, ...liveChapters]); } -// Add export to the validation function +// Replace validateSlugs with a more efficient implementation export async function validateSlugs(slugs: string[]): Promise { + // Skip validation in production for cached items + if (process.env.NODE_ENV === 'production') { + return slugs; + } + const validSlugs: string[] = []; const invalidSlugs: string[] = []; - // Process in small batches to avoid rate limiting - const batchSize = 3; + // Increase batch size for faster processing + const batchSize = 5; for (let i = 0; i < slugs.length; i += batchSize) { const batch = slugs.slice(i, i + batchSize); await Promise.all(batch.map(async (slug) => { try { - const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`, {}, 1); + // Reduced retries (1 instead of 3) and removed tachiyomi parameter + const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`, {}, 1); if (response.ok) { validSlugs.push(slug); } else { @@ -414,15 +415,10 @@ export async function validateSlugs(slugs: string[]): Promise { invalidSlugs.push(slug); } })); - - // Add a small delay between batches - if (i + batchSize < slugs.length) { - await new Promise(resolve => setTimeout(resolve, 300)); - } } if (invalidSlugs.length > 0) { - console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`); + console.warn(`Skipping invalid slugs: ${invalidSlugs.length} items`); } return validSlugs; diff --git a/src/lib/db.ts b/src/lib/db.ts index 7eabf97..f1fe845 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -106,24 +106,19 @@ async function createRedisClient(url: string): Promise { throw new Error("Redis URL not provided"); } - console.log(`Creating Redis client for ${getRedisName(url)}: ${url.slice(0, 20)}...`); const redisName = getRedisName(url); const client = new Redis(url, { retryStrategy: (times) => { if (times >= MAX_RETRIES) return null; - const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 1000); // Cap at 1s - console.log( - `${redisName} retry ${times}/${MAX_RETRIES}, delay ${delay}ms` - ); - return delay; + return Math.min(BASE_DELAY_MS * Math.pow(1.5, times), 1000); }, - maxRetriesPerRequest: null, + maxRetriesPerRequest: 2, enableOfflineQueue: true, enableReadyCheck: true, - connectTimeout: IS_VERCEL ? 5000 : 10000, // 5s for Vercel, 10s local - keepAlive: IS_VERCEL ? 2000 : 10000, // 2s ping for Vercel - commandTimeout: 5000, // Align with Vercel's timeout + connectTimeout: 3000, + keepAlive: 5000, + commandTimeout: 3000, reconnectOnError: (err) => ["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) => err.message.includes(e) @@ -714,11 +709,9 @@ export async function getBatchFeedMappings( const redis = await getRedisClient(); const pipeline = redis.pipeline(); - // Get all mappings in a single round trip + // Check regular keys first feedIds.forEach(id => { pipeline.get(id); - // Also check for compressed version - pipeline.exists(`${id}:compressed`); }); const results = await pipeline.exec(); @@ -727,52 +720,22 @@ export async function getBatchFeedMappings( // Process results const mappings: Record = {}; const ttlUpdates = redis.pipeline(); + const compressedChecks = redis.pipeline(); + const compressedIds: string[] = []; for (let i = 0; i < feedIds.length; i++) { const id = feedIds[i]; - const mappingResult = results[i * 2]; - const compressedResult = results[i * 2 + 1]; + const result = results[i]; - if (mappingResult[0]) { - console.error(`Error retrieving mapping for ${id}:`, mappingResult[0]); + if (result[0]) { mappings[id] = null; continue; } - if (compressedResult[0]) { - console.error(`Error checking compressed mapping for ${id}:`, compressedResult[0]); - mappings[id] = null; - continue; - } - - // Check if we have a compressed version - if (compressedResult[1]) { - // Fetch and decompress - const compressedData = await redis.get(`${id}:compressed`); - if (compressedData) { - try { - const buffer = Buffer.from(compressedData, 'base64'); - const decompressed = await gunzip(buffer); - const data = JSON.parse(decompressed.toString() as string); - - mappings[id] = { - slugs: data.slugs, - lang: data.lang - }; - - // Reset TTL - ttlUpdates.expire(`${id}:compressed`, DEFAULT_CACHE_TTL); - } catch (e) { - console.error(`Error decompressing data for ${id}:`, e); - mappings[id] = null; - } - } else { - mappings[id] = null; - } - } else if (mappingResult[1]) { + if (result[1]) { // Process regular mapping try { - const data = JSON.parse(mappingResult[1] as string); + const data = JSON.parse(result[1] as string); mappings[id] = { slugs: data.slugs, lang: data.lang @@ -781,16 +744,66 @@ export async function getBatchFeedMappings( // Reset TTL ttlUpdates.expire(id, DEFAULT_CACHE_TTL); } catch (e) { - console.error(`Error parsing feed mapping for ${id}:`, e); mappings[id] = null; } } else { - mappings[id] = null; + // Check for compressed version + compressedChecks.exists(`${id}:compressed`); + compressedIds.push(id); } } - // Execute all TTL updates - await ttlUpdates.exec(); + // Execute TTL updates in background + ttlUpdates.exec().catch(err => console.error('TTL update error:', err)); + + // Check compressed versions if needed + if (compressedIds.length > 0) { + const compressedResults = await compressedChecks.exec(); + + if (compressedResults) { + const compressedFetches = redis.pipeline(); + const compressedToFetch: string[] = []; + + compressedResults.forEach((result, index) => { + if (!result[0] && result[1]) { + const id = compressedIds[index]; + compressedFetches.get(`${id}:compressed`); + compressedToFetch.push(id); + } + }); + + if (compressedToFetch.length > 0) { + const fetchResults = await compressedFetches.exec(); + + if (fetchResults) { + for (let i = 0; i < compressedToFetch.length; i++) { + const id = compressedToFetch[i]; + const [err, compressedData] = fetchResults[i]; + + if (!err && compressedData) { + try { + const buffer = Buffer.from(compressedData as string, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL in background + redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {}); + } catch (e) { + mappings[id] = null; + } + } else { + mappings[id] = null; + } + } + } + } + } + } return mappings; } catch (error) {