From 9677cba078c8cb42c26e9d4cd4d17cd5b2466ab3 Mon Sep 17 00:00:00 2001 From: ld3zian <42660576+ld3z@users.noreply.github.com> Date: Sat, 22 Mar 2025 12:28:05 -0700 Subject: [PATCH 1/4] this will be slower --- package-lock.json | 44 ++++++++ package.json | 2 + src/lib/api.ts | 80 +++++++++++--- src/lib/db.ts | 276 ++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 360 insertions(+), 42 deletions(-) diff --git a/package-lock.json b/package-lock.json index 01d33cf..109fad9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,8 @@ "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", + "p-throttle": "^7.0.0", + "redis-modules-sdk": "^1.4.0", "tailwindcss": "^3.4.1" } }, @@ -4373,6 +4375,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-throttle": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-7.0.0.tgz", + "integrity": "sha512-aio0v+S0QVkH1O+9x4dHtD4dgCExACcL+3EtNaGqC01GBudS9ijMuUsmN8OVScyV4OOp0jqdLShZFuSlbL/AsA==", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-timeout": { "version": "6.1.4", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz", @@ -4737,6 +4750,37 @@ "node": ">=4" } }, + "node_modules/redis-modules-sdk": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/redis-modules-sdk/-/redis-modules-sdk-1.4.0.tgz", + "integrity": "sha512-HYCr3ThtyoMrR9MUGmJblPU5aua0G4sanIZHK5wz/QwhVACe3BBgIZpHeymidBaukiuC0uzKla9xJ8exRffjnA==", + "dependencies": { + "ioredis": "5.2.3" + } + }, + "node_modules/redis-modules-sdk/node_modules/ioredis": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz", + "integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.0.1", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/redis-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", diff --git a/package.json b/package.json index 15ad66e..1350193 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,8 @@ "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", + "p-throttle": "^7.0.0", + "redis-modules-sdk": "^1.4.0", "tailwindcss": "^3.4.1" } } diff --git a/src/lib/api.ts b/src/lib/api.ts index c22762a..3854e74 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -1,5 +1,6 @@ import type { Comic, Genre, ComicType, ChapterDetail } from './types'; import { getRedisClient, CHAPTER_CACHE_TTL } from './db'; +import pThrottle from 'p-throttle'; const languageMap = { "en": "English", @@ -26,6 +27,20 @@ let genreCache: Genre[] = []; const RETRY_COUNT = 3; const RETRY_DELAY = 1000; +// Rate limiting configuration - adjust these values if needed +const API_REQUESTS_PER_SECOND = 5; // Maximum requests per second +const API_CONCURRENT_REQUESTS = 3; // Maximum concurrent requests +const DELAY_AFTER_429 = 2000; // Wait 2 seconds after a rate limit hit + +// Create throttled fetch function to avoid rate limits +const throttledFetch = pThrottle({ + limit: API_REQUESTS_PER_SECOND, + interval: 1000, + concurrency: API_CONCURRENT_REQUESTS +} as any)((url: string, options: RequestInit = {}, retries = RETRY_COUNT) => { + return fetchWithRetry(url, options, retries); +}); + async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise { try { const response = await fetch(url, { @@ -37,6 +52,19 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries = }, }); + // Special handling for rate limit errors + if (response.status === 429) { + console.warn(`Rate limited on ${url}. Waiting ${DELAY_AFTER_429}ms before retry.`); + + // Wait for the specified delay + await new Promise(resolve => setTimeout(resolve, DELAY_AFTER_429)); + + // Retry with one less retry count but only if we have retries left + if (retries > 0) { + return fetchWithRetry(url, options, retries - 1); + } + } + if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } @@ -44,7 +72,10 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries = return response; } catch (error) { if (retries > 0) { - await new Promise(resolve => setTimeout(resolve, RETRY_DELAY)); + // For non-429 errors, use exponential backoff + const delay = Math.min(RETRY_DELAY * Math.pow(2, RETRY_COUNT - retries), 10000); + console.log(`Retrying after ${delay}ms. Retries left: ${retries-1}`); + await new Promise(resolve => setTimeout(resolve, delay)); return fetchWithRetry(url, options, retries - 1); } throw error; @@ -55,7 +86,8 @@ export async function fetchGenres(): Promise { if (genreCache.length > 0) return genreCache; try { - const response = await fetchWithRetry('https://api.comick.fun/genre'); + // No tachiyomi param for genre endpoint + const response = await throttledFetch('https://api.comick.fun/genre'); genreCache = await response.json(); return genreCache; } catch (error) { @@ -87,12 +119,12 @@ export async function fetchComics( comicType: ComicType = 'all', page: number = 1 ): Promise { - const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}`; + const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}&tachiyomi=true`; try { console.log('Fetching comics from:', apiUrl); - const response = await fetchWithRetry(apiUrl); + const response = await throttledFetch(apiUrl); const data = await response.json(); if (!Array.isArray(data)) { @@ -146,7 +178,7 @@ interface ChapterParams { export async function getComicBySlug(slug: string): Promise { try { - const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`); + const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`); const data = await response.json(); if (!data?.comic?.hid) { @@ -170,18 +202,19 @@ export async function getChaptersByHid( const queryParams = new URLSearchParams({ limit: params.limit?.toString() || '5', lang: params.lang || 'en', - ordering: '-created_at' + ordering: '-created_at', + tachiyomi: 'true' // Add tachiyomi parameter }); // Fetch comic details including cover - const comicResponse = await fetchWithRetry( - `https://api.comick.fun/comic/${hid}` + const comicResponse = await throttledFetch( + `https://api.comick.fun/comic/${hid}?tachiyomi=true` ); const comicData = await comicResponse.json(); const comicTitle = comicData?.comic?.title || 'Unknown Comic'; const covers = comicData?.comic?.md_covers || []; - const response = await fetchWithRetry( + const response = await throttledFetch( `https://api.comick.fun/comic/${hid}/chapters?${queryParams}` ); const data = await response.json(); @@ -364,18 +397,29 @@ export async function validateSlugs(slugs: string[]): Promise { const validSlugs: string[] = []; const invalidSlugs: string[] = []; - await Promise.all(slugs.map(async (slug) => { - try { - const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1); - if (response.ok) { - validSlugs.push(slug); - } else { + // Process in small batches to avoid rate limiting + const batchSize = 3; + for (let i = 0; i < slugs.length; i += batchSize) { + const batch = slugs.slice(i, i + batchSize); + + await Promise.all(batch.map(async (slug) => { + try { + const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`, {}, 1); + if (response.ok) { + validSlugs.push(slug); + } else { + invalidSlugs.push(slug); + } + } catch { invalidSlugs.push(slug); } - } catch { - invalidSlugs.push(slug); + })); + + // Add a small delay between batches + if (i + batchSize < slugs.length) { + await new Promise(resolve => setTimeout(resolve, 300)); } - })); + } if (invalidSlugs.length > 0) { console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`); diff --git a/src/lib/db.ts b/src/lib/db.ts index 32cea62..7eabf97 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -1,5 +1,8 @@ import Redis from "ioredis"; import type { FeedMapping } from "./types"; +import { ReJSON } from 'redis-modules-sdk'; +import * as zlib from 'zlib'; +import { promisify } from 'util'; // Configuration constants const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days @@ -55,6 +58,18 @@ const operationBuffer: RedisOperation[] = []; // Redis instance naming for logging const redisInstanceNames = new Map(); +// Add zlib for compression +const gzip = promisify(zlib.gzip); +const gunzip = promisify(zlib.gunzip); + +// Compression threshold (in bytes) +const COMPRESSION_THRESHOLD = 1024; // 1KB + +// Helper function to determine if compression should be used +function shouldCompress(data: string): boolean { + return Buffer.byteLength(data, 'utf8') > COMPRESSION_THRESHOLD; +} + /** * Initialize Redis instance names for better logging */ @@ -75,6 +90,14 @@ initializeRedisNames(); */ const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown"; +// Add at the top of your file, after the imports +declare module "ioredis" { + interface Redis { + json_set(key: string, path: string, json: any): Promise; + json_get(key: string, ...paths: string[]): Promise; + } +} + /** * Create a new Redis client with optimized settings */ @@ -455,15 +478,37 @@ export async function storeFeedMapping( const stringValue = JSON.stringify(mapping); try { - // First write to primary/active client - const redis = await getRedisClient(); // This already handles failover + // Get Redis client + const redis = await getRedisClient(); const activeUrl = redis.options.host || ''; - await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); - console.log(`Successfully stored mapping for feed ${feedId}`); + // Check if compression should be used + if (shouldCompress(stringValue)) { + console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`); + const compressed = await gzip(stringValue); + const base64Compressed = compressed.toString('base64'); + + // Store with compression flag + await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`); + + // Replicate to other instances + const replicas = await getAvailableReplicas(activeUrl); + for (const replica of replicas) { + await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + } + } else { + // Use standard set operations without compression + await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); + + // Replicate to other instances + const replicas = await getAvailableReplicas(activeUrl); + for (const replica of replicas) { + await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); + } + } - // Immediately replicate to other instances - await replicateOperation(activeUrl, feedId, stringValue, DEFAULT_CACHE_TTL); + console.log(`Successfully stored mapping for feed ${feedId}`); } catch (error) { console.error(`Error storing feed mapping for ${feedId}:`, error); throw error; @@ -477,29 +522,75 @@ export async function getFeedMapping( feedId: string ): Promise { try { - const redis = await getRedisClient(); // This already handles failover - const mapping = await redis.get(feedId); + // Try primary first + const redis = await getRedisClient(); - if (!mapping) return null; + // Check for compressed version first + const compressedKey = `${feedId}:compressed`; + const compressedData = await redis.get(compressedKey); + + if (compressedData) { + console.log(`Found compressed data for ${feedId}`); + try { + // Decompress and parse + const buffer = Buffer.from(compressedData, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + // Reset expiration + await redis.expire(compressedKey, DEFAULT_CACHE_TTL); + + return { + slugs: data.slugs, + lang: data.lang + }; + } catch (error) { + console.error(`Error decompressing data for ${feedId}:`, error); + // Fall through to try uncompressed version + } + } + + // If no compressed version, try regular key + let jsonString = await redis.get(feedId); + + // If not found in primary but we have replicas, check them + if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) { + console.log(`Key ${feedId} not found in primary, checking replicas...`); + + // Try each replica + for (const [url, replicaClient] of replicaClients.entries()) { + if (replicaClient.status === "ready") { + try { + jsonString = await replicaClient.get(feedId); + if (jsonString) { + console.log(`Found key ${feedId} in replica ${getRedisName(url)}`); + + // Sync this key back to primary for future requests + if (primaryClient && primaryClient.status === "ready") { + primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL) + .catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err)); + } + + break; + } + } catch (error) { + console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error); + } + } + } + } + + if (!jsonString) return null; // Reset expiration on access - const activeUrl = redis.options.host || ''; await redis.expire(feedId, DEFAULT_CACHE_TTL); - // Also reset TTL on other replicas - replicateOperation(activeUrl, feedId, mapping, DEFAULT_CACHE_TTL) - .catch(err => console.error(`Failed to reset TTL on replicas for ${feedId}:`, err)); - - try { - const data = JSON.parse(mapping); - return { - slugs: data.slugs, - lang: data.lang - }; - } catch (e) { - console.error(`Error parsing feed mapping for ${feedId}:`, e); - return null; - } + // Parse the result + const data = JSON.parse(jsonString as string); + return { + slugs: data.slugs, + lang: data.lang + }; } catch (error) { console.error(`Error retrieving feed mapping for ${feedId}:`, error); return null; @@ -571,6 +662,143 @@ export async function warmChapterCache( } } +// Add a new function for handling large feed mappings with compression +export async function storeLargeFeedMapping( + feedId: string, + slugs: string[], + lang: string +): Promise { + const mapping = { + slugs, + lang, + created_at: new Date().toISOString() + }; + + const jsonString = JSON.stringify(mapping); + + try { + const redis = await getRedisClient(); + const activeUrl = redis.options.host || ''; + + // Use compression for large mappings + if (shouldCompress(jsonString)) { + const compressed = await gzip(jsonString); + const base64Compressed = compressed.toString('base64'); + + // Store with compression flag + await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + + // Replicate to other instances + await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL); + + console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`); + } else { + // Use normal storage for small mappings + await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL); + await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL); + console.log(`Successfully stored mapping for feed ${feedId}`); + } + } catch (error) { + console.error(`Error storing feed mapping for ${feedId}:`, error); + throw error; + } +} + +// Batch get multiple feed mappings at once +export async function getBatchFeedMappings( + feedIds: string[] +): Promise> { + if (!feedIds.length) return {}; + + try { + const redis = await getRedisClient(); + const pipeline = redis.pipeline(); + + // Get all mappings in a single round trip + feedIds.forEach(id => { + pipeline.get(id); + // Also check for compressed version + pipeline.exists(`${id}:compressed`); + }); + + const results = await pipeline.exec(); + if (!results) return {}; + + // Process results + const mappings: Record = {}; + const ttlUpdates = redis.pipeline(); + + for (let i = 0; i < feedIds.length; i++) { + const id = feedIds[i]; + const mappingResult = results[i * 2]; + const compressedResult = results[i * 2 + 1]; + + if (mappingResult[0]) { + console.error(`Error retrieving mapping for ${id}:`, mappingResult[0]); + mappings[id] = null; + continue; + } + + if (compressedResult[0]) { + console.error(`Error checking compressed mapping for ${id}:`, compressedResult[0]); + mappings[id] = null; + continue; + } + + // Check if we have a compressed version + if (compressedResult[1]) { + // Fetch and decompress + const compressedData = await redis.get(`${id}:compressed`); + if (compressedData) { + try { + const buffer = Buffer.from(compressedData, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString() as string); + + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL + ttlUpdates.expire(`${id}:compressed`, DEFAULT_CACHE_TTL); + } catch (e) { + console.error(`Error decompressing data for ${id}:`, e); + mappings[id] = null; + } + } else { + mappings[id] = null; + } + } else if (mappingResult[1]) { + // Process regular mapping + try { + const data = JSON.parse(mappingResult[1] as string); + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL + ttlUpdates.expire(id, DEFAULT_CACHE_TTL); + } catch (e) { + console.error(`Error parsing feed mapping for ${id}:`, e); + mappings[id] = null; + } + } else { + mappings[id] = null; + } + } + + // Execute all TTL updates + await ttlUpdates.exec(); + + return mappings; + } catch (error) { + console.error(`Error retrieving batch feed mappings:`, error); + return {}; + } +} + // Initialize connections eagerly to establish connections as soon as possible if (!import.meta.env.SSR) { initializeAllClients() From eaeaeae7d29cc94384eef229860da4f67ff96a01 Mon Sep 17 00:00:00 2001 From: ld3zian <42660576+ld3z@users.noreply.github.com> Date: Sat, 22 Mar 2025 13:25:51 -0700 Subject: [PATCH 2/4] maybe its fixed? --- src/lib/api.ts | 170 ++++++++++++++++++++++++------------------------- src/lib/db.ts | 119 +++++++++++++++++++--------------- 2 files changed, 149 insertions(+), 140 deletions(-) diff --git a/src/lib/api.ts b/src/lib/api.ts index 3854e74..b0a4a49 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -28,9 +28,9 @@ const RETRY_COUNT = 3; const RETRY_DELAY = 1000; // Rate limiting configuration - adjust these values if needed -const API_REQUESTS_PER_SECOND = 5; // Maximum requests per second -const API_CONCURRENT_REQUESTS = 3; // Maximum concurrent requests -const DELAY_AFTER_429 = 2000; // Wait 2 seconds after a rate limit hit +const API_REQUESTS_PER_SECOND = 10; // Maximum requests per second +const API_CONCURRENT_REQUESTS = 6; // Maximum concurrent requests +const DELAY_AFTER_429 = 1000; // Wait 1 second after a rate limit hit // Create throttled fetch function to avoid rate limits const throttledFetch = pThrottle({ @@ -119,7 +119,7 @@ export async function fetchComics( comicType: ComicType = 'all', page: number = 1 ): Promise { - const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}&tachiyomi=true`; + const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}`; try { console.log('Fetching comics from:', apiUrl); @@ -178,7 +178,7 @@ interface ChapterParams { export async function getComicBySlug(slug: string): Promise { try { - const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`); + const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`); const data = await response.json(); if (!data?.comic?.hid) { @@ -202,13 +202,12 @@ export async function getChaptersByHid( const queryParams = new URLSearchParams({ limit: params.limit?.toString() || '5', lang: params.lang || 'en', - ordering: '-created_at', - tachiyomi: 'true' // Add tachiyomi parameter + ordering: '-created_at' }); // Fetch comic details including cover const comicResponse = await throttledFetch( - `https://api.comick.fun/comic/${hid}?tachiyomi=true` + `https://api.comick.fun/comic/${hid}` ); const comicData = await comicResponse.json(); const comicTitle = comicData?.comic?.title || 'Unknown Comic'; @@ -293,22 +292,25 @@ export async function getChaptersForSlugs( const uniqueSlugs = [...new Set(slugs)]; const pipeline = redis.pipeline(); - // Add cache key validation + // Check cache for all slugs uniqueSlugs.forEach(slug => { if (!slug.match(/^[a-z0-9-]+$/)) { console.error(`Invalid slug format: ${slug}`); + return; // Skip invalid slugs immediately } const cacheKey = `chapters:${slug}:${lang}`; pipeline.get(cacheKey); }); - const [cacheResults, slugsToFetch] = await Promise.all([ - pipeline.exec(), - validateSlugs(uniqueSlugs) // Add this new function - ]); - + const cacheResults = await pipeline.exec(); + + // Process cache results const cachedChapters: ChapterDetail[] = []; + const slugsToFetch: string[] = []; + + // Separate cached and uncached slugs in one pass cacheResults.forEach(([err, result], index) => { + const slug = uniqueSlugs[index]; if (!err && result) { try { const parsed = JSON.parse(result as string); @@ -316,95 +318,94 @@ export async function getChaptersForSlugs( cachedChapters.push(...parsed); } } catch (e) { - console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e); + console.error(`Cache parse error for ${slug}:`, e); + slugsToFetch.push(slug); } + } else { + slugsToFetch.push(slug); } }); - // Add progress tracking + // Skip validation if we have recent cache data const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug)); console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`); - console.log(`Fetching ${slugsToFetch.length} comics from API`); - - const fetchPromises = slugsToFetch.map(async (slug) => { - const timer = Date.now(); - try { - const comicDetail = await getComicBySlug(slug); - - if (!comicDetail?.comic?.hid) { - console.error(`❌ No HID found for ${slug}`); + + // Only fetch if we have slugs that need fetching + let liveChapters: ChapterDetail[] = []; + if (slugsToFetch.length > 0) { + console.log(`Fetching ${slugsToFetch.length} comics from API`); + + // Use Promise.all for concurrent fetching with higher concurrency + const fetchPromises = slugsToFetch.map(async (slug) => { + try { + const comicDetail = await getComicBySlug(slug); + + if (!comicDetail?.comic?.hid) { + return []; + } + + return getChaptersByHid(comicDetail.comic.hid, slug, { + limit: 5, + lang: lang + }); + } catch (error) { + console.error(`Error processing ${slug}:`, error); return []; } - - const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, { - limit: 5, - lang: lang + }); + + liveChapters = (await Promise.all(fetchPromises)).flat(); + + // Cache new results in batch if we have any + if (liveChapters.length > 0) { + const cachePipeline = redis.pipeline(); + const chaptersBySlug: Record = {}; + + liveChapters.forEach(chapter => { + const slug = chapter.md_comics.slug; + if (!chaptersBySlug[slug]) { + chaptersBySlug[slug] = []; + } + chaptersBySlug[slug].push(chapter); }); - - console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`); - return chapters; - } catch (error) { - console.error(`🚨 Error processing ${slug}:`, error); - return []; + + Object.entries(chaptersBySlug).forEach(([slug, chaps]) => { + const cacheKey = `chapters:${slug}:${lang}`; + cachePipeline.set( + cacheKey, + JSON.stringify(chaps), + 'EX', + CHAPTER_CACHE_TTL + ); + }); + + await cachePipeline.exec(); } - }); + } - const liveChapters = (await Promise.all(fetchPromises)).flat(); - - // Cache new results in batch - const cachePipeline = redis.pipeline(); - const chaptersBySlug: Record = {}; - - liveChapters.forEach(chapter => { - const slug = chapter.md_comics.slug; - if (!chaptersBySlug[slug]) { - chaptersBySlug[slug] = []; - } - chaptersBySlug[slug].push(chapter); - }); - - Object.entries(chaptersBySlug).forEach(([slug, chaps]) => { - const cacheKey = `chapters:${slug}:${lang}`; - cachePipeline.set( - cacheKey, - JSON.stringify(chaps), - 'EX', - CHAPTER_CACHE_TTL - ); - }); - - await cachePipeline.exec(); - - // Combine and sort all chapters - const allChapters = [...cachedChapters, ...liveChapters]; - const deduplicatedChapters = deduplicateChapters(allChapters); - return deduplicatedChapters.sort((a, b) => { - // Ensure the created_at fields are valid dates - const dateA = new Date(a.created_at).getTime(); - const dateB = new Date(b.created_at).getTime(); - - // First try to sort by chapter number - const chapDiff = parseFloat(b.chap) - parseFloat(a.chap); - if (!isNaN(chapDiff)) return chapDiff; - - // Fall back to date if chapter numbers are equal or invalid - return dateB - dateA; - }); + // Combine and deduplicate + return deduplicateChapters([...cachedChapters, ...liveChapters]); } -// Add export to the validation function +// Replace validateSlugs with a more efficient implementation export async function validateSlugs(slugs: string[]): Promise { + // Skip validation in production for cached items + if (process.env.NODE_ENV === 'production') { + return slugs; + } + const validSlugs: string[] = []; const invalidSlugs: string[] = []; - // Process in small batches to avoid rate limiting - const batchSize = 3; + // Increase batch size for faster processing + const batchSize = 5; for (let i = 0; i < slugs.length; i += batchSize) { const batch = slugs.slice(i, i + batchSize); await Promise.all(batch.map(async (slug) => { try { - const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`, {}, 1); + // Reduced retries (1 instead of 3) and removed tachiyomi parameter + const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`, {}, 1); if (response.ok) { validSlugs.push(slug); } else { @@ -414,15 +415,10 @@ export async function validateSlugs(slugs: string[]): Promise { invalidSlugs.push(slug); } })); - - // Add a small delay between batches - if (i + batchSize < slugs.length) { - await new Promise(resolve => setTimeout(resolve, 300)); - } } if (invalidSlugs.length > 0) { - console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`); + console.warn(`Skipping invalid slugs: ${invalidSlugs.length} items`); } return validSlugs; diff --git a/src/lib/db.ts b/src/lib/db.ts index 7eabf97..f1fe845 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -106,24 +106,19 @@ async function createRedisClient(url: string): Promise { throw new Error("Redis URL not provided"); } - console.log(`Creating Redis client for ${getRedisName(url)}: ${url.slice(0, 20)}...`); const redisName = getRedisName(url); const client = new Redis(url, { retryStrategy: (times) => { if (times >= MAX_RETRIES) return null; - const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 1000); // Cap at 1s - console.log( - `${redisName} retry ${times}/${MAX_RETRIES}, delay ${delay}ms` - ); - return delay; + return Math.min(BASE_DELAY_MS * Math.pow(1.5, times), 1000); }, - maxRetriesPerRequest: null, + maxRetriesPerRequest: 2, enableOfflineQueue: true, enableReadyCheck: true, - connectTimeout: IS_VERCEL ? 5000 : 10000, // 5s for Vercel, 10s local - keepAlive: IS_VERCEL ? 2000 : 10000, // 2s ping for Vercel - commandTimeout: 5000, // Align with Vercel's timeout + connectTimeout: 3000, + keepAlive: 5000, + commandTimeout: 3000, reconnectOnError: (err) => ["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) => err.message.includes(e) @@ -714,11 +709,9 @@ export async function getBatchFeedMappings( const redis = await getRedisClient(); const pipeline = redis.pipeline(); - // Get all mappings in a single round trip + // Check regular keys first feedIds.forEach(id => { pipeline.get(id); - // Also check for compressed version - pipeline.exists(`${id}:compressed`); }); const results = await pipeline.exec(); @@ -727,52 +720,22 @@ export async function getBatchFeedMappings( // Process results const mappings: Record = {}; const ttlUpdates = redis.pipeline(); + const compressedChecks = redis.pipeline(); + const compressedIds: string[] = []; for (let i = 0; i < feedIds.length; i++) { const id = feedIds[i]; - const mappingResult = results[i * 2]; - const compressedResult = results[i * 2 + 1]; + const result = results[i]; - if (mappingResult[0]) { - console.error(`Error retrieving mapping for ${id}:`, mappingResult[0]); + if (result[0]) { mappings[id] = null; continue; } - if (compressedResult[0]) { - console.error(`Error checking compressed mapping for ${id}:`, compressedResult[0]); - mappings[id] = null; - continue; - } - - // Check if we have a compressed version - if (compressedResult[1]) { - // Fetch and decompress - const compressedData = await redis.get(`${id}:compressed`); - if (compressedData) { - try { - const buffer = Buffer.from(compressedData, 'base64'); - const decompressed = await gunzip(buffer); - const data = JSON.parse(decompressed.toString() as string); - - mappings[id] = { - slugs: data.slugs, - lang: data.lang - }; - - // Reset TTL - ttlUpdates.expire(`${id}:compressed`, DEFAULT_CACHE_TTL); - } catch (e) { - console.error(`Error decompressing data for ${id}:`, e); - mappings[id] = null; - } - } else { - mappings[id] = null; - } - } else if (mappingResult[1]) { + if (result[1]) { // Process regular mapping try { - const data = JSON.parse(mappingResult[1] as string); + const data = JSON.parse(result[1] as string); mappings[id] = { slugs: data.slugs, lang: data.lang @@ -781,16 +744,66 @@ export async function getBatchFeedMappings( // Reset TTL ttlUpdates.expire(id, DEFAULT_CACHE_TTL); } catch (e) { - console.error(`Error parsing feed mapping for ${id}:`, e); mappings[id] = null; } } else { - mappings[id] = null; + // Check for compressed version + compressedChecks.exists(`${id}:compressed`); + compressedIds.push(id); } } - // Execute all TTL updates - await ttlUpdates.exec(); + // Execute TTL updates in background + ttlUpdates.exec().catch(err => console.error('TTL update error:', err)); + + // Check compressed versions if needed + if (compressedIds.length > 0) { + const compressedResults = await compressedChecks.exec(); + + if (compressedResults) { + const compressedFetches = redis.pipeline(); + const compressedToFetch: string[] = []; + + compressedResults.forEach((result, index) => { + if (!result[0] && result[1]) { + const id = compressedIds[index]; + compressedFetches.get(`${id}:compressed`); + compressedToFetch.push(id); + } + }); + + if (compressedToFetch.length > 0) { + const fetchResults = await compressedFetches.exec(); + + if (fetchResults) { + for (let i = 0; i < compressedToFetch.length; i++) { + const id = compressedToFetch[i]; + const [err, compressedData] = fetchResults[i]; + + if (!err && compressedData) { + try { + const buffer = Buffer.from(compressedData as string, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL in background + redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {}); + } catch (e) { + mappings[id] = null; + } + } else { + mappings[id] = null; + } + } + } + } + } + } return mappings; } catch (error) { From 17b910d5ab966dbc3a4a23bf81023acd3557cab3 Mon Sep 17 00:00:00 2001 From: ld3zian <42660576+ld3z@users.noreply.github.com> Date: Sat, 22 Mar 2025 13:58:20 -0700 Subject: [PATCH 3/4] format sluginput --- src/components/SlugInput.astro | 401 +++++++++++++++++++-------------- 1 file changed, 231 insertions(+), 170 deletions(-) diff --git a/src/components/SlugInput.astro b/src/components/SlugInput.astro index 43079a1..06ba25c 100644 --- a/src/components/SlugInput.astro +++ b/src/components/SlugInput.astro @@ -1,25 +1,17 @@ --- -import { getFeedMapping } from '../lib/db'; +import { getFeedMapping } from "../lib/db"; interface Props { currentSlugs?: string[]; } const { currentSlugs = [] } = Astro.props; -import { availableLanguages, languageMap } from '../lib/api'; +import { availableLanguages, languageMap } from "../lib/api"; -const languages = [ - { value: 'en', label: 'English' }, - { value: 'ja', label: 'Japanese' }, - { value: 'ko', label: 'Korean' }, - { value: 'zh', label: 'Chinese' }, - { value: 'fr', label: 'French' }, - { value: 'es', label: 'Spanish' }, - { value: 'it', label: 'Italian' }, - { value: 'de', label: 'German' }, - { value: 'tr', label: 'Turkish' }, - { value: 'pl', label: 'Polish' }, -]; +const languages = Object.entries(languageMap).map(([value, label]) => ({ + value, + label, +})); ---
@@ -37,9 +29,11 @@ const languages = [ id="lang-select" class="px-3 sm:px-4 py-2 bg-primary rounded-lg border border-hover/30 focus:outline-none focus:border-accent/50 transition-colors text-sm sm:text-base" > - {languages.map(lang => ( - - ))} + { + languages.map((lang) => ( + + )) + }
@@ -54,8 +48,16 @@ const languages = [ id="import-feed" class="px-4 py-2 bg-secondary hover:bg-hover transition-colors rounded-lg text-sm sm:text-base flex items-center gap-2" > - - + + Import Feed @@ -73,27 +75,46 @@ const languages = [ class="px-3 py-1.5 text-sm text-red-500 hover:text-red-600 hover:bg-red-500/10 rounded-lg transition-colors flex items-center gap-2" style="display: none;" > - - + + Clear All
- {currentSlugs.map(slug => ( -
- {slug} - -
- ))} + {slug} + +
+ )) + }
@@ -123,20 +144,26 @@ const languages = [
\ No newline at end of file + From caf6972cd215386468deee4244f7ae7ca22ca504 Mon Sep 17 00:00:00 2001 From: ld3zian <42660576+ld3z@users.noreply.github.com> Date: Sat, 22 Mar 2025 14:28:50 -0700 Subject: [PATCH 4/4] add more langs --- src/lib/api.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/api.ts b/src/lib/api.ts index b0a4a49..c0329e2 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -6,6 +6,7 @@ const languageMap = { "en": "English", "fr": "French", "es": "Spanish", + "es-419": "Spanish (Latin America)", "it": "Italian", "pl": "Polish", "tr": "Turkish", @@ -14,7 +15,10 @@ const languageMap = { "sv": "Swedish", "ar": "Arabic", "de": "German", - "ko": "Korean" + "ko": "Korean", + "pt": "Portuguese", + "pt-br": "Portuguese (Brazil)", + "pt-pt": "Portuguese (Portugal)" } as const; const languages = new Set(Object.keys(languageMap));