maybe its fixed?

This commit is contained in:
ld3zian 2025-03-22 13:25:51 -07:00
parent 9677cba078
commit eaeaeae7d2
2 changed files with 149 additions and 140 deletions

View file

@ -28,9 +28,9 @@ const RETRY_COUNT = 3;
const RETRY_DELAY = 1000; const RETRY_DELAY = 1000;
// Rate limiting configuration - adjust these values if needed // Rate limiting configuration - adjust these values if needed
const API_REQUESTS_PER_SECOND = 5; // Maximum requests per second const API_REQUESTS_PER_SECOND = 10; // Maximum requests per second
const API_CONCURRENT_REQUESTS = 3; // Maximum concurrent requests const API_CONCURRENT_REQUESTS = 6; // Maximum concurrent requests
const DELAY_AFTER_429 = 2000; // Wait 2 seconds after a rate limit hit const DELAY_AFTER_429 = 1000; // Wait 1 second after a rate limit hit
// Create throttled fetch function to avoid rate limits // Create throttled fetch function to avoid rate limits
const throttledFetch = pThrottle({ const throttledFetch = pThrottle({
@ -119,7 +119,7 @@ export async function fetchComics(
comicType: ComicType = 'all', comicType: ComicType = 'all',
page: number = 1 page: number = 1
): Promise<Comic[]> { ): Promise<Comic[]> {
const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}&tachiyomi=true`; const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}`;
try { try {
console.log('Fetching comics from:', apiUrl); console.log('Fetching comics from:', apiUrl);
@ -178,7 +178,7 @@ interface ChapterParams {
export async function getComicBySlug(slug: string): Promise<ComicDetail | null> { export async function getComicBySlug(slug: string): Promise<ComicDetail | null> {
try { try {
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`); const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`);
const data = await response.json(); const data = await response.json();
if (!data?.comic?.hid) { if (!data?.comic?.hid) {
@ -202,13 +202,12 @@ export async function getChaptersByHid(
const queryParams = new URLSearchParams({ const queryParams = new URLSearchParams({
limit: params.limit?.toString() || '5', limit: params.limit?.toString() || '5',
lang: params.lang || 'en', lang: params.lang || 'en',
ordering: '-created_at', ordering: '-created_at'
tachiyomi: 'true' // Add tachiyomi parameter
}); });
// Fetch comic details including cover // Fetch comic details including cover
const comicResponse = await throttledFetch( const comicResponse = await throttledFetch(
`https://api.comick.fun/comic/${hid}?tachiyomi=true` `https://api.comick.fun/comic/${hid}`
); );
const comicData = await comicResponse.json(); const comicData = await comicResponse.json();
const comicTitle = comicData?.comic?.title || 'Unknown Comic'; const comicTitle = comicData?.comic?.title || 'Unknown Comic';
@ -293,22 +292,25 @@ export async function getChaptersForSlugs(
const uniqueSlugs = [...new Set(slugs)]; const uniqueSlugs = [...new Set(slugs)];
const pipeline = redis.pipeline(); const pipeline = redis.pipeline();
// Add cache key validation // Check cache for all slugs
uniqueSlugs.forEach(slug => { uniqueSlugs.forEach(slug => {
if (!slug.match(/^[a-z0-9-]+$/)) { if (!slug.match(/^[a-z0-9-]+$/)) {
console.error(`Invalid slug format: ${slug}`); console.error(`Invalid slug format: ${slug}`);
return; // Skip invalid slugs immediately
} }
const cacheKey = `chapters:${slug}:${lang}`; const cacheKey = `chapters:${slug}:${lang}`;
pipeline.get(cacheKey); pipeline.get(cacheKey);
}); });
const [cacheResults, slugsToFetch] = await Promise.all([ const cacheResults = await pipeline.exec();
pipeline.exec(),
validateSlugs(uniqueSlugs) // Add this new function
]);
// Process cache results
const cachedChapters: ChapterDetail[] = []; const cachedChapters: ChapterDetail[] = [];
const slugsToFetch: string[] = [];
// Separate cached and uncached slugs in one pass
cacheResults.forEach(([err, result], index) => { cacheResults.forEach(([err, result], index) => {
const slug = uniqueSlugs[index];
if (!err && result) { if (!err && result) {
try { try {
const parsed = JSON.parse(result as string); const parsed = JSON.parse(result as string);
@ -316,42 +318,46 @@ export async function getChaptersForSlugs(
cachedChapters.push(...parsed); cachedChapters.push(...parsed);
} }
} catch (e) { } catch (e) {
console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e); console.error(`Cache parse error for ${slug}:`, e);
slugsToFetch.push(slug);
} }
} else {
slugsToFetch.push(slug);
} }
}); });
// Add progress tracking // Skip validation if we have recent cache data
const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug)); const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug));
console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`); console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`);
// Only fetch if we have slugs that need fetching
let liveChapters: ChapterDetail[] = [];
if (slugsToFetch.length > 0) {
console.log(`Fetching ${slugsToFetch.length} comics from API`); console.log(`Fetching ${slugsToFetch.length} comics from API`);
// Use Promise.all for concurrent fetching with higher concurrency
const fetchPromises = slugsToFetch.map(async (slug) => { const fetchPromises = slugsToFetch.map(async (slug) => {
const timer = Date.now();
try { try {
const comicDetail = await getComicBySlug(slug); const comicDetail = await getComicBySlug(slug);
if (!comicDetail?.comic?.hid) { if (!comicDetail?.comic?.hid) {
console.error(`❌ No HID found for ${slug}`);
return []; return [];
} }
const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, { return getChaptersByHid(comicDetail.comic.hid, slug, {
limit: 5, limit: 5,
lang: lang lang: lang
}); });
console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`);
return chapters;
} catch (error) { } catch (error) {
console.error(`🚨 Error processing ${slug}:`, error); console.error(`Error processing ${slug}:`, error);
return []; return [];
} }
}); });
const liveChapters = (await Promise.all(fetchPromises)).flat(); liveChapters = (await Promise.all(fetchPromises)).flat();
// Cache new results in batch // Cache new results in batch if we have any
if (liveChapters.length > 0) {
const cachePipeline = redis.pipeline(); const cachePipeline = redis.pipeline();
const chaptersBySlug: Record<string, ChapterDetail[]> = {}; const chaptersBySlug: Record<string, ChapterDetail[]> = {};
@ -374,37 +380,32 @@ export async function getChaptersForSlugs(
}); });
await cachePipeline.exec(); await cachePipeline.exec();
}
}
// Combine and sort all chapters // Combine and deduplicate
const allChapters = [...cachedChapters, ...liveChapters]; return deduplicateChapters([...cachedChapters, ...liveChapters]);
const deduplicatedChapters = deduplicateChapters(allChapters);
return deduplicatedChapters.sort((a, b) => {
// Ensure the created_at fields are valid dates
const dateA = new Date(a.created_at).getTime();
const dateB = new Date(b.created_at).getTime();
// First try to sort by chapter number
const chapDiff = parseFloat(b.chap) - parseFloat(a.chap);
if (!isNaN(chapDiff)) return chapDiff;
// Fall back to date if chapter numbers are equal or invalid
return dateB - dateA;
});
} }
// Add export to the validation function // Replace validateSlugs with a more efficient implementation
export async function validateSlugs(slugs: string[]): Promise<string[]> { export async function validateSlugs(slugs: string[]): Promise<string[]> {
// Skip validation in production for cached items
if (process.env.NODE_ENV === 'production') {
return slugs;
}
const validSlugs: string[] = []; const validSlugs: string[] = [];
const invalidSlugs: string[] = []; const invalidSlugs: string[] = [];
// Process in small batches to avoid rate limiting // Increase batch size for faster processing
const batchSize = 3; const batchSize = 5;
for (let i = 0; i < slugs.length; i += batchSize) { for (let i = 0; i < slugs.length; i += batchSize) {
const batch = slugs.slice(i, i + batchSize); const batch = slugs.slice(i, i + batchSize);
await Promise.all(batch.map(async (slug) => { await Promise.all(batch.map(async (slug) => {
try { try {
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`, {}, 1); // Reduced retries (1 instead of 3) and removed tachiyomi parameter
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`, {}, 1);
if (response.ok) { if (response.ok) {
validSlugs.push(slug); validSlugs.push(slug);
} else { } else {
@ -414,15 +415,10 @@ export async function validateSlugs(slugs: string[]): Promise<string[]> {
invalidSlugs.push(slug); invalidSlugs.push(slug);
} }
})); }));
// Add a small delay between batches
if (i + batchSize < slugs.length) {
await new Promise(resolve => setTimeout(resolve, 300));
}
} }
if (invalidSlugs.length > 0) { if (invalidSlugs.length > 0) {
console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`); console.warn(`Skipping invalid slugs: ${invalidSlugs.length} items`);
} }
return validSlugs; return validSlugs;

View file

@ -106,24 +106,19 @@ async function createRedisClient(url: string): Promise<Redis> {
throw new Error("Redis URL not provided"); throw new Error("Redis URL not provided");
} }
console.log(`Creating Redis client for ${getRedisName(url)}: ${url.slice(0, 20)}...`);
const redisName = getRedisName(url); const redisName = getRedisName(url);
const client = new Redis(url, { const client = new Redis(url, {
retryStrategy: (times) => { retryStrategy: (times) => {
if (times >= MAX_RETRIES) return null; if (times >= MAX_RETRIES) return null;
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 1000); // Cap at 1s return Math.min(BASE_DELAY_MS * Math.pow(1.5, times), 1000);
console.log(
`${redisName} retry ${times}/${MAX_RETRIES}, delay ${delay}ms`
);
return delay;
}, },
maxRetriesPerRequest: null, maxRetriesPerRequest: 2,
enableOfflineQueue: true, enableOfflineQueue: true,
enableReadyCheck: true, enableReadyCheck: true,
connectTimeout: IS_VERCEL ? 5000 : 10000, // 5s for Vercel, 10s local connectTimeout: 3000,
keepAlive: IS_VERCEL ? 2000 : 10000, // 2s ping for Vercel keepAlive: 5000,
commandTimeout: 5000, // Align with Vercel's timeout commandTimeout: 3000,
reconnectOnError: (err) => reconnectOnError: (err) =>
["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) => ["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) =>
err.message.includes(e) err.message.includes(e)
@ -714,11 +709,9 @@ export async function getBatchFeedMappings(
const redis = await getRedisClient(); const redis = await getRedisClient();
const pipeline = redis.pipeline(); const pipeline = redis.pipeline();
// Get all mappings in a single round trip // Check regular keys first
feedIds.forEach(id => { feedIds.forEach(id => {
pipeline.get(id); pipeline.get(id);
// Also check for compressed version
pipeline.exists(`${id}:compressed`);
}); });
const results = await pipeline.exec(); const results = await pipeline.exec();
@ -727,52 +720,22 @@ export async function getBatchFeedMappings(
// Process results // Process results
const mappings: Record<string, FeedMapping | null> = {}; const mappings: Record<string, FeedMapping | null> = {};
const ttlUpdates = redis.pipeline(); const ttlUpdates = redis.pipeline();
const compressedChecks = redis.pipeline();
const compressedIds: string[] = [];
for (let i = 0; i < feedIds.length; i++) { for (let i = 0; i < feedIds.length; i++) {
const id = feedIds[i]; const id = feedIds[i];
const mappingResult = results[i * 2]; const result = results[i];
const compressedResult = results[i * 2 + 1];
if (mappingResult[0]) { if (result[0]) {
console.error(`Error retrieving mapping for ${id}:`, mappingResult[0]);
mappings[id] = null; mappings[id] = null;
continue; continue;
} }
if (compressedResult[0]) { if (result[1]) {
console.error(`Error checking compressed mapping for ${id}:`, compressedResult[0]);
mappings[id] = null;
continue;
}
// Check if we have a compressed version
if (compressedResult[1]) {
// Fetch and decompress
const compressedData = await redis.get(`${id}:compressed`);
if (compressedData) {
try {
const buffer = Buffer.from(compressedData, 'base64');
const decompressed = await gunzip(buffer);
const data = JSON.parse(decompressed.toString() as string);
mappings[id] = {
slugs: data.slugs,
lang: data.lang
};
// Reset TTL
ttlUpdates.expire(`${id}:compressed`, DEFAULT_CACHE_TTL);
} catch (e) {
console.error(`Error decompressing data for ${id}:`, e);
mappings[id] = null;
}
} else {
mappings[id] = null;
}
} else if (mappingResult[1]) {
// Process regular mapping // Process regular mapping
try { try {
const data = JSON.parse(mappingResult[1] as string); const data = JSON.parse(result[1] as string);
mappings[id] = { mappings[id] = {
slugs: data.slugs, slugs: data.slugs,
lang: data.lang lang: data.lang
@ -781,16 +744,66 @@ export async function getBatchFeedMappings(
// Reset TTL // Reset TTL
ttlUpdates.expire(id, DEFAULT_CACHE_TTL); ttlUpdates.expire(id, DEFAULT_CACHE_TTL);
} catch (e) { } catch (e) {
console.error(`Error parsing feed mapping for ${id}:`, e); mappings[id] = null;
}
} else {
// Check for compressed version
compressedChecks.exists(`${id}:compressed`);
compressedIds.push(id);
}
}
// Execute TTL updates in background
ttlUpdates.exec().catch(err => console.error('TTL update error:', err));
// Check compressed versions if needed
if (compressedIds.length > 0) {
const compressedResults = await compressedChecks.exec();
if (compressedResults) {
const compressedFetches = redis.pipeline();
const compressedToFetch: string[] = [];
compressedResults.forEach((result, index) => {
if (!result[0] && result[1]) {
const id = compressedIds[index];
compressedFetches.get(`${id}:compressed`);
compressedToFetch.push(id);
}
});
if (compressedToFetch.length > 0) {
const fetchResults = await compressedFetches.exec();
if (fetchResults) {
for (let i = 0; i < compressedToFetch.length; i++) {
const id = compressedToFetch[i];
const [err, compressedData] = fetchResults[i];
if (!err && compressedData) {
try {
const buffer = Buffer.from(compressedData as string, 'base64');
const decompressed = await gunzip(buffer);
const data = JSON.parse(decompressed.toString());
mappings[id] = {
slugs: data.slugs,
lang: data.lang
};
// Reset TTL in background
redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {});
} catch (e) {
mappings[id] = null; mappings[id] = null;
} }
} else { } else {
mappings[id] = null; mappings[id] = null;
} }
} }
}
// Execute all TTL updates }
await ttlUpdates.exec(); }
}
return mappings; return mappings;
} catch (error) { } catch (error) {