mirror of
https://github.com/ld3z/new-manga.git
synced 2026-11-09 20:43:45 -07:00
maybe its fixed?
This commit is contained in:
ld3zian
parent
9677cba078
commit
eaeaeae7d2
2 changed files with 149 additions and 140 deletions
170
src/lib/api.ts
170
src/lib/api.ts
|
|
@ -28,9 +28,9 @@ const RETRY_COUNT = 3;
|
|||
const RETRY_DELAY = 1000;
|
||||
|
||||
// Rate limiting configuration - adjust these values if needed
|
||||
const API_REQUESTS_PER_SECOND = 5; // Maximum requests per second
|
||||
const API_CONCURRENT_REQUESTS = 3; // Maximum concurrent requests
|
||||
const DELAY_AFTER_429 = 2000; // Wait 2 seconds after a rate limit hit
|
||||
const API_REQUESTS_PER_SECOND = 10; // Maximum requests per second
|
||||
const API_CONCURRENT_REQUESTS = 6; // Maximum concurrent requests
|
||||
const DELAY_AFTER_429 = 1000; // Wait 1 second after a rate limit hit
|
||||
|
||||
// Create throttled fetch function to avoid rate limits
|
||||
const throttledFetch = pThrottle({
|
||||
|
|
@ -119,7 +119,7 @@ export async function fetchComics(
|
|||
comicType: ComicType = 'all',
|
||||
page: number = 1
|
||||
): Promise<Comic[]> {
|
||||
const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}&tachiyomi=true`;
|
||||
const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}`;
|
||||
|
||||
try {
|
||||
console.log('Fetching comics from:', apiUrl);
|
||||
|
|
@ -178,7 +178,7 @@ interface ChapterParams {
|
|||
|
||||
export async function getComicBySlug(slug: string): Promise<ComicDetail | null> {
|
||||
try {
|
||||
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`);
|
||||
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`);
|
||||
const data = await response.json();
|
||||
|
||||
if (!data?.comic?.hid) {
|
||||
|
|
@ -202,13 +202,12 @@ export async function getChaptersByHid(
|
|||
const queryParams = new URLSearchParams({
|
||||
limit: params.limit?.toString() || '5',
|
||||
lang: params.lang || 'en',
|
||||
ordering: '-created_at',
|
||||
tachiyomi: 'true' // Add tachiyomi parameter
|
||||
ordering: '-created_at'
|
||||
});
|
||||
|
||||
// Fetch comic details including cover
|
||||
const comicResponse = await throttledFetch(
|
||||
`https://api.comick.fun/comic/${hid}?tachiyomi=true`
|
||||
`https://api.comick.fun/comic/${hid}`
|
||||
);
|
||||
const comicData = await comicResponse.json();
|
||||
const comicTitle = comicData?.comic?.title || 'Unknown Comic';
|
||||
|
|
@ -293,22 +292,25 @@ export async function getChaptersForSlugs(
|
|||
const uniqueSlugs = [...new Set(slugs)];
|
||||
const pipeline = redis.pipeline();
|
||||
|
||||
// Add cache key validation
|
||||
// Check cache for all slugs
|
||||
uniqueSlugs.forEach(slug => {
|
||||
if (!slug.match(/^[a-z0-9-]+$/)) {
|
||||
console.error(`Invalid slug format: ${slug}`);
|
||||
return; // Skip invalid slugs immediately
|
||||
}
|
||||
const cacheKey = `chapters:${slug}:${lang}`;
|
||||
pipeline.get(cacheKey);
|
||||
});
|
||||
|
||||
const [cacheResults, slugsToFetch] = await Promise.all([
|
||||
pipeline.exec(),
|
||||
validateSlugs(uniqueSlugs) // Add this new function
|
||||
]);
|
||||
|
||||
const cacheResults = await pipeline.exec();
|
||||
|
||||
// Process cache results
|
||||
const cachedChapters: ChapterDetail[] = [];
|
||||
const slugsToFetch: string[] = [];
|
||||
|
||||
// Separate cached and uncached slugs in one pass
|
||||
cacheResults.forEach(([err, result], index) => {
|
||||
const slug = uniqueSlugs[index];
|
||||
if (!err && result) {
|
||||
try {
|
||||
const parsed = JSON.parse(result as string);
|
||||
|
|
@ -316,95 +318,94 @@ export async function getChaptersForSlugs(
|
|||
cachedChapters.push(...parsed);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e);
|
||||
console.error(`Cache parse error for ${slug}:`, e);
|
||||
slugsToFetch.push(slug);
|
||||
}
|
||||
} else {
|
||||
slugsToFetch.push(slug);
|
||||
}
|
||||
});
|
||||
|
||||
// Add progress tracking
|
||||
// Skip validation if we have recent cache data
|
||||
const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug));
|
||||
console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`);
|
||||
console.log(`Fetching ${slugsToFetch.length} comics from API`);
|
||||
|
||||
const fetchPromises = slugsToFetch.map(async (slug) => {
|
||||
const timer = Date.now();
|
||||
try {
|
||||
const comicDetail = await getComicBySlug(slug);
|
||||
|
||||
if (!comicDetail?.comic?.hid) {
|
||||
console.error(`❌ No HID found for ${slug}`);
|
||||
|
||||
// Only fetch if we have slugs that need fetching
|
||||
let liveChapters: ChapterDetail[] = [];
|
||||
if (slugsToFetch.length > 0) {
|
||||
console.log(`Fetching ${slugsToFetch.length} comics from API`);
|
||||
|
||||
// Use Promise.all for concurrent fetching with higher concurrency
|
||||
const fetchPromises = slugsToFetch.map(async (slug) => {
|
||||
try {
|
||||
const comicDetail = await getComicBySlug(slug);
|
||||
|
||||
if (!comicDetail?.comic?.hid) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return getChaptersByHid(comicDetail.comic.hid, slug, {
|
||||
limit: 5,
|
||||
lang: lang
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`Error processing ${slug}:`, error);
|
||||
return [];
|
||||
}
|
||||
|
||||
const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, {
|
||||
limit: 5,
|
||||
lang: lang
|
||||
});
|
||||
|
||||
liveChapters = (await Promise.all(fetchPromises)).flat();
|
||||
|
||||
// Cache new results in batch if we have any
|
||||
if (liveChapters.length > 0) {
|
||||
const cachePipeline = redis.pipeline();
|
||||
const chaptersBySlug: Record<string, ChapterDetail[]> = {};
|
||||
|
||||
liveChapters.forEach(chapter => {
|
||||
const slug = chapter.md_comics.slug;
|
||||
if (!chaptersBySlug[slug]) {
|
||||
chaptersBySlug[slug] = [];
|
||||
}
|
||||
chaptersBySlug[slug].push(chapter);
|
||||
});
|
||||
|
||||
console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`);
|
||||
return chapters;
|
||||
} catch (error) {
|
||||
console.error(`🚨 Error processing ${slug}:`, error);
|
||||
return [];
|
||||
|
||||
Object.entries(chaptersBySlug).forEach(([slug, chaps]) => {
|
||||
const cacheKey = `chapters:${slug}:${lang}`;
|
||||
cachePipeline.set(
|
||||
cacheKey,
|
||||
JSON.stringify(chaps),
|
||||
'EX',
|
||||
CHAPTER_CACHE_TTL
|
||||
);
|
||||
});
|
||||
|
||||
await cachePipeline.exec();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const liveChapters = (await Promise.all(fetchPromises)).flat();
|
||||
|
||||
// Cache new results in batch
|
||||
const cachePipeline = redis.pipeline();
|
||||
const chaptersBySlug: Record<string, ChapterDetail[]> = {};
|
||||
|
||||
liveChapters.forEach(chapter => {
|
||||
const slug = chapter.md_comics.slug;
|
||||
if (!chaptersBySlug[slug]) {
|
||||
chaptersBySlug[slug] = [];
|
||||
}
|
||||
chaptersBySlug[slug].push(chapter);
|
||||
});
|
||||
|
||||
Object.entries(chaptersBySlug).forEach(([slug, chaps]) => {
|
||||
const cacheKey = `chapters:${slug}:${lang}`;
|
||||
cachePipeline.set(
|
||||
cacheKey,
|
||||
JSON.stringify(chaps),
|
||||
'EX',
|
||||
CHAPTER_CACHE_TTL
|
||||
);
|
||||
});
|
||||
|
||||
await cachePipeline.exec();
|
||||
|
||||
// Combine and sort all chapters
|
||||
const allChapters = [...cachedChapters, ...liveChapters];
|
||||
const deduplicatedChapters = deduplicateChapters(allChapters);
|
||||
return deduplicatedChapters.sort((a, b) => {
|
||||
// Ensure the created_at fields are valid dates
|
||||
const dateA = new Date(a.created_at).getTime();
|
||||
const dateB = new Date(b.created_at).getTime();
|
||||
|
||||
// First try to sort by chapter number
|
||||
const chapDiff = parseFloat(b.chap) - parseFloat(a.chap);
|
||||
if (!isNaN(chapDiff)) return chapDiff;
|
||||
|
||||
// Fall back to date if chapter numbers are equal or invalid
|
||||
return dateB - dateA;
|
||||
});
|
||||
// Combine and deduplicate
|
||||
return deduplicateChapters([...cachedChapters, ...liveChapters]);
|
||||
}
|
||||
|
||||
// Add export to the validation function
|
||||
// Replace validateSlugs with a more efficient implementation
|
||||
export async function validateSlugs(slugs: string[]): Promise<string[]> {
|
||||
// Skip validation in production for cached items
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
return slugs;
|
||||
}
|
||||
|
||||
const validSlugs: string[] = [];
|
||||
const invalidSlugs: string[] = [];
|
||||
|
||||
// Process in small batches to avoid rate limiting
|
||||
const batchSize = 3;
|
||||
// Increase batch size for faster processing
|
||||
const batchSize = 5;
|
||||
for (let i = 0; i < slugs.length; i += batchSize) {
|
||||
const batch = slugs.slice(i, i + batchSize);
|
||||
|
||||
await Promise.all(batch.map(async (slug) => {
|
||||
try {
|
||||
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`, {}, 1);
|
||||
// Reduced retries (1 instead of 3) and removed tachiyomi parameter
|
||||
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`, {}, 1);
|
||||
if (response.ok) {
|
||||
validSlugs.push(slug);
|
||||
} else {
|
||||
|
|
@ -414,15 +415,10 @@ export async function validateSlugs(slugs: string[]): Promise<string[]> {
|
|||
invalidSlugs.push(slug);
|
||||
}
|
||||
}));
|
||||
|
||||
// Add a small delay between batches
|
||||
if (i + batchSize < slugs.length) {
|
||||
await new Promise(resolve => setTimeout(resolve, 300));
|
||||
}
|
||||
}
|
||||
|
||||
if (invalidSlugs.length > 0) {
|
||||
console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`);
|
||||
console.warn(`Skipping invalid slugs: ${invalidSlugs.length} items`);
|
||||
}
|
||||
|
||||
return validSlugs;
|
||||
|
|
|
|||
119
src/lib/db.ts
119
src/lib/db.ts
|
|
@ -106,24 +106,19 @@ async function createRedisClient(url: string): Promise<Redis> {
|
|||
throw new Error("Redis URL not provided");
|
||||
}
|
||||
|
||||
console.log(`Creating Redis client for ${getRedisName(url)}: ${url.slice(0, 20)}...`);
|
||||
const redisName = getRedisName(url);
|
||||
|
||||
const client = new Redis(url, {
|
||||
retryStrategy: (times) => {
|
||||
if (times >= MAX_RETRIES) return null;
|
||||
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 1000); // Cap at 1s
|
||||
console.log(
|
||||
`${redisName} retry ${times}/${MAX_RETRIES}, delay ${delay}ms`
|
||||
);
|
||||
return delay;
|
||||
return Math.min(BASE_DELAY_MS * Math.pow(1.5, times), 1000);
|
||||
},
|
||||
maxRetriesPerRequest: null,
|
||||
maxRetriesPerRequest: 2,
|
||||
enableOfflineQueue: true,
|
||||
enableReadyCheck: true,
|
||||
connectTimeout: IS_VERCEL ? 5000 : 10000, // 5s for Vercel, 10s local
|
||||
keepAlive: IS_VERCEL ? 2000 : 10000, // 2s ping for Vercel
|
||||
commandTimeout: 5000, // Align with Vercel's timeout
|
||||
connectTimeout: 3000,
|
||||
keepAlive: 5000,
|
||||
commandTimeout: 3000,
|
||||
reconnectOnError: (err) =>
|
||||
["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) =>
|
||||
err.message.includes(e)
|
||||
|
|
@ -714,11 +709,9 @@ export async function getBatchFeedMappings(
|
|||
const redis = await getRedisClient();
|
||||
const pipeline = redis.pipeline();
|
||||
|
||||
// Get all mappings in a single round trip
|
||||
// Check regular keys first
|
||||
feedIds.forEach(id => {
|
||||
pipeline.get(id);
|
||||
// Also check for compressed version
|
||||
pipeline.exists(`${id}:compressed`);
|
||||
});
|
||||
|
||||
const results = await pipeline.exec();
|
||||
|
|
@ -727,52 +720,22 @@ export async function getBatchFeedMappings(
|
|||
// Process results
|
||||
const mappings: Record<string, FeedMapping | null> = {};
|
||||
const ttlUpdates = redis.pipeline();
|
||||
const compressedChecks = redis.pipeline();
|
||||
const compressedIds: string[] = [];
|
||||
|
||||
for (let i = 0; i < feedIds.length; i++) {
|
||||
const id = feedIds[i];
|
||||
const mappingResult = results[i * 2];
|
||||
const compressedResult = results[i * 2 + 1];
|
||||
const result = results[i];
|
||||
|
||||
if (mappingResult[0]) {
|
||||
console.error(`Error retrieving mapping for ${id}:`, mappingResult[0]);
|
||||
if (result[0]) {
|
||||
mappings[id] = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (compressedResult[0]) {
|
||||
console.error(`Error checking compressed mapping for ${id}:`, compressedResult[0]);
|
||||
mappings[id] = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if we have a compressed version
|
||||
if (compressedResult[1]) {
|
||||
// Fetch and decompress
|
||||
const compressedData = await redis.get(`${id}:compressed`);
|
||||
if (compressedData) {
|
||||
try {
|
||||
const buffer = Buffer.from(compressedData, 'base64');
|
||||
const decompressed = await gunzip(buffer);
|
||||
const data = JSON.parse(decompressed.toString() as string);
|
||||
|
||||
mappings[id] = {
|
||||
slugs: data.slugs,
|
||||
lang: data.lang
|
||||
};
|
||||
|
||||
// Reset TTL
|
||||
ttlUpdates.expire(`${id}:compressed`, DEFAULT_CACHE_TTL);
|
||||
} catch (e) {
|
||||
console.error(`Error decompressing data for ${id}:`, e);
|
||||
mappings[id] = null;
|
||||
}
|
||||
} else {
|
||||
mappings[id] = null;
|
||||
}
|
||||
} else if (mappingResult[1]) {
|
||||
if (result[1]) {
|
||||
// Process regular mapping
|
||||
try {
|
||||
const data = JSON.parse(mappingResult[1] as string);
|
||||
const data = JSON.parse(result[1] as string);
|
||||
mappings[id] = {
|
||||
slugs: data.slugs,
|
||||
lang: data.lang
|
||||
|
|
@ -781,16 +744,66 @@ export async function getBatchFeedMappings(
|
|||
// Reset TTL
|
||||
ttlUpdates.expire(id, DEFAULT_CACHE_TTL);
|
||||
} catch (e) {
|
||||
console.error(`Error parsing feed mapping for ${id}:`, e);
|
||||
mappings[id] = null;
|
||||
}
|
||||
} else {
|
||||
mappings[id] = null;
|
||||
// Check for compressed version
|
||||
compressedChecks.exists(`${id}:compressed`);
|
||||
compressedIds.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
// Execute all TTL updates
|
||||
await ttlUpdates.exec();
|
||||
// Execute TTL updates in background
|
||||
ttlUpdates.exec().catch(err => console.error('TTL update error:', err));
|
||||
|
||||
// Check compressed versions if needed
|
||||
if (compressedIds.length > 0) {
|
||||
const compressedResults = await compressedChecks.exec();
|
||||
|
||||
if (compressedResults) {
|
||||
const compressedFetches = redis.pipeline();
|
||||
const compressedToFetch: string[] = [];
|
||||
|
||||
compressedResults.forEach((result, index) => {
|
||||
if (!result[0] && result[1]) {
|
||||
const id = compressedIds[index];
|
||||
compressedFetches.get(`${id}:compressed`);
|
||||
compressedToFetch.push(id);
|
||||
}
|
||||
});
|
||||
|
||||
if (compressedToFetch.length > 0) {
|
||||
const fetchResults = await compressedFetches.exec();
|
||||
|
||||
if (fetchResults) {
|
||||
for (let i = 0; i < compressedToFetch.length; i++) {
|
||||
const id = compressedToFetch[i];
|
||||
const [err, compressedData] = fetchResults[i];
|
||||
|
||||
if (!err && compressedData) {
|
||||
try {
|
||||
const buffer = Buffer.from(compressedData as string, 'base64');
|
||||
const decompressed = await gunzip(buffer);
|
||||
const data = JSON.parse(decompressed.toString());
|
||||
|
||||
mappings[id] = {
|
||||
slugs: data.slugs,
|
||||
lang: data.lang
|
||||
};
|
||||
|
||||
// Reset TTL in background
|
||||
redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {});
|
||||
} catch (e) {
|
||||
mappings[id] = null;
|
||||
}
|
||||
} else {
|
||||
mappings[id] = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return mappings;
|
||||
} catch (error) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue