ldez/new-manga
1
0
Fork 0
mirror of https://github.com/ld3z/new-manga.git synced 2026-11-09 20:53:45 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

this will be slower

Browse source
This commit is contained in:
ld3zian 2025-03-22 12:28:05 -07:00
parent 40221bad5d
commit 9677cba078
4 changed files with 360 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

44
package-lock.json generated
View file

@ -17,6 +17,8 @@
"astro": "^5.5.4",
"feed": "^4.2.2",
"ioredis": "^5.6.0",
"p-throttle": "^7.0.0",
"redis-modules-sdk": "^1.4.0",
"tailwindcss": "^3.4.1"
}
},
@ -4373,6 +4375,17 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/p-throttle": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-7.0.0.tgz",
"integrity": "sha512-aio0v+S0QVkH1O+9x4dHtD4dgCExACcL+3EtNaGqC01GBudS9ijMuUsmN8OVScyV4OOp0jqdLShZFuSlbL/AsA==",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/p-timeout": {
"version": "6.1.4",
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz",
@ -4737,6 +4750,37 @@
"node": ">=4"
}
},
"node_modules/redis-modules-sdk": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/redis-modules-sdk/-/redis-modules-sdk-1.4.0.tgz",
"integrity": "sha512-HYCr3ThtyoMrR9MUGmJblPU5aua0G4sanIZHK5wz/QwhVACe3BBgIZpHeymidBaukiuC0uzKla9xJ8exRffjnA==",
"dependencies": {
"ioredis": "5.2.3"
}
},
"node_modules/redis-modules-sdk/node_modules/ioredis": {
"version": "5.2.3",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz",
"integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==",
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.0.1",
"lodash.defaults": "^4.2.0",
"lodash.isarguments": "^3.1.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=12.22.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",

2
package.json
View file

@ -20,6 +20,8 @@
"astro": "^5.5.4",
"feed": "^4.2.2",
"ioredis": "^5.6.0",
"p-throttle": "^7.0.0",
"redis-modules-sdk": "^1.4.0",
"tailwindcss": "^3.4.1"
}
}

80
src/lib/api.ts
View file

@ -1,5 +1,6 @@
import type { Comic, Genre, ComicType, ChapterDetail } from './types';
import { getRedisClient, CHAPTER_CACHE_TTL } from './db';
import pThrottle from 'p-throttle';
const languageMap = {
"en": "English",
@ -26,6 +27,20 @@ let genreCache: Genre[] = [];
const RETRY_COUNT = 3;
const RETRY_DELAY = 1000;
// Rate limiting configuration - adjust these values if needed
const API_REQUESTS_PER_SECOND = 5; // Maximum requests per second
const API_CONCURRENT_REQUESTS = 3; // Maximum concurrent requests
const DELAY_AFTER_429 = 2000; // Wait 2 seconds after a rate limit hit
// Create throttled fetch function to avoid rate limits
const throttledFetch = pThrottle({
limit: API_REQUESTS_PER_SECOND,
interval: 1000,
concurrency: API_CONCURRENT_REQUESTS
} as any)((url: string, options: RequestInit = {}, retries = RETRY_COUNT) => {
return fetchWithRetry(url, options, retries);
});
async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise<Response> {
try {
const response = await fetch(url, {
@ -37,6 +52,19 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries =
},
});
// Special handling for rate limit errors
if (response.status === 429) {
console.warn(`Rate limited on ${url}. Waiting ${DELAY_AFTER_429}ms before retry.`);
// Wait for the specified delay
await new Promise(resolve => setTimeout(resolve, DELAY_AFTER_429));
// Retry with one less retry count but only if we have retries left
if (retries > 0) {
return fetchWithRetry(url, options, retries - 1);
}
}
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
@ -44,7 +72,10 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries =
return response;
} catch (error) {
if (retries > 0) {
await new Promise(resolve => setTimeout(resolve, RETRY_DELAY));
// For non-429 errors, use exponential backoff
const delay = Math.min(RETRY_DELAY * Math.pow(2, RETRY_COUNT - retries), 10000);
console.log(`Retrying after ${delay}ms. Retries left: ${retries-1}`);
await new Promise(resolve => setTimeout(resolve, delay));
return fetchWithRetry(url, options, retries - 1);
}
throw error;
@ -55,7 +86,8 @@ export async function fetchGenres(): Promise<Genre[]> {
if (genreCache.length > 0) return genreCache;
try {
const response = await fetchWithRetry('https://api.comick.fun/genre');
// No tachiyomi param for genre endpoint
const response = await throttledFetch('https://api.comick.fun/genre');
genreCache = await response.json();
return genreCache;
} catch (error) {
@ -87,12 +119,12 @@ export async function fetchComics(
comicType: ComicType = 'all',
page: number = 1
): Promise<Comic[]> {
const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}`;
const apiUrl = `${urlBase}?lang=${language}&page=${page}&order=new&accept_erotic_content=${contentType === 'nsfw'}${comicType === 'all' ? '' : `&type=${comicType}`}&tachiyomi=true`;
try {
console.log('Fetching comics from:', apiUrl);
const response = await fetchWithRetry(apiUrl);
const response = await throttledFetch(apiUrl);
const data = await response.json();
if (!Array.isArray(data)) {
@ -146,7 +178,7 @@ interface ChapterParams {
export async function getComicBySlug(slug: string): Promise<ComicDetail | null> {
try {
const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`);
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`);
const data = await response.json();
if (!data?.comic?.hid) {
@ -170,18 +202,19 @@ export async function getChaptersByHid(
const queryParams = new URLSearchParams({
limit: params.limit?.toString() || '5',
lang: params.lang || 'en',
ordering: '-created_at'
ordering: '-created_at',
tachiyomi: 'true' // Add tachiyomi parameter
});
// Fetch comic details including cover
const comicResponse = await fetchWithRetry(
`https://api.comick.fun/comic/${hid}`
const comicResponse = await throttledFetch(
`https://api.comick.fun/comic/${hid}?tachiyomi=true`
);
const comicData = await comicResponse.json();
const comicTitle = comicData?.comic?.title || 'Unknown Comic';
const covers = comicData?.comic?.md_covers || [];
const response = await fetchWithRetry(
const response = await throttledFetch(
`https://api.comick.fun/comic/${hid}/chapters?${queryParams}`
);
const data = await response.json();
@ -364,18 +397,29 @@ export async function validateSlugs(slugs: string[]): Promise<string[]> {
const validSlugs: string[] = [];
const invalidSlugs: string[] = [];
await Promise.all(slugs.map(async (slug) => {
try {
const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1);
if (response.ok) {
validSlugs.push(slug);
} else {
// Process in small batches to avoid rate limiting
const batchSize = 3;
for (let i = 0; i < slugs.length; i += batchSize) {
const batch = slugs.slice(i, i + batchSize);
await Promise.all(batch.map(async (slug) => {
try {
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}?tachiyomi=true`, {}, 1);
if (response.ok) {
validSlugs.push(slug);
} else {
invalidSlugs.push(slug);
}
} catch {
invalidSlugs.push(slug);
}
} catch {
invalidSlugs.push(slug);
}));
// Add a small delay between batches
if (i + batchSize < slugs.length) {
await new Promise(resolve => setTimeout(resolve, 300));
}
}));
}
if (invalidSlugs.length > 0) {
console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`);

276
src/lib/db.ts
View file

@ -1,5 +1,8 @@
import Redis from "ioredis";
import type { FeedMapping } from "./types";
import { ReJSON } from 'redis-modules-sdk';
import * as zlib from 'zlib';
import { promisify } from 'util';
// Configuration constants
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
@ -55,6 +58,18 @@ const operationBuffer: RedisOperation[] = [];
// Redis instance naming for logging
const redisInstanceNames = new Map<string, string>();
// Add zlib for compression
const gzip = promisify(zlib.gzip);
const gunzip = promisify(zlib.gunzip);
// Compression threshold (in bytes)
const COMPRESSION_THRESHOLD = 1024; // 1KB
// Helper function to determine if compression should be used
function shouldCompress(data: string): boolean {
return Buffer.byteLength(data, 'utf8') > COMPRESSION_THRESHOLD;
}
/**
* Initialize Redis instance names for better logging
*/
@ -75,6 +90,14 @@ initializeRedisNames();
*/
const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown";
// Add at the top of your file, after the imports
declare module "ioredis" {
interface Redis {
json_set(key: string, path: string, json: any): Promise<string>;
json_get(key: string, ...paths: string[]): Promise<string>;
}
}
/**
* Create a new Redis client with optimized settings
*/
@ -455,15 +478,37 @@ export async function storeFeedMapping(
const stringValue = JSON.stringify(mapping);
try {
// First write to primary/active client
const redis = await getRedisClient(); // This already handles failover
// Get Redis client
const redis = await getRedisClient();
const activeUrl = redis.options.host || '';
await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
console.log(`Successfully stored mapping for feed ${feedId}`);
// Check if compression should be used
if (shouldCompress(stringValue)) {
console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`);
const compressed = await gzip(stringValue);
const base64Compressed = compressed.toString('base64');
// Store with compression flag
await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`);
// Replicate to other instances
const replicas = await getAvailableReplicas(activeUrl);
for (const replica of replicas) {
await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
}
} else {
// Use standard set operations without compression
await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
// Replicate to other instances
const replicas = await getAvailableReplicas(activeUrl);
for (const replica of replicas) {
await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
}
}
// Immediately replicate to other instances
await replicateOperation(activeUrl, feedId, stringValue, DEFAULT_CACHE_TTL);
console.log(`Successfully stored mapping for feed ${feedId}`);
} catch (error) {
console.error(`Error storing feed mapping for ${feedId}:`, error);
throw error;
@ -477,29 +522,75 @@ export async function getFeedMapping(
feedId: string
): Promise<FeedMapping | null> {
try {
const redis = await getRedisClient(); // This already handles failover
const mapping = await redis.get(feedId);
// Try primary first
const redis = await getRedisClient();
if (!mapping) return null;
// Check for compressed version first
const compressedKey = `${feedId}:compressed`;
const compressedData = await redis.get(compressedKey);
if (compressedData) {
console.log(`Found compressed data for ${feedId}`);
try {
// Decompress and parse
const buffer = Buffer.from(compressedData, 'base64');
const decompressed = await gunzip(buffer);
const data = JSON.parse(decompressed.toString());
// Reset expiration
await redis.expire(compressedKey, DEFAULT_CACHE_TTL);
return {
slugs: data.slugs,
lang: data.lang
};
} catch (error) {
console.error(`Error decompressing data for ${feedId}:`, error);
// Fall through to try uncompressed version
}
}
// If no compressed version, try regular key
let jsonString = await redis.get(feedId);
// If not found in primary but we have replicas, check them
if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) {
console.log(`Key ${feedId} not found in primary, checking replicas...`);
// Try each replica
for (const [url, replicaClient] of replicaClients.entries()) {
if (replicaClient.status === "ready") {
try {
jsonString = await replicaClient.get(feedId);
if (jsonString) {
console.log(`Found key ${feedId} in replica ${getRedisName(url)}`);
// Sync this key back to primary for future requests
if (primaryClient && primaryClient.status === "ready") {
primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL)
.catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err));
}
break;
}
} catch (error) {
console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error);
}
}
}
}
if (!jsonString) return null;
// Reset expiration on access
const activeUrl = redis.options.host || '';
await redis.expire(feedId, DEFAULT_CACHE_TTL);
// Also reset TTL on other replicas
replicateOperation(activeUrl, feedId, mapping, DEFAULT_CACHE_TTL)
.catch(err => console.error(`Failed to reset TTL on replicas for ${feedId}:`, err));
try {
const data = JSON.parse(mapping);
return {
slugs: data.slugs,
lang: data.lang
};
} catch (e) {
console.error(`Error parsing feed mapping for ${feedId}:`, e);
return null;
}
// Parse the result
const data = JSON.parse(jsonString as string);
return {
slugs: data.slugs,
lang: data.lang
};
} catch (error) {
console.error(`Error retrieving feed mapping for ${feedId}:`, error);
return null;
@ -571,6 +662,143 @@ export async function warmChapterCache(
}
}
// Add a new function for handling large feed mappings with compression
export async function storeLargeFeedMapping(
feedId: string,
slugs: string[],
lang: string
): Promise<void> {
const mapping = {
slugs,
lang,
created_at: new Date().toISOString()
};
const jsonString = JSON.stringify(mapping);
try {
const redis = await getRedisClient();
const activeUrl = redis.options.host || '';
// Use compression for large mappings
if (shouldCompress(jsonString)) {
const compressed = await gzip(jsonString);
const base64Compressed = compressed.toString('base64');
// Store with compression flag
await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
// Replicate to other instances
await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL);
console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`);
} else {
// Use normal storage for small mappings
await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL);
await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL);
console.log(`Successfully stored mapping for feed ${feedId}`);
}
} catch (error) {
console.error(`Error storing feed mapping for ${feedId}:`, error);
throw error;
}
}
// Batch get multiple feed mappings at once
export async function getBatchFeedMappings(
feedIds: string[]
): Promise<Record<string, FeedMapping | null>> {
if (!feedIds.length) return {};
try {
const redis = await getRedisClient();
const pipeline = redis.pipeline();
// Get all mappings in a single round trip
feedIds.forEach(id => {
pipeline.get(id);
// Also check for compressed version
pipeline.exists(`${id}:compressed`);
});
const results = await pipeline.exec();
if (!results) return {};
// Process results
const mappings: Record<string, FeedMapping | null> = {};
const ttlUpdates = redis.pipeline();
for (let i = 0; i < feedIds.length; i++) {
const id = feedIds[i];
const mappingResult = results[i * 2];
const compressedResult = results[i * 2 + 1];
if (mappingResult[0]) {
console.error(`Error retrieving mapping for ${id}:`, mappingResult[0]);
mappings[id] = null;
continue;
}
if (compressedResult[0]) {
console.error(`Error checking compressed mapping for ${id}:`, compressedResult[0]);
mappings[id] = null;
continue;
}
// Check if we have a compressed version
if (compressedResult[1]) {
// Fetch and decompress
const compressedData = await redis.get(`${id}:compressed`);
if (compressedData) {
try {
const buffer = Buffer.from(compressedData, 'base64');
const decompressed = await gunzip(buffer);
const data = JSON.parse(decompressed.toString() as string);
mappings[id] = {
slugs: data.slugs,
lang: data.lang
};
// Reset TTL
ttlUpdates.expire(`${id}:compressed`, DEFAULT_CACHE_TTL);
} catch (e) {
console.error(`Error decompressing data for ${id}:`, e);
mappings[id] = null;
}
} else {
mappings[id] = null;
}
} else if (mappingResult[1]) {
// Process regular mapping
try {
const data = JSON.parse(mappingResult[1] as string);
mappings[id] = {
slugs: data.slugs,
lang: data.lang
};
// Reset TTL
ttlUpdates.expire(id, DEFAULT_CACHE_TTL);
} catch (e) {
console.error(`Error parsing feed mapping for ${id}:`, e);
mappings[id] = null;
}
} else {
mappings[id] = null;
}
}
// Execute all TTL updates
await ttlUpdates.exec();
return mappings;
} catch (error) {
console.error(`Error retrieving batch feed mappings:`, error);
return {};
}
}
// Initialize connections eagerly to establish connections as soon as possible
if (!import.meta.env.SSR) {
initializeAllClients()