Compare commits

...

4 commits

Author SHA1 Message Date
ld3zian
01734e8ce5 update astro 2025-02-04 22:10:14 -07:00
ld3zian
66e9dd2a06 Fix errors maybe? 2025-02-04 19:29:03 -07:00
ld3zian
22c3e665e8 fix errors 2025-02-04 19:17:28 -07:00
ld3zian
0daac66b26 add better feed handling 2025-02-04 19:10:22 -07:00
7 changed files with 309 additions and 207 deletions

View file

@ -9,4 +9,9 @@ export default defineConfig({
remotePatterns: [{ protocol: "https" }], remotePatterns: [{ protocol: "https" }],
}, },
integrations: [tailwind()], integrations: [tailwind()],
vite: {
define: {
'process.env.REDIS_URL': JSON.stringify(process.env.REDIS_URL)
}
}
}); });

56
package-lock.json generated
View file

@ -12,9 +12,10 @@
"@astrojs/tailwind": "^6.0.0", "@astrojs/tailwind": "^6.0.0",
"@astrojs/vercel": "^8.0.5", "@astrojs/vercel": "^8.0.5",
"@floating-ui/dom": "^1.6.13", "@floating-ui/dom": "^1.6.13",
"@types/node": "^22.13.1",
"@vercel/analytics": "^1.4.1", "@vercel/analytics": "^1.4.1",
"@vercel/kv": "^3.0.0", "@vercel/kv": "^3.0.0",
"astro": "^5.2.3", "astro": "^5.2.5",
"feed": "^4.2.2", "feed": "^4.2.2",
"ioredis": "^5.4.2", "ioredis": "^5.4.2",
"tailwindcss": "^3.4.1" "tailwindcss": "^3.4.1"
@ -1462,6 +1463,14 @@
"@types/unist": "*" "@types/unist": "*"
} }
}, },
"node_modules/@types/node": {
"version": "22.13.1",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.1.tgz",
"integrity": "sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==",
"dependencies": {
"undici-types": "~6.20.0"
}
},
"node_modules/@types/unist": { "node_modules/@types/unist": {
"version": "3.0.3", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz",
@ -1744,12 +1753,12 @@
} }
}, },
"node_modules/astro": { "node_modules/astro": {
"version": "5.2.3", "version": "5.2.5",
"resolved": "https://registry.npmjs.org/astro/-/astro-5.2.3.tgz", "resolved": "https://registry.npmjs.org/astro/-/astro-5.2.5.tgz",
"integrity": "sha512-04AceE/gVihtO3V28Ss0+tPPgLHGoulXrm1E7fUVPYyQu7y6w8+oXfYyzh1lpy+uEYiebjH7AFlJoz73anBmZA==", "integrity": "sha512-AYXyYkc+c5xbKTm48FyQA91y81nXyNPAaoyafR0LUugE4lAwuvIUcXDBfMzmbuP1lGRvsE33G2oypv6gbGaPFg==",
"dependencies": { "dependencies": {
"@astrojs/compiler": "^2.10.3", "@astrojs/compiler": "^2.10.3",
"@astrojs/internal-helpers": "0.5.0", "@astrojs/internal-helpers": "0.5.1",
"@astrojs/markdown-remark": "6.1.0", "@astrojs/markdown-remark": "6.1.0",
"@astrojs/telemetry": "3.2.0", "@astrojs/telemetry": "3.2.0",
"@oslojs/encoding": "^1.1.0", "@oslojs/encoding": "^1.1.0",
@ -1787,10 +1796,10 @@
"neotraverse": "^0.6.18", "neotraverse": "^0.6.18",
"p-limit": "^6.2.0", "p-limit": "^6.2.0",
"p-queue": "^8.1.0", "p-queue": "^8.1.0",
"preferred-pm": "^4.1.0", "preferred-pm": "^4.1.1",
"prompts": "^2.4.2", "prompts": "^2.4.2",
"rehype": "^13.0.2", "rehype": "^13.0.2",
"semver": "^7.7.0", "semver": "^7.7.1",
"shiki": "^1.29.2", "shiki": "^1.29.2",
"tinyexec": "^0.3.2", "tinyexec": "^0.3.2",
"tsconfck": "^3.1.4", "tsconfck": "^3.1.4",
@ -1800,7 +1809,7 @@
"vfile": "^6.0.3", "vfile": "^6.0.3",
"vite": "^6.0.11", "vite": "^6.0.11",
"vitefu": "^1.0.5", "vitefu": "^1.0.5",
"which-pm": "^3.0.0", "which-pm": "^3.0.1",
"xxhash-wasm": "^1.1.0", "xxhash-wasm": "^1.1.0",
"yargs-parser": "^21.1.1", "yargs-parser": "^21.1.1",
"yocto-spinner": "^0.2.0", "yocto-spinner": "^0.2.0",
@ -1825,9 +1834,9 @@
} }
}, },
"node_modules/astro/node_modules/@astrojs/internal-helpers": { "node_modules/astro/node_modules/@astrojs/internal-helpers": {
"version": "0.5.0", "version": "0.5.1",
"resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.5.0.tgz", "resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.5.1.tgz",
"integrity": "sha512-CgB5ZaZO1PFG+rbjF3HnA7G6gIBjJ070xb7bUjeu5Gqqufma+t6fpuRWMXnK2iEO3zVyX7e/xplPlqtFKy/lvw==" "integrity": "sha512-M7rAge1n2+aOSxNvKUFa0u/KFn0W+sZy7EW91KOSERotm2Ti8qs+1K0xx3zbOxtAVrmJb5/J98eohVvvEqtNkw=="
}, },
"node_modules/astro/node_modules/@vercel/kv": { "node_modules/astro/node_modules/@vercel/kv": {
"version": "1.0.1", "version": "1.0.1",
@ -4789,13 +4798,13 @@
"integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==" "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
}, },
"node_modules/preferred-pm": { "node_modules/preferred-pm": {
"version": "4.1.0", "version": "4.1.1",
"resolved": "https://registry.npmjs.org/preferred-pm/-/preferred-pm-4.1.0.tgz", "resolved": "https://registry.npmjs.org/preferred-pm/-/preferred-pm-4.1.1.tgz",
"integrity": "sha512-cNKMVcSvE3hZBRukdRsvPtlUaM2fYLFbIvBz620XaDA5SjHsCRsAjNE2baI90Hh7QvJIjCHJYnpPtyqJ1/kOjQ==", "integrity": "sha512-rU+ZAv1Ur9jAUZtGPebQVQPzdGhNzaEiQ7VL9+cjsAWPHFYOccNXPNiev1CCDSOg/2j7UujM7ojNhpkuILEVNQ==",
"dependencies": { "dependencies": {
"find-up-simple": "^1.0.0", "find-up-simple": "^1.0.0",
"find-yarn-workspace-root2": "1.2.16", "find-yarn-workspace-root2": "1.2.16",
"which-pm": "^3.0.0" "which-pm": "^3.0.1"
}, },
"engines": { "engines": {
"node": ">=18.12" "node": ">=18.12"
@ -5288,9 +5297,9 @@
"integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==" "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg=="
}, },
"node_modules/semver": { "node_modules/semver": {
"version": "7.7.0", "version": "7.7.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.0.tgz", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
"integrity": "sha512-DrfFnPzblFmNrIZzg5RzHegbiRWg7KMR7btwi2yjHwx06zsUbO5g613sVwEV7FTwmzJu+Io0lJe2GJ3LxqpvBQ==", "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==",
"bin": { "bin": {
"semver": "bin/semver.js" "semver": "bin/semver.js"
}, },
@ -5811,6 +5820,11 @@
"resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz", "resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz",
"integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==" "integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q=="
}, },
"node_modules/undici-types": {
"version": "6.20.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
},
"node_modules/unenv": { "node_modules/unenv": {
"version": "1.10.0", "version": "1.10.0",
"resolved": "https://registry.npmjs.org/unenv/-/unenv-1.10.0.tgz", "resolved": "https://registry.npmjs.org/unenv/-/unenv-1.10.0.tgz",
@ -6162,9 +6176,9 @@
} }
}, },
"node_modules/which-pm": { "node_modules/which-pm": {
"version": "3.0.0", "version": "3.0.1",
"resolved": "https://registry.npmjs.org/which-pm/-/which-pm-3.0.0.tgz", "resolved": "https://registry.npmjs.org/which-pm/-/which-pm-3.0.1.tgz",
"integrity": "sha512-ysVYmw6+ZBhx3+ZkcPwRuJi38ZOTLJJ33PSHaitLxSKUMsh0LkKd0nC69zZCwt5D+AYUcMK2hhw4yWny20vSGg==", "integrity": "sha512-v2JrMq0waAI4ju1xU5x3blsxBBMgdgZve580iYMN5frDaLGjbA24fok7wKCsya8KLVO19Ju4XDc5+zTZCJkQfg==",
"dependencies": { "dependencies": {
"load-yaml-file": "^0.2.0" "load-yaml-file": "^0.2.0"
}, },

View file

@ -15,9 +15,10 @@
"@astrojs/tailwind": "^6.0.0", "@astrojs/tailwind": "^6.0.0",
"@astrojs/vercel": "^8.0.5", "@astrojs/vercel": "^8.0.5",
"@floating-ui/dom": "^1.6.13", "@floating-ui/dom": "^1.6.13",
"@types/node": "^22.13.1",
"@vercel/analytics": "^1.4.1", "@vercel/analytics": "^1.4.1",
"@vercel/kv": "^3.0.0", "@vercel/kv": "^3.0.0",
"astro": "^5.2.3", "astro": "^5.2.5",
"feed": "^4.2.2", "feed": "^4.2.2",
"ioredis": "^5.4.2", "ioredis": "^5.4.2",
"tailwindcss": "^3.4.1" "tailwindcss": "^3.4.1"

View file

@ -209,39 +209,31 @@ export async function getChaptersByHid(
const now = new Date(); const now = new Date();
const chapters = data.chapters const chapters: ChapterDetail[] = [];
.filter(chapter => { const rawChapters = data.chapters || [];
// Basic validation
const isValid = chapter &&
chapter.chap &&
chapter.hid;
if (!isValid) return false; for (const chap of rawChapters) {
try {
// Check publish time if (!chap.id || !chap.chap || !chap.updated_at) {
if (chapter.publish_at) { console.warn(`Skipping chapter with missing fields in ${hid}:`, chap);
const publishTime = new Date(chapter.publish_at); continue;
if (publishTime > now) {
console.log(`Chapter ${chapter.chap} will be published at ${publishTime}`);
return false;
}
} }
// Use the earliest available timestamp chapters.push({
const hasValidTimestamp = chapter.publish_at || chapter.created_at || chapter.updated_at; id: chap.id,
return Boolean(hasValidTimestamp); chap: chap.chap.toString(),
}) title: chap.title || `Chapter ${chap.chap}`,
.map(chapter => ({ updated_at: chap.updated_at,
id: chapter.hid,
chap: chapter.chap,
title: chapter.title || `Chapter ${chapter.chap}`,
updated_at: chapter.publish_at || chapter.created_at || chapter.updated_at,
md_comics: { md_comics: {
title: comicTitle, title: comicTitle,
slug: slug, slug: slug,
md_covers: covers md_covers: covers
} }
})); });
} catch (e) {
console.error(`Error processing chapter ${chap.id} for ${hid}:`, e);
}
}
console.log(`Successfully processed ${chapters.length} chapters for ${comicTitle}`); console.log(`Successfully processed ${chapters.length} chapters for ${comicTitle}`);
return chapters; return chapters;
@ -251,88 +243,140 @@ export async function getChaptersByHid(
} }
} }
// Add deduplication function
function deduplicateChapters(chapters: ChapterDetail[]): ChapterDetail[] {
const seen = new Set<string>();
return chapters.filter(chapter => {
const key = `${chapter.md_comics.slug}-${chapter.chap}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
});
}
export async function getChaptersForSlugs( export async function getChaptersForSlugs(
slugs: string[], slugs: string[],
lang: string = 'en' lang: string = 'en'
): Promise<ChapterDetail[]> { ): Promise<ChapterDetail[]> {
console.log(`Fetching chapters for slugs:`, slugs, `in language: ${lang}`); console.log(`Fetching chapters for ${slugs.length} comics`);
const chapters: ChapterDetail[] = [];
const redis = await getRedisClient(); const redis = await getRedisClient();
// Cache results for 1 hour (3600 seconds)
const CACHE_TTL = 60 * 60; const CACHE_TTL = 60 * 60;
for (const slug of slugs) { const uniqueSlugs = [...new Set(slugs)];
console.log(`Processing slug: ${slug}`); const pipeline = redis.pipeline();
// Add cache key validation
uniqueSlugs.forEach(slug => {
if (!slug.match(/^[a-z0-9-]+$/)) {
console.error(`Invalid slug format: ${slug}`);
}
const cacheKey = `chapters:${slug}:${lang}`; const cacheKey = `chapters:${slug}:${lang}`;
pipeline.get(cacheKey);
});
const [cacheResults, slugsToFetch] = await Promise.all([
pipeline.exec(),
validateSlugs(uniqueSlugs) // Add this new function
]);
const cachedChapters: ChapterDetail[] = [];
cacheResults.forEach(([err, result], index) => {
if (!err && result) {
try { try {
// Try to get cached chapters const parsed = JSON.parse(result);
const cachedChapters = await redis.get(cacheKey); if (Array.isArray(parsed)) {
if (cachedChapters) { cachedChapters.push(...parsed);
console.log(`Using cached chapters for ${slug}`);
chapters.push(...JSON.parse(cachedChapters));
continue;
} }
} catch (error) { } catch (e) {
console.error('Redis cache read error:', error); console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e);
} }
}
});
// Add progress tracking
const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug));
console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`);
console.log(`Fetching ${slugsToFetch.length} comics from API`);
const fetchPromises = slugsToFetch.map(async (slug) => {
const timer = Date.now();
try {
const comicDetail = await getComicBySlug(slug); const comicDetail = await getComicBySlug(slug);
if (!comicDetail) { if (!comicDetail?.comic?.hid) {
console.error(`Failed to get comic details for slug: ${slug}`); console.error(`❌ No HID found for ${slug}`);
continue; return [];
} }
if (!comicDetail.comic?.hid) { const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, {
console.error(`No HID found for slug ${slug}`, comicDetail);
continue;
}
console.log(`Found HID ${comicDetail.comic.hid} for slug ${slug}`);
const comicChapters = await getChaptersByHid(
comicDetail.comic.hid,
slug,
{
limit: 5, limit: 5,
lang: lang lang: lang
});
console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`);
return chapters;
} catch (error) {
console.error(`🚨 Error processing ${slug}:`, error);
return [];
} }
); });
console.log(`Found ${comicChapters.length} chapters for ${slug}`);
if (comicChapters.length > 0) { const liveChapters = (await Promise.all(fetchPromises)).flat();
chapters.push(...comicChapters);
// Cache the results // Cache new results in batch
try { const cachePipeline = redis.pipeline();
await redis.set( const chaptersBySlug: Record<string, ChapterDetail[]> = {};
liveChapters.forEach(chapter => {
const slug = chapter.md_comics.slug;
if (!chaptersBySlug[slug]) {
chaptersBySlug[slug] = [];
}
chaptersBySlug[slug].push(chapter);
});
Object.entries(chaptersBySlug).forEach(([slug, chaps]) => {
const cacheKey = `chapters:${slug}:${lang}`;
cachePipeline.set(
cacheKey, cacheKey,
JSON.stringify(comicChapters), JSON.stringify(chaps),
'EX', 'EX',
CACHE_TTL CACHE_TTL
); );
console.log(`Cached chapters for ${slug}`);
} catch (error) {
console.error('Redis cache write error:', error);
}
}
}
// Sort by chapter number (descending) first, then by date
const sortedChapters = chapters.sort((a, b) => {
// Convert chapter numbers to floats for proper numerical sorting
const chapA = parseFloat(a.chap);
const chapB = parseFloat(b.chap);
if (chapB !== chapA) {
return chapB - chapA; // Sort by chapter number first
}
// If chapters are the same, sort by date
return new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime();
}); });
console.log(`Total chapters found: ${sortedChapters.length}`); await cachePipeline.exec();
return sortedChapters;
// Combine and sort all chapters
const allChapters = [...cachedChapters, ...liveChapters];
const deduplicatedChapters = deduplicateChapters(allChapters);
return deduplicatedChapters.sort((a, b) =>
parseFloat(b.chap) - parseFloat(a.chap) ||
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime()
);
}
// Add export to the validation function
export async function validateSlugs(slugs: string[]): Promise<string[]> {
const validSlugs: string[] = [];
const invalidSlugs: string[] = [];
await Promise.all(slugs.map(async (slug) => {
try {
const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1);
if (response.ok) {
validSlugs.push(slug);
} else {
invalidSlugs.push(slug);
}
} catch {
invalidSlugs.push(slug);
}
}));
if (invalidSlugs.length > 0) {
console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`);
}
return validSlugs;
} }

View file

@ -1,104 +1,115 @@
import Redis from 'ioredis'; import Redis from 'ioredis';
import type { FeedMapping } from './types'; import type { FeedMapping } from './types';
import { getChaptersForSlugs } from './api';
const REDIS_URL = import.meta.env.REDIS_URL || process.env.REDIS_URL; const REDIS_URL = import.meta.env.REDIS_URL || process.env.REDIS_URL;
let redis: Redis | null = null; let redisClient: Redis | null = null;
let redisRetryCount = 0;
const MAX_RETRIES = 3;
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
const MAX_RETRIES = 5;
const BASE_DELAY_MS = 100;
try { // Improved connection handling with exponential backoff
console.log('Environment check:', { async function createRedisClient(): Promise<Redis> {
hasEnvVar: !!REDIS_URL, if (!REDIS_URL) throw new Error('REDIS_URL environment variable is not set');
envVarStart: REDIS_URL?.substring(0, 20) + '...',
nodeEnv: process.env.NODE_ENV
});
if (!REDIS_URL) { return new Redis(REDIS_URL, {
throw new Error('REDIS_URL environment variable is not set'); retryStrategy: (times) => {
} if (times >= MAX_RETRIES) return null;
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000);
console.log('Attempting to connect to Redis...'); console.log(`Redis retry attempt ${times}, next try in ${delay}ms`);
return delay + Math.random() * 200; // Add jitter
redis = new Redis(REDIS_URL, {
retryStrategy(times) {
const delay = Math.min(times * 50, 2000);
console.log(`Retry attempt ${times} with delay ${delay}ms`);
return delay;
}, },
maxRetriesPerRequest: 2, maxRetriesPerRequest: null, // Allow retries for queued commands
enableOfflineQueue: true, // Enable command queuing
enableReadyCheck: true, enableReadyCheck: true,
reconnectOnError: (err) => { reconnectOnError: (err) => {
const targetError = 'READONLY'; return err.message.includes('READONLY');
if (err.message.includes(targetError)) {
return true;
}
return false;
} }
}); });
redis.on('error', (error) => {
console.error('Redis connection error:', error.message);
});
redis.on('connect', () => {
console.log('Successfully connected to Redis');
});
redis.on('ready', () => {
console.log('Redis client is ready');
});
redis.on('reconnecting', () => {
console.log('Redis client is reconnecting');
});
} catch (error) {
console.error('Failed to initialize Redis:', error);
if (error instanceof Error) {
console.error('Error details:', error.message);
console.error('Stack trace:', error.stack);
}
} }
// Singleton pattern with connection pooling
export async function getRedisClient(): Promise<Redis> { export async function getRedisClient(): Promise<Redis> {
if (redis?.status === 'ready') return redis; if (redisClient?.status === 'ready') {
// Validate connection with a quick ping
if (redisRetryCount >= MAX_RETRIES) {
throw new Error('Failed to connect to Redis after multiple attempts');
}
try { try {
redis = new Redis(REDIS_URL, { await redisClient.ping();
retryStrategy(times) { return redisClient;
const delay = Math.min(times * 50, 2000); } catch {
console.log(`Retry attempt ${times} with delay ${delay}ms`); // Fall through to reconnect
return delay;
},
maxRetriesPerRequest: 2,
enableReadyCheck: true,
reconnectOnError: (err) => {
const targetError = 'READONLY';
if (err.message.includes(targetError)) {
return true;
} }
return false;
} }
console.log('Initializing new Redis connection...');
redisClient = await createRedisClient();
// Add event listeners for better monitoring
redisClient
.on('ready', () => console.log('Redis connection ready'))
.on('error', (err) => console.error('Redis error:', err))
.on('reconnecting', () => console.log('Redis reconnecting'))
.on('end', () => console.log('Redis connection closed'));
return redisClient;
}
// Batch operations for better performance
export async function storeFeedMappings(mappings: Array<{ feedId: string, slugs: string[], lang: string }>): Promise<void> {
const redis = await getRedisClient();
const pipeline = redis.pipeline();
mappings.forEach(({ feedId, slugs, lang }) => {
const mapping = {
slugs,
lang,
created_at: new Date().toISOString()
};
pipeline.set(feedId, JSON.stringify(mapping), 'EX', DEFAULT_CACHE_TTL);
}); });
redisRetryCount = 0;
return redis; await pipeline.exec();
} catch (error) {
redisRetryCount++;
throw error;
} }
// Update warmChapterCache to prevent empty cache entries
export async function warmChapterCache(slugs: string[], lang: string): Promise<void> {
const redis = await getRedisClient();
const needsCache = await checkCacheStatus(slugs, lang);
if (needsCache.length > 0) {
console.log(`Pre-warming cache for ${needsCache.length} comics`);
const pipeline = redis.pipeline();
// Replace empty array with actual chapter fetching
await Promise.all(needsCache.map(async slug => {
try {
const chapters = await getChaptersForSlugs([slug], lang);
const cacheKey = `chapters:${slug}:${lang}`;
pipeline.set(cacheKey, JSON.stringify(chapters), 'EX', 60 * 60);
} catch (error) {
console.error(`Failed to warm cache for ${slug}:`, error);
}
}));
await pipeline.exec();
}
}
// Add batch cache checking
export async function checkCacheStatus(slugs: string[], lang: string): Promise<string[]> {
const redis = await getRedisClient();
const pipeline = redis.pipeline();
slugs.forEach(slug => {
const cacheKey = `chapters:${slug}:${lang}`;
pipeline.exists(cacheKey);
});
const results = await pipeline.exec();
return slugs.filter((_, index) => results![index][1] === 0);
} }
export async function storeFeedMapping(feedId: string, slugs: string[], lang: string): Promise<void> { export async function storeFeedMapping(feedId: string, slugs: string[], lang: string): Promise<void> {
if (!redis) { const redis = await getRedisClient();
console.error('Redis client not initialized');
throw new Error('Storage system unavailable');
}
const mapping = { const mapping = {
slugs, slugs,
lang, lang,
@ -119,10 +130,7 @@ export async function storeFeedMapping(feedId: string, slugs: string[], lang: st
} }
export async function getFeedMapping(feedId: string): Promise<FeedMapping | null> { export async function getFeedMapping(feedId: string): Promise<FeedMapping | null> {
if (!redis) { const redis = await getRedisClient();
console.error('Redis client not initialized');
throw new Error('Storage system unavailable');
}
try { try {
const mapping = await redis.get(feedId); const mapping = await redis.get(feedId);
@ -142,3 +150,14 @@ export async function getFeedMapping(feedId: string): Promise<FeedMapping | null
} }
// No need for manual cleanup as we're using TTL (expire) // No need for manual cleanup as we're using TTL (expire)
// Warm connection on startup
(async () => {
try {
const client = await getRedisClient();
await client.ping();
console.log('Redis pre-connected successfully');
} catch (error) {
console.error('Redis pre-connection failed:', error);
}
})();

View file

@ -0,0 +1,17 @@
import type { APIRoute } from "astro";
import { validateSlugs } from "../../lib/api";
export const POST: APIRoute = async ({ request }) => {
try {
const { slugs } = await request.json();
const validSlugs = await validateSlugs(slugs);
return new Response(JSON.stringify({ validSlugs }), {
status: 200,
headers: { 'Content-Type': 'application/json' }
});
} catch (error) {
console.error('Validation error:', error);
return new Response("Validation failed", { status: 500 });
}
};

View file

@ -1,6 +1,6 @@
import { Feed } from 'feed'; import { Feed } from 'feed';
import { getChaptersForSlugs, isValidLanguage } from "../../lib/api"; import { getChaptersForSlugs, isValidLanguage } from "../../lib/api";
import { getFeedMapping } from "../../lib/db"; import { getFeedMapping, warmChapterCache } from "../../lib/db";
import type { APIRoute } from "astro"; import type { APIRoute } from "astro";
export const GET: APIRoute = async ({ params, request }) => { export const GET: APIRoute = async ({ params, request }) => {
@ -22,6 +22,8 @@ export const GET: APIRoute = async ({ params, request }) => {
return new Response("Invalid language", { status: 400 }); return new Response("Invalid language", { status: 400 });
} }
await warmChapterCache(slugs, lang);
const chapters = await getChaptersForSlugs(slugs, lang); const chapters = await getChaptersForSlugs(slugs, lang);
if (chapters.length === 0) { if (chapters.length === 0) {