mirror of
https://github.com/ld3z/new-manga.git
synced 2026-11-09 21:23:45 -07:00
Compare commits
4 commits
b8499d90c6
...
01734e8ce5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
01734e8ce5 | ||
|
|
66e9dd2a06 | ||
|
|
22c3e665e8 | ||
|
|
0daac66b26 |
7 changed files with 309 additions and 207 deletions
|
|
@ -9,4 +9,9 @@ export default defineConfig({
|
||||||
remotePatterns: [{ protocol: "https" }],
|
remotePatterns: [{ protocol: "https" }],
|
||||||
},
|
},
|
||||||
integrations: [tailwind()],
|
integrations: [tailwind()],
|
||||||
|
vite: {
|
||||||
|
define: {
|
||||||
|
'process.env.REDIS_URL': JSON.stringify(process.env.REDIS_URL)
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
56
package-lock.json
generated
56
package-lock.json
generated
|
|
@ -12,9 +12,10 @@
|
||||||
"@astrojs/tailwind": "^6.0.0",
|
"@astrojs/tailwind": "^6.0.0",
|
||||||
"@astrojs/vercel": "^8.0.5",
|
"@astrojs/vercel": "^8.0.5",
|
||||||
"@floating-ui/dom": "^1.6.13",
|
"@floating-ui/dom": "^1.6.13",
|
||||||
|
"@types/node": "^22.13.1",
|
||||||
"@vercel/analytics": "^1.4.1",
|
"@vercel/analytics": "^1.4.1",
|
||||||
"@vercel/kv": "^3.0.0",
|
"@vercel/kv": "^3.0.0",
|
||||||
"astro": "^5.2.3",
|
"astro": "^5.2.5",
|
||||||
"feed": "^4.2.2",
|
"feed": "^4.2.2",
|
||||||
"ioredis": "^5.4.2",
|
"ioredis": "^5.4.2",
|
||||||
"tailwindcss": "^3.4.1"
|
"tailwindcss": "^3.4.1"
|
||||||
|
|
@ -1462,6 +1463,14 @@
|
||||||
"@types/unist": "*"
|
"@types/unist": "*"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/node": {
|
||||||
|
"version": "22.13.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.1.tgz",
|
||||||
|
"integrity": "sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==",
|
||||||
|
"dependencies": {
|
||||||
|
"undici-types": "~6.20.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@types/unist": {
|
"node_modules/@types/unist": {
|
||||||
"version": "3.0.3",
|
"version": "3.0.3",
|
||||||
"resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz",
|
"resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz",
|
||||||
|
|
@ -1744,12 +1753,12 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/astro": {
|
"node_modules/astro": {
|
||||||
"version": "5.2.3",
|
"version": "5.2.5",
|
||||||
"resolved": "https://registry.npmjs.org/astro/-/astro-5.2.3.tgz",
|
"resolved": "https://registry.npmjs.org/astro/-/astro-5.2.5.tgz",
|
||||||
"integrity": "sha512-04AceE/gVihtO3V28Ss0+tPPgLHGoulXrm1E7fUVPYyQu7y6w8+oXfYyzh1lpy+uEYiebjH7AFlJoz73anBmZA==",
|
"integrity": "sha512-AYXyYkc+c5xbKTm48FyQA91y81nXyNPAaoyafR0LUugE4lAwuvIUcXDBfMzmbuP1lGRvsE33G2oypv6gbGaPFg==",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@astrojs/compiler": "^2.10.3",
|
"@astrojs/compiler": "^2.10.3",
|
||||||
"@astrojs/internal-helpers": "0.5.0",
|
"@astrojs/internal-helpers": "0.5.1",
|
||||||
"@astrojs/markdown-remark": "6.1.0",
|
"@astrojs/markdown-remark": "6.1.0",
|
||||||
"@astrojs/telemetry": "3.2.0",
|
"@astrojs/telemetry": "3.2.0",
|
||||||
"@oslojs/encoding": "^1.1.0",
|
"@oslojs/encoding": "^1.1.0",
|
||||||
|
|
@ -1787,10 +1796,10 @@
|
||||||
"neotraverse": "^0.6.18",
|
"neotraverse": "^0.6.18",
|
||||||
"p-limit": "^6.2.0",
|
"p-limit": "^6.2.0",
|
||||||
"p-queue": "^8.1.0",
|
"p-queue": "^8.1.0",
|
||||||
"preferred-pm": "^4.1.0",
|
"preferred-pm": "^4.1.1",
|
||||||
"prompts": "^2.4.2",
|
"prompts": "^2.4.2",
|
||||||
"rehype": "^13.0.2",
|
"rehype": "^13.0.2",
|
||||||
"semver": "^7.7.0",
|
"semver": "^7.7.1",
|
||||||
"shiki": "^1.29.2",
|
"shiki": "^1.29.2",
|
||||||
"tinyexec": "^0.3.2",
|
"tinyexec": "^0.3.2",
|
||||||
"tsconfck": "^3.1.4",
|
"tsconfck": "^3.1.4",
|
||||||
|
|
@ -1800,7 +1809,7 @@
|
||||||
"vfile": "^6.0.3",
|
"vfile": "^6.0.3",
|
||||||
"vite": "^6.0.11",
|
"vite": "^6.0.11",
|
||||||
"vitefu": "^1.0.5",
|
"vitefu": "^1.0.5",
|
||||||
"which-pm": "^3.0.0",
|
"which-pm": "^3.0.1",
|
||||||
"xxhash-wasm": "^1.1.0",
|
"xxhash-wasm": "^1.1.0",
|
||||||
"yargs-parser": "^21.1.1",
|
"yargs-parser": "^21.1.1",
|
||||||
"yocto-spinner": "^0.2.0",
|
"yocto-spinner": "^0.2.0",
|
||||||
|
|
@ -1825,9 +1834,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/astro/node_modules/@astrojs/internal-helpers": {
|
"node_modules/astro/node_modules/@astrojs/internal-helpers": {
|
||||||
"version": "0.5.0",
|
"version": "0.5.1",
|
||||||
"resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.5.0.tgz",
|
"resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.5.1.tgz",
|
||||||
"integrity": "sha512-CgB5ZaZO1PFG+rbjF3HnA7G6gIBjJ070xb7bUjeu5Gqqufma+t6fpuRWMXnK2iEO3zVyX7e/xplPlqtFKy/lvw=="
|
"integrity": "sha512-M7rAge1n2+aOSxNvKUFa0u/KFn0W+sZy7EW91KOSERotm2Ti8qs+1K0xx3zbOxtAVrmJb5/J98eohVvvEqtNkw=="
|
||||||
},
|
},
|
||||||
"node_modules/astro/node_modules/@vercel/kv": {
|
"node_modules/astro/node_modules/@vercel/kv": {
|
||||||
"version": "1.0.1",
|
"version": "1.0.1",
|
||||||
|
|
@ -4789,13 +4798,13 @@
|
||||||
"integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
|
"integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
|
||||||
},
|
},
|
||||||
"node_modules/preferred-pm": {
|
"node_modules/preferred-pm": {
|
||||||
"version": "4.1.0",
|
"version": "4.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/preferred-pm/-/preferred-pm-4.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/preferred-pm/-/preferred-pm-4.1.1.tgz",
|
||||||
"integrity": "sha512-cNKMVcSvE3hZBRukdRsvPtlUaM2fYLFbIvBz620XaDA5SjHsCRsAjNE2baI90Hh7QvJIjCHJYnpPtyqJ1/kOjQ==",
|
"integrity": "sha512-rU+ZAv1Ur9jAUZtGPebQVQPzdGhNzaEiQ7VL9+cjsAWPHFYOccNXPNiev1CCDSOg/2j7UujM7ojNhpkuILEVNQ==",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"find-up-simple": "^1.0.0",
|
"find-up-simple": "^1.0.0",
|
||||||
"find-yarn-workspace-root2": "1.2.16",
|
"find-yarn-workspace-root2": "1.2.16",
|
||||||
"which-pm": "^3.0.0"
|
"which-pm": "^3.0.1"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=18.12"
|
"node": ">=18.12"
|
||||||
|
|
@ -5288,9 +5297,9 @@
|
||||||
"integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg=="
|
"integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg=="
|
||||||
},
|
},
|
||||||
"node_modules/semver": {
|
"node_modules/semver": {
|
||||||
"version": "7.7.0",
|
"version": "7.7.1",
|
||||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.0.tgz",
|
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
|
||||||
"integrity": "sha512-DrfFnPzblFmNrIZzg5RzHegbiRWg7KMR7btwi2yjHwx06zsUbO5g613sVwEV7FTwmzJu+Io0lJe2GJ3LxqpvBQ==",
|
"integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==",
|
||||||
"bin": {
|
"bin": {
|
||||||
"semver": "bin/semver.js"
|
"semver": "bin/semver.js"
|
||||||
},
|
},
|
||||||
|
|
@ -5811,6 +5820,11 @@
|
||||||
"resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz",
|
"resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz",
|
||||||
"integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q=="
|
"integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q=="
|
||||||
},
|
},
|
||||||
|
"node_modules/undici-types": {
|
||||||
|
"version": "6.20.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
|
||||||
|
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
|
||||||
|
},
|
||||||
"node_modules/unenv": {
|
"node_modules/unenv": {
|
||||||
"version": "1.10.0",
|
"version": "1.10.0",
|
||||||
"resolved": "https://registry.npmjs.org/unenv/-/unenv-1.10.0.tgz",
|
"resolved": "https://registry.npmjs.org/unenv/-/unenv-1.10.0.tgz",
|
||||||
|
|
@ -6162,9 +6176,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/which-pm": {
|
"node_modules/which-pm": {
|
||||||
"version": "3.0.0",
|
"version": "3.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/which-pm/-/which-pm-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/which-pm/-/which-pm-3.0.1.tgz",
|
||||||
"integrity": "sha512-ysVYmw6+ZBhx3+ZkcPwRuJi38ZOTLJJ33PSHaitLxSKUMsh0LkKd0nC69zZCwt5D+AYUcMK2hhw4yWny20vSGg==",
|
"integrity": "sha512-v2JrMq0waAI4ju1xU5x3blsxBBMgdgZve580iYMN5frDaLGjbA24fok7wKCsya8KLVO19Ju4XDc5+zTZCJkQfg==",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"load-yaml-file": "^0.2.0"
|
"load-yaml-file": "^0.2.0"
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -15,9 +15,10 @@
|
||||||
"@astrojs/tailwind": "^6.0.0",
|
"@astrojs/tailwind": "^6.0.0",
|
||||||
"@astrojs/vercel": "^8.0.5",
|
"@astrojs/vercel": "^8.0.5",
|
||||||
"@floating-ui/dom": "^1.6.13",
|
"@floating-ui/dom": "^1.6.13",
|
||||||
|
"@types/node": "^22.13.1",
|
||||||
"@vercel/analytics": "^1.4.1",
|
"@vercel/analytics": "^1.4.1",
|
||||||
"@vercel/kv": "^3.0.0",
|
"@vercel/kv": "^3.0.0",
|
||||||
"astro": "^5.2.3",
|
"astro": "^5.2.5",
|
||||||
"feed": "^4.2.2",
|
"feed": "^4.2.2",
|
||||||
"ioredis": "^5.4.2",
|
"ioredis": "^5.4.2",
|
||||||
"tailwindcss": "^3.4.1"
|
"tailwindcss": "^3.4.1"
|
||||||
|
|
|
||||||
206
src/lib/api.ts
206
src/lib/api.ts
|
|
@ -209,39 +209,31 @@ export async function getChaptersByHid(
|
||||||
|
|
||||||
const now = new Date();
|
const now = new Date();
|
||||||
|
|
||||||
const chapters = data.chapters
|
const chapters: ChapterDetail[] = [];
|
||||||
.filter(chapter => {
|
const rawChapters = data.chapters || [];
|
||||||
// Basic validation
|
|
||||||
const isValid = chapter &&
|
|
||||||
chapter.chap &&
|
|
||||||
chapter.hid;
|
|
||||||
|
|
||||||
if (!isValid) return false;
|
for (const chap of rawChapters) {
|
||||||
|
try {
|
||||||
// Check publish time
|
if (!chap.id || !chap.chap || !chap.updated_at) {
|
||||||
if (chapter.publish_at) {
|
console.warn(`Skipping chapter with missing fields in ${hid}:`, chap);
|
||||||
const publishTime = new Date(chapter.publish_at);
|
continue;
|
||||||
if (publishTime > now) {
|
|
||||||
console.log(`Chapter ${chapter.chap} will be published at ${publishTime}`);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use the earliest available timestamp
|
chapters.push({
|
||||||
const hasValidTimestamp = chapter.publish_at || chapter.created_at || chapter.updated_at;
|
id: chap.id,
|
||||||
return Boolean(hasValidTimestamp);
|
chap: chap.chap.toString(),
|
||||||
})
|
title: chap.title || `Chapter ${chap.chap}`,
|
||||||
.map(chapter => ({
|
updated_at: chap.updated_at,
|
||||||
id: chapter.hid,
|
|
||||||
chap: chapter.chap,
|
|
||||||
title: chapter.title || `Chapter ${chapter.chap}`,
|
|
||||||
updated_at: chapter.publish_at || chapter.created_at || chapter.updated_at,
|
|
||||||
md_comics: {
|
md_comics: {
|
||||||
title: comicTitle,
|
title: comicTitle,
|
||||||
slug: slug,
|
slug: slug,
|
||||||
md_covers: covers
|
md_covers: covers
|
||||||
}
|
}
|
||||||
}));
|
});
|
||||||
|
} catch (e) {
|
||||||
|
console.error(`Error processing chapter ${chap.id} for ${hid}:`, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
console.log(`Successfully processed ${chapters.length} chapters for ${comicTitle}`);
|
console.log(`Successfully processed ${chapters.length} chapters for ${comicTitle}`);
|
||||||
return chapters;
|
return chapters;
|
||||||
|
|
@ -251,88 +243,140 @@ export async function getChaptersByHid(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add deduplication function
|
||||||
|
function deduplicateChapters(chapters: ChapterDetail[]): ChapterDetail[] {
|
||||||
|
const seen = new Set<string>();
|
||||||
|
return chapters.filter(chapter => {
|
||||||
|
const key = `${chapter.md_comics.slug}-${chapter.chap}`;
|
||||||
|
if (seen.has(key)) return false;
|
||||||
|
seen.add(key);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export async function getChaptersForSlugs(
|
export async function getChaptersForSlugs(
|
||||||
slugs: string[],
|
slugs: string[],
|
||||||
lang: string = 'en'
|
lang: string = 'en'
|
||||||
): Promise<ChapterDetail[]> {
|
): Promise<ChapterDetail[]> {
|
||||||
console.log(`Fetching chapters for slugs:`, slugs, `in language: ${lang}`);
|
console.log(`Fetching chapters for ${slugs.length} comics`);
|
||||||
const chapters: ChapterDetail[] = [];
|
|
||||||
const redis = await getRedisClient();
|
const redis = await getRedisClient();
|
||||||
|
|
||||||
// Cache results for 1 hour (3600 seconds)
|
|
||||||
const CACHE_TTL = 60 * 60;
|
const CACHE_TTL = 60 * 60;
|
||||||
|
|
||||||
for (const slug of slugs) {
|
const uniqueSlugs = [...new Set(slugs)];
|
||||||
console.log(`Processing slug: ${slug}`);
|
const pipeline = redis.pipeline();
|
||||||
|
|
||||||
|
// Add cache key validation
|
||||||
|
uniqueSlugs.forEach(slug => {
|
||||||
|
if (!slug.match(/^[a-z0-9-]+$/)) {
|
||||||
|
console.error(`Invalid slug format: ${slug}`);
|
||||||
|
}
|
||||||
const cacheKey = `chapters:${slug}:${lang}`;
|
const cacheKey = `chapters:${slug}:${lang}`;
|
||||||
|
pipeline.get(cacheKey);
|
||||||
|
});
|
||||||
|
|
||||||
|
const [cacheResults, slugsToFetch] = await Promise.all([
|
||||||
|
pipeline.exec(),
|
||||||
|
validateSlugs(uniqueSlugs) // Add this new function
|
||||||
|
]);
|
||||||
|
|
||||||
|
const cachedChapters: ChapterDetail[] = [];
|
||||||
|
cacheResults.forEach(([err, result], index) => {
|
||||||
|
if (!err && result) {
|
||||||
try {
|
try {
|
||||||
// Try to get cached chapters
|
const parsed = JSON.parse(result);
|
||||||
const cachedChapters = await redis.get(cacheKey);
|
if (Array.isArray(parsed)) {
|
||||||
if (cachedChapters) {
|
cachedChapters.push(...parsed);
|
||||||
console.log(`Using cached chapters for ${slug}`);
|
|
||||||
chapters.push(...JSON.parse(cachedChapters));
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (e) {
|
||||||
console.error('Redis cache read error:', error);
|
console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Add progress tracking
|
||||||
|
const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug));
|
||||||
|
console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`);
|
||||||
|
console.log(`Fetching ${slugsToFetch.length} comics from API`);
|
||||||
|
|
||||||
|
const fetchPromises = slugsToFetch.map(async (slug) => {
|
||||||
|
const timer = Date.now();
|
||||||
|
try {
|
||||||
const comicDetail = await getComicBySlug(slug);
|
const comicDetail = await getComicBySlug(slug);
|
||||||
|
|
||||||
if (!comicDetail) {
|
if (!comicDetail?.comic?.hid) {
|
||||||
console.error(`Failed to get comic details for slug: ${slug}`);
|
console.error(`❌ No HID found for ${slug}`);
|
||||||
continue;
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!comicDetail.comic?.hid) {
|
const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, {
|
||||||
console.error(`No HID found for slug ${slug}`, comicDetail);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Found HID ${comicDetail.comic.hid} for slug ${slug}`);
|
|
||||||
const comicChapters = await getChaptersByHid(
|
|
||||||
comicDetail.comic.hid,
|
|
||||||
slug,
|
|
||||||
{
|
|
||||||
limit: 5,
|
limit: 5,
|
||||||
lang: lang
|
lang: lang
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`);
|
||||||
|
return chapters;
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`🚨 Error processing ${slug}:`, error);
|
||||||
|
return [];
|
||||||
}
|
}
|
||||||
);
|
});
|
||||||
console.log(`Found ${comicChapters.length} chapters for ${slug}`);
|
|
||||||
|
|
||||||
if (comicChapters.length > 0) {
|
const liveChapters = (await Promise.all(fetchPromises)).flat();
|
||||||
chapters.push(...comicChapters);
|
|
||||||
|
|
||||||
// Cache the results
|
// Cache new results in batch
|
||||||
try {
|
const cachePipeline = redis.pipeline();
|
||||||
await redis.set(
|
const chaptersBySlug: Record<string, ChapterDetail[]> = {};
|
||||||
|
|
||||||
|
liveChapters.forEach(chapter => {
|
||||||
|
const slug = chapter.md_comics.slug;
|
||||||
|
if (!chaptersBySlug[slug]) {
|
||||||
|
chaptersBySlug[slug] = [];
|
||||||
|
}
|
||||||
|
chaptersBySlug[slug].push(chapter);
|
||||||
|
});
|
||||||
|
|
||||||
|
Object.entries(chaptersBySlug).forEach(([slug, chaps]) => {
|
||||||
|
const cacheKey = `chapters:${slug}:${lang}`;
|
||||||
|
cachePipeline.set(
|
||||||
cacheKey,
|
cacheKey,
|
||||||
JSON.stringify(comicChapters),
|
JSON.stringify(chaps),
|
||||||
'EX',
|
'EX',
|
||||||
CACHE_TTL
|
CACHE_TTL
|
||||||
);
|
);
|
||||||
console.log(`Cached chapters for ${slug}`);
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Redis cache write error:', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort by chapter number (descending) first, then by date
|
|
||||||
const sortedChapters = chapters.sort((a, b) => {
|
|
||||||
// Convert chapter numbers to floats for proper numerical sorting
|
|
||||||
const chapA = parseFloat(a.chap);
|
|
||||||
const chapB = parseFloat(b.chap);
|
|
||||||
|
|
||||||
if (chapB !== chapA) {
|
|
||||||
return chapB - chapA; // Sort by chapter number first
|
|
||||||
}
|
|
||||||
|
|
||||||
// If chapters are the same, sort by date
|
|
||||||
return new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(`Total chapters found: ${sortedChapters.length}`);
|
await cachePipeline.exec();
|
||||||
return sortedChapters;
|
|
||||||
|
// Combine and sort all chapters
|
||||||
|
const allChapters = [...cachedChapters, ...liveChapters];
|
||||||
|
const deduplicatedChapters = deduplicateChapters(allChapters);
|
||||||
|
return deduplicatedChapters.sort((a, b) =>
|
||||||
|
parseFloat(b.chap) - parseFloat(a.chap) ||
|
||||||
|
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add export to the validation function
|
||||||
|
export async function validateSlugs(slugs: string[]): Promise<string[]> {
|
||||||
|
const validSlugs: string[] = [];
|
||||||
|
const invalidSlugs: string[] = [];
|
||||||
|
|
||||||
|
await Promise.all(slugs.map(async (slug) => {
|
||||||
|
try {
|
||||||
|
const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1);
|
||||||
|
if (response.ok) {
|
||||||
|
validSlugs.push(slug);
|
||||||
|
} else {
|
||||||
|
invalidSlugs.push(slug);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
invalidSlugs.push(slug);
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
if (invalidSlugs.length > 0) {
|
||||||
|
console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return validSlugs;
|
||||||
}
|
}
|
||||||
177
src/lib/db.ts
177
src/lib/db.ts
|
|
@ -1,104 +1,115 @@
|
||||||
import Redis from 'ioredis';
|
import Redis from 'ioredis';
|
||||||
import type { FeedMapping } from './types';
|
import type { FeedMapping } from './types';
|
||||||
|
import { getChaptersForSlugs } from './api';
|
||||||
|
|
||||||
const REDIS_URL = import.meta.env.REDIS_URL || process.env.REDIS_URL;
|
const REDIS_URL = import.meta.env.REDIS_URL || process.env.REDIS_URL;
|
||||||
let redis: Redis | null = null;
|
let redisClient: Redis | null = null;
|
||||||
let redisRetryCount = 0;
|
|
||||||
const MAX_RETRIES = 3;
|
|
||||||
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
|
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
|
||||||
|
const MAX_RETRIES = 5;
|
||||||
|
const BASE_DELAY_MS = 100;
|
||||||
|
|
||||||
try {
|
// Improved connection handling with exponential backoff
|
||||||
console.log('Environment check:', {
|
async function createRedisClient(): Promise<Redis> {
|
||||||
hasEnvVar: !!REDIS_URL,
|
if (!REDIS_URL) throw new Error('REDIS_URL environment variable is not set');
|
||||||
envVarStart: REDIS_URL?.substring(0, 20) + '...',
|
|
||||||
nodeEnv: process.env.NODE_ENV
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!REDIS_URL) {
|
return new Redis(REDIS_URL, {
|
||||||
throw new Error('REDIS_URL environment variable is not set');
|
retryStrategy: (times) => {
|
||||||
}
|
if (times >= MAX_RETRIES) return null;
|
||||||
|
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000);
|
||||||
console.log('Attempting to connect to Redis...');
|
console.log(`Redis retry attempt ${times}, next try in ${delay}ms`);
|
||||||
|
return delay + Math.random() * 200; // Add jitter
|
||||||
redis = new Redis(REDIS_URL, {
|
|
||||||
retryStrategy(times) {
|
|
||||||
const delay = Math.min(times * 50, 2000);
|
|
||||||
console.log(`Retry attempt ${times} with delay ${delay}ms`);
|
|
||||||
return delay;
|
|
||||||
},
|
},
|
||||||
maxRetriesPerRequest: 2,
|
maxRetriesPerRequest: null, // Allow retries for queued commands
|
||||||
|
enableOfflineQueue: true, // Enable command queuing
|
||||||
enableReadyCheck: true,
|
enableReadyCheck: true,
|
||||||
reconnectOnError: (err) => {
|
reconnectOnError: (err) => {
|
||||||
const targetError = 'READONLY';
|
return err.message.includes('READONLY');
|
||||||
if (err.message.includes(targetError)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
redis.on('error', (error) => {
|
// Singleton pattern with connection pooling
|
||||||
console.error('Redis connection error:', error.message);
|
export async function getRedisClient(): Promise<Redis> {
|
||||||
|
if (redisClient?.status === 'ready') {
|
||||||
|
// Validate connection with a quick ping
|
||||||
|
try {
|
||||||
|
await redisClient.ping();
|
||||||
|
return redisClient;
|
||||||
|
} catch {
|
||||||
|
// Fall through to reconnect
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Initializing new Redis connection...');
|
||||||
|
redisClient = await createRedisClient();
|
||||||
|
|
||||||
|
// Add event listeners for better monitoring
|
||||||
|
redisClient
|
||||||
|
.on('ready', () => console.log('Redis connection ready'))
|
||||||
|
.on('error', (err) => console.error('Redis error:', err))
|
||||||
|
.on('reconnecting', () => console.log('Redis reconnecting'))
|
||||||
|
.on('end', () => console.log('Redis connection closed'));
|
||||||
|
|
||||||
|
return redisClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch operations for better performance
|
||||||
|
export async function storeFeedMappings(mappings: Array<{ feedId: string, slugs: string[], lang: string }>): Promise<void> {
|
||||||
|
const redis = await getRedisClient();
|
||||||
|
const pipeline = redis.pipeline();
|
||||||
|
|
||||||
|
mappings.forEach(({ feedId, slugs, lang }) => {
|
||||||
|
const mapping = {
|
||||||
|
slugs,
|
||||||
|
lang,
|
||||||
|
created_at: new Date().toISOString()
|
||||||
|
};
|
||||||
|
pipeline.set(feedId, JSON.stringify(mapping), 'EX', DEFAULT_CACHE_TTL);
|
||||||
});
|
});
|
||||||
|
|
||||||
redis.on('connect', () => {
|
await pipeline.exec();
|
||||||
console.log('Successfully connected to Redis');
|
}
|
||||||
});
|
|
||||||
|
|
||||||
redis.on('ready', () => {
|
// Update warmChapterCache to prevent empty cache entries
|
||||||
console.log('Redis client is ready');
|
export async function warmChapterCache(slugs: string[], lang: string): Promise<void> {
|
||||||
});
|
const redis = await getRedisClient();
|
||||||
|
const needsCache = await checkCacheStatus(slugs, lang);
|
||||||
|
|
||||||
redis.on('reconnecting', () => {
|
if (needsCache.length > 0) {
|
||||||
console.log('Redis client is reconnecting');
|
console.log(`Pre-warming cache for ${needsCache.length} comics`);
|
||||||
});
|
const pipeline = redis.pipeline();
|
||||||
|
|
||||||
} catch (error) {
|
// Replace empty array with actual chapter fetching
|
||||||
console.error('Failed to initialize Redis:', error);
|
await Promise.all(needsCache.map(async slug => {
|
||||||
if (error instanceof Error) {
|
try {
|
||||||
console.error('Error details:', error.message);
|
const chapters = await getChaptersForSlugs([slug], lang);
|
||||||
console.error('Stack trace:', error.stack);
|
const cacheKey = `chapters:${slug}:${lang}`;
|
||||||
|
pipeline.set(cacheKey, JSON.stringify(chapters), 'EX', 60 * 60);
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Failed to warm cache for ${slug}:`, error);
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
await pipeline.exec();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getRedisClient(): Promise<Redis> {
|
// Add batch cache checking
|
||||||
if (redis?.status === 'ready') return redis;
|
export async function checkCacheStatus(slugs: string[], lang: string): Promise<string[]> {
|
||||||
|
const redis = await getRedisClient();
|
||||||
|
const pipeline = redis.pipeline();
|
||||||
|
|
||||||
if (redisRetryCount >= MAX_RETRIES) {
|
slugs.forEach(slug => {
|
||||||
throw new Error('Failed to connect to Redis after multiple attempts');
|
const cacheKey = `chapters:${slug}:${lang}`;
|
||||||
}
|
pipeline.exists(cacheKey);
|
||||||
|
|
||||||
try {
|
|
||||||
redis = new Redis(REDIS_URL, {
|
|
||||||
retryStrategy(times) {
|
|
||||||
const delay = Math.min(times * 50, 2000);
|
|
||||||
console.log(`Retry attempt ${times} with delay ${delay}ms`);
|
|
||||||
return delay;
|
|
||||||
},
|
|
||||||
maxRetriesPerRequest: 2,
|
|
||||||
enableReadyCheck: true,
|
|
||||||
reconnectOnError: (err) => {
|
|
||||||
const targetError = 'READONLY';
|
|
||||||
if (err.message.includes(targetError)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
redisRetryCount = 0;
|
|
||||||
return redis;
|
const results = await pipeline.exec();
|
||||||
} catch (error) {
|
return slugs.filter((_, index) => results![index][1] === 0);
|
||||||
redisRetryCount++;
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function storeFeedMapping(feedId: string, slugs: string[], lang: string): Promise<void> {
|
export async function storeFeedMapping(feedId: string, slugs: string[], lang: string): Promise<void> {
|
||||||
if (!redis) {
|
const redis = await getRedisClient();
|
||||||
console.error('Redis client not initialized');
|
|
||||||
throw new Error('Storage system unavailable');
|
|
||||||
}
|
|
||||||
|
|
||||||
const mapping = {
|
const mapping = {
|
||||||
slugs,
|
slugs,
|
||||||
lang,
|
lang,
|
||||||
|
|
@ -119,10 +130,7 @@ export async function storeFeedMapping(feedId: string, slugs: string[], lang: st
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getFeedMapping(feedId: string): Promise<FeedMapping | null> {
|
export async function getFeedMapping(feedId: string): Promise<FeedMapping | null> {
|
||||||
if (!redis) {
|
const redis = await getRedisClient();
|
||||||
console.error('Redis client not initialized');
|
|
||||||
throw new Error('Storage system unavailable');
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const mapping = await redis.get(feedId);
|
const mapping = await redis.get(feedId);
|
||||||
|
|
@ -142,3 +150,14 @@ export async function getFeedMapping(feedId: string): Promise<FeedMapping | null
|
||||||
}
|
}
|
||||||
|
|
||||||
// No need for manual cleanup as we're using TTL (expire)
|
// No need for manual cleanup as we're using TTL (expire)
|
||||||
|
|
||||||
|
// Warm connection on startup
|
||||||
|
(async () => {
|
||||||
|
try {
|
||||||
|
const client = await getRedisClient();
|
||||||
|
await client.ping();
|
||||||
|
console.log('Redis pre-connected successfully');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Redis pre-connection failed:', error);
|
||||||
|
}
|
||||||
|
})();
|
||||||
17
src/pages/api/validate-slugs.ts
Normal file
17
src/pages/api/validate-slugs.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
import type { APIRoute } from "astro";
|
||||||
|
import { validateSlugs } from "../../lib/api";
|
||||||
|
|
||||||
|
export const POST: APIRoute = async ({ request }) => {
|
||||||
|
try {
|
||||||
|
const { slugs } = await request.json();
|
||||||
|
const validSlugs = await validateSlugs(slugs);
|
||||||
|
|
||||||
|
return new Response(JSON.stringify({ validSlugs }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' }
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Validation error:', error);
|
||||||
|
return new Response("Validation failed", { status: 500 });
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { Feed } from 'feed';
|
import { Feed } from 'feed';
|
||||||
import { getChaptersForSlugs, isValidLanguage } from "../../lib/api";
|
import { getChaptersForSlugs, isValidLanguage } from "../../lib/api";
|
||||||
import { getFeedMapping } from "../../lib/db";
|
import { getFeedMapping, warmChapterCache } from "../../lib/db";
|
||||||
import type { APIRoute } from "astro";
|
import type { APIRoute } from "astro";
|
||||||
|
|
||||||
export const GET: APIRoute = async ({ params, request }) => {
|
export const GET: APIRoute = async ({ params, request }) => {
|
||||||
|
|
@ -22,6 +22,8 @@ export const GET: APIRoute = async ({ params, request }) => {
|
||||||
return new Response("Invalid language", { status: 400 });
|
return new Response("Invalid language", { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await warmChapterCache(slugs, lang);
|
||||||
|
|
||||||
const chapters = await getChaptersForSlugs(slugs, lang);
|
const chapters = await getChaptersForSlugs(slugs, lang);
|
||||||
|
|
||||||
if (chapters.length === 0) {
|
if (chapters.length === 0) {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue