mirror of
https://github.com/ld3z/new-manga.git
synced 2026-11-09 21:03:45 -07:00
Compare commits
4 commits
40221bad5d
...
caf6972cd2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
caf6972cd2 | ||
|
|
17b910d5ab | ||
|
|
eaeaeae7d2 | ||
|
|
9677cba078 |
5 changed files with 684 additions and 292 deletions
44
package-lock.json
generated
44
package-lock.json
generated
|
|
@ -17,6 +17,8 @@
|
||||||
"astro": "^5.5.4",
|
"astro": "^5.5.4",
|
||||||
"feed": "^4.2.2",
|
"feed": "^4.2.2",
|
||||||
"ioredis": "^5.6.0",
|
"ioredis": "^5.6.0",
|
||||||
|
"p-throttle": "^7.0.0",
|
||||||
|
"redis-modules-sdk": "^1.4.0",
|
||||||
"tailwindcss": "^3.4.1"
|
"tailwindcss": "^3.4.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -4373,6 +4375,17 @@
|
||||||
"url": "https://github.com/sponsors/sindresorhus"
|
"url": "https://github.com/sponsors/sindresorhus"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/p-throttle": {
|
||||||
|
"version": "7.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-7.0.0.tgz",
|
||||||
|
"integrity": "sha512-aio0v+S0QVkH1O+9x4dHtD4dgCExACcL+3EtNaGqC01GBudS9ijMuUsmN8OVScyV4OOp0jqdLShZFuSlbL/AsA==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/sponsors/sindresorhus"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/p-timeout": {
|
"node_modules/p-timeout": {
|
||||||
"version": "6.1.4",
|
"version": "6.1.4",
|
||||||
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz",
|
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz",
|
||||||
|
|
@ -4737,6 +4750,37 @@
|
||||||
"node": ">=4"
|
"node": ">=4"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/redis-modules-sdk": {
|
||||||
|
"version": "1.4.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/redis-modules-sdk/-/redis-modules-sdk-1.4.0.tgz",
|
||||||
|
"integrity": "sha512-HYCr3ThtyoMrR9MUGmJblPU5aua0G4sanIZHK5wz/QwhVACe3BBgIZpHeymidBaukiuC0uzKla9xJ8exRffjnA==",
|
||||||
|
"dependencies": {
|
||||||
|
"ioredis": "5.2.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/redis-modules-sdk/node_modules/ioredis": {
|
||||||
|
"version": "5.2.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz",
|
||||||
|
"integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==",
|
||||||
|
"dependencies": {
|
||||||
|
"@ioredis/commands": "^1.1.1",
|
||||||
|
"cluster-key-slot": "^1.1.0",
|
||||||
|
"debug": "^4.3.4",
|
||||||
|
"denque": "^2.0.1",
|
||||||
|
"lodash.defaults": "^4.2.0",
|
||||||
|
"lodash.isarguments": "^3.1.0",
|
||||||
|
"redis-errors": "^1.2.0",
|
||||||
|
"redis-parser": "^3.0.0",
|
||||||
|
"standard-as-callback": "^2.1.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12.22.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "opencollective",
|
||||||
|
"url": "https://opencollective.com/ioredis"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/redis-parser": {
|
"node_modules/redis-parser": {
|
||||||
"version": "3.0.0",
|
"version": "3.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,8 @@
|
||||||
"astro": "^5.5.4",
|
"astro": "^5.5.4",
|
||||||
"feed": "^4.2.2",
|
"feed": "^4.2.2",
|
||||||
"ioredis": "^5.6.0",
|
"ioredis": "^5.6.0",
|
||||||
|
"p-throttle": "^7.0.0",
|
||||||
|
"redis-modules-sdk": "^1.4.0",
|
||||||
"tailwindcss": "^3.4.1"
|
"tailwindcss": "^3.4.1"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,25 +1,17 @@
|
||||||
---
|
---
|
||||||
import { getFeedMapping } from '../lib/db';
|
import { getFeedMapping } from "../lib/db";
|
||||||
interface Props {
|
interface Props {
|
||||||
currentSlugs?: string[];
|
currentSlugs?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
const { currentSlugs = [] } = Astro.props;
|
const { currentSlugs = [] } = Astro.props;
|
||||||
|
|
||||||
import { availableLanguages, languageMap } from '../lib/api';
|
import { availableLanguages, languageMap } from "../lib/api";
|
||||||
|
|
||||||
const languages = [
|
const languages = Object.entries(languageMap).map(([value, label]) => ({
|
||||||
{ value: 'en', label: 'English' },
|
value,
|
||||||
{ value: 'ja', label: 'Japanese' },
|
label,
|
||||||
{ value: 'ko', label: 'Korean' },
|
}));
|
||||||
{ value: 'zh', label: 'Chinese' },
|
|
||||||
{ value: 'fr', label: 'French' },
|
|
||||||
{ value: 'es', label: 'Spanish' },
|
|
||||||
{ value: 'it', label: 'Italian' },
|
|
||||||
{ value: 'de', label: 'German' },
|
|
||||||
{ value: 'tr', label: 'Turkish' },
|
|
||||||
{ value: 'pl', label: 'Polish' },
|
|
||||||
];
|
|
||||||
---
|
---
|
||||||
|
|
||||||
<div class="space-y-4 sm:space-y-6">
|
<div class="space-y-4 sm:space-y-6">
|
||||||
|
|
@ -37,9 +29,11 @@ const languages = [
|
||||||
id="lang-select"
|
id="lang-select"
|
||||||
class="px-3 sm:px-4 py-2 bg-primary rounded-lg border border-hover/30 focus:outline-none focus:border-accent/50 transition-colors text-sm sm:text-base"
|
class="px-3 sm:px-4 py-2 bg-primary rounded-lg border border-hover/30 focus:outline-none focus:border-accent/50 transition-colors text-sm sm:text-base"
|
||||||
>
|
>
|
||||||
{languages.map(lang => (
|
{
|
||||||
|
languages.map((lang) => (
|
||||||
<option value={lang.value}>{lang.label}</option>
|
<option value={lang.value}>{lang.label}</option>
|
||||||
))}
|
))
|
||||||
|
}
|
||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
@ -54,8 +48,16 @@ const languages = [
|
||||||
id="import-feed"
|
id="import-feed"
|
||||||
class="px-4 py-2 bg-secondary hover:bg-hover transition-colors rounded-lg text-sm sm:text-base flex items-center gap-2"
|
class="px-4 py-2 bg-secondary hover:bg-hover transition-colors rounded-lg text-sm sm:text-base flex items-center gap-2"
|
||||||
>
|
>
|
||||||
<svg class="w-4 h-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
<svg
|
||||||
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4M7 10l5 5 5-5M12 15V3"/>
|
class="w-4 h-4"
|
||||||
|
viewBox="0 0 24 24"
|
||||||
|
fill="none"
|
||||||
|
stroke="currentColor"
|
||||||
|
stroke-width="2"
|
||||||
|
>
|
||||||
|
<path
|
||||||
|
d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4M7 10l5 5 5-5M12 15V3"
|
||||||
|
></path>
|
||||||
</svg>
|
</svg>
|
||||||
Import Feed
|
Import Feed
|
||||||
</button>
|
</button>
|
||||||
|
|
@ -73,27 +75,46 @@ const languages = [
|
||||||
class="px-3 py-1.5 text-sm text-red-500 hover:text-red-600 hover:bg-red-500/10 rounded-lg transition-colors flex items-center gap-2"
|
class="px-3 py-1.5 text-sm text-red-500 hover:text-red-600 hover:bg-red-500/10 rounded-lg transition-colors flex items-center gap-2"
|
||||||
style="display: none;"
|
style="display: none;"
|
||||||
>
|
>
|
||||||
<svg class="w-4 h-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
<svg
|
||||||
<path d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"/>
|
class="w-4 h-4"
|
||||||
|
viewBox="0 0 24 24"
|
||||||
|
fill="none"
|
||||||
|
stroke="currentColor"
|
||||||
|
stroke-width="2"
|
||||||
|
>
|
||||||
|
<path
|
||||||
|
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"
|
||||||
|
></path>
|
||||||
</svg>
|
</svg>
|
||||||
Clear All
|
Clear All
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div id="slug-list" class="space-y-2">
|
<div id="slug-list" class="space-y-2">
|
||||||
{currentSlugs.map(slug => (
|
{
|
||||||
<div class="flex items-center justify-between p-3 bg-primary rounded-lg border border-hover/30" data-slug={slug}>
|
currentSlugs.map((slug) => (
|
||||||
|
<div
|
||||||
|
class="flex items-center justify-between p-3 bg-primary rounded-lg border border-hover/30"
|
||||||
|
data-slug={slug}
|
||||||
|
>
|
||||||
<span class="text-primary truncate pr-2">{slug}</span>
|
<span class="text-primary truncate pr-2">{slug}</span>
|
||||||
<button
|
<button
|
||||||
class="text-red-500 hover:text-red-600 transition-colors flex-shrink-0 pl-2"
|
class="text-red-500 hover:text-red-600 transition-colors flex-shrink-0 pl-2"
|
||||||
data-remove={slug}
|
data-remove={slug}
|
||||||
>
|
>
|
||||||
<svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
<svg
|
||||||
<path d="M6 18L18 6M6 6l12 12"></path>
|
class="w-5 h-5"
|
||||||
|
viewBox="0 0 24 24"
|
||||||
|
fill="none"
|
||||||
|
stroke="currentColor"
|
||||||
|
stroke-width="2"
|
||||||
|
>
|
||||||
|
<path d="M6 18L18 6M6 6l12 12" />
|
||||||
</svg>
|
</svg>
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
))}
|
))
|
||||||
|
}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="flex justify-center">
|
<div class="flex justify-center">
|
||||||
|
|
@ -123,20 +144,26 @@ const languages = [
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
const searchInput = document.getElementById('search-input') as HTMLInputElement;
|
const searchInput = document.getElementById(
|
||||||
const searchResults = document.getElementById('search-results');
|
"search-input"
|
||||||
const slugList = document.getElementById('slug-list');
|
) as HTMLInputElement;
|
||||||
const generateRSSLink = document.getElementById('generate-rss');
|
const searchResults = document.getElementById("search-results");
|
||||||
const langSelect = document.getElementById('lang-select') as HTMLSelectElement;
|
const slugList = document.getElementById("slug-list");
|
||||||
const feedIdInput = document.getElementById('feed-id-input') as HTMLInputElement;
|
const generateRSSLink = document.getElementById("generate-rss");
|
||||||
const importFeedButton = document.getElementById('import-feed');
|
const langSelect = document.getElementById(
|
||||||
const clearAllButton = document.getElementById('clear-all');
|
"lang-select"
|
||||||
|
) as HTMLSelectElement;
|
||||||
|
const feedIdInput = document.getElementById(
|
||||||
|
"feed-id-input"
|
||||||
|
) as HTMLInputElement;
|
||||||
|
const importFeedButton = document.getElementById("import-feed");
|
||||||
|
const clearAllButton = document.getElementById("clear-all");
|
||||||
|
|
||||||
// Load saved slugs from localStorage
|
// Load saved slugs from localStorage
|
||||||
let slugs = new Set(JSON.parse(localStorage.getItem('comic-slugs') || '[]'));
|
let slugs = new Set(JSON.parse(localStorage.getItem("comic-slugs") || "[]"));
|
||||||
|
|
||||||
function updateSlugs() {
|
function updateSlugs() {
|
||||||
localStorage.setItem('comic-slugs', JSON.stringify([...slugs]));
|
localStorage.setItem("comic-slugs", JSON.stringify([...slugs]));
|
||||||
updateRSSLink();
|
updateRSSLink();
|
||||||
renderSlugs();
|
renderSlugs();
|
||||||
updateClearAllButton();
|
updateClearAllButton();
|
||||||
|
|
@ -144,13 +171,17 @@ const languages = [
|
||||||
|
|
||||||
function updateClearAllButton() {
|
function updateClearAllButton() {
|
||||||
if (clearAllButton) {
|
if (clearAllButton) {
|
||||||
clearAllButton.style.display = slugs.size > 0 ? 'flex' : 'none';
|
clearAllButton.style.display = slugs.size > 0 ? "flex" : "none";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clear all functionality
|
// Clear all functionality
|
||||||
clearAllButton?.addEventListener('click', () => {
|
clearAllButton?.addEventListener("click", () => {
|
||||||
if (confirm('Are you sure you want to clear all comics? This cannot be undone.')) {
|
if (
|
||||||
|
confirm(
|
||||||
|
"Are you sure you want to clear all comics? This cannot be undone."
|
||||||
|
)
|
||||||
|
) {
|
||||||
slugs.clear();
|
slugs.clear();
|
||||||
updateSlugs();
|
updateSlugs();
|
||||||
}
|
}
|
||||||
|
|
@ -162,7 +193,9 @@ const languages = [
|
||||||
// Show loading state
|
// Show loading state
|
||||||
slugList.innerHTML = `
|
slugList.innerHTML = `
|
||||||
<div class="animate-pulse space-y-2">
|
<div class="animate-pulse space-y-2">
|
||||||
${[...slugs].map(() => `
|
${[...slugs]
|
||||||
|
.map(
|
||||||
|
() => `
|
||||||
<div class="flex items-center gap-4 p-3 bg-primary rounded-lg border border-hover/30">
|
<div class="flex items-center gap-4 p-3 bg-primary rounded-lg border border-hover/30">
|
||||||
<div class="w-12 h-16 bg-hover/30 rounded"></div>
|
<div class="w-12 h-16 bg-hover/30 rounded"></div>
|
||||||
<div class="flex-1">
|
<div class="flex-1">
|
||||||
|
|
@ -170,7 +203,9 @@ const languages = [
|
||||||
<div class="h-4 bg-hover/30 rounded w-1/2"></div>
|
<div class="h-4 bg-hover/30 rounded w-1/2"></div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
`).join('')}
|
`
|
||||||
|
)
|
||||||
|
.join("")}
|
||||||
</div>
|
</div>
|
||||||
`;
|
`;
|
||||||
|
|
||||||
|
|
@ -181,62 +216,71 @@ const languages = [
|
||||||
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
||||||
|
|
||||||
const response = await fetch(`/api/comic/${encodeURIComponent(slug)}`, {
|
const response = await fetch(`/api/comic/${encodeURIComponent(slug)}`, {
|
||||||
signal: controller.signal
|
signal: controller.signal,
|
||||||
});
|
});
|
||||||
clearTimeout(timeoutId);
|
clearTimeout(timeoutId);
|
||||||
|
|
||||||
if (!response.ok) throw new Error('Comic not found');
|
if (!response.ok) throw new Error("Comic not found");
|
||||||
|
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
|
|
||||||
const div = document.createElement('div');
|
const div = document.createElement("div");
|
||||||
div.className = 'flex items-center gap-4 p-4 bg-primary rounded-lg border border-hover/30 hover:border-accent/50 transition-all duration-300 group relative overflow-hidden';
|
div.className =
|
||||||
|
"flex items-center gap-4 p-4 bg-primary rounded-lg border border-hover/30 hover:border-accent/50 transition-all duration-300 group relative overflow-hidden";
|
||||||
|
|
||||||
// Add a link to the comic
|
// Add a link to the comic
|
||||||
const link = document.createElement('a');
|
const link = document.createElement("a");
|
||||||
link.href = `https://comick.io/comic/${slug}`;
|
link.href = `https://comick.io/comic/${slug}`;
|
||||||
link.target = '_blank';
|
link.target = "_blank";
|
||||||
link.rel = 'noopener noreferrer';
|
link.rel = "noopener noreferrer";
|
||||||
link.className = 'flex items-center gap-4 flex-1 min-w-0';
|
link.className = "flex items-center gap-4 flex-1 min-w-0";
|
||||||
|
|
||||||
// Handle cover image
|
// Handle cover image
|
||||||
if (data.comic?.md_covers?.[0]?.b2key) {
|
if (data.comic?.md_covers?.[0]?.b2key) {
|
||||||
const img = document.createElement('img');
|
const img = document.createElement("img");
|
||||||
img.src = `https://meo.comick.pictures/${data.comic.md_covers[0].b2key}`;
|
img.src = `https://meo.comick.pictures/${data.comic.md_covers[0].b2key}`;
|
||||||
img.className = 'w-16 h-24 object-cover rounded shadow-md group-hover:scale-105 transition-transform duration-300';
|
img.className =
|
||||||
|
"w-16 h-24 object-cover rounded shadow-md group-hover:scale-105 transition-transform duration-300";
|
||||||
img.alt = data.comic.title || slug;
|
img.alt = data.comic.title || slug;
|
||||||
img.onerror = () => {
|
img.onerror = () => {
|
||||||
img.src = '/placeholder.png'; // Fallback image
|
img.src = "/placeholder.png"; // Fallback image
|
||||||
img.onerror = null;
|
img.onerror = null;
|
||||||
};
|
};
|
||||||
link.appendChild(img);
|
link.appendChild(img);
|
||||||
} else {
|
} else {
|
||||||
// Placeholder if no image
|
// Placeholder if no image
|
||||||
const placeholder = document.createElement('div');
|
const placeholder = document.createElement("div");
|
||||||
placeholder.className = 'w-16 h-24 bg-hover/30 rounded flex items-center justify-center text-primary/30';
|
placeholder.className =
|
||||||
placeholder.innerHTML = '<svg class="w-8 h-8" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"></path></svg>';
|
"w-16 h-24 bg-hover/30 rounded flex items-center justify-center text-primary/30";
|
||||||
|
placeholder.innerHTML =
|
||||||
|
'<svg class="w-8 h-8" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"></path></svg>';
|
||||||
link.appendChild(placeholder);
|
link.appendChild(placeholder);
|
||||||
}
|
}
|
||||||
|
|
||||||
const info = document.createElement('div');
|
const info = document.createElement("div");
|
||||||
info.className = 'flex-1 min-w-0 truncate';
|
info.className = "flex-1 min-w-0 truncate";
|
||||||
info.innerHTML = `
|
info.innerHTML = `
|
||||||
<div class="font-medium text-lg truncate group-hover:text-accent transition-colors">
|
<div class="font-medium text-lg truncate group-hover:text-accent transition-colors">
|
||||||
${data.comic?.title || slug}
|
${data.comic?.title || slug}
|
||||||
</div>
|
</div>
|
||||||
<div class="text-sm text-primary/60 truncate">${slug}</div>
|
<div class="text-sm text-primary/60 truncate">${slug}</div>
|
||||||
${data.comic?.last_chapter ? `
|
${
|
||||||
|
data.comic?.last_chapter
|
||||||
|
? `
|
||||||
<div class="text-xs text-accent mt-1">
|
<div class="text-xs text-accent mt-1">
|
||||||
Latest: Chapter ${data.comic.last_chapter}
|
Latest: Chapter ${data.comic.last_chapter}
|
||||||
</div>
|
</div>
|
||||||
` : ''}
|
`
|
||||||
|
: ""
|
||||||
|
}
|
||||||
`;
|
`;
|
||||||
|
|
||||||
link.appendChild(info);
|
link.appendChild(info);
|
||||||
div.appendChild(link);
|
div.appendChild(link);
|
||||||
|
|
||||||
const removeBtn = document.createElement('button');
|
const removeBtn = document.createElement("button");
|
||||||
removeBtn.className = 'text-red-500 hover:text-red-600 transition-colors p-2 hover:bg-red-500/10 rounded-full flex-shrink-0';
|
removeBtn.className =
|
||||||
|
"text-red-500 hover:text-red-600 transition-colors p-2 hover:bg-red-500/10 rounded-full flex-shrink-0";
|
||||||
removeBtn.innerHTML = `
|
removeBtn.innerHTML = `
|
||||||
<svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
<svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||||
<path d="M6 18L18 6M6 6l12 12"></path>
|
<path d="M6 18L18 6M6 6l12 12"></path>
|
||||||
|
|
@ -245,7 +289,7 @@ const languages = [
|
||||||
removeBtn.onclick = (e) => {
|
removeBtn.onclick = (e) => {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
e.stopPropagation();
|
e.stopPropagation();
|
||||||
div.classList.add('scale-95', 'opacity-0');
|
div.classList.add("scale-95", "opacity-0");
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
slugs.delete(slug);
|
slugs.delete(slug);
|
||||||
updateSlugs();
|
updateSlugs();
|
||||||
|
|
@ -257,8 +301,9 @@ const languages = [
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Failed to fetch comic info for ${slug}:`, error);
|
console.error(`Failed to fetch comic info for ${slug}:`, error);
|
||||||
// Improved error state
|
// Improved error state
|
||||||
const div = document.createElement('div');
|
const div = document.createElement("div");
|
||||||
div.className = 'flex items-center justify-between p-4 bg-primary rounded-lg border border-red-500/30 group';
|
div.className =
|
||||||
|
"flex items-center justify-between p-4 bg-primary rounded-lg border border-red-500/30 group";
|
||||||
div.innerHTML = `
|
div.innerHTML = `
|
||||||
<div class="flex items-center gap-4 flex-1">
|
<div class="flex items-center gap-4 flex-1">
|
||||||
<div class="w-16 h-24 bg-red-500/10 rounded flex items-center justify-center text-red-500/50">
|
<div class="w-16 h-24 bg-red-500/10 rounded flex items-center justify-center text-red-500/50">
|
||||||
|
|
@ -280,12 +325,12 @@ const languages = [
|
||||||
</button>
|
</button>
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const removeButton = div.querySelector('button');
|
const removeButton = div.querySelector("button");
|
||||||
if (removeButton) {
|
if (removeButton) {
|
||||||
removeButton.onclick = (e) => {
|
removeButton.onclick = (e) => {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
e.stopPropagation();
|
e.stopPropagation();
|
||||||
div.classList.add('scale-95', 'opacity-0');
|
div.classList.add("scale-95", "opacity-0");
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
slugs.delete(slug);
|
slugs.delete(slug);
|
||||||
updateSlugs();
|
updateSlugs();
|
||||||
|
|
@ -296,21 +341,23 @@ const languages = [
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
Promise.all(promises).then(elements => {
|
Promise.all(promises)
|
||||||
slugList.innerHTML = '';
|
.then((elements) => {
|
||||||
elements.forEach(element => slugList.appendChild(element));
|
slugList.innerHTML = "";
|
||||||
}).catch(error => {
|
elements.forEach((element) => slugList.appendChild(element));
|
||||||
console.error('Failed to render slugs:', error);
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
console.error("Failed to render slugs:", error);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let searchTimeout: NodeJS.Timeout;
|
let searchTimeout: NodeJS.Timeout;
|
||||||
searchInput?.addEventListener('input', () => {
|
searchInput?.addEventListener("input", () => {
|
||||||
clearTimeout(searchTimeout);
|
clearTimeout(searchTimeout);
|
||||||
searchTimeout = setTimeout(async () => {
|
searchTimeout = setTimeout(async () => {
|
||||||
const query = searchInput.value.trim();
|
const query = searchInput.value.trim();
|
||||||
if (query.length < 2) {
|
if (query.length < 2) {
|
||||||
searchResults!.classList.add('hidden');
|
searchResults!.classList.add("hidden");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -320,14 +367,14 @@ const languages = [
|
||||||
Searching...
|
Searching...
|
||||||
</div>
|
</div>
|
||||||
`;
|
`;
|
||||||
searchResults!.classList.remove('hidden');
|
searchResults!.classList.remove("hidden");
|
||||||
|
|
||||||
const response = await fetch('/api/search', {
|
const response = await fetch("/api/search", {
|
||||||
method: 'POST',
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
"Content-Type": "application/json",
|
||||||
},
|
},
|
||||||
body: JSON.stringify({ query })
|
body: JSON.stringify({ query }),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
|
|
@ -345,27 +392,28 @@ const languages = [
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
searchResults!.innerHTML = '';
|
searchResults!.innerHTML = "";
|
||||||
|
|
||||||
data.forEach((comic: any) => {
|
data.forEach((comic: any) => {
|
||||||
const div = document.createElement('div');
|
const div = document.createElement("div");
|
||||||
div.className = 'flex items-center gap-3 p-3 bg-primary rounded-lg border border-hover/30 cursor-pointer hover:border-accent/50 transition-colors';
|
div.className =
|
||||||
|
"flex items-center gap-3 p-3 bg-primary rounded-lg border border-hover/30 cursor-pointer hover:border-accent/50 transition-colors";
|
||||||
div.onclick = () => {
|
div.onclick = () => {
|
||||||
slugs.add(comic.slug);
|
slugs.add(comic.slug);
|
||||||
updateSlugs();
|
updateSlugs();
|
||||||
searchInput.value = '';
|
searchInput.value = "";
|
||||||
searchResults!.classList.add('hidden');
|
searchResults!.classList.add("hidden");
|
||||||
};
|
};
|
||||||
|
|
||||||
if (comic.md_covers && comic.md_covers[0]) {
|
if (comic.md_covers && comic.md_covers[0]) {
|
||||||
const img = document.createElement('img');
|
const img = document.createElement("img");
|
||||||
img.src = `https://meo.comick.pictures/${comic.md_covers[0].b2key}`;
|
img.src = `https://meo.comick.pictures/${comic.md_covers[0].b2key}`;
|
||||||
img.className = 'w-12 h-16 object-cover rounded';
|
img.className = "w-12 h-16 object-cover rounded";
|
||||||
img.alt = comic.title;
|
img.alt = comic.title;
|
||||||
div.appendChild(img);
|
div.appendChild(img);
|
||||||
}
|
}
|
||||||
|
|
||||||
const info = document.createElement('div');
|
const info = document.createElement("div");
|
||||||
info.innerHTML = `
|
info.innerHTML = `
|
||||||
<div class="font-medium">${comic.title}</div>
|
<div class="font-medium">${comic.title}</div>
|
||||||
<div class="text-sm text-primary/60">${comic.slug}</div>
|
<div class="text-sm text-primary/60">${comic.slug}</div>
|
||||||
|
|
@ -375,7 +423,7 @@ const languages = [
|
||||||
searchResults!.appendChild(div);
|
searchResults!.appendChild(div);
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to search comics:', error);
|
console.error("Failed to search comics:", error);
|
||||||
searchResults!.innerHTML = `
|
searchResults!.innerHTML = `
|
||||||
<div class="p-3 bg-primary rounded-lg border border-red-500/30 text-red-500">
|
<div class="p-3 bg-primary rounded-lg border border-red-500/30 text-red-500">
|
||||||
Failed to search comics. Please try again.
|
Failed to search comics. Please try again.
|
||||||
|
|
@ -386,8 +434,8 @@ const languages = [
|
||||||
});
|
});
|
||||||
|
|
||||||
async function updateRSSLink() {
|
async function updateRSSLink() {
|
||||||
const slugString = [...slugs].join(',');
|
const slugString = [...slugs].join(",");
|
||||||
const lang = langSelect?.value || 'en';
|
const lang = langSelect?.value || "en";
|
||||||
|
|
||||||
if (slugString) {
|
if (slugString) {
|
||||||
try {
|
try {
|
||||||
|
|
@ -402,18 +450,18 @@ const languages = [
|
||||||
`;
|
`;
|
||||||
generateRSSLink!.onclick = null;
|
generateRSSLink!.onclick = null;
|
||||||
|
|
||||||
const response = await fetch('/api/create-feed', {
|
const response = await fetch("/api/create-feed", {
|
||||||
method: 'POST',
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
"Content-Type": "application/json",
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
slugs: [...slugs],
|
slugs: [...slugs],
|
||||||
lang
|
lang,
|
||||||
})
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!response.ok) throw new Error('Failed to create feed');
|
if (!response.ok) throw new Error("Failed to create feed");
|
||||||
|
|
||||||
const { feedId } = await response.json();
|
const { feedId } = await response.json();
|
||||||
generateRSSLink!.href = `/rss/${feedId}.xml`;
|
generateRSSLink!.href = `/rss/${feedId}.xml`;
|
||||||
|
|
@ -431,7 +479,7 @@ const languages = [
|
||||||
`;
|
`;
|
||||||
generateRSSLink!.onclick = null; // Remove any existing click handlers
|
generateRSSLink!.onclick = null; // Remove any existing click handlers
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to generate feed:', error);
|
console.error("Failed to generate feed:", error);
|
||||||
generateRSSLink!.innerHTML = `
|
generateRSSLink!.innerHTML = `
|
||||||
<span class="flex items-center gap-2">
|
<span class="flex items-center gap-2">
|
||||||
<svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
<svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||||
|
|
@ -443,7 +491,7 @@ const languages = [
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Reset to default state when no slugs
|
// Reset to default state when no slugs
|
||||||
generateRSSLink!.href = '#';
|
generateRSSLink!.href = "#";
|
||||||
generateRSSLink!.innerHTML = `
|
generateRSSLink!.innerHTML = `
|
||||||
<span class="flex items-center gap-2">
|
<span class="flex items-center gap-2">
|
||||||
<svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
<svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||||
|
|
@ -462,7 +510,7 @@ const languages = [
|
||||||
}
|
}
|
||||||
|
|
||||||
// Import feed functionality
|
// Import feed functionality
|
||||||
importFeedButton?.addEventListener('click', async () => {
|
importFeedButton?.addEventListener("click", async () => {
|
||||||
const feedId = feedIdInput.value.trim();
|
const feedId = feedIdInput.value.trim();
|
||||||
if (!feedId) return;
|
if (!feedId) return;
|
||||||
|
|
||||||
|
|
@ -478,20 +526,28 @@ const languages = [
|
||||||
const response = await fetch(`/api/feed/${feedId}`);
|
const response = await fetch(`/api/feed/${feedId}`);
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const errorText = await response.text();
|
const errorText = await response.text();
|
||||||
console.error(`Feed import failed with status ${response.status}: ${errorText}`);
|
console.error(
|
||||||
|
`Feed import failed with status ${response.status}: ${errorText}`
|
||||||
|
);
|
||||||
|
|
||||||
// Display a more user-friendly error based on the status code
|
// Display a more user-friendly error based on the status code
|
||||||
let userFriendlyError = 'Failed to import feed.';
|
let userFriendlyError = "Failed to import feed.";
|
||||||
|
|
||||||
if (response.status === 404) {
|
if (response.status === 404) {
|
||||||
userFriendlyError = `Feed ID "${feedId}" not found. Please check the ID and try again.`;
|
userFriendlyError = `Feed ID "${feedId}" not found. Please check the ID and try again.`;
|
||||||
} else if (response.status === 500) {
|
} else if (response.status === 500) {
|
||||||
if (errorText.includes('ETIMEDOUT') || errorText.includes('timeout')) {
|
if (
|
||||||
userFriendlyError = 'Connection to database timed out. This is common on the first request. Please try again in a few seconds.';
|
errorText.includes("ETIMEDOUT") ||
|
||||||
} else if (errorText.includes('ECONNREFUSED')) {
|
errorText.includes("timeout")
|
||||||
userFriendlyError = 'Database connection refused. Please try again later.';
|
) {
|
||||||
|
userFriendlyError =
|
||||||
|
"Connection to database timed out. This is common on the first request. Please try again in a few seconds.";
|
||||||
|
} else if (errorText.includes("ECONNREFUSED")) {
|
||||||
|
userFriendlyError =
|
||||||
|
"Database connection refused. Please try again later.";
|
||||||
} else {
|
} else {
|
||||||
userFriendlyError = 'Server error while importing feed. Please try again later.';
|
userFriendlyError =
|
||||||
|
"Server error while importing feed. Please try again later.";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -499,11 +555,11 @@ const languages = [
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
console.log('Feed import data:', data); // Diagnostic log
|
console.log("Feed import data:", data); // Diagnostic log
|
||||||
|
|
||||||
// Validate response structure
|
// Validate response structure
|
||||||
if (!data || typeof data !== 'object') {
|
if (!data || typeof data !== "object") {
|
||||||
throw new Error('Invalid feed data format: not an object');
|
throw new Error("Invalid feed data format: not an object");
|
||||||
}
|
}
|
||||||
|
|
||||||
const feedSlugs = data.slugs;
|
const feedSlugs = data.slugs;
|
||||||
|
|
@ -511,15 +567,17 @@ const languages = [
|
||||||
|
|
||||||
// Validate slugs array
|
// Validate slugs array
|
||||||
if (!feedSlugs || !Array.isArray(feedSlugs)) {
|
if (!feedSlugs || !Array.isArray(feedSlugs)) {
|
||||||
throw new Error('Invalid feed data: missing or invalid slugs array');
|
throw new Error("Invalid feed data: missing or invalid slugs array");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate language
|
// Validate language
|
||||||
if (!lang || typeof lang !== 'string') {
|
if (!lang || typeof lang !== "string") {
|
||||||
console.warn('Feed has missing or invalid language, using default');
|
console.warn("Feed has missing or invalid language, using default");
|
||||||
} else if (langSelect) {
|
} else if (langSelect) {
|
||||||
// Only update language if it's valid
|
// Only update language if it's valid
|
||||||
const validLanguages = Array.from(langSelect.options).map(opt => opt.value);
|
const validLanguages = Array.from(langSelect.options).map(
|
||||||
|
(opt) => opt.value
|
||||||
|
);
|
||||||
if (validLanguages.includes(lang)) {
|
if (validLanguages.includes(lang)) {
|
||||||
langSelect.value = lang;
|
langSelect.value = lang;
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -530,32 +588,32 @@ const languages = [
|
||||||
// Add new slugs to existing collection
|
// Add new slugs to existing collection
|
||||||
let importCount = 0;
|
let importCount = 0;
|
||||||
feedSlugs.forEach((slug: string) => {
|
feedSlugs.forEach((slug: string) => {
|
||||||
if (typeof slug === 'string' && slug.trim()) {
|
if (typeof slug === "string" && slug.trim()) {
|
||||||
slugs.add(slug.trim());
|
slugs.add(slug.trim());
|
||||||
importCount++;
|
importCount++;
|
||||||
} else {
|
} else {
|
||||||
console.warn('Skipping invalid slug in feed:', slug);
|
console.warn("Skipping invalid slug in feed:", slug);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Update UI
|
// Update UI
|
||||||
updateSlugs();
|
updateSlugs();
|
||||||
feedIdInput.value = '';
|
feedIdInput.value = "";
|
||||||
|
|
||||||
// Show success message
|
// Show success message
|
||||||
const successDiv = document.createElement('div');
|
const successDiv = document.createElement("div");
|
||||||
successDiv.className = 'text-sm text-green-500 mt-1';
|
successDiv.className = "text-sm text-green-500 mt-1";
|
||||||
successDiv.textContent = `Feed imported successfully! Added ${importCount} comics.`;
|
successDiv.textContent = `Feed imported successfully! Added ${importCount} comics.`;
|
||||||
feedIdInput.parentNode?.appendChild(successDiv);
|
feedIdInput.parentNode?.appendChild(successDiv);
|
||||||
setTimeout(() => successDiv.remove(), 3000);
|
setTimeout(() => successDiv.remove(), 3000);
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to import feed:', error);
|
console.error("Failed to import feed:", error);
|
||||||
const errorDiv = document.createElement('div');
|
const errorDiv = document.createElement("div");
|
||||||
errorDiv.className = 'text-sm text-red-500 mt-1';
|
errorDiv.className = "text-sm text-red-500 mt-1";
|
||||||
errorDiv.textContent = error instanceof Error
|
errorDiv.textContent =
|
||||||
|
error instanceof Error
|
||||||
? `Import failed: ${error.message}`
|
? `Import failed: ${error.message}`
|
||||||
: 'Failed to import feed. Please check the feed ID and try again.';
|
: "Failed to import feed. Please check the feed ID and try again.";
|
||||||
feedIdInput.parentNode?.appendChild(errorDiv);
|
feedIdInput.parentNode?.appendChild(errorDiv);
|
||||||
setTimeout(() => errorDiv.remove(), 5000);
|
setTimeout(() => errorDiv.remove(), 5000);
|
||||||
} finally {
|
} finally {
|
||||||
|
|
@ -575,12 +633,15 @@ const languages = [
|
||||||
updateClearAllButton();
|
updateClearAllButton();
|
||||||
|
|
||||||
// Update RSS link when language changes
|
// Update RSS link when language changes
|
||||||
langSelect?.addEventListener('change', () => updateRSSLink());
|
langSelect?.addEventListener("change", () => updateRSSLink());
|
||||||
|
|
||||||
// Close search results when clicking outside
|
// Close search results when clicking outside
|
||||||
document.addEventListener('click', (e) => {
|
document.addEventListener("click", (e) => {
|
||||||
if (!searchResults?.contains(e.target as Node) && !searchInput?.contains(e.target as Node)) {
|
if (
|
||||||
searchResults?.classList.add('hidden');
|
!searchResults?.contains(e.target as Node) &&
|
||||||
|
!searchInput?.contains(e.target as Node)
|
||||||
|
) {
|
||||||
|
searchResults?.classList.add("hidden");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
130
src/lib/api.ts
130
src/lib/api.ts
|
|
@ -1,10 +1,12 @@
|
||||||
import type { Comic, Genre, ComicType, ChapterDetail } from './types';
|
import type { Comic, Genre, ComicType, ChapterDetail } from './types';
|
||||||
import { getRedisClient, CHAPTER_CACHE_TTL } from './db';
|
import { getRedisClient, CHAPTER_CACHE_TTL } from './db';
|
||||||
|
import pThrottle from 'p-throttle';
|
||||||
|
|
||||||
const languageMap = {
|
const languageMap = {
|
||||||
"en": "English",
|
"en": "English",
|
||||||
"fr": "French",
|
"fr": "French",
|
||||||
"es": "Spanish",
|
"es": "Spanish",
|
||||||
|
"es-419": "Spanish (Latin America)",
|
||||||
"it": "Italian",
|
"it": "Italian",
|
||||||
"pl": "Polish",
|
"pl": "Polish",
|
||||||
"tr": "Turkish",
|
"tr": "Turkish",
|
||||||
|
|
@ -13,7 +15,10 @@ const languageMap = {
|
||||||
"sv": "Swedish",
|
"sv": "Swedish",
|
||||||
"ar": "Arabic",
|
"ar": "Arabic",
|
||||||
"de": "German",
|
"de": "German",
|
||||||
"ko": "Korean"
|
"ko": "Korean",
|
||||||
|
"pt": "Portuguese",
|
||||||
|
"pt-br": "Portuguese (Brazil)",
|
||||||
|
"pt-pt": "Portuguese (Portugal)"
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
const languages = new Set(Object.keys(languageMap));
|
const languages = new Set(Object.keys(languageMap));
|
||||||
|
|
@ -26,6 +31,20 @@ let genreCache: Genre[] = [];
|
||||||
const RETRY_COUNT = 3;
|
const RETRY_COUNT = 3;
|
||||||
const RETRY_DELAY = 1000;
|
const RETRY_DELAY = 1000;
|
||||||
|
|
||||||
|
// Rate limiting configuration - adjust these values if needed
|
||||||
|
const API_REQUESTS_PER_SECOND = 10; // Maximum requests per second
|
||||||
|
const API_CONCURRENT_REQUESTS = 6; // Maximum concurrent requests
|
||||||
|
const DELAY_AFTER_429 = 1000; // Wait 1 second after a rate limit hit
|
||||||
|
|
||||||
|
// Create throttled fetch function to avoid rate limits
|
||||||
|
const throttledFetch = pThrottle({
|
||||||
|
limit: API_REQUESTS_PER_SECOND,
|
||||||
|
interval: 1000,
|
||||||
|
concurrency: API_CONCURRENT_REQUESTS
|
||||||
|
} as any)((url: string, options: RequestInit = {}, retries = RETRY_COUNT) => {
|
||||||
|
return fetchWithRetry(url, options, retries);
|
||||||
|
});
|
||||||
|
|
||||||
async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise<Response> {
|
async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise<Response> {
|
||||||
try {
|
try {
|
||||||
const response = await fetch(url, {
|
const response = await fetch(url, {
|
||||||
|
|
@ -37,6 +56,19 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries =
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Special handling for rate limit errors
|
||||||
|
if (response.status === 429) {
|
||||||
|
console.warn(`Rate limited on ${url}. Waiting ${DELAY_AFTER_429}ms before retry.`);
|
||||||
|
|
||||||
|
// Wait for the specified delay
|
||||||
|
await new Promise(resolve => setTimeout(resolve, DELAY_AFTER_429));
|
||||||
|
|
||||||
|
// Retry with one less retry count but only if we have retries left
|
||||||
|
if (retries > 0) {
|
||||||
|
return fetchWithRetry(url, options, retries - 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`HTTP error! status: ${response.status}`);
|
throw new Error(`HTTP error! status: ${response.status}`);
|
||||||
}
|
}
|
||||||
|
|
@ -44,7 +76,10 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries =
|
||||||
return response;
|
return response;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (retries > 0) {
|
if (retries > 0) {
|
||||||
await new Promise(resolve => setTimeout(resolve, RETRY_DELAY));
|
// For non-429 errors, use exponential backoff
|
||||||
|
const delay = Math.min(RETRY_DELAY * Math.pow(2, RETRY_COUNT - retries), 10000);
|
||||||
|
console.log(`Retrying after ${delay}ms. Retries left: ${retries-1}`);
|
||||||
|
await new Promise(resolve => setTimeout(resolve, delay));
|
||||||
return fetchWithRetry(url, options, retries - 1);
|
return fetchWithRetry(url, options, retries - 1);
|
||||||
}
|
}
|
||||||
throw error;
|
throw error;
|
||||||
|
|
@ -55,7 +90,8 @@ export async function fetchGenres(): Promise<Genre[]> {
|
||||||
if (genreCache.length > 0) return genreCache;
|
if (genreCache.length > 0) return genreCache;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetchWithRetry('https://api.comick.fun/genre');
|
// No tachiyomi param for genre endpoint
|
||||||
|
const response = await throttledFetch('https://api.comick.fun/genre');
|
||||||
genreCache = await response.json();
|
genreCache = await response.json();
|
||||||
return genreCache;
|
return genreCache;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -92,7 +128,7 @@ export async function fetchComics(
|
||||||
try {
|
try {
|
||||||
console.log('Fetching comics from:', apiUrl);
|
console.log('Fetching comics from:', apiUrl);
|
||||||
|
|
||||||
const response = await fetchWithRetry(apiUrl);
|
const response = await throttledFetch(apiUrl);
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
|
|
||||||
if (!Array.isArray(data)) {
|
if (!Array.isArray(data)) {
|
||||||
|
|
@ -146,7 +182,7 @@ interface ChapterParams {
|
||||||
|
|
||||||
export async function getComicBySlug(slug: string): Promise<ComicDetail | null> {
|
export async function getComicBySlug(slug: string): Promise<ComicDetail | null> {
|
||||||
try {
|
try {
|
||||||
const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`);
|
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`);
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
|
|
||||||
if (!data?.comic?.hid) {
|
if (!data?.comic?.hid) {
|
||||||
|
|
@ -174,14 +210,14 @@ export async function getChaptersByHid(
|
||||||
});
|
});
|
||||||
|
|
||||||
// Fetch comic details including cover
|
// Fetch comic details including cover
|
||||||
const comicResponse = await fetchWithRetry(
|
const comicResponse = await throttledFetch(
|
||||||
`https://api.comick.fun/comic/${hid}`
|
`https://api.comick.fun/comic/${hid}`
|
||||||
);
|
);
|
||||||
const comicData = await comicResponse.json();
|
const comicData = await comicResponse.json();
|
||||||
const comicTitle = comicData?.comic?.title || 'Unknown Comic';
|
const comicTitle = comicData?.comic?.title || 'Unknown Comic';
|
||||||
const covers = comicData?.comic?.md_covers || [];
|
const covers = comicData?.comic?.md_covers || [];
|
||||||
|
|
||||||
const response = await fetchWithRetry(
|
const response = await throttledFetch(
|
||||||
`https://api.comick.fun/comic/${hid}/chapters?${queryParams}`
|
`https://api.comick.fun/comic/${hid}/chapters?${queryParams}`
|
||||||
);
|
);
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
|
|
@ -260,22 +296,25 @@ export async function getChaptersForSlugs(
|
||||||
const uniqueSlugs = [...new Set(slugs)];
|
const uniqueSlugs = [...new Set(slugs)];
|
||||||
const pipeline = redis.pipeline();
|
const pipeline = redis.pipeline();
|
||||||
|
|
||||||
// Add cache key validation
|
// Check cache for all slugs
|
||||||
uniqueSlugs.forEach(slug => {
|
uniqueSlugs.forEach(slug => {
|
||||||
if (!slug.match(/^[a-z0-9-]+$/)) {
|
if (!slug.match(/^[a-z0-9-]+$/)) {
|
||||||
console.error(`Invalid slug format: ${slug}`);
|
console.error(`Invalid slug format: ${slug}`);
|
||||||
|
return; // Skip invalid slugs immediately
|
||||||
}
|
}
|
||||||
const cacheKey = `chapters:${slug}:${lang}`;
|
const cacheKey = `chapters:${slug}:${lang}`;
|
||||||
pipeline.get(cacheKey);
|
pipeline.get(cacheKey);
|
||||||
});
|
});
|
||||||
|
|
||||||
const [cacheResults, slugsToFetch] = await Promise.all([
|
const cacheResults = await pipeline.exec();
|
||||||
pipeline.exec(),
|
|
||||||
validateSlugs(uniqueSlugs) // Add this new function
|
|
||||||
]);
|
|
||||||
|
|
||||||
|
// Process cache results
|
||||||
const cachedChapters: ChapterDetail[] = [];
|
const cachedChapters: ChapterDetail[] = [];
|
||||||
|
const slugsToFetch: string[] = [];
|
||||||
|
|
||||||
|
// Separate cached and uncached slugs in one pass
|
||||||
cacheResults.forEach(([err, result], index) => {
|
cacheResults.forEach(([err, result], index) => {
|
||||||
|
const slug = uniqueSlugs[index];
|
||||||
if (!err && result) {
|
if (!err && result) {
|
||||||
try {
|
try {
|
||||||
const parsed = JSON.parse(result as string);
|
const parsed = JSON.parse(result as string);
|
||||||
|
|
@ -283,42 +322,46 @@ export async function getChaptersForSlugs(
|
||||||
cachedChapters.push(...parsed);
|
cachedChapters.push(...parsed);
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e);
|
console.error(`Cache parse error for ${slug}:`, e);
|
||||||
|
slugsToFetch.push(slug);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
slugsToFetch.push(slug);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Add progress tracking
|
// Skip validation if we have recent cache data
|
||||||
const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug));
|
const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug));
|
||||||
console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`);
|
console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`);
|
||||||
|
|
||||||
|
// Only fetch if we have slugs that need fetching
|
||||||
|
let liveChapters: ChapterDetail[] = [];
|
||||||
|
if (slugsToFetch.length > 0) {
|
||||||
console.log(`Fetching ${slugsToFetch.length} comics from API`);
|
console.log(`Fetching ${slugsToFetch.length} comics from API`);
|
||||||
|
|
||||||
|
// Use Promise.all for concurrent fetching with higher concurrency
|
||||||
const fetchPromises = slugsToFetch.map(async (slug) => {
|
const fetchPromises = slugsToFetch.map(async (slug) => {
|
||||||
const timer = Date.now();
|
|
||||||
try {
|
try {
|
||||||
const comicDetail = await getComicBySlug(slug);
|
const comicDetail = await getComicBySlug(slug);
|
||||||
|
|
||||||
if (!comicDetail?.comic?.hid) {
|
if (!comicDetail?.comic?.hid) {
|
||||||
console.error(`❌ No HID found for ${slug}`);
|
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, {
|
return getChaptersByHid(comicDetail.comic.hid, slug, {
|
||||||
limit: 5,
|
limit: 5,
|
||||||
lang: lang
|
lang: lang
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`);
|
|
||||||
return chapters;
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`🚨 Error processing ${slug}:`, error);
|
console.error(`Error processing ${slug}:`, error);
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
const liveChapters = (await Promise.all(fetchPromises)).flat();
|
liveChapters = (await Promise.all(fetchPromises)).flat();
|
||||||
|
|
||||||
// Cache new results in batch
|
// Cache new results in batch if we have any
|
||||||
|
if (liveChapters.length > 0) {
|
||||||
const cachePipeline = redis.pipeline();
|
const cachePipeline = redis.pipeline();
|
||||||
const chaptersBySlug: Record<string, ChapterDetail[]> = {};
|
const chaptersBySlug: Record<string, ChapterDetail[]> = {};
|
||||||
|
|
||||||
|
|
@ -341,32 +384,32 @@ export async function getChaptersForSlugs(
|
||||||
});
|
});
|
||||||
|
|
||||||
await cachePipeline.exec();
|
await cachePipeline.exec();
|
||||||
|
}
|
||||||
// Combine and sort all chapters
|
|
||||||
const allChapters = [...cachedChapters, ...liveChapters];
|
|
||||||
const deduplicatedChapters = deduplicateChapters(allChapters);
|
|
||||||
return deduplicatedChapters.sort((a, b) => {
|
|
||||||
// Ensure the created_at fields are valid dates
|
|
||||||
const dateA = new Date(a.created_at).getTime();
|
|
||||||
const dateB = new Date(b.created_at).getTime();
|
|
||||||
|
|
||||||
// First try to sort by chapter number
|
|
||||||
const chapDiff = parseFloat(b.chap) - parseFloat(a.chap);
|
|
||||||
if (!isNaN(chapDiff)) return chapDiff;
|
|
||||||
|
|
||||||
// Fall back to date if chapter numbers are equal or invalid
|
|
||||||
return dateB - dateA;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add export to the validation function
|
// Combine and deduplicate
|
||||||
|
return deduplicateChapters([...cachedChapters, ...liveChapters]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Replace validateSlugs with a more efficient implementation
|
||||||
export async function validateSlugs(slugs: string[]): Promise<string[]> {
|
export async function validateSlugs(slugs: string[]): Promise<string[]> {
|
||||||
|
// Skip validation in production for cached items
|
||||||
|
if (process.env.NODE_ENV === 'production') {
|
||||||
|
return slugs;
|
||||||
|
}
|
||||||
|
|
||||||
const validSlugs: string[] = [];
|
const validSlugs: string[] = [];
|
||||||
const invalidSlugs: string[] = [];
|
const invalidSlugs: string[] = [];
|
||||||
|
|
||||||
await Promise.all(slugs.map(async (slug) => {
|
// Increase batch size for faster processing
|
||||||
|
const batchSize = 5;
|
||||||
|
for (let i = 0; i < slugs.length; i += batchSize) {
|
||||||
|
const batch = slugs.slice(i, i + batchSize);
|
||||||
|
|
||||||
|
await Promise.all(batch.map(async (slug) => {
|
||||||
try {
|
try {
|
||||||
const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1);
|
// Reduced retries (1 instead of 3) and removed tachiyomi parameter
|
||||||
|
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`, {}, 1);
|
||||||
if (response.ok) {
|
if (response.ok) {
|
||||||
validSlugs.push(slug);
|
validSlugs.push(slug);
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -376,9 +419,10 @@ export async function validateSlugs(slugs: string[]): Promise<string[]> {
|
||||||
invalidSlugs.push(slug);
|
invalidSlugs.push(slug);
|
||||||
}
|
}
|
||||||
}));
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
if (invalidSlugs.length > 0) {
|
if (invalidSlugs.length > 0) {
|
||||||
console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`);
|
console.warn(`Skipping invalid slugs: ${invalidSlugs.length} items`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return validSlugs;
|
return validSlugs;
|
||||||
|
|
|
||||||
303
src/lib/db.ts
303
src/lib/db.ts
|
|
@ -1,5 +1,8 @@
|
||||||
import Redis from "ioredis";
|
import Redis from "ioredis";
|
||||||
import type { FeedMapping } from "./types";
|
import type { FeedMapping } from "./types";
|
||||||
|
import { ReJSON } from 'redis-modules-sdk';
|
||||||
|
import * as zlib from 'zlib';
|
||||||
|
import { promisify } from 'util';
|
||||||
|
|
||||||
// Configuration constants
|
// Configuration constants
|
||||||
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
|
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
|
||||||
|
|
@ -55,6 +58,18 @@ const operationBuffer: RedisOperation[] = [];
|
||||||
// Redis instance naming for logging
|
// Redis instance naming for logging
|
||||||
const redisInstanceNames = new Map<string, string>();
|
const redisInstanceNames = new Map<string, string>();
|
||||||
|
|
||||||
|
// Add zlib for compression
|
||||||
|
const gzip = promisify(zlib.gzip);
|
||||||
|
const gunzip = promisify(zlib.gunzip);
|
||||||
|
|
||||||
|
// Compression threshold (in bytes)
|
||||||
|
const COMPRESSION_THRESHOLD = 1024; // 1KB
|
||||||
|
|
||||||
|
// Helper function to determine if compression should be used
|
||||||
|
function shouldCompress(data: string): boolean {
|
||||||
|
return Buffer.byteLength(data, 'utf8') > COMPRESSION_THRESHOLD;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize Redis instance names for better logging
|
* Initialize Redis instance names for better logging
|
||||||
*/
|
*/
|
||||||
|
|
@ -75,6 +90,14 @@ initializeRedisNames();
|
||||||
*/
|
*/
|
||||||
const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown";
|
const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown";
|
||||||
|
|
||||||
|
// Add at the top of your file, after the imports
|
||||||
|
declare module "ioredis" {
|
||||||
|
interface Redis {
|
||||||
|
json_set(key: string, path: string, json: any): Promise<string>;
|
||||||
|
json_get(key: string, ...paths: string[]): Promise<string>;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a new Redis client with optimized settings
|
* Create a new Redis client with optimized settings
|
||||||
*/
|
*/
|
||||||
|
|
@ -83,24 +106,19 @@ async function createRedisClient(url: string): Promise<Redis> {
|
||||||
throw new Error("Redis URL not provided");
|
throw new Error("Redis URL not provided");
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`Creating Redis client for ${getRedisName(url)}: ${url.slice(0, 20)}...`);
|
|
||||||
const redisName = getRedisName(url);
|
const redisName = getRedisName(url);
|
||||||
|
|
||||||
const client = new Redis(url, {
|
const client = new Redis(url, {
|
||||||
retryStrategy: (times) => {
|
retryStrategy: (times) => {
|
||||||
if (times >= MAX_RETRIES) return null;
|
if (times >= MAX_RETRIES) return null;
|
||||||
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 1000); // Cap at 1s
|
return Math.min(BASE_DELAY_MS * Math.pow(1.5, times), 1000);
|
||||||
console.log(
|
|
||||||
`${redisName} retry ${times}/${MAX_RETRIES}, delay ${delay}ms`
|
|
||||||
);
|
|
||||||
return delay;
|
|
||||||
},
|
},
|
||||||
maxRetriesPerRequest: null,
|
maxRetriesPerRequest: 2,
|
||||||
enableOfflineQueue: true,
|
enableOfflineQueue: true,
|
||||||
enableReadyCheck: true,
|
enableReadyCheck: true,
|
||||||
connectTimeout: IS_VERCEL ? 5000 : 10000, // 5s for Vercel, 10s local
|
connectTimeout: 3000,
|
||||||
keepAlive: IS_VERCEL ? 2000 : 10000, // 2s ping for Vercel
|
keepAlive: 5000,
|
||||||
commandTimeout: 5000, // Align with Vercel's timeout
|
commandTimeout: 3000,
|
||||||
reconnectOnError: (err) =>
|
reconnectOnError: (err) =>
|
||||||
["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) =>
|
["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) =>
|
||||||
err.message.includes(e)
|
err.message.includes(e)
|
||||||
|
|
@ -455,15 +473,37 @@ export async function storeFeedMapping(
|
||||||
const stringValue = JSON.stringify(mapping);
|
const stringValue = JSON.stringify(mapping);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// First write to primary/active client
|
// Get Redis client
|
||||||
const redis = await getRedisClient(); // This already handles failover
|
const redis = await getRedisClient();
|
||||||
const activeUrl = redis.options.host || '';
|
const activeUrl = redis.options.host || '';
|
||||||
|
|
||||||
await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
|
// Check if compression should be used
|
||||||
console.log(`Successfully stored mapping for feed ${feedId}`);
|
if (shouldCompress(stringValue)) {
|
||||||
|
console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`);
|
||||||
|
const compressed = await gzip(stringValue);
|
||||||
|
const base64Compressed = compressed.toString('base64');
|
||||||
|
|
||||||
// Immediately replicate to other instances
|
// Store with compression flag
|
||||||
await replicateOperation(activeUrl, feedId, stringValue, DEFAULT_CACHE_TTL);
|
await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
|
||||||
|
console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`);
|
||||||
|
|
||||||
|
// Replicate to other instances
|
||||||
|
const replicas = await getAvailableReplicas(activeUrl);
|
||||||
|
for (const replica of replicas) {
|
||||||
|
await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Use standard set operations without compression
|
||||||
|
await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
|
||||||
|
|
||||||
|
// Replicate to other instances
|
||||||
|
const replicas = await getAvailableReplicas(activeUrl);
|
||||||
|
for (const replica of replicas) {
|
||||||
|
await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Successfully stored mapping for feed ${feedId}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error storing feed mapping for ${feedId}:`, error);
|
console.error(`Error storing feed mapping for ${feedId}:`, error);
|
||||||
throw error;
|
throw error;
|
||||||
|
|
@ -477,29 +517,75 @@ export async function getFeedMapping(
|
||||||
feedId: string
|
feedId: string
|
||||||
): Promise<FeedMapping | null> {
|
): Promise<FeedMapping | null> {
|
||||||
try {
|
try {
|
||||||
const redis = await getRedisClient(); // This already handles failover
|
// Try primary first
|
||||||
const mapping = await redis.get(feedId);
|
const redis = await getRedisClient();
|
||||||
|
|
||||||
if (!mapping) return null;
|
// Check for compressed version first
|
||||||
|
const compressedKey = `${feedId}:compressed`;
|
||||||
// Reset expiration on access
|
const compressedData = await redis.get(compressedKey);
|
||||||
const activeUrl = redis.options.host || '';
|
|
||||||
await redis.expire(feedId, DEFAULT_CACHE_TTL);
|
|
||||||
|
|
||||||
// Also reset TTL on other replicas
|
|
||||||
replicateOperation(activeUrl, feedId, mapping, DEFAULT_CACHE_TTL)
|
|
||||||
.catch(err => console.error(`Failed to reset TTL on replicas for ${feedId}:`, err));
|
|
||||||
|
|
||||||
|
if (compressedData) {
|
||||||
|
console.log(`Found compressed data for ${feedId}`);
|
||||||
try {
|
try {
|
||||||
const data = JSON.parse(mapping);
|
// Decompress and parse
|
||||||
|
const buffer = Buffer.from(compressedData, 'base64');
|
||||||
|
const decompressed = await gunzip(buffer);
|
||||||
|
const data = JSON.parse(decompressed.toString());
|
||||||
|
|
||||||
|
// Reset expiration
|
||||||
|
await redis.expire(compressedKey, DEFAULT_CACHE_TTL);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
slugs: data.slugs,
|
slugs: data.slugs,
|
||||||
lang: data.lang
|
lang: data.lang
|
||||||
};
|
};
|
||||||
} catch (e) {
|
} catch (error) {
|
||||||
console.error(`Error parsing feed mapping for ${feedId}:`, e);
|
console.error(`Error decompressing data for ${feedId}:`, error);
|
||||||
return null;
|
// Fall through to try uncompressed version
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no compressed version, try regular key
|
||||||
|
let jsonString = await redis.get(feedId);
|
||||||
|
|
||||||
|
// If not found in primary but we have replicas, check them
|
||||||
|
if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) {
|
||||||
|
console.log(`Key ${feedId} not found in primary, checking replicas...`);
|
||||||
|
|
||||||
|
// Try each replica
|
||||||
|
for (const [url, replicaClient] of replicaClients.entries()) {
|
||||||
|
if (replicaClient.status === "ready") {
|
||||||
|
try {
|
||||||
|
jsonString = await replicaClient.get(feedId);
|
||||||
|
if (jsonString) {
|
||||||
|
console.log(`Found key ${feedId} in replica ${getRedisName(url)}`);
|
||||||
|
|
||||||
|
// Sync this key back to primary for future requests
|
||||||
|
if (primaryClient && primaryClient.status === "ready") {
|
||||||
|
primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL)
|
||||||
|
.catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err));
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!jsonString) return null;
|
||||||
|
|
||||||
|
// Reset expiration on access
|
||||||
|
await redis.expire(feedId, DEFAULT_CACHE_TTL);
|
||||||
|
|
||||||
|
// Parse the result
|
||||||
|
const data = JSON.parse(jsonString as string);
|
||||||
|
return {
|
||||||
|
slugs: data.slugs,
|
||||||
|
lang: data.lang
|
||||||
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error retrieving feed mapping for ${feedId}:`, error);
|
console.error(`Error retrieving feed mapping for ${feedId}:`, error);
|
||||||
return null;
|
return null;
|
||||||
|
|
@ -571,6 +657,161 @@ export async function warmChapterCache(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add a new function for handling large feed mappings with compression
|
||||||
|
export async function storeLargeFeedMapping(
|
||||||
|
feedId: string,
|
||||||
|
slugs: string[],
|
||||||
|
lang: string
|
||||||
|
): Promise<void> {
|
||||||
|
const mapping = {
|
||||||
|
slugs,
|
||||||
|
lang,
|
||||||
|
created_at: new Date().toISOString()
|
||||||
|
};
|
||||||
|
|
||||||
|
const jsonString = JSON.stringify(mapping);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const redis = await getRedisClient();
|
||||||
|
const activeUrl = redis.options.host || '';
|
||||||
|
|
||||||
|
// Use compression for large mappings
|
||||||
|
if (shouldCompress(jsonString)) {
|
||||||
|
const compressed = await gzip(jsonString);
|
||||||
|
const base64Compressed = compressed.toString('base64');
|
||||||
|
|
||||||
|
// Store with compression flag
|
||||||
|
await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
|
||||||
|
|
||||||
|
// Replicate to other instances
|
||||||
|
await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL);
|
||||||
|
|
||||||
|
console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`);
|
||||||
|
} else {
|
||||||
|
// Use normal storage for small mappings
|
||||||
|
await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL);
|
||||||
|
await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL);
|
||||||
|
console.log(`Successfully stored mapping for feed ${feedId}`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error storing feed mapping for ${feedId}:`, error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch get multiple feed mappings at once
|
||||||
|
export async function getBatchFeedMappings(
|
||||||
|
feedIds: string[]
|
||||||
|
): Promise<Record<string, FeedMapping | null>> {
|
||||||
|
if (!feedIds.length) return {};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const redis = await getRedisClient();
|
||||||
|
const pipeline = redis.pipeline();
|
||||||
|
|
||||||
|
// Check regular keys first
|
||||||
|
feedIds.forEach(id => {
|
||||||
|
pipeline.get(id);
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = await pipeline.exec();
|
||||||
|
if (!results) return {};
|
||||||
|
|
||||||
|
// Process results
|
||||||
|
const mappings: Record<string, FeedMapping | null> = {};
|
||||||
|
const ttlUpdates = redis.pipeline();
|
||||||
|
const compressedChecks = redis.pipeline();
|
||||||
|
const compressedIds: string[] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < feedIds.length; i++) {
|
||||||
|
const id = feedIds[i];
|
||||||
|
const result = results[i];
|
||||||
|
|
||||||
|
if (result[0]) {
|
||||||
|
mappings[id] = null;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result[1]) {
|
||||||
|
// Process regular mapping
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(result[1] as string);
|
||||||
|
mappings[id] = {
|
||||||
|
slugs: data.slugs,
|
||||||
|
lang: data.lang
|
||||||
|
};
|
||||||
|
|
||||||
|
// Reset TTL
|
||||||
|
ttlUpdates.expire(id, DEFAULT_CACHE_TTL);
|
||||||
|
} catch (e) {
|
||||||
|
mappings[id] = null;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Check for compressed version
|
||||||
|
compressedChecks.exists(`${id}:compressed`);
|
||||||
|
compressedIds.push(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute TTL updates in background
|
||||||
|
ttlUpdates.exec().catch(err => console.error('TTL update error:', err));
|
||||||
|
|
||||||
|
// Check compressed versions if needed
|
||||||
|
if (compressedIds.length > 0) {
|
||||||
|
const compressedResults = await compressedChecks.exec();
|
||||||
|
|
||||||
|
if (compressedResults) {
|
||||||
|
const compressedFetches = redis.pipeline();
|
||||||
|
const compressedToFetch: string[] = [];
|
||||||
|
|
||||||
|
compressedResults.forEach((result, index) => {
|
||||||
|
if (!result[0] && result[1]) {
|
||||||
|
const id = compressedIds[index];
|
||||||
|
compressedFetches.get(`${id}:compressed`);
|
||||||
|
compressedToFetch.push(id);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (compressedToFetch.length > 0) {
|
||||||
|
const fetchResults = await compressedFetches.exec();
|
||||||
|
|
||||||
|
if (fetchResults) {
|
||||||
|
for (let i = 0; i < compressedToFetch.length; i++) {
|
||||||
|
const id = compressedToFetch[i];
|
||||||
|
const [err, compressedData] = fetchResults[i];
|
||||||
|
|
||||||
|
if (!err && compressedData) {
|
||||||
|
try {
|
||||||
|
const buffer = Buffer.from(compressedData as string, 'base64');
|
||||||
|
const decompressed = await gunzip(buffer);
|
||||||
|
const data = JSON.parse(decompressed.toString());
|
||||||
|
|
||||||
|
mappings[id] = {
|
||||||
|
slugs: data.slugs,
|
||||||
|
lang: data.lang
|
||||||
|
};
|
||||||
|
|
||||||
|
// Reset TTL in background
|
||||||
|
redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {});
|
||||||
|
} catch (e) {
|
||||||
|
mappings[id] = null;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
mappings[id] = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return mappings;
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error retrieving batch feed mappings:`, error);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Initialize connections eagerly to establish connections as soon as possible
|
// Initialize connections eagerly to establish connections as soon as possible
|
||||||
if (!import.meta.env.SSR) {
|
if (!import.meta.env.SSR) {
|
||||||
initializeAllClients()
|
initializeAllClients()
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue