mirror of
https://github.com/ld3z/new-manga.git
synced 2026-11-09 20:53:45 -07:00
Compare commits
4 commits
40221bad5d
...
caf6972cd2
| Author | SHA1 | Date | |
|---|---|---|---|
|
caf6972cd2 | ||
|
|
17b910d5ab | ||
|
|
eaeaeae7d2 | ||
|
|
9677cba078 |
5 changed files with 684 additions and 292 deletions
44
package-lock.json
generated
44
package-lock.json
generated
|
|
@ -17,6 +17,8 @@
|
|||
"astro": "^5.5.4",
|
||||
"feed": "^4.2.2",
|
||||
"ioredis": "^5.6.0",
|
||||
"p-throttle": "^7.0.0",
|
||||
"redis-modules-sdk": "^1.4.0",
|
||||
"tailwindcss": "^3.4.1"
|
||||
}
|
||||
},
|
||||
|
|
@ -4373,6 +4375,17 @@
|
|||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/p-throttle": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-7.0.0.tgz",
|
||||
"integrity": "sha512-aio0v+S0QVkH1O+9x4dHtD4dgCExACcL+3EtNaGqC01GBudS9ijMuUsmN8OVScyV4OOp0jqdLShZFuSlbL/AsA==",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/p-timeout": {
|
||||
"version": "6.1.4",
|
||||
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz",
|
||||
|
|
@ -4737,6 +4750,37 @@
|
|||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/redis-modules-sdk": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-modules-sdk/-/redis-modules-sdk-1.4.0.tgz",
|
||||
"integrity": "sha512-HYCr3ThtyoMrR9MUGmJblPU5aua0G4sanIZHK5wz/QwhVACe3BBgIZpHeymidBaukiuC0uzKla9xJ8exRffjnA==",
|
||||
"dependencies": {
|
||||
"ioredis": "5.2.3"
|
||||
}
|
||||
},
|
||||
"node_modules/redis-modules-sdk/node_modules/ioredis": {
|
||||
"version": "5.2.3",
|
||||
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz",
|
||||
"integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==",
|
||||
"dependencies": {
|
||||
"@ioredis/commands": "^1.1.1",
|
||||
"cluster-key-slot": "^1.1.0",
|
||||
"debug": "^4.3.4",
|
||||
"denque": "^2.0.1",
|
||||
"lodash.defaults": "^4.2.0",
|
||||
"lodash.isarguments": "^3.1.0",
|
||||
"redis-errors": "^1.2.0",
|
||||
"redis-parser": "^3.0.0",
|
||||
"standard-as-callback": "^2.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.22.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/ioredis"
|
||||
}
|
||||
},
|
||||
"node_modules/redis-parser": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@
|
|||
"astro": "^5.5.4",
|
||||
"feed": "^4.2.2",
|
||||
"ioredis": "^5.6.0",
|
||||
"p-throttle": "^7.0.0",
|
||||
"redis-modules-sdk": "^1.4.0",
|
||||
"tailwindcss": "^3.4.1"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,25 +1,17 @@
|
|||
---
|
||||
import { getFeedMapping } from '../lib/db';
|
||||
import { getFeedMapping } from "../lib/db";
|
||||
interface Props {
|
||||
currentSlugs?: string[];
|
||||
}
|
||||
|
||||
const { currentSlugs = [] } = Astro.props;
|
||||
|
||||
import { availableLanguages, languageMap } from '../lib/api';
|
||||
import { availableLanguages, languageMap } from "../lib/api";
|
||||
|
||||
const languages = [
|
||||
{ value: 'en', label: 'English' },
|
||||
{ value: 'ja', label: 'Japanese' },
|
||||
{ value: 'ko', label: 'Korean' },
|
||||
{ value: 'zh', label: 'Chinese' },
|
||||
{ value: 'fr', label: 'French' },
|
||||
{ value: 'es', label: 'Spanish' },
|
||||
{ value: 'it', label: 'Italian' },
|
||||
{ value: 'de', label: 'German' },
|
||||
{ value: 'tr', label: 'Turkish' },
|
||||
{ value: 'pl', label: 'Polish' },
|
||||
];
|
||||
const languages = Object.entries(languageMap).map(([value, label]) => ({
|
||||
value,
|
||||
label,
|
||||
}));
|
||||
---
|
||||
|
||||
<div class="space-y-4 sm:space-y-6">
|
||||
|
|
@ -37,9 +29,11 @@ const languages = [
|
|||
id="lang-select"
|
||||
class="px-3 sm:px-4 py-2 bg-primary rounded-lg border border-hover/30 focus:outline-none focus:border-accent/50 transition-colors text-sm sm:text-base"
|
||||
>
|
||||
{languages.map(lang => (
|
||||
{
|
||||
languages.map((lang) => (
|
||||
<option value={lang.value}>{lang.label}</option>
|
||||
))}
|
||||
))
|
||||
}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
|
|
@ -54,8 +48,16 @@ const languages = [
|
|||
id="import-feed"
|
||||
class="px-4 py-2 bg-secondary hover:bg-hover transition-colors rounded-lg text-sm sm:text-base flex items-center gap-2"
|
||||
>
|
||||
<svg class="w-4 h-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4M7 10l5 5 5-5M12 15V3"/>
|
||||
<svg
|
||||
class="w-4 h-4"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
>
|
||||
<path
|
||||
d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4M7 10l5 5 5-5M12 15V3"
|
||||
></path>
|
||||
</svg>
|
||||
Import Feed
|
||||
</button>
|
||||
|
|
@ -73,27 +75,46 @@ const languages = [
|
|||
class="px-3 py-1.5 text-sm text-red-500 hover:text-red-600 hover:bg-red-500/10 rounded-lg transition-colors flex items-center gap-2"
|
||||
style="display: none;"
|
||||
>
|
||||
<svg class="w-4 h-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"/>
|
||||
<svg
|
||||
class="w-4 h-4"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
>
|
||||
<path
|
||||
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"
|
||||
></path>
|
||||
</svg>
|
||||
Clear All
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div id="slug-list" class="space-y-2">
|
||||
{currentSlugs.map(slug => (
|
||||
<div class="flex items-center justify-between p-3 bg-primary rounded-lg border border-hover/30" data-slug={slug}>
|
||||
{
|
||||
currentSlugs.map((slug) => (
|
||||
<div
|
||||
class="flex items-center justify-between p-3 bg-primary rounded-lg border border-hover/30"
|
||||
data-slug={slug}
|
||||
>
|
||||
<span class="text-primary truncate pr-2">{slug}</span>
|
||||
<button
|
||||
class="text-red-500 hover:text-red-600 transition-colors flex-shrink-0 pl-2"
|
||||
data-remove={slug}
|
||||
>
|
||||
<svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path d="M6 18L18 6M6 6l12 12"></path>
|
||||
<svg
|
||||
class="w-5 h-5"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
>
|
||||
<path d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
))
|
||||
}
|
||||
</div>
|
||||
|
||||
<div class="flex justify-center">
|
||||
|
|
@ -123,20 +144,26 @@ const languages = [
|
|||
</div>
|
||||
|
||||
<script>
|
||||
const searchInput = document.getElementById('search-input') as HTMLInputElement;
|
||||
const searchResults = document.getElementById('search-results');
|
||||
const slugList = document.getElementById('slug-list');
|
||||
const generateRSSLink = document.getElementById('generate-rss');
|
||||
const langSelect = document.getElementById('lang-select') as HTMLSelectElement;
|
||||
const feedIdInput = document.getElementById('feed-id-input') as HTMLInputElement;
|
||||
const importFeedButton = document.getElementById('import-feed');
|
||||
const clearAllButton = document.getElementById('clear-all');
|
||||
const searchInput = document.getElementById(
|
||||
"search-input"
|
||||
) as HTMLInputElement;
|
||||
const searchResults = document.getElementById("search-results");
|
||||
const slugList = document.getElementById("slug-list");
|
||||
const generateRSSLink = document.getElementById("generate-rss");
|
||||
const langSelect = document.getElementById(
|
||||
"lang-select"
|
||||
) as HTMLSelectElement;
|
||||
const feedIdInput = document.getElementById(
|
||||
"feed-id-input"
|
||||
) as HTMLInputElement;
|
||||
const importFeedButton = document.getElementById("import-feed");
|
||||
const clearAllButton = document.getElementById("clear-all");
|
||||
|
||||
// Load saved slugs from localStorage
|
||||
let slugs = new Set(JSON.parse(localStorage.getItem('comic-slugs') || '[]'));
|
||||
let slugs = new Set(JSON.parse(localStorage.getItem("comic-slugs") || "[]"));
|
||||
|
||||
function updateSlugs() {
|
||||
localStorage.setItem('comic-slugs', JSON.stringify([...slugs]));
|
||||
localStorage.setItem("comic-slugs", JSON.stringify([...slugs]));
|
||||
updateRSSLink();
|
||||
renderSlugs();
|
||||
updateClearAllButton();
|
||||
|
|
@ -144,13 +171,17 @@ const languages = [
|
|||
|
||||
function updateClearAllButton() {
|
||||
if (clearAllButton) {
|
||||
clearAllButton.style.display = slugs.size > 0 ? 'flex' : 'none';
|
||||
clearAllButton.style.display = slugs.size > 0 ? "flex" : "none";
|
||||
}
|
||||
}
|
||||
|
||||
// Clear all functionality
|
||||
clearAllButton?.addEventListener('click', () => {
|
||||
if (confirm('Are you sure you want to clear all comics? This cannot be undone.')) {
|
||||
clearAllButton?.addEventListener("click", () => {
|
||||
if (
|
||||
confirm(
|
||||
"Are you sure you want to clear all comics? This cannot be undone."
|
||||
)
|
||||
) {
|
||||
slugs.clear();
|
||||
updateSlugs();
|
||||
}
|
||||
|
|
@ -162,7 +193,9 @@ const languages = [
|
|||
// Show loading state
|
||||
slugList.innerHTML = `
|
||||
<div class="animate-pulse space-y-2">
|
||||
${[...slugs].map(() => `
|
||||
${[...slugs]
|
||||
.map(
|
||||
() => `
|
||||
<div class="flex items-center gap-4 p-3 bg-primary rounded-lg border border-hover/30">
|
||||
<div class="w-12 h-16 bg-hover/30 rounded"></div>
|
||||
<div class="flex-1">
|
||||
|
|
@ -170,7 +203,9 @@ const languages = [
|
|||
<div class="h-4 bg-hover/30 rounded w-1/2"></div>
|
||||
</div>
|
||||
</div>
|
||||
`).join('')}
|
||||
`
|
||||
)
|
||||
.join("")}
|
||||
</div>
|
||||
`;
|
||||
|
||||
|
|
@ -181,62 +216,71 @@ const languages = [
|
|||
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
||||
|
||||
const response = await fetch(`/api/comic/${encodeURIComponent(slug)}`, {
|
||||
signal: controller.signal
|
||||
signal: controller.signal,
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) throw new Error('Comic not found');
|
||||
if (!response.ok) throw new Error("Comic not found");
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
const div = document.createElement('div');
|
||||
div.className = 'flex items-center gap-4 p-4 bg-primary rounded-lg border border-hover/30 hover:border-accent/50 transition-all duration-300 group relative overflow-hidden';
|
||||
const div = document.createElement("div");
|
||||
div.className =
|
||||
"flex items-center gap-4 p-4 bg-primary rounded-lg border border-hover/30 hover:border-accent/50 transition-all duration-300 group relative overflow-hidden";
|
||||
|
||||
// Add a link to the comic
|
||||
const link = document.createElement('a');
|
||||
const link = document.createElement("a");
|
||||
link.href = `https://comick.io/comic/${slug}`;
|
||||
link.target = '_blank';
|
||||
link.rel = 'noopener noreferrer';
|
||||
link.className = 'flex items-center gap-4 flex-1 min-w-0';
|
||||
link.target = "_blank";
|
||||
link.rel = "noopener noreferrer";
|
||||
link.className = "flex items-center gap-4 flex-1 min-w-0";
|
||||
|
||||
// Handle cover image
|
||||
if (data.comic?.md_covers?.[0]?.b2key) {
|
||||
const img = document.createElement('img');
|
||||
const img = document.createElement("img");
|
||||
img.src = `https://meo.comick.pictures/${data.comic.md_covers[0].b2key}`;
|
||||
img.className = 'w-16 h-24 object-cover rounded shadow-md group-hover:scale-105 transition-transform duration-300';
|
||||
img.className =
|
||||
"w-16 h-24 object-cover rounded shadow-md group-hover:scale-105 transition-transform duration-300";
|
||||
img.alt = data.comic.title || slug;
|
||||
img.onerror = () => {
|
||||
img.src = '/placeholder.png'; // Fallback image
|
||||
img.src = "/placeholder.png"; // Fallback image
|
||||
img.onerror = null;
|
||||
};
|
||||
link.appendChild(img);
|
||||
} else {
|
||||
// Placeholder if no image
|
||||
const placeholder = document.createElement('div');
|
||||
placeholder.className = 'w-16 h-24 bg-hover/30 rounded flex items-center justify-center text-primary/30';
|
||||
placeholder.innerHTML = '<svg class="w-8 h-8" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"></path></svg>';
|
||||
const placeholder = document.createElement("div");
|
||||
placeholder.className =
|
||||
"w-16 h-24 bg-hover/30 rounded flex items-center justify-center text-primary/30";
|
||||
placeholder.innerHTML =
|
||||
'<svg class="w-8 h-8" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"></path></svg>';
|
||||
link.appendChild(placeholder);
|
||||
}
|
||||
|
||||
const info = document.createElement('div');
|
||||
info.className = 'flex-1 min-w-0 truncate';
|
||||
const info = document.createElement("div");
|
||||
info.className = "flex-1 min-w-0 truncate";
|
||||
info.innerHTML = `
|
||||
<div class="font-medium text-lg truncate group-hover:text-accent transition-colors">
|
||||
${data.comic?.title || slug}
|
||||
</div>
|
||||
<div class="text-sm text-primary/60 truncate">${slug}</div>
|
||||
${data.comic?.last_chapter ? `
|
||||
${
|
||||
data.comic?.last_chapter
|
||||
? `
|
||||
<div class="text-xs text-accent mt-1">
|
||||
Latest: Chapter ${data.comic.last_chapter}
|
||||
</div>
|
||||
` : ''}
|
||||
`
|
||||
: ""
|
||||
}
|
||||
`;
|
||||
|
||||
link.appendChild(info);
|
||||
div.appendChild(link);
|
||||
|
||||
const removeBtn = document.createElement('button');
|
||||
removeBtn.className = 'text-red-500 hover:text-red-600 transition-colors p-2 hover:bg-red-500/10 rounded-full flex-shrink-0';
|
||||
const removeBtn = document.createElement("button");
|
||||
removeBtn.className =
|
||||
"text-red-500 hover:text-red-600 transition-colors p-2 hover:bg-red-500/10 rounded-full flex-shrink-0";
|
||||
removeBtn.innerHTML = `
|
||||
<svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path d="M6 18L18 6M6 6l12 12"></path>
|
||||
|
|
@ -245,7 +289,7 @@ const languages = [
|
|||
removeBtn.onclick = (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
div.classList.add('scale-95', 'opacity-0');
|
||||
div.classList.add("scale-95", "opacity-0");
|
||||
setTimeout(() => {
|
||||
slugs.delete(slug);
|
||||
updateSlugs();
|
||||
|
|
@ -257,8 +301,9 @@ const languages = [
|
|||
} catch (error) {
|
||||
console.error(`Failed to fetch comic info for ${slug}:`, error);
|
||||
// Improved error state
|
||||
const div = document.createElement('div');
|
||||
div.className = 'flex items-center justify-between p-4 bg-primary rounded-lg border border-red-500/30 group';
|
||||
const div = document.createElement("div");
|
||||
div.className =
|
||||
"flex items-center justify-between p-4 bg-primary rounded-lg border border-red-500/30 group";
|
||||
div.innerHTML = `
|
||||
<div class="flex items-center gap-4 flex-1">
|
||||
<div class="w-16 h-24 bg-red-500/10 rounded flex items-center justify-center text-red-500/50">
|
||||
|
|
@ -280,12 +325,12 @@ const languages = [
|
|||
</button>
|
||||
`;
|
||||
|
||||
const removeButton = div.querySelector('button');
|
||||
const removeButton = div.querySelector("button");
|
||||
if (removeButton) {
|
||||
removeButton.onclick = (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
div.classList.add('scale-95', 'opacity-0');
|
||||
div.classList.add("scale-95", "opacity-0");
|
||||
setTimeout(() => {
|
||||
slugs.delete(slug);
|
||||
updateSlugs();
|
||||
|
|
@ -296,21 +341,23 @@ const languages = [
|
|||
}
|
||||
});
|
||||
|
||||
Promise.all(promises).then(elements => {
|
||||
slugList.innerHTML = '';
|
||||
elements.forEach(element => slugList.appendChild(element));
|
||||
}).catch(error => {
|
||||
console.error('Failed to render slugs:', error);
|
||||
Promise.all(promises)
|
||||
.then((elements) => {
|
||||
slugList.innerHTML = "";
|
||||
elements.forEach((element) => slugList.appendChild(element));
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error("Failed to render slugs:", error);
|
||||
});
|
||||
}
|
||||
|
||||
let searchTimeout: NodeJS.Timeout;
|
||||
searchInput?.addEventListener('input', () => {
|
||||
searchInput?.addEventListener("input", () => {
|
||||
clearTimeout(searchTimeout);
|
||||
searchTimeout = setTimeout(async () => {
|
||||
const query = searchInput.value.trim();
|
||||
if (query.length < 2) {
|
||||
searchResults!.classList.add('hidden');
|
||||
searchResults!.classList.add("hidden");
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -320,14 +367,14 @@ const languages = [
|
|||
Searching...
|
||||
</div>
|
||||
`;
|
||||
searchResults!.classList.remove('hidden');
|
||||
searchResults!.classList.remove("hidden");
|
||||
|
||||
const response = await fetch('/api/search', {
|
||||
method: 'POST',
|
||||
const response = await fetch("/api/search", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ query })
|
||||
body: JSON.stringify({ query }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
|
@ -345,27 +392,28 @@ const languages = [
|
|||
return;
|
||||
}
|
||||
|
||||
searchResults!.innerHTML = '';
|
||||
searchResults!.innerHTML = "";
|
||||
|
||||
data.forEach((comic: any) => {
|
||||
const div = document.createElement('div');
|
||||
div.className = 'flex items-center gap-3 p-3 bg-primary rounded-lg border border-hover/30 cursor-pointer hover:border-accent/50 transition-colors';
|
||||
const div = document.createElement("div");
|
||||
div.className =
|
||||
"flex items-center gap-3 p-3 bg-primary rounded-lg border border-hover/30 cursor-pointer hover:border-accent/50 transition-colors";
|
||||
div.onclick = () => {
|
||||
slugs.add(comic.slug);
|
||||
updateSlugs();
|
||||
searchInput.value = '';
|
||||
searchResults!.classList.add('hidden');
|
||||
searchInput.value = "";
|
||||
searchResults!.classList.add("hidden");
|
||||
};
|
||||
|
||||
if (comic.md_covers && comic.md_covers[0]) {
|
||||
const img = document.createElement('img');
|
||||
const img = document.createElement("img");
|
||||
img.src = `https://meo.comick.pictures/${comic.md_covers[0].b2key}`;
|
||||
img.className = 'w-12 h-16 object-cover rounded';
|
||||
img.className = "w-12 h-16 object-cover rounded";
|
||||
img.alt = comic.title;
|
||||
div.appendChild(img);
|
||||
}
|
||||
|
||||
const info = document.createElement('div');
|
||||
const info = document.createElement("div");
|
||||
info.innerHTML = `
|
||||
<div class="font-medium">${comic.title}</div>
|
||||
<div class="text-sm text-primary/60">${comic.slug}</div>
|
||||
|
|
@ -375,7 +423,7 @@ const languages = [
|
|||
searchResults!.appendChild(div);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to search comics:', error);
|
||||
console.error("Failed to search comics:", error);
|
||||
searchResults!.innerHTML = `
|
||||
<div class="p-3 bg-primary rounded-lg border border-red-500/30 text-red-500">
|
||||
Failed to search comics. Please try again.
|
||||
|
|
@ -386,8 +434,8 @@ const languages = [
|
|||
});
|
||||
|
||||
async function updateRSSLink() {
|
||||
const slugString = [...slugs].join(',');
|
||||
const lang = langSelect?.value || 'en';
|
||||
const slugString = [...slugs].join(",");
|
||||
const lang = langSelect?.value || "en";
|
||||
|
||||
if (slugString) {
|
||||
try {
|
||||
|
|
@ -402,18 +450,18 @@ const languages = [
|
|||
`;
|
||||
generateRSSLink!.onclick = null;
|
||||
|
||||
const response = await fetch('/api/create-feed', {
|
||||
method: 'POST',
|
||||
const response = await fetch("/api/create-feed", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
slugs: [...slugs],
|
||||
lang
|
||||
})
|
||||
lang,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) throw new Error('Failed to create feed');
|
||||
if (!response.ok) throw new Error("Failed to create feed");
|
||||
|
||||
const { feedId } = await response.json();
|
||||
generateRSSLink!.href = `/rss/${feedId}.xml`;
|
||||
|
|
@ -431,7 +479,7 @@ const languages = [
|
|||
`;
|
||||
generateRSSLink!.onclick = null; // Remove any existing click handlers
|
||||
} catch (error) {
|
||||
console.error('Failed to generate feed:', error);
|
||||
console.error("Failed to generate feed:", error);
|
||||
generateRSSLink!.innerHTML = `
|
||||
<span class="flex items-center gap-2">
|
||||
<svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
|
|
@ -443,7 +491,7 @@ const languages = [
|
|||
}
|
||||
} else {
|
||||
// Reset to default state when no slugs
|
||||
generateRSSLink!.href = '#';
|
||||
generateRSSLink!.href = "#";
|
||||
generateRSSLink!.innerHTML = `
|
||||
<span class="flex items-center gap-2">
|
||||
<svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
|
|
@ -462,7 +510,7 @@ const languages = [
|
|||
}
|
||||
|
||||
// Import feed functionality
|
||||
importFeedButton?.addEventListener('click', async () => {
|
||||
importFeedButton?.addEventListener("click", async () => {
|
||||
const feedId = feedIdInput.value.trim();
|
||||
if (!feedId) return;
|
||||
|
||||
|
|
@ -478,20 +526,28 @@ const languages = [
|
|||
const response = await fetch(`/api/feed/${feedId}`);
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
console.error(`Feed import failed with status ${response.status}: ${errorText}`);
|
||||
console.error(
|
||||
`Feed import failed with status ${response.status}: ${errorText}`
|
||||
);
|
||||
|
||||
// Display a more user-friendly error based on the status code
|
||||
let userFriendlyError = 'Failed to import feed.';
|
||||
let userFriendlyError = "Failed to import feed.";
|
||||
|
||||
if (response.status === 404) {
|
||||
userFriendlyError = `Feed ID "${feedId}" not found. Please check the ID and try again.`;
|
||||
} else if (response.status === 500) {
|
||||
if (errorText.includes('ETIMEDOUT') || errorText.includes('timeout')) {
|
||||
userFriendlyError = 'Connection to database timed out. This is common on the first request. Please try again in a few seconds.';
|
||||
} else if (errorText.includes('ECONNREFUSED')) {
|
||||
userFriendlyError = 'Database connection refused. Please try again later.';
|
||||
if (
|
||||
errorText.includes("ETIMEDOUT") ||
|
||||
errorText.includes("timeout")
|
||||
) {
|
||||
userFriendlyError =
|
||||
"Connection to database timed out. This is common on the first request. Please try again in a few seconds.";
|
||||
} else if (errorText.includes("ECONNREFUSED")) {
|
||||
userFriendlyError =
|
||||
"Database connection refused. Please try again later.";
|
||||
} else {
|
||||
userFriendlyError = 'Server error while importing feed. Please try again later.';
|
||||
userFriendlyError =
|
||||
"Server error while importing feed. Please try again later.";
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -499,11 +555,11 @@ const languages = [
|
|||
}
|
||||
|
||||
const data = await response.json();
|
||||
console.log('Feed import data:', data); // Diagnostic log
|
||||
console.log("Feed import data:", data); // Diagnostic log
|
||||
|
||||
// Validate response structure
|
||||
if (!data || typeof data !== 'object') {
|
||||
throw new Error('Invalid feed data format: not an object');
|
||||
if (!data || typeof data !== "object") {
|
||||
throw new Error("Invalid feed data format: not an object");
|
||||
}
|
||||
|
||||
const feedSlugs = data.slugs;
|
||||
|
|
@ -511,15 +567,17 @@ const languages = [
|
|||
|
||||
// Validate slugs array
|
||||
if (!feedSlugs || !Array.isArray(feedSlugs)) {
|
||||
throw new Error('Invalid feed data: missing or invalid slugs array');
|
||||
throw new Error("Invalid feed data: missing or invalid slugs array");
|
||||
}
|
||||
|
||||
// Validate language
|
||||
if (!lang || typeof lang !== 'string') {
|
||||
console.warn('Feed has missing or invalid language, using default');
|
||||
if (!lang || typeof lang !== "string") {
|
||||
console.warn("Feed has missing or invalid language, using default");
|
||||
} else if (langSelect) {
|
||||
// Only update language if it's valid
|
||||
const validLanguages = Array.from(langSelect.options).map(opt => opt.value);
|
||||
const validLanguages = Array.from(langSelect.options).map(
|
||||
(opt) => opt.value
|
||||
);
|
||||
if (validLanguages.includes(lang)) {
|
||||
langSelect.value = lang;
|
||||
} else {
|
||||
|
|
@ -530,32 +588,32 @@ const languages = [
|
|||
// Add new slugs to existing collection
|
||||
let importCount = 0;
|
||||
feedSlugs.forEach((slug: string) => {
|
||||
if (typeof slug === 'string' && slug.trim()) {
|
||||
if (typeof slug === "string" && slug.trim()) {
|
||||
slugs.add(slug.trim());
|
||||
importCount++;
|
||||
} else {
|
||||
console.warn('Skipping invalid slug in feed:', slug);
|
||||
console.warn("Skipping invalid slug in feed:", slug);
|
||||
}
|
||||
});
|
||||
|
||||
// Update UI
|
||||
updateSlugs();
|
||||
feedIdInput.value = '';
|
||||
feedIdInput.value = "";
|
||||
|
||||
// Show success message
|
||||
const successDiv = document.createElement('div');
|
||||
successDiv.className = 'text-sm text-green-500 mt-1';
|
||||
const successDiv = document.createElement("div");
|
||||
successDiv.className = "text-sm text-green-500 mt-1";
|
||||
successDiv.textContent = `Feed imported successfully! Added ${importCount} comics.`;
|
||||
feedIdInput.parentNode?.appendChild(successDiv);
|
||||
setTimeout(() => successDiv.remove(), 3000);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to import feed:', error);
|
||||
const errorDiv = document.createElement('div');
|
||||
errorDiv.className = 'text-sm text-red-500 mt-1';
|
||||
errorDiv.textContent = error instanceof Error
|
||||
console.error("Failed to import feed:", error);
|
||||
const errorDiv = document.createElement("div");
|
||||
errorDiv.className = "text-sm text-red-500 mt-1";
|
||||
errorDiv.textContent =
|
||||
error instanceof Error
|
||||
? `Import failed: ${error.message}`
|
||||
: 'Failed to import feed. Please check the feed ID and try again.';
|
||||
: "Failed to import feed. Please check the feed ID and try again.";
|
||||
feedIdInput.parentNode?.appendChild(errorDiv);
|
||||
setTimeout(() => errorDiv.remove(), 5000);
|
||||
} finally {
|
||||
|
|
@ -575,12 +633,15 @@ const languages = [
|
|||
updateClearAllButton();
|
||||
|
||||
// Update RSS link when language changes
|
||||
langSelect?.addEventListener('change', () => updateRSSLink());
|
||||
langSelect?.addEventListener("change", () => updateRSSLink());
|
||||
|
||||
// Close search results when clicking outside
|
||||
document.addEventListener('click', (e) => {
|
||||
if (!searchResults?.contains(e.target as Node) && !searchInput?.contains(e.target as Node)) {
|
||||
searchResults?.classList.add('hidden');
|
||||
document.addEventListener("click", (e) => {
|
||||
if (
|
||||
!searchResults?.contains(e.target as Node) &&
|
||||
!searchInput?.contains(e.target as Node)
|
||||
) {
|
||||
searchResults?.classList.add("hidden");
|
||||
}
|
||||
});
|
||||
</script>
|
||||
130
src/lib/api.ts
130
src/lib/api.ts
|
|
@ -1,10 +1,12 @@
|
|||
import type { Comic, Genre, ComicType, ChapterDetail } from './types';
|
||||
import { getRedisClient, CHAPTER_CACHE_TTL } from './db';
|
||||
import pThrottle from 'p-throttle';
|
||||
|
||||
const languageMap = {
|
||||
"en": "English",
|
||||
"fr": "French",
|
||||
"es": "Spanish",
|
||||
"es-419": "Spanish (Latin America)",
|
||||
"it": "Italian",
|
||||
"pl": "Polish",
|
||||
"tr": "Turkish",
|
||||
|
|
@ -13,7 +15,10 @@ const languageMap = {
|
|||
"sv": "Swedish",
|
||||
"ar": "Arabic",
|
||||
"de": "German",
|
||||
"ko": "Korean"
|
||||
"ko": "Korean",
|
||||
"pt": "Portuguese",
|
||||
"pt-br": "Portuguese (Brazil)",
|
||||
"pt-pt": "Portuguese (Portugal)"
|
||||
} as const;
|
||||
|
||||
const languages = new Set(Object.keys(languageMap));
|
||||
|
|
@ -26,6 +31,20 @@ let genreCache: Genre[] = [];
|
|||
const RETRY_COUNT = 3;
|
||||
const RETRY_DELAY = 1000;
|
||||
|
||||
// Rate limiting configuration - adjust these values if needed
|
||||
const API_REQUESTS_PER_SECOND = 10; // Maximum requests per second
|
||||
const API_CONCURRENT_REQUESTS = 6; // Maximum concurrent requests
|
||||
const DELAY_AFTER_429 = 1000; // Wait 1 second after a rate limit hit
|
||||
|
||||
// Create throttled fetch function to avoid rate limits
|
||||
const throttledFetch = pThrottle({
|
||||
limit: API_REQUESTS_PER_SECOND,
|
||||
interval: 1000,
|
||||
concurrency: API_CONCURRENT_REQUESTS
|
||||
} as any)((url: string, options: RequestInit = {}, retries = RETRY_COUNT) => {
|
||||
return fetchWithRetry(url, options, retries);
|
||||
});
|
||||
|
||||
async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise<Response> {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
|
|
@ -37,6 +56,19 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries =
|
|||
},
|
||||
});
|
||||
|
||||
// Special handling for rate limit errors
|
||||
if (response.status === 429) {
|
||||
console.warn(`Rate limited on ${url}. Waiting ${DELAY_AFTER_429}ms before retry.`);
|
||||
|
||||
// Wait for the specified delay
|
||||
await new Promise(resolve => setTimeout(resolve, DELAY_AFTER_429));
|
||||
|
||||
// Retry with one less retry count but only if we have retries left
|
||||
if (retries > 0) {
|
||||
return fetchWithRetry(url, options, retries - 1);
|
||||
}
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
|
@ -44,7 +76,10 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries =
|
|||
return response;
|
||||
} catch (error) {
|
||||
if (retries > 0) {
|
||||
await new Promise(resolve => setTimeout(resolve, RETRY_DELAY));
|
||||
// For non-429 errors, use exponential backoff
|
||||
const delay = Math.min(RETRY_DELAY * Math.pow(2, RETRY_COUNT - retries), 10000);
|
||||
console.log(`Retrying after ${delay}ms. Retries left: ${retries-1}`);
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
return fetchWithRetry(url, options, retries - 1);
|
||||
}
|
||||
throw error;
|
||||
|
|
@ -55,7 +90,8 @@ export async function fetchGenres(): Promise<Genre[]> {
|
|||
if (genreCache.length > 0) return genreCache;
|
||||
|
||||
try {
|
||||
const response = await fetchWithRetry('https://api.comick.fun/genre');
|
||||
// No tachiyomi param for genre endpoint
|
||||
const response = await throttledFetch('https://api.comick.fun/genre');
|
||||
genreCache = await response.json();
|
||||
return genreCache;
|
||||
} catch (error) {
|
||||
|
|
@ -92,7 +128,7 @@ export async function fetchComics(
|
|||
try {
|
||||
console.log('Fetching comics from:', apiUrl);
|
||||
|
||||
const response = await fetchWithRetry(apiUrl);
|
||||
const response = await throttledFetch(apiUrl);
|
||||
const data = await response.json();
|
||||
|
||||
if (!Array.isArray(data)) {
|
||||
|
|
@ -146,7 +182,7 @@ interface ChapterParams {
|
|||
|
||||
export async function getComicBySlug(slug: string): Promise<ComicDetail | null> {
|
||||
try {
|
||||
const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`);
|
||||
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`);
|
||||
const data = await response.json();
|
||||
|
||||
if (!data?.comic?.hid) {
|
||||
|
|
@ -174,14 +210,14 @@ export async function getChaptersByHid(
|
|||
});
|
||||
|
||||
// Fetch comic details including cover
|
||||
const comicResponse = await fetchWithRetry(
|
||||
const comicResponse = await throttledFetch(
|
||||
`https://api.comick.fun/comic/${hid}`
|
||||
);
|
||||
const comicData = await comicResponse.json();
|
||||
const comicTitle = comicData?.comic?.title || 'Unknown Comic';
|
||||
const covers = comicData?.comic?.md_covers || [];
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
const response = await throttledFetch(
|
||||
`https://api.comick.fun/comic/${hid}/chapters?${queryParams}`
|
||||
);
|
||||
const data = await response.json();
|
||||
|
|
@ -260,22 +296,25 @@ export async function getChaptersForSlugs(
|
|||
const uniqueSlugs = [...new Set(slugs)];
|
||||
const pipeline = redis.pipeline();
|
||||
|
||||
// Add cache key validation
|
||||
// Check cache for all slugs
|
||||
uniqueSlugs.forEach(slug => {
|
||||
if (!slug.match(/^[a-z0-9-]+$/)) {
|
||||
console.error(`Invalid slug format: ${slug}`);
|
||||
return; // Skip invalid slugs immediately
|
||||
}
|
||||
const cacheKey = `chapters:${slug}:${lang}`;
|
||||
pipeline.get(cacheKey);
|
||||
});
|
||||
|
||||
const [cacheResults, slugsToFetch] = await Promise.all([
|
||||
pipeline.exec(),
|
||||
validateSlugs(uniqueSlugs) // Add this new function
|
||||
]);
|
||||
const cacheResults = await pipeline.exec();
|
||||
|
||||
// Process cache results
|
||||
const cachedChapters: ChapterDetail[] = [];
|
||||
const slugsToFetch: string[] = [];
|
||||
|
||||
// Separate cached and uncached slugs in one pass
|
||||
cacheResults.forEach(([err, result], index) => {
|
||||
const slug = uniqueSlugs[index];
|
||||
if (!err && result) {
|
||||
try {
|
||||
const parsed = JSON.parse(result as string);
|
||||
|
|
@ -283,42 +322,46 @@ export async function getChaptersForSlugs(
|
|||
cachedChapters.push(...parsed);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e);
|
||||
console.error(`Cache parse error for ${slug}:`, e);
|
||||
slugsToFetch.push(slug);
|
||||
}
|
||||
} else {
|
||||
slugsToFetch.push(slug);
|
||||
}
|
||||
});
|
||||
|
||||
// Add progress tracking
|
||||
// Skip validation if we have recent cache data
|
||||
const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug));
|
||||
console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`);
|
||||
|
||||
// Only fetch if we have slugs that need fetching
|
||||
let liveChapters: ChapterDetail[] = [];
|
||||
if (slugsToFetch.length > 0) {
|
||||
console.log(`Fetching ${slugsToFetch.length} comics from API`);
|
||||
|
||||
// Use Promise.all for concurrent fetching with higher concurrency
|
||||
const fetchPromises = slugsToFetch.map(async (slug) => {
|
||||
const timer = Date.now();
|
||||
try {
|
||||
const comicDetail = await getComicBySlug(slug);
|
||||
|
||||
if (!comicDetail?.comic?.hid) {
|
||||
console.error(`❌ No HID found for ${slug}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, {
|
||||
return getChaptersByHid(comicDetail.comic.hid, slug, {
|
||||
limit: 5,
|
||||
lang: lang
|
||||
});
|
||||
|
||||
console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`);
|
||||
return chapters;
|
||||
} catch (error) {
|
||||
console.error(`🚨 Error processing ${slug}:`, error);
|
||||
console.error(`Error processing ${slug}:`, error);
|
||||
return [];
|
||||
}
|
||||
});
|
||||
|
||||
const liveChapters = (await Promise.all(fetchPromises)).flat();
|
||||
liveChapters = (await Promise.all(fetchPromises)).flat();
|
||||
|
||||
// Cache new results in batch
|
||||
// Cache new results in batch if we have any
|
||||
if (liveChapters.length > 0) {
|
||||
const cachePipeline = redis.pipeline();
|
||||
const chaptersBySlug: Record<string, ChapterDetail[]> = {};
|
||||
|
||||
|
|
@ -341,32 +384,32 @@ export async function getChaptersForSlugs(
|
|||
});
|
||||
|
||||
await cachePipeline.exec();
|
||||
|
||||
// Combine and sort all chapters
|
||||
const allChapters = [...cachedChapters, ...liveChapters];
|
||||
const deduplicatedChapters = deduplicateChapters(allChapters);
|
||||
return deduplicatedChapters.sort((a, b) => {
|
||||
// Ensure the created_at fields are valid dates
|
||||
const dateA = new Date(a.created_at).getTime();
|
||||
const dateB = new Date(b.created_at).getTime();
|
||||
|
||||
// First try to sort by chapter number
|
||||
const chapDiff = parseFloat(b.chap) - parseFloat(a.chap);
|
||||
if (!isNaN(chapDiff)) return chapDiff;
|
||||
|
||||
// Fall back to date if chapter numbers are equal or invalid
|
||||
return dateB - dateA;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Add export to the validation function
|
||||
// Combine and deduplicate
|
||||
return deduplicateChapters([...cachedChapters, ...liveChapters]);
|
||||
}
|
||||
|
||||
// Replace validateSlugs with a more efficient implementation
|
||||
export async function validateSlugs(slugs: string[]): Promise<string[]> {
|
||||
// Skip validation in production for cached items
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
return slugs;
|
||||
}
|
||||
|
||||
const validSlugs: string[] = [];
|
||||
const invalidSlugs: string[] = [];
|
||||
|
||||
await Promise.all(slugs.map(async (slug) => {
|
||||
// Increase batch size for faster processing
|
||||
const batchSize = 5;
|
||||
for (let i = 0; i < slugs.length; i += batchSize) {
|
||||
const batch = slugs.slice(i, i + batchSize);
|
||||
|
||||
await Promise.all(batch.map(async (slug) => {
|
||||
try {
|
||||
const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1);
|
||||
// Reduced retries (1 instead of 3) and removed tachiyomi parameter
|
||||
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`, {}, 1);
|
||||
if (response.ok) {
|
||||
validSlugs.push(slug);
|
||||
} else {
|
||||
|
|
@ -376,9 +419,10 @@ export async function validateSlugs(slugs: string[]): Promise<string[]> {
|
|||
invalidSlugs.push(slug);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
if (invalidSlugs.length > 0) {
|
||||
console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`);
|
||||
console.warn(`Skipping invalid slugs: ${invalidSlugs.length} items`);
|
||||
}
|
||||
|
||||
return validSlugs;
|
||||
|
|
|
|||
303
src/lib/db.ts
303
src/lib/db.ts
|
|
@ -1,5 +1,8 @@
|
|||
import Redis from "ioredis";
|
||||
import type { FeedMapping } from "./types";
|
||||
import { ReJSON } from 'redis-modules-sdk';
|
||||
import * as zlib from 'zlib';
|
||||
import { promisify } from 'util';
|
||||
|
||||
// Configuration constants
|
||||
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
|
||||
|
|
@ -55,6 +58,18 @@ const operationBuffer: RedisOperation[] = [];
|
|||
// Redis instance naming for logging
|
||||
const redisInstanceNames = new Map<string, string>();
|
||||
|
||||
// Add zlib for compression
|
||||
const gzip = promisify(zlib.gzip);
|
||||
const gunzip = promisify(zlib.gunzip);
|
||||
|
||||
// Compression threshold (in bytes)
|
||||
const COMPRESSION_THRESHOLD = 1024; // 1KB
|
||||
|
||||
// Helper function to determine if compression should be used
|
||||
function shouldCompress(data: string): boolean {
|
||||
return Buffer.byteLength(data, 'utf8') > COMPRESSION_THRESHOLD;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize Redis instance names for better logging
|
||||
*/
|
||||
|
|
@ -75,6 +90,14 @@ initializeRedisNames();
|
|||
*/
|
||||
const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown";
|
||||
|
||||
// Add at the top of your file, after the imports
|
||||
declare module "ioredis" {
|
||||
interface Redis {
|
||||
json_set(key: string, path: string, json: any): Promise<string>;
|
||||
json_get(key: string, ...paths: string[]): Promise<string>;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new Redis client with optimized settings
|
||||
*/
|
||||
|
|
@ -83,24 +106,19 @@ async function createRedisClient(url: string): Promise<Redis> {
|
|||
throw new Error("Redis URL not provided");
|
||||
}
|
||||
|
||||
console.log(`Creating Redis client for ${getRedisName(url)}: ${url.slice(0, 20)}...`);
|
||||
const redisName = getRedisName(url);
|
||||
|
||||
const client = new Redis(url, {
|
||||
retryStrategy: (times) => {
|
||||
if (times >= MAX_RETRIES) return null;
|
||||
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 1000); // Cap at 1s
|
||||
console.log(
|
||||
`${redisName} retry ${times}/${MAX_RETRIES}, delay ${delay}ms`
|
||||
);
|
||||
return delay;
|
||||
return Math.min(BASE_DELAY_MS * Math.pow(1.5, times), 1000);
|
||||
},
|
||||
maxRetriesPerRequest: null,
|
||||
maxRetriesPerRequest: 2,
|
||||
enableOfflineQueue: true,
|
||||
enableReadyCheck: true,
|
||||
connectTimeout: IS_VERCEL ? 5000 : 10000, // 5s for Vercel, 10s local
|
||||
keepAlive: IS_VERCEL ? 2000 : 10000, // 2s ping for Vercel
|
||||
commandTimeout: 5000, // Align with Vercel's timeout
|
||||
connectTimeout: 3000,
|
||||
keepAlive: 5000,
|
||||
commandTimeout: 3000,
|
||||
reconnectOnError: (err) =>
|
||||
["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) =>
|
||||
err.message.includes(e)
|
||||
|
|
@ -455,15 +473,37 @@ export async function storeFeedMapping(
|
|||
const stringValue = JSON.stringify(mapping);
|
||||
|
||||
try {
|
||||
// First write to primary/active client
|
||||
const redis = await getRedisClient(); // This already handles failover
|
||||
// Get Redis client
|
||||
const redis = await getRedisClient();
|
||||
const activeUrl = redis.options.host || '';
|
||||
|
||||
await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
|
||||
console.log(`Successfully stored mapping for feed ${feedId}`);
|
||||
// Check if compression should be used
|
||||
if (shouldCompress(stringValue)) {
|
||||
console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`);
|
||||
const compressed = await gzip(stringValue);
|
||||
const base64Compressed = compressed.toString('base64');
|
||||
|
||||
// Immediately replicate to other instances
|
||||
await replicateOperation(activeUrl, feedId, stringValue, DEFAULT_CACHE_TTL);
|
||||
// Store with compression flag
|
||||
await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
|
||||
console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`);
|
||||
|
||||
// Replicate to other instances
|
||||
const replicas = await getAvailableReplicas(activeUrl);
|
||||
for (const replica of replicas) {
|
||||
await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
|
||||
}
|
||||
} else {
|
||||
// Use standard set operations without compression
|
||||
await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
|
||||
|
||||
// Replicate to other instances
|
||||
const replicas = await getAvailableReplicas(activeUrl);
|
||||
for (const replica of replicas) {
|
||||
await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Successfully stored mapping for feed ${feedId}`);
|
||||
} catch (error) {
|
||||
console.error(`Error storing feed mapping for ${feedId}:`, error);
|
||||
throw error;
|
||||
|
|
@ -477,29 +517,75 @@ export async function getFeedMapping(
|
|||
feedId: string
|
||||
): Promise<FeedMapping | null> {
|
||||
try {
|
||||
const redis = await getRedisClient(); // This already handles failover
|
||||
const mapping = await redis.get(feedId);
|
||||
// Try primary first
|
||||
const redis = await getRedisClient();
|
||||
|
||||
if (!mapping) return null;
|
||||
|
||||
// Reset expiration on access
|
||||
const activeUrl = redis.options.host || '';
|
||||
await redis.expire(feedId, DEFAULT_CACHE_TTL);
|
||||
|
||||
// Also reset TTL on other replicas
|
||||
replicateOperation(activeUrl, feedId, mapping, DEFAULT_CACHE_TTL)
|
||||
.catch(err => console.error(`Failed to reset TTL on replicas for ${feedId}:`, err));
|
||||
// Check for compressed version first
|
||||
const compressedKey = `${feedId}:compressed`;
|
||||
const compressedData = await redis.get(compressedKey);
|
||||
|
||||
if (compressedData) {
|
||||
console.log(`Found compressed data for ${feedId}`);
|
||||
try {
|
||||
const data = JSON.parse(mapping);
|
||||
// Decompress and parse
|
||||
const buffer = Buffer.from(compressedData, 'base64');
|
||||
const decompressed = await gunzip(buffer);
|
||||
const data = JSON.parse(decompressed.toString());
|
||||
|
||||
// Reset expiration
|
||||
await redis.expire(compressedKey, DEFAULT_CACHE_TTL);
|
||||
|
||||
return {
|
||||
slugs: data.slugs,
|
||||
lang: data.lang
|
||||
};
|
||||
} catch (e) {
|
||||
console.error(`Error parsing feed mapping for ${feedId}:`, e);
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.error(`Error decompressing data for ${feedId}:`, error);
|
||||
// Fall through to try uncompressed version
|
||||
}
|
||||
}
|
||||
|
||||
// If no compressed version, try regular key
|
||||
let jsonString = await redis.get(feedId);
|
||||
|
||||
// If not found in primary but we have replicas, check them
|
||||
if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) {
|
||||
console.log(`Key ${feedId} not found in primary, checking replicas...`);
|
||||
|
||||
// Try each replica
|
||||
for (const [url, replicaClient] of replicaClients.entries()) {
|
||||
if (replicaClient.status === "ready") {
|
||||
try {
|
||||
jsonString = await replicaClient.get(feedId);
|
||||
if (jsonString) {
|
||||
console.log(`Found key ${feedId} in replica ${getRedisName(url)}`);
|
||||
|
||||
// Sync this key back to primary for future requests
|
||||
if (primaryClient && primaryClient.status === "ready") {
|
||||
primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL)
|
||||
.catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err));
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!jsonString) return null;
|
||||
|
||||
// Reset expiration on access
|
||||
await redis.expire(feedId, DEFAULT_CACHE_TTL);
|
||||
|
||||
// Parse the result
|
||||
const data = JSON.parse(jsonString as string);
|
||||
return {
|
||||
slugs: data.slugs,
|
||||
lang: data.lang
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error retrieving feed mapping for ${feedId}:`, error);
|
||||
return null;
|
||||
|
|
@ -571,6 +657,161 @@ export async function warmChapterCache(
|
|||
}
|
||||
}
|
||||
|
||||
// Add a new function for handling large feed mappings with compression
|
||||
export async function storeLargeFeedMapping(
|
||||
feedId: string,
|
||||
slugs: string[],
|
||||
lang: string
|
||||
): Promise<void> {
|
||||
const mapping = {
|
||||
slugs,
|
||||
lang,
|
||||
created_at: new Date().toISOString()
|
||||
};
|
||||
|
||||
const jsonString = JSON.stringify(mapping);
|
||||
|
||||
try {
|
||||
const redis = await getRedisClient();
|
||||
const activeUrl = redis.options.host || '';
|
||||
|
||||
// Use compression for large mappings
|
||||
if (shouldCompress(jsonString)) {
|
||||
const compressed = await gzip(jsonString);
|
||||
const base64Compressed = compressed.toString('base64');
|
||||
|
||||
// Store with compression flag
|
||||
await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
|
||||
|
||||
// Replicate to other instances
|
||||
await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL);
|
||||
|
||||
console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`);
|
||||
} else {
|
||||
// Use normal storage for small mappings
|
||||
await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL);
|
||||
await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL);
|
||||
console.log(`Successfully stored mapping for feed ${feedId}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error storing feed mapping for ${feedId}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Batch get multiple feed mappings at once
|
||||
export async function getBatchFeedMappings(
|
||||
feedIds: string[]
|
||||
): Promise<Record<string, FeedMapping | null>> {
|
||||
if (!feedIds.length) return {};
|
||||
|
||||
try {
|
||||
const redis = await getRedisClient();
|
||||
const pipeline = redis.pipeline();
|
||||
|
||||
// Check regular keys first
|
||||
feedIds.forEach(id => {
|
||||
pipeline.get(id);
|
||||
});
|
||||
|
||||
const results = await pipeline.exec();
|
||||
if (!results) return {};
|
||||
|
||||
// Process results
|
||||
const mappings: Record<string, FeedMapping | null> = {};
|
||||
const ttlUpdates = redis.pipeline();
|
||||
const compressedChecks = redis.pipeline();
|
||||
const compressedIds: string[] = [];
|
||||
|
||||
for (let i = 0; i < feedIds.length; i++) {
|
||||
const id = feedIds[i];
|
||||
const result = results[i];
|
||||
|
||||
if (result[0]) {
|
||||
mappings[id] = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (result[1]) {
|
||||
// Process regular mapping
|
||||
try {
|
||||
const data = JSON.parse(result[1] as string);
|
||||
mappings[id] = {
|
||||
slugs: data.slugs,
|
||||
lang: data.lang
|
||||
};
|
||||
|
||||
// Reset TTL
|
||||
ttlUpdates.expire(id, DEFAULT_CACHE_TTL);
|
||||
} catch (e) {
|
||||
mappings[id] = null;
|
||||
}
|
||||
} else {
|
||||
// Check for compressed version
|
||||
compressedChecks.exists(`${id}:compressed`);
|
||||
compressedIds.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
// Execute TTL updates in background
|
||||
ttlUpdates.exec().catch(err => console.error('TTL update error:', err));
|
||||
|
||||
// Check compressed versions if needed
|
||||
if (compressedIds.length > 0) {
|
||||
const compressedResults = await compressedChecks.exec();
|
||||
|
||||
if (compressedResults) {
|
||||
const compressedFetches = redis.pipeline();
|
||||
const compressedToFetch: string[] = [];
|
||||
|
||||
compressedResults.forEach((result, index) => {
|
||||
if (!result[0] && result[1]) {
|
||||
const id = compressedIds[index];
|
||||
compressedFetches.get(`${id}:compressed`);
|
||||
compressedToFetch.push(id);
|
||||
}
|
||||
});
|
||||
|
||||
if (compressedToFetch.length > 0) {
|
||||
const fetchResults = await compressedFetches.exec();
|
||||
|
||||
if (fetchResults) {
|
||||
for (let i = 0; i < compressedToFetch.length; i++) {
|
||||
const id = compressedToFetch[i];
|
||||
const [err, compressedData] = fetchResults[i];
|
||||
|
||||
if (!err && compressedData) {
|
||||
try {
|
||||
const buffer = Buffer.from(compressedData as string, 'base64');
|
||||
const decompressed = await gunzip(buffer);
|
||||
const data = JSON.parse(decompressed.toString());
|
||||
|
||||
mappings[id] = {
|
||||
slugs: data.slugs,
|
||||
lang: data.lang
|
||||
};
|
||||
|
||||
// Reset TTL in background
|
||||
redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {});
|
||||
} catch (e) {
|
||||
mappings[id] = null;
|
||||
}
|
||||
} else {
|
||||
mappings[id] = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return mappings;
|
||||
} catch (error) {
|
||||
console.error(`Error retrieving batch feed mappings:`, error);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize connections eagerly to establish connections as soon as possible
|
||||
if (!import.meta.env.SSR) {
|
||||
initializeAllClients()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue