ldez/new-manga
1
0
Fork 0
mirror of https://github.com/ld3z/new-manga.git synced 2026-11-09 21:13:46 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Compare commits

base: ldez:caf6972cd215386468deee4244f7ae7ca22ca504
Branches Tags
ldez:main
..
compare: ldez:40221bad5d4680b69503f433a988fff007499318
Branches Tags
ldez:main

No commits in common. "caf6972cd215386468deee4244f7ae7ca22ca504" and "40221bad5d4680b69503f433a988fff007499318" have entirely different histories.
caf6972cd2 ... 40221bad5d

5 changed files with 292 additions and 684 deletions
Download patch file Download diff file Expand all files Collapse all files

44
package-lock.json generated
View file

@ -17,8 +17,6 @@
"astro": "^5.5.4", "astro": "^5.5.4",
"feed": "^4.2.2", "feed": "^4.2.2",
"ioredis": "^5.6.0", "ioredis": "^5.6.0",
"p-throttle": "^7.0.0",
"redis-modules-sdk": "^1.4.0",
"tailwindcss": "^3.4.1" "tailwindcss": "^3.4.1"
} }
}, },
@ -4375,17 +4373,6 @@
"url": "https://github.com/sponsors/sindresorhus" "url": "https://github.com/sponsors/sindresorhus"
} }
}, },
"node_modules/p-throttle": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-7.0.0.tgz",
"integrity": "sha512-aio0v+S0QVkH1O+9x4dHtD4dgCExACcL+3EtNaGqC01GBudS9ijMuUsmN8OVScyV4OOp0jqdLShZFuSlbL/AsA==",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/p-timeout": { "node_modules/p-timeout": {
"version": "6.1.4", "version": "6.1.4",
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.4.tgz",
@ -4750,37 +4737,6 @@
"node": ">=4" "node": ">=4"
} }
}, },
"node_modules/redis-modules-sdk": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/redis-modules-sdk/-/redis-modules-sdk-1.4.0.tgz",
"integrity": "sha512-HYCr3ThtyoMrR9MUGmJblPU5aua0G4sanIZHK5wz/QwhVACe3BBgIZpHeymidBaukiuC0uzKla9xJ8exRffjnA==",
"dependencies": {
"ioredis": "5.2.3"
}
},
"node_modules/redis-modules-sdk/node_modules/ioredis": {
"version": "5.2.3",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz",
"integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==",
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.0.1",
"lodash.defaults": "^4.2.0",
"lodash.isarguments": "^3.1.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=12.22.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/redis-parser": { "node_modules/redis-parser": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",

2
package.json
View file

@ -20,8 +20,6 @@
"astro": "^5.5.4", "astro": "^5.5.4",
"feed": "^4.2.2", "feed": "^4.2.2",
"ioredis": "^5.6.0", "ioredis": "^5.6.0",
"p-throttle": "^7.0.0",
"redis-modules-sdk": "^1.4.0",
"tailwindcss": "^3.4.1" "tailwindcss": "^3.4.1"
} }
} }

313
src/components/SlugInput.astro
View file

@ -1,17 +1,25 @@
--- ---
import { getFeedMapping } from "../lib/db"; import { getFeedMapping } from '../lib/db';
interface Props { interface Props {
currentSlugs?: string[]; currentSlugs?: string[];
} }
const { currentSlugs = [] } = Astro.props; const { currentSlugs = [] } = Astro.props;
import { availableLanguages, languageMap } from "../lib/api"; import { availableLanguages, languageMap } from '../lib/api';
const languages = Object.entries(languageMap).map(([value, label]) => ({ const languages = [
value, { value: 'en', label: 'English' },
label, { value: 'ja', label: 'Japanese' },
})); { value: 'ko', label: 'Korean' },
{ value: 'zh', label: 'Chinese' },
{ value: 'fr', label: 'French' },
{ value: 'es', label: 'Spanish' },
{ value: 'it', label: 'Italian' },
{ value: 'de', label: 'German' },
{ value: 'tr', label: 'Turkish' },
{ value: 'pl', label: 'Polish' },
];
--- ---
<div class="space-y-4 sm:space-y-6"> <div class="space-y-4 sm:space-y-6">
@ -29,11 +37,9 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
id="lang-select" id="lang-select"
class="px-3 sm:px-4 py-2 bg-primary rounded-lg border border-hover/30 focus:outline-none focus:border-accent/50 transition-colors text-sm sm:text-base" class="px-3 sm:px-4 py-2 bg-primary rounded-lg border border-hover/30 focus:outline-none focus:border-accent/50 transition-colors text-sm sm:text-base"
> >
{ {languages.map(lang => (
languages.map((lang) => (
<option value={lang.value}>{lang.label}</option> <option value={lang.value}>{lang.label}</option>
)) ))}
}
</select> </select>
</div> </div>
@ -48,16 +54,8 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
id="import-feed" id="import-feed"
class="px-4 py-2 bg-secondary hover:bg-hover transition-colors rounded-lg text-sm sm:text-base flex items-center gap-2" class="px-4 py-2 bg-secondary hover:bg-hover transition-colors rounded-lg text-sm sm:text-base flex items-center gap-2"
> >
<svg <svg class="w-4 h-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
class="w-4 h-4" <path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4M7 10l5 5 5-5M12 15V3"/>
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
>
<path
d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4M7 10l5 5 5-5M12 15V3"
></path>
</svg> </svg>
Import Feed Import Feed
</button> </button>
@ -75,46 +73,27 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
class="px-3 py-1.5 text-sm text-red-500 hover:text-red-600 hover:bg-red-500/10 rounded-lg transition-colors flex items-center gap-2" class="px-3 py-1.5 text-sm text-red-500 hover:text-red-600 hover:bg-red-500/10 rounded-lg transition-colors flex items-center gap-2"
style="display: none;" style="display: none;"
> >
<svg <svg class="w-4 h-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
class="w-4 h-4" <path d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"/>
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
>
<path
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"
></path>
</svg> </svg>
Clear All Clear All
</button> </button>
</div> </div>
<div id="slug-list" class="space-y-2"> <div id="slug-list" class="space-y-2">
{ {currentSlugs.map(slug => (
currentSlugs.map((slug) => ( <div class="flex items-center justify-between p-3 bg-primary rounded-lg border border-hover/30" data-slug={slug}>
<div
class="flex items-center justify-between p-3 bg-primary rounded-lg border border-hover/30"
data-slug={slug}
>
<span class="text-primary truncate pr-2">{slug}</span> <span class="text-primary truncate pr-2">{slug}</span>
<button <button
class="text-red-500 hover:text-red-600 transition-colors flex-shrink-0 pl-2" class="text-red-500 hover:text-red-600 transition-colors flex-shrink-0 pl-2"
data-remove={slug} data-remove={slug}
> >
<svg <svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
class="w-5 h-5" <path d="M6 18L18 6M6 6l12 12"></path>
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
>
<path d="M6 18L18 6M6 6l12 12" />
</svg> </svg>
</button> </button>
</div> </div>
)) ))}
}
</div> </div>
<div class="flex justify-center"> <div class="flex justify-center">
@ -144,26 +123,20 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
</div> </div>
<script> <script>
const searchInput = document.getElementById( const searchInput = document.getElementById('search-input') as HTMLInputElement;
"search-input" const searchResults = document.getElementById('search-results');
) as HTMLInputElement; const slugList = document.getElementById('slug-list');
const searchResults = document.getElementById("search-results"); const generateRSSLink = document.getElementById('generate-rss');
const slugList = document.getElementById("slug-list"); const langSelect = document.getElementById('lang-select') as HTMLSelectElement;
const generateRSSLink = document.getElementById("generate-rss"); const feedIdInput = document.getElementById('feed-id-input') as HTMLInputElement;
const langSelect = document.getElementById( const importFeedButton = document.getElementById('import-feed');
"lang-select" const clearAllButton = document.getElementById('clear-all');
) as HTMLSelectElement;
const feedIdInput = document.getElementById(
"feed-id-input"
) as HTMLInputElement;
const importFeedButton = document.getElementById("import-feed");
const clearAllButton = document.getElementById("clear-all");
// Load saved slugs from localStorage // Load saved slugs from localStorage
let slugs = new Set(JSON.parse(localStorage.getItem("comic-slugs") || "[]")); let slugs = new Set(JSON.parse(localStorage.getItem('comic-slugs') || '[]'));
function updateSlugs() { function updateSlugs() {
localStorage.setItem("comic-slugs", JSON.stringify([...slugs])); localStorage.setItem('comic-slugs', JSON.stringify([...slugs]));
updateRSSLink(); updateRSSLink();
renderSlugs(); renderSlugs();
updateClearAllButton(); updateClearAllButton();
@ -171,17 +144,13 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
function updateClearAllButton() { function updateClearAllButton() {
if (clearAllButton) { if (clearAllButton) {
clearAllButton.style.display = slugs.size > 0 ? "flex" : "none"; clearAllButton.style.display = slugs.size > 0 ? 'flex' : 'none';
} }
} }
// Clear all functionality // Clear all functionality
clearAllButton?.addEventListener("click", () => { clearAllButton?.addEventListener('click', () => {
if ( if (confirm('Are you sure you want to clear all comics? This cannot be undone.')) {
confirm(
"Are you sure you want to clear all comics? This cannot be undone."
)
) {
slugs.clear(); slugs.clear();
updateSlugs(); updateSlugs();
} }
@ -193,9 +162,7 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
// Show loading state // Show loading state
slugList.innerHTML = ` slugList.innerHTML = `
<div class="animate-pulse space-y-2"> <div class="animate-pulse space-y-2">
${[...slugs] ${[...slugs].map(() => `
.map(
() => `
<div class="flex items-center gap-4 p-3 bg-primary rounded-lg border border-hover/30"> <div class="flex items-center gap-4 p-3 bg-primary rounded-lg border border-hover/30">
<div class="w-12 h-16 bg-hover/30 rounded"></div> <div class="w-12 h-16 bg-hover/30 rounded"></div>
<div class="flex-1"> <div class="flex-1">
@ -203,9 +170,7 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
<div class="h-4 bg-hover/30 rounded w-1/2"></div> <div class="h-4 bg-hover/30 rounded w-1/2"></div>
</div> </div>
</div> </div>
` `).join('')}
)
.join("")}
</div> </div>
`; `;
@ -216,71 +181,62 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
const timeoutId = setTimeout(() => controller.abort(), 5000); const timeoutId = setTimeout(() => controller.abort(), 5000);
const response = await fetch(`/api/comic/${encodeURIComponent(slug)}`, { const response = await fetch(`/api/comic/${encodeURIComponent(slug)}`, {
signal: controller.signal, signal: controller.signal
}); });
clearTimeout(timeoutId); clearTimeout(timeoutId);
if (!response.ok) throw new Error("Comic not found"); if (!response.ok) throw new Error('Comic not found');
const data = await response.json(); const data = await response.json();
const div = document.createElement("div"); const div = document.createElement('div');
div.className = div.className = 'flex items-center gap-4 p-4 bg-primary rounded-lg border border-hover/30 hover:border-accent/50 transition-all duration-300 group relative overflow-hidden';
"flex items-center gap-4 p-4 bg-primary rounded-lg border border-hover/30 hover:border-accent/50 transition-all duration-300 group relative overflow-hidden";
// Add a link to the comic // Add a link to the comic
const link = document.createElement("a"); const link = document.createElement('a');
link.href = `https://comick.io/comic/${slug}`; link.href = `https://comick.io/comic/${slug}`;
link.target = "_blank"; link.target = '_blank';
link.rel = "noopener noreferrer"; link.rel = 'noopener noreferrer';
link.className = "flex items-center gap-4 flex-1 min-w-0"; link.className = 'flex items-center gap-4 flex-1 min-w-0';
// Handle cover image // Handle cover image
if (data.comic?.md_covers?.[0]?.b2key) { if (data.comic?.md_covers?.[0]?.b2key) {
const img = document.createElement("img"); const img = document.createElement('img');
img.src = `https://meo.comick.pictures/${data.comic.md_covers[0].b2key}`; img.src = `https://meo.comick.pictures/${data.comic.md_covers[0].b2key}`;
img.className = img.className = 'w-16 h-24 object-cover rounded shadow-md group-hover:scale-105 transition-transform duration-300';
"w-16 h-24 object-cover rounded shadow-md group-hover:scale-105 transition-transform duration-300";
img.alt = data.comic.title || slug; img.alt = data.comic.title || slug;
img.onerror = () => { img.onerror = () => {
img.src = "/placeholder.png"; // Fallback image img.src = '/placeholder.png'; // Fallback image
img.onerror = null; img.onerror = null;
}; };
link.appendChild(img); link.appendChild(img);
} else { } else {
// Placeholder if no image // Placeholder if no image
const placeholder = document.createElement("div"); const placeholder = document.createElement('div');
placeholder.className = placeholder.className = 'w-16 h-24 bg-hover/30 rounded flex items-center justify-center text-primary/30';
"w-16 h-24 bg-hover/30 rounded flex items-center justify-center text-primary/30"; placeholder.innerHTML = '<svg class="w-8 h-8" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"></path></svg>';
placeholder.innerHTML =
'<svg class="w-8 h-8" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"></path></svg>';
link.appendChild(placeholder); link.appendChild(placeholder);
} }
const info = document.createElement("div"); const info = document.createElement('div');
info.className = "flex-1 min-w-0 truncate"; info.className = 'flex-1 min-w-0 truncate';
info.innerHTML = ` info.innerHTML = `
<div class="font-medium text-lg truncate group-hover:text-accent transition-colors"> <div class="font-medium text-lg truncate group-hover:text-accent transition-colors">
${data.comic?.title || slug} ${data.comic?.title || slug}
</div> </div>
<div class="text-sm text-primary/60 truncate">${slug}</div> <div class="text-sm text-primary/60 truncate">${slug}</div>
${ ${data.comic?.last_chapter ? `
data.comic?.last_chapter
? `
<div class="text-xs text-accent mt-1"> <div class="text-xs text-accent mt-1">
Latest: Chapter ${data.comic.last_chapter} Latest: Chapter ${data.comic.last_chapter}
</div> </div>
` ` : ''}
: ""
}
`; `;
link.appendChild(info); link.appendChild(info);
div.appendChild(link); div.appendChild(link);
const removeBtn = document.createElement("button"); const removeBtn = document.createElement('button');
removeBtn.className = removeBtn.className = 'text-red-500 hover:text-red-600 transition-colors p-2 hover:bg-red-500/10 rounded-full flex-shrink-0';
"text-red-500 hover:text-red-600 transition-colors p-2 hover:bg-red-500/10 rounded-full flex-shrink-0";
removeBtn.innerHTML = ` removeBtn.innerHTML = `
<svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"> <svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M6 18L18 6M6 6l12 12"></path> <path d="M6 18L18 6M6 6l12 12"></path>
@ -289,7 +245,7 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
removeBtn.onclick = (e) => { removeBtn.onclick = (e) => {
e.preventDefault(); e.preventDefault();
e.stopPropagation(); e.stopPropagation();
div.classList.add("scale-95", "opacity-0"); div.classList.add('scale-95', 'opacity-0');
setTimeout(() => { setTimeout(() => {
slugs.delete(slug); slugs.delete(slug);
updateSlugs(); updateSlugs();
@ -301,9 +257,8 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
} catch (error) { } catch (error) {
console.error(`Failed to fetch comic info for ${slug}:`, error); console.error(`Failed to fetch comic info for ${slug}:`, error);
// Improved error state // Improved error state
const div = document.createElement("div"); const div = document.createElement('div');
div.className = div.className = 'flex items-center justify-between p-4 bg-primary rounded-lg border border-red-500/30 group';
"flex items-center justify-between p-4 bg-primary rounded-lg border border-red-500/30 group";
div.innerHTML = ` div.innerHTML = `
<div class="flex items-center gap-4 flex-1"> <div class="flex items-center gap-4 flex-1">
<div class="w-16 h-24 bg-red-500/10 rounded flex items-center justify-center text-red-500/50"> <div class="w-16 h-24 bg-red-500/10 rounded flex items-center justify-center text-red-500/50">
@ -325,12 +280,12 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
</button> </button>
`; `;
const removeButton = div.querySelector("button"); const removeButton = div.querySelector('button');
if (removeButton) { if (removeButton) {
removeButton.onclick = (e) => { removeButton.onclick = (e) => {
e.preventDefault(); e.preventDefault();
e.stopPropagation(); e.stopPropagation();
div.classList.add("scale-95", "opacity-0"); div.classList.add('scale-95', 'opacity-0');
setTimeout(() => { setTimeout(() => {
slugs.delete(slug); slugs.delete(slug);
updateSlugs(); updateSlugs();
@ -341,23 +296,21 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
} }
}); });
Promise.all(promises) Promise.all(promises).then(elements => {
.then((elements) => { slugList.innerHTML = '';
slugList.innerHTML = ""; elements.forEach(element => slugList.appendChild(element));
elements.forEach((element) => slugList.appendChild(element)); }).catch(error => {
}) console.error('Failed to render slugs:', error);
.catch((error) => {
console.error("Failed to render slugs:", error);
}); });
} }
let searchTimeout: NodeJS.Timeout; let searchTimeout: NodeJS.Timeout;
searchInput?.addEventListener("input", () => { searchInput?.addEventListener('input', () => {
clearTimeout(searchTimeout); clearTimeout(searchTimeout);
searchTimeout = setTimeout(async () => { searchTimeout = setTimeout(async () => {
const query = searchInput.value.trim(); const query = searchInput.value.trim();
if (query.length < 2) { if (query.length < 2) {
searchResults!.classList.add("hidden"); searchResults!.classList.add('hidden');
return; return;
} }
@ -367,14 +320,14 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
Searching... Searching...
</div> </div>
`; `;
searchResults!.classList.remove("hidden"); searchResults!.classList.remove('hidden');
const response = await fetch("/api/search", { const response = await fetch('/api/search', {
method: "POST", method: 'POST',
headers: { headers: {
"Content-Type": "application/json", 'Content-Type': 'application/json',
}, },
body: JSON.stringify({ query }), body: JSON.stringify({ query })
}); });
if (!response.ok) { if (!response.ok) {
@ -392,28 +345,27 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
return; return;
} }
searchResults!.innerHTML = ""; searchResults!.innerHTML = '';
data.forEach((comic: any) => { data.forEach((comic: any) => {
const div = document.createElement("div"); const div = document.createElement('div');
div.className = div.className = 'flex items-center gap-3 p-3 bg-primary rounded-lg border border-hover/30 cursor-pointer hover:border-accent/50 transition-colors';
"flex items-center gap-3 p-3 bg-primary rounded-lg border border-hover/30 cursor-pointer hover:border-accent/50 transition-colors";
div.onclick = () => { div.onclick = () => {
slugs.add(comic.slug); slugs.add(comic.slug);
updateSlugs(); updateSlugs();
searchInput.value = ""; searchInput.value = '';
searchResults!.classList.add("hidden"); searchResults!.classList.add('hidden');
}; };
if (comic.md_covers && comic.md_covers[0]) { if (comic.md_covers && comic.md_covers[0]) {
const img = document.createElement("img"); const img = document.createElement('img');
img.src = `https://meo.comick.pictures/${comic.md_covers[0].b2key}`; img.src = `https://meo.comick.pictures/${comic.md_covers[0].b2key}`;
img.className = "w-12 h-16 object-cover rounded"; img.className = 'w-12 h-16 object-cover rounded';
img.alt = comic.title; img.alt = comic.title;
div.appendChild(img); div.appendChild(img);
} }
const info = document.createElement("div"); const info = document.createElement('div');
info.innerHTML = ` info.innerHTML = `
<div class="font-medium">${comic.title}</div> <div class="font-medium">${comic.title}</div>
<div class="text-sm text-primary/60">${comic.slug}</div> <div class="text-sm text-primary/60">${comic.slug}</div>
@ -423,7 +375,7 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
searchResults!.appendChild(div); searchResults!.appendChild(div);
}); });
} catch (error) { } catch (error) {
console.error("Failed to search comics:", error); console.error('Failed to search comics:', error);
searchResults!.innerHTML = ` searchResults!.innerHTML = `
<div class="p-3 bg-primary rounded-lg border border-red-500/30 text-red-500"> <div class="p-3 bg-primary rounded-lg border border-red-500/30 text-red-500">
Failed to search comics. Please try again. Failed to search comics. Please try again.
@ -434,8 +386,8 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
}); });
async function updateRSSLink() { async function updateRSSLink() {
const slugString = [...slugs].join(","); const slugString = [...slugs].join(',');
const lang = langSelect?.value || "en"; const lang = langSelect?.value || 'en';
if (slugString) { if (slugString) {
try { try {
@ -450,18 +402,18 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
`; `;
generateRSSLink!.onclick = null; generateRSSLink!.onclick = null;
const response = await fetch("/api/create-feed", { const response = await fetch('/api/create-feed', {
method: "POST", method: 'POST',
headers: { headers: {
"Content-Type": "application/json", 'Content-Type': 'application/json',
}, },
body: JSON.stringify({ body: JSON.stringify({
slugs: [...slugs], slugs: [...slugs],
lang, lang
}), })
}); });
if (!response.ok) throw new Error("Failed to create feed"); if (!response.ok) throw new Error('Failed to create feed');
const { feedId } = await response.json(); const { feedId } = await response.json();
generateRSSLink!.href = `/rss/${feedId}.xml`; generateRSSLink!.href = `/rss/${feedId}.xml`;
@ -479,7 +431,7 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
`; `;
generateRSSLink!.onclick = null; // Remove any existing click handlers generateRSSLink!.onclick = null; // Remove any existing click handlers
} catch (error) { } catch (error) {
console.error("Failed to generate feed:", error); console.error('Failed to generate feed:', error);
generateRSSLink!.innerHTML = ` generateRSSLink!.innerHTML = `
<span class="flex items-center gap-2"> <span class="flex items-center gap-2">
<svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"> <svg class="w-5 h-5" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
@ -491,7 +443,7 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
} }
} else { } else {
// Reset to default state when no slugs // Reset to default state when no slugs
generateRSSLink!.href = "#"; generateRSSLink!.href = '#';
generateRSSLink!.innerHTML = ` generateRSSLink!.innerHTML = `
<span class="flex items-center gap-2"> <span class="flex items-center gap-2">
<svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"> <svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
@ -510,7 +462,7 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
} }
// Import feed functionality // Import feed functionality
importFeedButton?.addEventListener("click", async () => { importFeedButton?.addEventListener('click', async () => {
const feedId = feedIdInput.value.trim(); const feedId = feedIdInput.value.trim();
if (!feedId) return; if (!feedId) return;
@ -526,28 +478,20 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
const response = await fetch(`/api/feed/${feedId}`); const response = await fetch(`/api/feed/${feedId}`);
if (!response.ok) { if (!response.ok) {
const errorText = await response.text(); const errorText = await response.text();
console.error( console.error(`Feed import failed with status ${response.status}: ${errorText}`);
`Feed import failed with status ${response.status}: ${errorText}`
);
// Display a more user-friendly error based on the status code // Display a more user-friendly error based on the status code
let userFriendlyError = "Failed to import feed."; let userFriendlyError = 'Failed to import feed.';
if (response.status === 404) { if (response.status === 404) {
userFriendlyError = `Feed ID "${feedId}" not found. Please check the ID and try again.`; userFriendlyError = `Feed ID "${feedId}" not found. Please check the ID and try again.`;
} else if (response.status === 500) { } else if (response.status === 500) {
if ( if (errorText.includes('ETIMEDOUT') || errorText.includes('timeout')) {
errorText.includes("ETIMEDOUT") || userFriendlyError = 'Connection to database timed out. This is common on the first request. Please try again in a few seconds.';
errorText.includes("timeout") } else if (errorText.includes('ECONNREFUSED')) {
) { userFriendlyError = 'Database connection refused. Please try again later.';
userFriendlyError =
"Connection to database timed out. This is common on the first request. Please try again in a few seconds.";
} else if (errorText.includes("ECONNREFUSED")) {
userFriendlyError =
"Database connection refused. Please try again later.";
} else { } else {
userFriendlyError = userFriendlyError = 'Server error while importing feed. Please try again later.';
"Server error while importing feed. Please try again later.";
} }
} }
@ -555,11 +499,11 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
} }
const data = await response.json(); const data = await response.json();
console.log("Feed import data:", data); // Diagnostic log console.log('Feed import data:', data); // Diagnostic log
// Validate response structure // Validate response structure
if (!data || typeof data !== "object") { if (!data || typeof data !== 'object') {
throw new Error("Invalid feed data format: not an object"); throw new Error('Invalid feed data format: not an object');
} }
const feedSlugs = data.slugs; const feedSlugs = data.slugs;
@ -567,17 +511,15 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
// Validate slugs array // Validate slugs array
if (!feedSlugs || !Array.isArray(feedSlugs)) { if (!feedSlugs || !Array.isArray(feedSlugs)) {
throw new Error("Invalid feed data: missing or invalid slugs array"); throw new Error('Invalid feed data: missing or invalid slugs array');
} }
// Validate language // Validate language
if (!lang || typeof lang !== "string") { if (!lang || typeof lang !== 'string') {
console.warn("Feed has missing or invalid language, using default"); console.warn('Feed has missing or invalid language, using default');
} else if (langSelect) { } else if (langSelect) {
// Only update language if it's valid // Only update language if it's valid
const validLanguages = Array.from(langSelect.options).map( const validLanguages = Array.from(langSelect.options).map(opt => opt.value);
(opt) => opt.value
);
if (validLanguages.includes(lang)) { if (validLanguages.includes(lang)) {
langSelect.value = lang; langSelect.value = lang;
} else { } else {
@ -588,32 +530,32 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
// Add new slugs to existing collection // Add new slugs to existing collection
let importCount = 0; let importCount = 0;
feedSlugs.forEach((slug: string) => { feedSlugs.forEach((slug: string) => {
if (typeof slug === "string" && slug.trim()) { if (typeof slug === 'string' && slug.trim()) {
slugs.add(slug.trim()); slugs.add(slug.trim());
importCount++; importCount++;
} else { } else {
console.warn("Skipping invalid slug in feed:", slug); console.warn('Skipping invalid slug in feed:', slug);
} }
}); });
// Update UI // Update UI
updateSlugs(); updateSlugs();
feedIdInput.value = ""; feedIdInput.value = '';
// Show success message // Show success message
const successDiv = document.createElement("div"); const successDiv = document.createElement('div');
successDiv.className = "text-sm text-green-500 mt-1"; successDiv.className = 'text-sm text-green-500 mt-1';
successDiv.textContent = `Feed imported successfully! Added ${importCount} comics.`; successDiv.textContent = `Feed imported successfully! Added ${importCount} comics.`;
feedIdInput.parentNode?.appendChild(successDiv); feedIdInput.parentNode?.appendChild(successDiv);
setTimeout(() => successDiv.remove(), 3000); setTimeout(() => successDiv.remove(), 3000);
} catch (error) { } catch (error) {
console.error("Failed to import feed:", error); console.error('Failed to import feed:', error);
const errorDiv = document.createElement("div"); const errorDiv = document.createElement('div');
errorDiv.className = "text-sm text-red-500 mt-1"; errorDiv.className = 'text-sm text-red-500 mt-1';
errorDiv.textContent = errorDiv.textContent = error instanceof Error
error instanceof Error
? `Import failed: ${error.message}` ? `Import failed: ${error.message}`
: "Failed to import feed. Please check the feed ID and try again."; : 'Failed to import feed. Please check the feed ID and try again.';
feedIdInput.parentNode?.appendChild(errorDiv); feedIdInput.parentNode?.appendChild(errorDiv);
setTimeout(() => errorDiv.remove(), 5000); setTimeout(() => errorDiv.remove(), 5000);
} finally { } finally {
@ -633,15 +575,12 @@ const languages = Object.entries(languageMap).map(([value, label]) => ({
updateClearAllButton(); updateClearAllButton();
// Update RSS link when language changes // Update RSS link when language changes
langSelect?.addEventListener("change", () => updateRSSLink()); langSelect?.addEventListener('change', () => updateRSSLink());
// Close search results when clicking outside // Close search results when clicking outside
document.addEventListener("click", (e) => { document.addEventListener('click', (e) => {
if ( if (!searchResults?.contains(e.target as Node) && !searchInput?.contains(e.target as Node)) {
!searchResults?.contains(e.target as Node) && searchResults?.classList.add('hidden');
!searchInput?.contains(e.target as Node)
) {
searchResults?.classList.add("hidden");
} }
}); });
</script> </script>

130
src/lib/api.ts
View file

@ -1,12 +1,10 @@
import type { Comic, Genre, ComicType, ChapterDetail } from './types'; import type { Comic, Genre, ComicType, ChapterDetail } from './types';
import { getRedisClient, CHAPTER_CACHE_TTL } from './db'; import { getRedisClient, CHAPTER_CACHE_TTL } from './db';
import pThrottle from 'p-throttle';
const languageMap = { const languageMap = {
"en": "English", "en": "English",
"fr": "French", "fr": "French",
"es": "Spanish", "es": "Spanish",
"es-419": "Spanish (Latin America)",
"it": "Italian", "it": "Italian",
"pl": "Polish", "pl": "Polish",
"tr": "Turkish", "tr": "Turkish",
@ -15,10 +13,7 @@ const languageMap = {
"sv": "Swedish", "sv": "Swedish",
"ar": "Arabic", "ar": "Arabic",
"de": "German", "de": "German",
"ko": "Korean", "ko": "Korean"
"pt": "Portuguese",
"pt-br": "Portuguese (Brazil)",
"pt-pt": "Portuguese (Portugal)"
} as const; } as const;
const languages = new Set(Object.keys(languageMap)); const languages = new Set(Object.keys(languageMap));
@ -31,20 +26,6 @@ let genreCache: Genre[] = [];
const RETRY_COUNT = 3; const RETRY_COUNT = 3;
const RETRY_DELAY = 1000; const RETRY_DELAY = 1000;
// Rate limiting configuration - adjust these values if needed
const API_REQUESTS_PER_SECOND = 10; // Maximum requests per second
const API_CONCURRENT_REQUESTS = 6; // Maximum concurrent requests
const DELAY_AFTER_429 = 1000; // Wait 1 second after a rate limit hit
// Create throttled fetch function to avoid rate limits
const throttledFetch = pThrottle({
limit: API_REQUESTS_PER_SECOND,
interval: 1000,
concurrency: API_CONCURRENT_REQUESTS
} as any)((url: string, options: RequestInit = {}, retries = RETRY_COUNT) => {
return fetchWithRetry(url, options, retries);
});
async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise<Response> { async function fetchWithRetry(url: string, options: RequestInit = {}, retries = RETRY_COUNT): Promise<Response> {
try { try {
const response = await fetch(url, { const response = await fetch(url, {
@ -56,19 +37,6 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries =
}, },
}); });
// Special handling for rate limit errors
if (response.status === 429) {
console.warn(`Rate limited on ${url}. Waiting ${DELAY_AFTER_429}ms before retry.`);
// Wait for the specified delay
await new Promise(resolve => setTimeout(resolve, DELAY_AFTER_429));
// Retry with one less retry count but only if we have retries left
if (retries > 0) {
return fetchWithRetry(url, options, retries - 1);
}
}
if (!response.ok) { if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`); throw new Error(`HTTP error! status: ${response.status}`);
} }
@ -76,10 +44,7 @@ async function fetchWithRetry(url: string, options: RequestInit = {}, retries =
return response; return response;
} catch (error) { } catch (error) {
if (retries > 0) { if (retries > 0) {
// For non-429 errors, use exponential backoff await new Promise(resolve => setTimeout(resolve, RETRY_DELAY));
const delay = Math.min(RETRY_DELAY * Math.pow(2, RETRY_COUNT - retries), 10000);
console.log(`Retrying after ${delay}ms. Retries left: ${retries-1}`);
await new Promise(resolve => setTimeout(resolve, delay));
return fetchWithRetry(url, options, retries - 1); return fetchWithRetry(url, options, retries - 1);
} }
throw error; throw error;
@ -90,8 +55,7 @@ export async function fetchGenres(): Promise<Genre[]> {
if (genreCache.length > 0) return genreCache; if (genreCache.length > 0) return genreCache;
try { try {
// No tachiyomi param for genre endpoint const response = await fetchWithRetry('https://api.comick.fun/genre');
const response = await throttledFetch('https://api.comick.fun/genre');
genreCache = await response.json(); genreCache = await response.json();
return genreCache; return genreCache;
} catch (error) { } catch (error) {
@ -128,7 +92,7 @@ export async function fetchComics(
try { try {
console.log('Fetching comics from:', apiUrl); console.log('Fetching comics from:', apiUrl);
const response = await throttledFetch(apiUrl); const response = await fetchWithRetry(apiUrl);
const data = await response.json(); const data = await response.json();
if (!Array.isArray(data)) { if (!Array.isArray(data)) {
@ -182,7 +146,7 @@ interface ChapterParams {
export async function getComicBySlug(slug: string): Promise<ComicDetail | null> { export async function getComicBySlug(slug: string): Promise<ComicDetail | null> {
try { try {
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`); const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`);
const data = await response.json(); const data = await response.json();
if (!data?.comic?.hid) { if (!data?.comic?.hid) {
@ -210,14 +174,14 @@ export async function getChaptersByHid(
}); });
// Fetch comic details including cover // Fetch comic details including cover
const comicResponse = await throttledFetch( const comicResponse = await fetchWithRetry(
`https://api.comick.fun/comic/${hid}` `https://api.comick.fun/comic/${hid}`
); );
const comicData = await comicResponse.json(); const comicData = await comicResponse.json();
const comicTitle = comicData?.comic?.title || 'Unknown Comic'; const comicTitle = comicData?.comic?.title || 'Unknown Comic';
const covers = comicData?.comic?.md_covers || []; const covers = comicData?.comic?.md_covers || [];
const response = await throttledFetch( const response = await fetchWithRetry(
`https://api.comick.fun/comic/${hid}/chapters?${queryParams}` `https://api.comick.fun/comic/${hid}/chapters?${queryParams}`
); );
const data = await response.json(); const data = await response.json();
@ -296,25 +260,22 @@ export async function getChaptersForSlugs(
const uniqueSlugs = [...new Set(slugs)]; const uniqueSlugs = [...new Set(slugs)];
const pipeline = redis.pipeline(); const pipeline = redis.pipeline();
// Check cache for all slugs // Add cache key validation
uniqueSlugs.forEach(slug => { uniqueSlugs.forEach(slug => {
if (!slug.match(/^[a-z0-9-]+$/)) { if (!slug.match(/^[a-z0-9-]+$/)) {
console.error(`Invalid slug format: ${slug}`); console.error(`Invalid slug format: ${slug}`);
return; // Skip invalid slugs immediately
} }
const cacheKey = `chapters:${slug}:${lang}`; const cacheKey = `chapters:${slug}:${lang}`;
pipeline.get(cacheKey); pipeline.get(cacheKey);
}); });
const cacheResults = await pipeline.exec(); const [cacheResults, slugsToFetch] = await Promise.all([
pipeline.exec(),
validateSlugs(uniqueSlugs) // Add this new function
]);
// Process cache results
const cachedChapters: ChapterDetail[] = []; const cachedChapters: ChapterDetail[] = [];
const slugsToFetch: string[] = [];
// Separate cached and uncached slugs in one pass
cacheResults.forEach(([err, result], index) => { cacheResults.forEach(([err, result], index) => {
const slug = uniqueSlugs[index];
if (!err && result) { if (!err && result) {
try { try {
const parsed = JSON.parse(result as string); const parsed = JSON.parse(result as string);
@ -322,46 +283,42 @@ export async function getChaptersForSlugs(
cachedChapters.push(...parsed); cachedChapters.push(...parsed);
} }
} catch (e) { } catch (e) {
console.error(`Cache parse error for ${slug}:`, e); console.error(`Cache parse error for ${uniqueSlugs[index]}:`, e);
slugsToFetch.push(slug);
} }
} else {
slugsToFetch.push(slug);
} }
}); });
// Skip validation if we have recent cache data // Add progress tracking
const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug)); const cachedSlugs = new Set(cachedChapters.map(c => c.md_comics.slug));
console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`); console.log(`Cache hits: ${cachedChapters.length} chapters from ${cachedSlugs.size} comics`);
// Only fetch if we have slugs that need fetching
let liveChapters: ChapterDetail[] = [];
if (slugsToFetch.length > 0) {
console.log(`Fetching ${slugsToFetch.length} comics from API`); console.log(`Fetching ${slugsToFetch.length} comics from API`);
// Use Promise.all for concurrent fetching with higher concurrency
const fetchPromises = slugsToFetch.map(async (slug) => { const fetchPromises = slugsToFetch.map(async (slug) => {
const timer = Date.now();
try { try {
const comicDetail = await getComicBySlug(slug); const comicDetail = await getComicBySlug(slug);
if (!comicDetail?.comic?.hid) { if (!comicDetail?.comic?.hid) {
console.error(`❌ No HID found for ${slug}`);
return []; return [];
} }
return getChaptersByHid(comicDetail.comic.hid, slug, { const chapters = await getChaptersByHid(comicDetail.comic.hid, slug, {
limit: 5, limit: 5,
lang: lang lang: lang
}); });
console.log(`✅ ${slug} (${comicDetail.comic.hid}) returned ${chapters.length} chapters in ${Date.now() - timer}ms`);
return chapters;
} catch (error) { } catch (error) {
console.error(`Error processing ${slug}:`, error); console.error(`🚨 Error processing ${slug}:`, error);
return []; return [];
} }
}); });
liveChapters = (await Promise.all(fetchPromises)).flat(); const liveChapters = (await Promise.all(fetchPromises)).flat();
// Cache new results in batch if we have any // Cache new results in batch
if (liveChapters.length > 0) {
const cachePipeline = redis.pipeline(); const cachePipeline = redis.pipeline();
const chaptersBySlug: Record<string, ChapterDetail[]> = {}; const chaptersBySlug: Record<string, ChapterDetail[]> = {};
@ -384,32 +341,32 @@ export async function getChaptersForSlugs(
}); });
await cachePipeline.exec(); await cachePipeline.exec();
}
// Combine and sort all chapters
const allChapters = [...cachedChapters, ...liveChapters];
const deduplicatedChapters = deduplicateChapters(allChapters);
return deduplicatedChapters.sort((a, b) => {
// Ensure the created_at fields are valid dates
const dateA = new Date(a.created_at).getTime();
const dateB = new Date(b.created_at).getTime();
// First try to sort by chapter number
const chapDiff = parseFloat(b.chap) - parseFloat(a.chap);
if (!isNaN(chapDiff)) return chapDiff;
// Fall back to date if chapter numbers are equal or invalid
return dateB - dateA;
});
} }
// Combine and deduplicate // Add export to the validation function
return deduplicateChapters([...cachedChapters, ...liveChapters]);
}
// Replace validateSlugs with a more efficient implementation
export async function validateSlugs(slugs: string[]): Promise<string[]> { export async function validateSlugs(slugs: string[]): Promise<string[]> {
// Skip validation in production for cached items
if (process.env.NODE_ENV === 'production') {
return slugs;
}
const validSlugs: string[] = []; const validSlugs: string[] = [];
const invalidSlugs: string[] = []; const invalidSlugs: string[] = [];
// Increase batch size for faster processing await Promise.all(slugs.map(async (slug) => {
const batchSize = 5;
for (let i = 0; i < slugs.length; i += batchSize) {
const batch = slugs.slice(i, i + batchSize);
await Promise.all(batch.map(async (slug) => {
try { try {
// Reduced retries (1 instead of 3) and removed tachiyomi parameter const response = await fetchWithRetry(`https://api.comick.fun/comic/${slug}`, {}, 1);
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`, {}, 1);
if (response.ok) { if (response.ok) {
validSlugs.push(slug); validSlugs.push(slug);
} else { } else {
@ -419,10 +376,9 @@ export async function validateSlugs(slugs: string[]): Promise<string[]> {
invalidSlugs.push(slug); invalidSlugs.push(slug);
} }
})); }));
}
if (invalidSlugs.length > 0) { if (invalidSlugs.length > 0) {
console.warn(`Skipping invalid slugs: ${invalidSlugs.length} items`); console.error(`Invalid/Unavailable slugs: ${invalidSlugs.join(', ')}`);
} }
return validSlugs; return validSlugs;

299
src/lib/db.ts
View file

@ -1,8 +1,5 @@
import Redis from "ioredis"; import Redis from "ioredis";
import type { FeedMapping } from "./types"; import type { FeedMapping } from "./types";
import { ReJSON } from 'redis-modules-sdk';
import * as zlib from 'zlib';
import { promisify } from 'util';
// Configuration constants // Configuration constants
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
@ -58,18 +55,6 @@ const operationBuffer: RedisOperation[] = [];
// Redis instance naming for logging // Redis instance naming for logging
const redisInstanceNames = new Map<string, string>(); const redisInstanceNames = new Map<string, string>();
// Add zlib for compression
const gzip = promisify(zlib.gzip);
const gunzip = promisify(zlib.gunzip);
// Compression threshold (in bytes)
const COMPRESSION_THRESHOLD = 1024; // 1KB
// Helper function to determine if compression should be used
function shouldCompress(data: string): boolean {
return Buffer.byteLength(data, 'utf8') > COMPRESSION_THRESHOLD;
}
/** /**
* Initialize Redis instance names for better logging * Initialize Redis instance names for better logging
*/ */
@ -90,14 +75,6 @@ initializeRedisNames();
*/ */
const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown"; const getRedisName = (url: string) => redisInstanceNames.get(url) || "Unknown";
// Add at the top of your file, after the imports
declare module "ioredis" {
interface Redis {
json_set(key: string, path: string, json: any): Promise<string>;
json_get(key: string, ...paths: string[]): Promise<string>;
}
}
/** /**
* Create a new Redis client with optimized settings * Create a new Redis client with optimized settings
*/ */
@ -106,19 +83,24 @@ async function createRedisClient(url: string): Promise<Redis> {
throw new Error("Redis URL not provided"); throw new Error("Redis URL not provided");
} }
console.log(`Creating Redis client for ${getRedisName(url)}: ${url.slice(0, 20)}...`);
const redisName = getRedisName(url); const redisName = getRedisName(url);
const client = new Redis(url, { const client = new Redis(url, {
retryStrategy: (times) => { retryStrategy: (times) => {
if (times >= MAX_RETRIES) return null; if (times >= MAX_RETRIES) return null;
return Math.min(BASE_DELAY_MS * Math.pow(1.5, times), 1000); const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 1000); // Cap at 1s
console.log(
`${redisName} retry ${times}/${MAX_RETRIES}, delay ${delay}ms`
);
return delay;
}, },
maxRetriesPerRequest: 2, maxRetriesPerRequest: null,
enableOfflineQueue: true, enableOfflineQueue: true,
enableReadyCheck: true, enableReadyCheck: true,
connectTimeout: 3000, connectTimeout: IS_VERCEL ? 5000 : 10000, // 5s for Vercel, 10s local
keepAlive: 5000, keepAlive: IS_VERCEL ? 2000 : 10000, // 2s ping for Vercel
commandTimeout: 3000, commandTimeout: 5000, // Align with Vercel's timeout
reconnectOnError: (err) => reconnectOnError: (err) =>
["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) => ["READONLY", "ETIMEDOUT", "ECONNRESET"].some((e) =>
err.message.includes(e) err.message.includes(e)
@ -473,37 +455,15 @@ export async function storeFeedMapping(
const stringValue = JSON.stringify(mapping); const stringValue = JSON.stringify(mapping);
try { try {
// Get Redis client // First write to primary/active client
const redis = await getRedisClient(); const redis = await getRedisClient(); // This already handles failover
const activeUrl = redis.options.host || ''; const activeUrl = redis.options.host || '';
// Check if compression should be used
if (shouldCompress(stringValue)) {
console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`);
const compressed = await gzip(stringValue);
const base64Compressed = compressed.toString('base64');
// Store with compression flag
await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`);
// Replicate to other instances
const replicas = await getAvailableReplicas(activeUrl);
for (const replica of replicas) {
await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
}
} else {
// Use standard set operations without compression
await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
// Replicate to other instances
const replicas = await getAvailableReplicas(activeUrl);
for (const replica of replicas) {
await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
}
}
console.log(`Successfully stored mapping for feed ${feedId}`); console.log(`Successfully stored mapping for feed ${feedId}`);
// Immediately replicate to other instances
await replicateOperation(activeUrl, feedId, stringValue, DEFAULT_CACHE_TTL);
} catch (error) { } catch (error) {
console.error(`Error storing feed mapping for ${feedId}:`, error); console.error(`Error storing feed mapping for ${feedId}:`, error);
throw error; throw error;
@ -517,75 +477,29 @@ export async function getFeedMapping(
feedId: string feedId: string
): Promise<FeedMapping | null> { ): Promise<FeedMapping | null> {
try { try {
// Try primary first const redis = await getRedisClient(); // This already handles failover
const redis = await getRedisClient(); const mapping = await redis.get(feedId);
// Check for compressed version first if (!mapping) return null;
const compressedKey = `${feedId}:compressed`;
const compressedData = await redis.get(compressedKey);
if (compressedData) {
console.log(`Found compressed data for ${feedId}`);
try {
// Decompress and parse
const buffer = Buffer.from(compressedData, 'base64');
const decompressed = await gunzip(buffer);
const data = JSON.parse(decompressed.toString());
// Reset expiration
await redis.expire(compressedKey, DEFAULT_CACHE_TTL);
return {
slugs: data.slugs,
lang: data.lang
};
} catch (error) {
console.error(`Error decompressing data for ${feedId}:`, error);
// Fall through to try uncompressed version
}
}
// If no compressed version, try regular key
let jsonString = await redis.get(feedId);
// If not found in primary but we have replicas, check them
if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) {
console.log(`Key ${feedId} not found in primary, checking replicas...`);
// Try each replica
for (const [url, replicaClient] of replicaClients.entries()) {
if (replicaClient.status === "ready") {
try {
jsonString = await replicaClient.get(feedId);
if (jsonString) {
console.log(`Found key ${feedId} in replica ${getRedisName(url)}`);
// Sync this key back to primary for future requests
if (primaryClient && primaryClient.status === "ready") {
primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL)
.catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err));
}
break;
}
} catch (error) {
console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error);
}
}
}
}
if (!jsonString) return null;
// Reset expiration on access // Reset expiration on access
const activeUrl = redis.options.host || '';
await redis.expire(feedId, DEFAULT_CACHE_TTL); await redis.expire(feedId, DEFAULT_CACHE_TTL);
// Parse the result // Also reset TTL on other replicas
const data = JSON.parse(jsonString as string); replicateOperation(activeUrl, feedId, mapping, DEFAULT_CACHE_TTL)
.catch(err => console.error(`Failed to reset TTL on replicas for ${feedId}:`, err));
try {
const data = JSON.parse(mapping);
return { return {
slugs: data.slugs, slugs: data.slugs,
lang: data.lang lang: data.lang
}; };
} catch (e) {
console.error(`Error parsing feed mapping for ${feedId}:`, e);
return null;
}
} catch (error) { } catch (error) {
console.error(`Error retrieving feed mapping for ${feedId}:`, error); console.error(`Error retrieving feed mapping for ${feedId}:`, error);
return null; return null;
@ -657,161 +571,6 @@ export async function warmChapterCache(
} }
} }
// Add a new function for handling large feed mappings with compression
export async function storeLargeFeedMapping(
feedId: string,
slugs: string[],
lang: string
): Promise<void> {
const mapping = {
slugs,
lang,
created_at: new Date().toISOString()
};
const jsonString = JSON.stringify(mapping);
try {
const redis = await getRedisClient();
const activeUrl = redis.options.host || '';
// Use compression for large mappings
if (shouldCompress(jsonString)) {
const compressed = await gzip(jsonString);
const base64Compressed = compressed.toString('base64');
// Store with compression flag
await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL);
// Replicate to other instances
await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL);
console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`);
} else {
// Use normal storage for small mappings
await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL);
await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL);
console.log(`Successfully stored mapping for feed ${feedId}`);
}
} catch (error) {
console.error(`Error storing feed mapping for ${feedId}:`, error);
throw error;
}
}
// Batch get multiple feed mappings at once
export async function getBatchFeedMappings(
feedIds: string[]
): Promise<Record<string, FeedMapping | null>> {
if (!feedIds.length) return {};
try {
const redis = await getRedisClient();
const pipeline = redis.pipeline();
// Check regular keys first
feedIds.forEach(id => {
pipeline.get(id);
});
const results = await pipeline.exec();
if (!results) return {};
// Process results
const mappings: Record<string, FeedMapping | null> = {};
const ttlUpdates = redis.pipeline();
const compressedChecks = redis.pipeline();
const compressedIds: string[] = [];
for (let i = 0; i < feedIds.length; i++) {
const id = feedIds[i];
const result = results[i];
if (result[0]) {
mappings[id] = null;
continue;
}
if (result[1]) {
// Process regular mapping
try {
const data = JSON.parse(result[1] as string);
mappings[id] = {
slugs: data.slugs,
lang: data.lang
};
// Reset TTL
ttlUpdates.expire(id, DEFAULT_CACHE_TTL);
} catch (e) {
mappings[id] = null;
}
} else {
// Check for compressed version
compressedChecks.exists(`${id}:compressed`);
compressedIds.push(id);
}
}
// Execute TTL updates in background
ttlUpdates.exec().catch(err => console.error('TTL update error:', err));
// Check compressed versions if needed
if (compressedIds.length > 0) {
const compressedResults = await compressedChecks.exec();
if (compressedResults) {
const compressedFetches = redis.pipeline();
const compressedToFetch: string[] = [];
compressedResults.forEach((result, index) => {
if (!result[0] && result[1]) {
const id = compressedIds[index];
compressedFetches.get(`${id}:compressed`);
compressedToFetch.push(id);
}
});
if (compressedToFetch.length > 0) {
const fetchResults = await compressedFetches.exec();
if (fetchResults) {
for (let i = 0; i < compressedToFetch.length; i++) {
const id = compressedToFetch[i];
const [err, compressedData] = fetchResults[i];
if (!err && compressedData) {
try {
const buffer = Buffer.from(compressedData as string, 'base64');
const decompressed = await gunzip(buffer);
const data = JSON.parse(decompressed.toString());
mappings[id] = {
slugs: data.slugs,
lang: data.lang
};
// Reset TTL in background
redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {});
} catch (e) {
mappings[id] = null;
}
} else {
mappings[id] = null;
}
}
}
}
}
}
return mappings;
} catch (error) {
console.error(`Error retrieving batch feed mappings:`, error);
return {};
}
}
// Initialize connections eagerly to establish connections as soon as possible // Initialize connections eagerly to establish connections as soon as possible
if (!import.meta.env.SSR) { if (!import.meta.env.SSR) {
initializeAllClients() initializeAllClients()