push change

This commit is contained in:
ld3zian 2025-05-04 12:19:02 -07:00
parent bc0f10ab1d
commit b103a24643
9 changed files with 557 additions and 3 deletions

View file

@ -3,6 +3,7 @@ import vercel from "@astrojs/vercel";
import tailwind from "@astrojs/tailwind";
export default defineConfig({
site: process.env.ASTRO_SITE || 'https://manga-ldez.vercel.app',
output: "server",
adapter: vercel(),
image: {
@ -14,7 +15,8 @@ export default defineConfig({
'process.env.PRIMARY_REDIS_URL': JSON.stringify(process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL),
'process.env.REPLICA_REDIS_URL_1': JSON.stringify(process.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL),
'process.env.REPLICA_REDIS_URL_2': JSON.stringify(process.env.REPLICA_REDIS_URL_2),
'process.env.REPLICA_REDIS_URL_3': JSON.stringify(process.env.REPLICA_REDIS_URL_3)
'process.env.REPLICA_REDIS_URL_3': JSON.stringify(process.env.REPLICA_REDIS_URL_3),
'process.env.SCRAPE_API_BASE_URL': JSON.stringify(process.env.SCRAPE_API_BASE_URL)
}
}
});

115
package-lock.json generated
View file

@ -19,6 +19,7 @@
"ioredis": "^5.6.1",
"p-throttle": "^7.0.0",
"redis-modules-sdk": "^1.4.0",
"sanitize-html": "^2.16.0",
"tailwindcss": "^3.4.1"
}
},
@ -2535,6 +2536,14 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/deepmerge": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
"integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/defu": {
"version": "6.1.4",
"resolved": "https://registry.npmjs.org/defu/-/defu-6.1.4.tgz",
@ -2620,6 +2629,57 @@
"resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz",
"integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA=="
},
"node_modules/dom-serializer": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
"dependencies": {
"domelementtype": "^2.3.0",
"domhandler": "^5.0.2",
"entities": "^4.2.0"
},
"funding": {
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
}
},
"node_modules/domelementtype": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
]
},
"node_modules/domhandler": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
"dependencies": {
"domelementtype": "^2.3.0"
},
"engines": {
"node": ">= 4"
},
"funding": {
"url": "https://github.com/fb55/domhandler?sponsor=1"
}
},
"node_modules/domutils": {
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
"integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
"dependencies": {
"dom-serializer": "^2.0.0",
"domelementtype": "^2.3.0",
"domhandler": "^5.0.3"
},
"funding": {
"url": "https://github.com/fb55/domutils?sponsor=1"
}
},
"node_modules/dset": {
"version": "3.1.4",
"resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz",
@ -3172,6 +3232,24 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/htmlparser2": {
"version": "8.0.2",
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz",
"integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==",
"funding": [
"https://github.com/fb55/htmlparser2?sponsor=1",
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
],
"dependencies": {
"domelementtype": "^2.3.0",
"domhandler": "^5.0.3",
"domutils": "^3.0.1",
"entities": "^4.4.0"
}
},
"node_modules/http-cache-semantics": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
@ -3337,6 +3415,14 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/is-plain-object": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
"integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/is-wsl": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.0.tgz",
@ -4565,6 +4651,11 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/parse-srcset": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/parse-srcset/-/parse-srcset-1.0.2.tgz",
"integrity": "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q=="
},
"node_modules/parse5": {
"version": "7.2.1",
"resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz",
@ -5259,6 +5350,30 @@
"queue-microtask": "^1.2.2"
}
},
"node_modules/sanitize-html": {
"version": "2.16.0",
"resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.16.0.tgz",
"integrity": "sha512-0s4caLuHHaZFVxFTG74oW91+j6vW7gKbGD6CD2+miP73CE6z6YtOBN0ArtLd2UGyi4IC7K47v3ENUbQX4jV3Mg==",
"dependencies": {
"deepmerge": "^4.2.2",
"escape-string-regexp": "^4.0.0",
"htmlparser2": "^8.0.0",
"is-plain-object": "^5.0.0",
"parse-srcset": "^1.0.2",
"postcss": "^8.3.11"
}
},
"node_modules/sanitize-html/node_modules/escape-string-regexp": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
"integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/sax": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz",

View file

@ -22,6 +22,7 @@
"ioredis": "^5.6.1",
"p-throttle": "^7.0.0",
"redis-modules-sdk": "^1.4.0",
"sanitize-html": "^2.16.0",
"tailwindcss": "^3.4.1"
}
}

View file

@ -26,6 +26,9 @@ const contentTypes = new Set(["sfw", "nsfw"]);
const comicTypes = new Set(['manga', 'manhwa', 'manhua', 'all']);
const urlBase = "https://api.comick.fun/chapter";
// Use import.meta.env for Astro environment variables
const SCRAPE_API_BASE_URL = import.meta.env.SCRAPE_API_BASE_URL || 'http://localhost:8000';
let genreCache: Genre[] = [];
const RETRY_COUNT = 3;
@ -180,6 +183,31 @@ interface ChapterParams {
lang?: string;
}
interface UserLastReadChapter {
hid: string;
chap: string;
lang: string;
vol: string | null;
}
interface ComicInfo {
slug: string;
user_last_read_chapter: UserLastReadChapter | null;
comic_latest_chapter: number;
user_read_at: string;
}
interface ScrapeData {
user_id: string;
limit: number;
comics: ComicInfo[];
}
interface ScrapeApiResponse {
success: boolean;
data: ScrapeData;
}
export async function getComicBySlug(slug: string): Promise<ComicDetail | null> {
try {
const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`);
@ -293,7 +321,7 @@ export async function getChaptersForSlugs(
console.log(`Fetching chapters for ${slugs.length} comics`);
const { client: redis } = await getRedisClient();
const uniqueSlugs = [...new Set(slugs)];
const uniqueSlugs = Array.from(new Set(slugs));
const pipeline = redis.pipeline();
// Check cache for all slugs
@ -426,4 +454,40 @@ export async function validateSlugs(slugs: string[]): Promise<string[]> {
}
return validSlugs;
}
/**
* Fetches the comic list for a specific user from the scrape endpoint.
* @param userId The ID of the user.
* @param limit The maximum number of comics to fetch.
* @returns A promise that resolves to the scrape API response data.
*/
export async function fetchUserComics(userId: string, limit: number = 10): Promise<ScrapeData> {
const url = `${SCRAPE_API_BASE_URL}/scrape/${userId}?limit=${limit}`;
try {
// Using the existing throttledFetch for consistency, though rate limits might differ for local API
const response = await throttledFetch(url);
const result: ScrapeApiResponse = await response.json();
if (!result.success) {
// Handle API-level failure indication
throw new Error(`Scrape API indicated failure for user ${userId}`);
}
return result.data;
} catch (error) {
console.error(`Failed to fetch user comics for ${userId}:`, error);
// Re-throw or return a default/error state depending on your error handling strategy
if (error instanceof Error) {
throw new APIError(
`Failed to fetch user comics: ${error.message}`,
error instanceof Response ? error.status : undefined, // Use Response type guard if available
url
);
} else {
throw new APIError(
'An unknown error occurred while fetching user comics',
undefined,
url
);
}
}
}

View file

@ -1113,6 +1113,60 @@ async function syncFromReplicaToPrimary(replicaUrl: string): Promise<void> {
}
}
// Add user feed mapping functions
interface UserFeedMapping {
userId: string;
created_at: string;
}
export async function storeUserFeedMapping(
feedId: string,
userId: string
): Promise<void> {
const mapping: UserFeedMapping = {
userId,
created_at: new Date().toISOString()
};
const stringValue = JSON.stringify(mapping);
const regularKey = `user:${feedId}`;
try {
const { client: redis, url: activeUrl } = await getRedisClient();
const clientName = getRedisName(activeUrl);
await redis.set(regularKey, stringValue, "EX", DEFAULT_CACHE_TTL);
await ensurePrimaryAndReplicate(activeUrl, regularKey, stringValue, DEFAULT_CACHE_TTL);
} catch (error) {
console.error(`Error storing user feed mapping for ${feedId}:`, error);
throw error;
}
}
export async function getUserFeedMapping(
feedId: string
): Promise<UserFeedMapping | null> {
try {
const { client } = await getRedisClient();
const regularKey = `user:${feedId}`;
const jsonString = await client.get(regularKey);
if (!jsonString) {
return null;
}
const data = JSON.parse(jsonString);
return {
userId: data.userId,
created_at: data.created_at
};
} catch (error) {
console.error(`Error retrieving user feed mapping for ${feedId}:`, error);
return null;
}
}
// Initialize connections eagerly to establish connections as soon as possible
if (!import.meta.env.SSR) {
// Trigger initialization but don't block module loading

View file

@ -0,0 +1,28 @@
import type { APIRoute } from "astro";
import { createHash } from 'crypto';
import { storeUserFeedMapping } from '../../lib/db';
export const POST: APIRoute = async ({ request }) => {
try {
const { userId } = await request.json();
if (!userId || typeof userId !== 'string') {
return new Response("Invalid user ID", { status: 400 });
}
// Create a unique hash based on userId
const hash = createHash('md5').update(userId).digest('hex').slice(0, 8);
// Store the mapping in Redis
await storeUserFeedMapping(hash, userId);
return new Response(JSON.stringify({ feedId: hash }), {
status: 200,
headers: {
'Content-Type': 'application/json'
}
});
} catch (error) {
return new Response("Internal server error", { status: 500 });
}
};

View file

@ -35,10 +35,24 @@ const description = "Create a custom RSS feed for your favorite comics";
<SlugInput />
</div>
<!-- Add section for User Feed link -->
<div class="mt-8 bg-secondary/50 p-3 sm:p-6 rounded-lg shadow-lg text-center">
<h2 class="text-xl font-semibold mb-3">Generate Feed from User List</h2>
<p class="mb-4 text-primary/80">
Alternatively, generate an RSS feed based on a specific Comick user's reading list.
</p>
<a
href="/user-feed"
class="inline-block px-6 py-2 bg-accent text-accent-text rounded-md hover:bg-accent-dark transition-colors font-medium"
>
Go to User Feed Generator
</a>
</div>
<div class="mt-4 sm:mt-8 text-center text-primary/60 text-sm sm:text-base">
<p>Search for your favorite comics using the search box above, or add their slugs manually.</p>
<p class="mt-2">Manual example: For <a href="https://comick.io/comic/01-one-piece" class="text-accent hover:underline">https://comick.io/comic/01-one-piece</a>, the slug is "01-one-piece"</p>
</div>
</main>
</div>
</Layout>
</Layout>

View file

@ -0,0 +1,140 @@
import { Feed } from 'feed';
import type { APIRoute } from 'astro';
import { fetchUserComics, getComicBySlug, APIError } from '../../../lib/api';
import { getUserFeedMapping } from '../../../lib/db';
import sanitizeHtml from 'sanitize-html';
export const GET: APIRoute = async ({ params, site }) => {
const feedId = params.userId?.replace('.xml', '');
if (!feedId) {
return new Response("Feed ID not provided", { status: 400 });
}
if (!site) {
return new Response("Site configuration is missing", { status: 500 });
}
try {
const mapping = await getUserFeedMapping(feedId);
if (!mapping) {
return new Response("Feed not found", { status: 404 });
}
const userId = mapping.userId;
const userData = await fetchUserComics(userId);
const now = new Date();
// Store all items first so we can sort them
const feedItems = [];
let mostRecentDate = new Date(0);
const comicDetailsPromises = userData.comics
.filter(comic => comic.user_last_read_chapter !== null)
.map(async (comic) => {
try {
const details = await getComicBySlug(comic.slug);
if (comic.user_read_at) {
const readDate = new Date(comic.user_read_at);
if (!isNaN(readDate.getTime()) && readDate > mostRecentDate) {
mostRecentDate = readDate;
}
}
return { comic, details };
} catch (error) {
return { comic, details: null };
}
});
const results = await Promise.all(comicDetailsPromises);
const feed = new Feed({
title: `Comick User Read Feed - ${userId}`,
description: `Recently read chapters by Comick user ${userId}`,
id: `${site}rss/user/${feedId}.xml`,
link: site.toString(),
language: "en",
copyright: `All rights reserved`,
updated: mostRecentDate > new Date(0) ? mostRecentDate : now,
generator: "ComicK RSS Generator",
feedLinks: {
rss2: `${site}rss/user/${feedId}.xml`
},
});
for (const { comic, details } of results) {
if (comic.user_last_read_chapter && details?.comic) {
const comicTitle = details.comic.title;
const chapterNumber = comic.user_last_read_chapter.chap;
const itemTitle = sanitizeHtml(comicTitle);
const itemDescription = `User read chapter ${chapterNumber}`;
const itemLink = `https://comick.fun/comic/${comic.slug}`;
let readDate;
if (comic.user_read_at) {
readDate = new Date(comic.user_read_at);
if (isNaN(readDate.getTime())) {
readDate = now;
}
} else {
readDate = now;
}
feedItems.push({
title: itemTitle,
id: `${itemLink}#read-chapter-${chapterNumber}-${readDate.getTime()}`,
link: itemLink,
description: itemDescription,
date: readDate,
pubDate: readDate
});
}
}
feedItems.sort((a, b) => b.date.getTime() - a.date.getTime());
feedItems.forEach(item => feed.addItem(item));
return new Response(feed.rss2(), {
status: 200,
headers: {
'Content-Type': 'application/xml; charset=utf-8',
'Cache-Control': 'public, max-age=600'
}
});
} catch (error) {
let status = 500;
let message = 'Failed to generate RSS feed.';
if (error instanceof APIError) {
message = error.message;
status = error.statusCode === 404 ? 404 : (error.statusCode || 500);
if (status === 404) message = 'Comick user not found or scrape failed.';
} else if (error instanceof Error) {
message = error.message;
}
const errorFeed = new Feed({
title: "Error Generating Feed",
description: message,
id: site.toString(),
link: site.toString(),
language: "en",
copyright: ""
});
errorFeed.addItem({
title: `Error ${status}`,
description: message,
link: site.toString(),
date: new Date(),
id: `${site}error/${status}/${new Date().getTime()}`
});
return new Response(errorFeed.rss2(), {
status: status,
headers: { 'Content-Type': 'application/xml; charset=utf-8' },
});
}
};

136
src/pages/user-feed.astro Normal file
View file

@ -0,0 +1,136 @@
---
import Layout from '../layouts/Layout.astro';
---
<Layout title="Generate User RSS Feed">
<main class="container mx-auto px-4 py-8">
<!-- Back button -->
<a
href="/custom"
class="inline-flex items-center gap-2 mb-6 px-4 py-2 bg-secondary hover:bg-hover rounded-md transition-colors font-medium"
>
<svg
xmlns="http://www.w3.org/2000/svg"
class="w-5 h-5"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
>
<path d="M19 12H5M12 19l-7-7 7-7"/>
</svg>
Back to Custom Feed
</a>
<div class="mx-auto px-3 sm:px-4 lg:px-8 max-w-[800px] py-4 sm:py-8">
<h1 class="text-2xl sm:text-3xl lg:text-4xl font-bold text-center mb-4 sm:mb-8">Generate Comick User RSS Feed</h1>
<p class="mb-6 text-center text-primary/80">
Enter a Comick User ID below to generate an RSS feed link for their recently read chapters.
</p>
<div class="max-w-2xl mx-auto bg-secondary/50 p-6 rounded-lg shadow-lg">
<label for="userIdInput" class="block text-sm font-medium text-primary mb-2">Comick User ID:</label>
<input
type="text"
id="userIdInput"
placeholder="e.g., 4e996622-f879-4fd1-ba1e-dade02600555"
class="w-full px-3 py-2 bg-secondary border border-hover rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-accent/50 focus:border-accent mb-4 text-primary placeholder-primary/60"
/>
<button
id="generateButton"
class="w-full bg-accent text-accent-text font-medium py-2 px-4 rounded hover:bg-accent-dark transition-colors focus:outline-none focus:ring-2 focus:ring-accent/50"
>
Generate Feed Link
</button>
<div id="feedLinkContainer" class="mt-4 text-center" style="display: none;">
<p class="text-sm text-primary/80 mb-2">Your RSS Feed Link:</p>
<a id="feedLink" href="#" target="_blank" class="text-accent hover:underline break-all"></a>
<button
id="copyButton"
class="ml-2 px-3 py-1 text-sm bg-secondary hover:bg-hover rounded transition-colors focus:outline-none focus:ring-2 focus:ring-accent/50"
>
Copy
</button>
<p id="copyMessage" class="text-xs text-green-500 mt-2" style="display: none;">Copied to clipboard!</p>
</div>
<p id="errorMessage" class="text-red-500 text-sm mt-2 text-center" style="display: none;"></p>
</div>
<div class="mt-6 text-center text-primary/60 text-sm">
<p>This will generate an RSS feed of the user's recently read chapters.</p>
<p class="mt-2">The User ID can be found in their Comick profile URL.</p>
</div>
</div>
</main>
<script>
const userIdInput = document.getElementById('userIdInput') as HTMLInputElement;
const generateButton = document.getElementById('generateButton') as HTMLButtonElement;
const feedLinkContainer = document.getElementById('feedLinkContainer')!;
const feedLink = document.getElementById('feedLink') as HTMLAnchorElement;
const copyButton = document.getElementById('copyButton') as HTMLButtonElement;
const copyMessage = document.getElementById('copyMessage')!;
const errorMessage = document.getElementById('errorMessage')!;
const userIdRegex = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
generateButton.addEventListener('click', async () => {
const userId = userIdInput.value.trim();
errorMessage.style.display = 'none';
feedLinkContainer.style.display = 'none';
if (!userId) {
errorMessage.textContent = 'Please enter a User ID.';
errorMessage.style.display = 'block';
return;
}
if (!userIdRegex.test(userId)) {
errorMessage.textContent = 'Invalid User ID format. Please enter a valid UUID.';
errorMessage.style.display = 'block';
return;
}
try {
const response = await fetch('/api/create-user-feed', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ userId }),
});
if (!response.ok) {
throw new Error('Failed to generate feed');
}
const data = await response.json();
const feedUrl = `${window.location.origin}/rss/user/${data.feedId}.xml`;
feedLink.href = feedUrl;
feedLink.textContent = feedUrl;
feedLinkContainer.style.display = 'block';
copyMessage.style.display = 'none';
} catch (error) {
errorMessage.textContent = 'Failed to generate feed. Please try again.';
errorMessage.style.display = 'block';
}
});
copyButton.addEventListener('click', async () => {
try {
await navigator.clipboard.writeText(feedLink.href);
copyMessage.style.display = 'block';
setTimeout(() => {
copyMessage.style.display = 'none';
}, 2000);
} catch (err) {
errorMessage.textContent = 'Failed to copy to clipboard';
errorMessage.style.display = 'block';
}
});
</script>
</Layout>