diff --git a/astro.config.mjs b/astro.config.mjs index 778d936..6e6e063 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -3,6 +3,7 @@ import vercel from "@astrojs/vercel"; import tailwind from "@astrojs/tailwind"; export default defineConfig({ + site: process.env.ASTRO_SITE || 'https://manga-ldez.vercel.app', output: "server", adapter: vercel(), image: { @@ -14,7 +15,8 @@ export default defineConfig({ 'process.env.PRIMARY_REDIS_URL': JSON.stringify(process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL), 'process.env.REPLICA_REDIS_URL_1': JSON.stringify(process.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL), 'process.env.REPLICA_REDIS_URL_2': JSON.stringify(process.env.REPLICA_REDIS_URL_2), - 'process.env.REPLICA_REDIS_URL_3': JSON.stringify(process.env.REPLICA_REDIS_URL_3) + 'process.env.REPLICA_REDIS_URL_3': JSON.stringify(process.env.REPLICA_REDIS_URL_3), + 'process.env.SCRAPE_API_BASE_URL': JSON.stringify(process.env.SCRAPE_API_BASE_URL) } } }); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 25c7bd5..0b4d04d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "ioredis": "^5.6.1", "p-throttle": "^7.0.0", "redis-modules-sdk": "^1.4.0", + "sanitize-html": "^2.16.0", "tailwindcss": "^3.4.1" } }, @@ -2535,6 +2536,14 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/defu": { "version": "6.1.4", "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.4.tgz", @@ -2620,6 +2629,57 @@ "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==" }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dset": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", @@ -3172,6 +3232,24 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "node_modules/http-cache-semantics": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", @@ -3337,6 +3415,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-wsl": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.0.tgz", @@ -4565,6 +4651,11 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/parse-srcset": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/parse-srcset/-/parse-srcset-1.0.2.tgz", + "integrity": "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==" + }, "node_modules/parse5": { "version": "7.2.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", @@ -5259,6 +5350,30 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/sanitize-html": { + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.16.0.tgz", + "integrity": "sha512-0s4caLuHHaZFVxFTG74oW91+j6vW7gKbGD6CD2+miP73CE6z6YtOBN0ArtLd2UGyi4IC7K47v3ENUbQX4jV3Mg==", + "dependencies": { + "deepmerge": "^4.2.2", + "escape-string-regexp": "^4.0.0", + "htmlparser2": "^8.0.0", + "is-plain-object": "^5.0.0", + "parse-srcset": "^1.0.2", + "postcss": "^8.3.11" + } + }, + "node_modules/sanitize-html/node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/sax": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", diff --git a/package.json b/package.json index f237618..7c34bf1 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "ioredis": "^5.6.1", "p-throttle": "^7.0.0", "redis-modules-sdk": "^1.4.0", + "sanitize-html": "^2.16.0", "tailwindcss": "^3.4.1" } } diff --git a/src/lib/api.ts b/src/lib/api.ts index b77f572..397e61c 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -26,6 +26,9 @@ const contentTypes = new Set(["sfw", "nsfw"]); const comicTypes = new Set(['manga', 'manhwa', 'manhua', 'all']); const urlBase = "https://api.comick.fun/chapter"; +// Use import.meta.env for Astro environment variables +const SCRAPE_API_BASE_URL = import.meta.env.SCRAPE_API_BASE_URL || 'http://localhost:8000'; + let genreCache: Genre[] = []; const RETRY_COUNT = 3; @@ -180,6 +183,31 @@ interface ChapterParams { lang?: string; } +interface UserLastReadChapter { + hid: string; + chap: string; + lang: string; + vol: string | null; +} + +interface ComicInfo { + slug: string; + user_last_read_chapter: UserLastReadChapter | null; + comic_latest_chapter: number; + user_read_at: string; +} + +interface ScrapeData { + user_id: string; + limit: number; + comics: ComicInfo[]; +} + +interface ScrapeApiResponse { + success: boolean; + data: ScrapeData; +} + export async function getComicBySlug(slug: string): Promise { try { const response = await throttledFetch(`https://api.comick.fun/comic/${slug}`); @@ -293,7 +321,7 @@ export async function getChaptersForSlugs( console.log(`Fetching chapters for ${slugs.length} comics`); const { client: redis } = await getRedisClient(); - const uniqueSlugs = [...new Set(slugs)]; + const uniqueSlugs = Array.from(new Set(slugs)); const pipeline = redis.pipeline(); // Check cache for all slugs @@ -426,4 +454,40 @@ export async function validateSlugs(slugs: string[]): Promise { } return validSlugs; +} + +/** + * Fetches the comic list for a specific user from the scrape endpoint. + * @param userId The ID of the user. + * @param limit The maximum number of comics to fetch. + * @returns A promise that resolves to the scrape API response data. + */ +export async function fetchUserComics(userId: string, limit: number = 10): Promise { + const url = `${SCRAPE_API_BASE_URL}/scrape/${userId}?limit=${limit}`; + try { + // Using the existing throttledFetch for consistency, though rate limits might differ for local API + const response = await throttledFetch(url); + const result: ScrapeApiResponse = await response.json(); + if (!result.success) { + // Handle API-level failure indication + throw new Error(`Scrape API indicated failure for user ${userId}`); + } + return result.data; + } catch (error) { + console.error(`Failed to fetch user comics for ${userId}:`, error); + // Re-throw or return a default/error state depending on your error handling strategy + if (error instanceof Error) { + throw new APIError( + `Failed to fetch user comics: ${error.message}`, + error instanceof Response ? error.status : undefined, // Use Response type guard if available + url + ); + } else { + throw new APIError( + 'An unknown error occurred while fetching user comics', + undefined, + url + ); + } + } } \ No newline at end of file diff --git a/src/lib/db.ts b/src/lib/db.ts index caee75c..d766e80 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -1113,6 +1113,60 @@ async function syncFromReplicaToPrimary(replicaUrl: string): Promise { } } +// Add user feed mapping functions +interface UserFeedMapping { + userId: string; + created_at: string; +} + +export async function storeUserFeedMapping( + feedId: string, + userId: string +): Promise { + const mapping: UserFeedMapping = { + userId, + created_at: new Date().toISOString() + }; + + const stringValue = JSON.stringify(mapping); + const regularKey = `user:${feedId}`; + + try { + const { client: redis, url: activeUrl } = await getRedisClient(); + const clientName = getRedisName(activeUrl); + + await redis.set(regularKey, stringValue, "EX", DEFAULT_CACHE_TTL); + await ensurePrimaryAndReplicate(activeUrl, regularKey, stringValue, DEFAULT_CACHE_TTL); + + } catch (error) { + console.error(`Error storing user feed mapping for ${feedId}:`, error); + throw error; + } +} + +export async function getUserFeedMapping( + feedId: string +): Promise { + try { + const { client } = await getRedisClient(); + const regularKey = `user:${feedId}`; + const jsonString = await client.get(regularKey); + + if (!jsonString) { + return null; + } + + const data = JSON.parse(jsonString); + return { + userId: data.userId, + created_at: data.created_at + }; + } catch (error) { + console.error(`Error retrieving user feed mapping for ${feedId}:`, error); + return null; + } +} + // Initialize connections eagerly to establish connections as soon as possible if (!import.meta.env.SSR) { // Trigger initialization but don't block module loading diff --git a/src/pages/api/create-user-feed.ts b/src/pages/api/create-user-feed.ts new file mode 100644 index 0000000..c32ea34 --- /dev/null +++ b/src/pages/api/create-user-feed.ts @@ -0,0 +1,28 @@ +import type { APIRoute } from "astro"; +import { createHash } from 'crypto'; +import { storeUserFeedMapping } from '../../lib/db'; + +export const POST: APIRoute = async ({ request }) => { + try { + const { userId } = await request.json(); + + if (!userId || typeof userId !== 'string') { + return new Response("Invalid user ID", { status: 400 }); + } + + // Create a unique hash based on userId + const hash = createHash('md5').update(userId).digest('hex').slice(0, 8); + + // Store the mapping in Redis + await storeUserFeedMapping(hash, userId); + + return new Response(JSON.stringify({ feedId: hash }), { + status: 200, + headers: { + 'Content-Type': 'application/json' + } + }); + } catch (error) { + return new Response("Internal server error", { status: 500 }); + } +}; \ No newline at end of file diff --git a/src/pages/custom.astro b/src/pages/custom.astro index a99e96e..9315a49 100644 --- a/src/pages/custom.astro +++ b/src/pages/custom.astro @@ -35,10 +35,24 @@ const description = "Create a custom RSS feed for your favorite comics"; + +
+

Generate Feed from User List

+

+ Alternatively, generate an RSS feed based on a specific Comick user's reading list. +

+ + Go to User Feed Generator + +
+

Search for your favorite comics using the search box above, or add their slugs manually.

Manual example: For https://comick.io/comic/01-one-piece, the slug is "01-one-piece"

- \ No newline at end of file + \ No newline at end of file diff --git a/src/pages/rss/user/[userId].xml.ts b/src/pages/rss/user/[userId].xml.ts new file mode 100644 index 0000000..68cef36 --- /dev/null +++ b/src/pages/rss/user/[userId].xml.ts @@ -0,0 +1,140 @@ +import { Feed } from 'feed'; +import type { APIRoute } from 'astro'; +import { fetchUserComics, getComicBySlug, APIError } from '../../../lib/api'; +import { getUserFeedMapping } from '../../../lib/db'; +import sanitizeHtml from 'sanitize-html'; + +export const GET: APIRoute = async ({ params, site }) => { + const feedId = params.userId?.replace('.xml', ''); + + if (!feedId) { + return new Response("Feed ID not provided", { status: 400 }); + } + + if (!site) { + return new Response("Site configuration is missing", { status: 500 }); + } + + try { + const mapping = await getUserFeedMapping(feedId); + + if (!mapping) { + return new Response("Feed not found", { status: 404 }); + } + + const userId = mapping.userId; + const userData = await fetchUserComics(userId); + const now = new Date(); + + // Store all items first so we can sort them + const feedItems = []; + let mostRecentDate = new Date(0); + + const comicDetailsPromises = userData.comics + .filter(comic => comic.user_last_read_chapter !== null) + .map(async (comic) => { + try { + const details = await getComicBySlug(comic.slug); + if (comic.user_read_at) { + const readDate = new Date(comic.user_read_at); + if (!isNaN(readDate.getTime()) && readDate > mostRecentDate) { + mostRecentDate = readDate; + } + } + return { comic, details }; + } catch (error) { + return { comic, details: null }; + } + }); + + const results = await Promise.all(comicDetailsPromises); + + const feed = new Feed({ + title: `Comick User Read Feed - ${userId}`, + description: `Recently read chapters by Comick user ${userId}`, + id: `${site}rss/user/${feedId}.xml`, + link: site.toString(), + language: "en", + copyright: `All rights reserved`, + updated: mostRecentDate > new Date(0) ? mostRecentDate : now, + generator: "ComicK RSS Generator", + feedLinks: { + rss2: `${site}rss/user/${feedId}.xml` + }, + }); + + for (const { comic, details } of results) { + if (comic.user_last_read_chapter && details?.comic) { + const comicTitle = details.comic.title; + const chapterNumber = comic.user_last_read_chapter.chap; + const itemTitle = sanitizeHtml(comicTitle); + const itemDescription = `User read chapter ${chapterNumber}`; + const itemLink = `https://comick.fun/comic/${comic.slug}`; + + let readDate; + if (comic.user_read_at) { + readDate = new Date(comic.user_read_at); + if (isNaN(readDate.getTime())) { + readDate = now; + } + } else { + readDate = now; + } + + feedItems.push({ + title: itemTitle, + id: `${itemLink}#read-chapter-${chapterNumber}-${readDate.getTime()}`, + link: itemLink, + description: itemDescription, + date: readDate, + pubDate: readDate + }); + } + } + + feedItems.sort((a, b) => b.date.getTime() - a.date.getTime()); + feedItems.forEach(item => feed.addItem(item)); + + return new Response(feed.rss2(), { + status: 200, + headers: { + 'Content-Type': 'application/xml; charset=utf-8', + 'Cache-Control': 'public, max-age=600' + } + }); + + } catch (error) { + let status = 500; + let message = 'Failed to generate RSS feed.'; + + if (error instanceof APIError) { + message = error.message; + status = error.statusCode === 404 ? 404 : (error.statusCode || 500); + if (status === 404) message = 'Comick user not found or scrape failed.'; + } else if (error instanceof Error) { + message = error.message; + } + + const errorFeed = new Feed({ + title: "Error Generating Feed", + description: message, + id: site.toString(), + link: site.toString(), + language: "en", + copyright: "" + }); + + errorFeed.addItem({ + title: `Error ${status}`, + description: message, + link: site.toString(), + date: new Date(), + id: `${site}error/${status}/${new Date().getTime()}` + }); + + return new Response(errorFeed.rss2(), { + status: status, + headers: { 'Content-Type': 'application/xml; charset=utf-8' }, + }); + } +}; diff --git a/src/pages/user-feed.astro b/src/pages/user-feed.astro new file mode 100644 index 0000000..3383358 --- /dev/null +++ b/src/pages/user-feed.astro @@ -0,0 +1,136 @@ +--- +import Layout from '../layouts/Layout.astro'; +--- + + +
+ + + + + + Back to Custom Feed + + +
+

Generate Comick User RSS Feed

+

+ Enter a Comick User ID below to generate an RSS feed link for their recently read chapters. +

+ +
+ + + + + + +
+ +
+

This will generate an RSS feed of the user's recently read chapters.

+

The User ID can be found in their Comick profile URL.

+
+
+
+ + +