import type { APIRoute } from "astro"; import chromium from 'chrome-aws-lambda'; import puppeteerCore from 'puppeteer-core'; // Check if running on Vercel const IS_VERCEL = process.env.VERCEL === '1' || process.env.VERCEL === 'true'; // Import puppeteer dynamically for ESM compatibility import puppeteer from 'puppeteer'; // Decide which Puppeteer to use based on environment const puppeteerInstance = IS_VERCEL ? puppeteerCore : puppeteer; // Helper function to extract comic slugs from a ComicK user list page async function scrapeComicList(userId: string, limit: number = 10): Promise { let browser; if (IS_VERCEL) { // Use chrome-aws-lambda for Vercel environment console.log('Running on Vercel - using chrome-aws-lambda'); browser = await puppeteerCore.launch({ args: chromium.args, defaultViewport: chromium.defaultViewport, executablePath: await chromium.executablePath, headless: chromium.headless, ignoreHTTPSErrors: true, }); } else { // Use regular Puppeteer for local development browser = await puppeteerInstance.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] }); } try { const page = await browser.newPage(); // Set a user agent to mimic a real browser await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36'); console.log(`Navigating to ComicK user page: ${userId}`); // Navigate to the user's list page await page.goto(`https://comick.io/user/${userId}/list`, { waitUntil: 'networkidle0', // Wait until there are no network connections for at least 500ms timeout: 60000 // Longer timeout to ensure everything loads }); // Wait directly for the Next.js data script to load (the most reliable approach) console.log('Waiting for Next.js data to load...'); try { await page.waitForFunction( () => document.getElementById('__NEXT_DATA__') !== null, { timeout: 10000 } ); console.log('Next.js data script found!'); } catch (e) { console.log('Next.js data script not found, page might not be fully loaded'); } // Minimal page interaction - Next.js will load the data we need console.log('Ensuring Next.js data is fully loaded...'); // Wait a bit for any dynamic data to populate await new Promise(resolve => setTimeout(resolve, 3000)); // Verify Next.js data is available and complete const dataLoaded = await page.evaluate(() => { const script = document.getElementById('__NEXT_DATA__'); if (!script) return false; try { const data = JSON.parse(script.textContent || '{}'); // Check if the follows data is present return !!data?.props?.pageProps?.follows; } catch (e) { return false; } }); if (!dataLoaded) { console.log('Next.js data not fully loaded, waiting a bit longer...'); await new Promise(resolve => setTimeout(resolve, 3000)); } else { console.log('Next.js data confirmed loaded!'); } console.log('Page loaded, extracting comic data...'); // Extract the HTML structure for debugging const pageHtml = await page.content(); console.log('Page HTML length:', pageHtml.length); // Extract comic slugs using only Next.js data - no CSS selectors const slugs = await page.evaluate(() => { console.log('Extracting comics directly from Next.js data structure...'); try { // Find the __NEXT_DATA__ script which contains all the comic data const nextDataScript = document.getElementById('__NEXT_DATA__'); if (!nextDataScript) { console.log('ERROR: __NEXT_DATA__ script not found!'); return []; } console.log('Found Next.js data script!'); const nextData = JSON.parse(nextDataScript.textContent || '{}'); // Navigate to the follows data in the structure const follows = nextData?.props?.pageProps?.follows || []; console.log(`Found ${follows.length} comics in Next.js data`); if (follows.length > 0) { // Extract slugs from the follows array const comicSlugs = follows.map(follow => { const comic = follow.md_comics; if (comic && comic.slug) { console.log(`Found comic: ${comic.title} with slug: ${comic.slug}`); return comic.slug; } return null; }).filter(Boolean); console.log(`Successfully extracted ${comicSlugs.length} comics from Next.js data`); return comicSlugs; } else { console.log('No follows data found in Next.js data'); // Try to locate comics in a different location in the data structure const alternative = nextData?.props?.pageProps?.user_info?.user_comic_follow_count; if (alternative) { console.log(`Found user has ${alternative} comics according to user_info`); } return []; } } catch (e) { console.log('Error extracting from Next.js data:', e); return []; } }); // Remove duplicates by using a Set const uniqueSlugs = Array.from(new Set(slugs)); console.log(`Extracted ${slugs.length} comics from page (${uniqueSlugs.length} unique)`); // Limit the number of comics to the specified limit const limitedSlugs = uniqueSlugs.slice(0, limit); console.log(`Returning ${limitedSlugs.length} comics (limit: ${limit})`); return limitedSlugs as string[]; } finally { await browser.close(); } } // Define timeout for Vercel environment const VERCEL_TIMEOUT = 10000; // 10 seconds for Vercel functions const LOCAL_TIMEOUT = 60000; // 60 seconds for local development const FUNCTION_TIMEOUT = IS_VERCEL ? VERCEL_TIMEOUT : LOCAL_TIMEOUT; export const GET: APIRoute = async ({ request }) => { // Ensure the function doesn't exceed Vercel's timeout limits let timeoutId: NodeJS.Timeout; // Create a promise that rejects after the timeout const timeoutPromise = new Promise((_, reject) => { timeoutId = setTimeout(() => { reject(new Error(`Function timed out after ${FUNCTION_TIMEOUT}ms`)); }, FUNCTION_TIMEOUT); }); try { const url = new URL(request.url); const userId = url.searchParams.get('userId'); // Get limit parameter from query string (default to 10 if not provided) const limitParam = url.searchParams.get('limit'); const limit = limitParam ? parseInt(limitParam, 10) : 10; // Ensure limit is a valid number const validLimit = isNaN(limit) || limit <= 0 ? 10 : Math.min(limit, 100); if (!userId) { return new Response(JSON.stringify({ error: 'Missing user ID' }), { status: 400, headers: { 'Content-Type': 'application/json' } }); } // Extract user ID from URL if full URL was provided let cleanUserId = userId; if (userId.includes('comick.io/user/')) { const match = userId.match(/comick\.io\/user\/([a-zA-Z0-9-]+)/); if (match && match[1]) { cleanUserId = match[1]; } } // Validate the UUID format const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; if (!uuidRegex.test(cleanUserId)) { return new Response(JSON.stringify({ error: 'Invalid user ID format. Must be a UUID like: 8da7d6cf-a2ac-4972-a465-770ba975ee2a' }), { status: 400, headers: { 'Content-Type': 'application/json' } }); } console.log(`Scraping comics for user: ${cleanUserId} with limit: ${validLimit}`); // Use Puppeteer to scrape the user's comic list with the specified limit const comics = await scrapeComicList(cleanUserId, validLimit); // Calculate total const totalComics = comics.length; // Check if the list is empty if (totalComics === 0) { console.log('No comics found for user, returning empty list'); const response = new Response(JSON.stringify({ success: false, error: "No comics found in this user's list. Make sure the user ID is correct and the list is not empty.", data: { comics: [], total: 0, limit: validLimit, userId: cleanUserId } }), { status: 404, headers: { 'Content-Type': 'application/json' } }); // Clear the timeout to prevent memory leaks clearTimeout(timeoutId); return response; } console.log(`Returning ${totalComics} comics found for user ${cleanUserId}`); const response = new Response(JSON.stringify({ success: true, data: { comics, total: totalComics, limit: validLimit, userId: cleanUserId } }), { status: 200, headers: { 'Content-Type': 'application/json', 'Cache-Control': 'max-age=3600' // Cache for 1 hour } }); // Clear the timeout to prevent memory leaks clearTimeout(timeoutId); return response; } catch (error) { console.error('Error fetching user comics:', error); // Try to generate a helpful error message based on the error let errorMessage = 'Failed to fetch user comics'; let statusCode = 500; if (error instanceof Error) { // Handle specific error types if (error.message.includes('Failed to fetch') || error.message.includes('network') || error.message.includes('ECONNREFUSED')) { errorMessage = 'Could not connect to ComicK API. Please try again later.'; } else if (error.message.includes('timeout')) { errorMessage = 'Request timed out. The ComicK API might be experiencing high load.'; } else if (error.message.includes('parsing')) { errorMessage = 'Received invalid data from ComicK API. Their API structure might have changed.'; } } // Clear the timeout to prevent memory leaks clearTimeout(timeoutId); return new Response( JSON.stringify({ success: false, error: errorMessage, details: error instanceof Error ? error.message : String(error) }), { status: statusCode, headers: { 'Content-Type': 'application/json' } } ); } };