mirror of
https://github.com/ld3z/new-manga.git
synced 2026-11-09 20:04:00 -07:00
bruh
This commit is contained in:
ld3zian
parent
2d3eeeaeba
commit
94c02124f2
3 changed files with 819 additions and 125 deletions
|
|
@ -1,12 +1,37 @@
|
|||
import type { APIRoute } from "astro";
|
||||
import chromium from 'chrome-aws-lambda';
|
||||
import puppeteerCore from 'puppeteer-core';
|
||||
|
||||
// Check if running on Vercel
|
||||
const IS_VERCEL = process.env.VERCEL === '1' || process.env.VERCEL === 'true';
|
||||
|
||||
// Import puppeteer dynamically for ESM compatibility
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
// Decide which Puppeteer to use based on environment
|
||||
const puppeteerInstance = IS_VERCEL ? puppeteerCore : puppeteer;
|
||||
|
||||
// Helper function to extract comic slugs from a ComicK user list page
|
||||
async function scrapeComicList(userId: string, limit: number = 10): Promise<string[]> {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox']
|
||||
});
|
||||
let browser;
|
||||
|
||||
if (IS_VERCEL) {
|
||||
// Use chrome-aws-lambda for Vercel environment
|
||||
console.log('Running on Vercel - using chrome-aws-lambda');
|
||||
browser = await puppeteerCore.launch({
|
||||
args: chromium.args,
|
||||
defaultViewport: chromium.defaultViewport,
|
||||
executablePath: await chromium.executablePath,
|
||||
headless: chromium.headless,
|
||||
ignoreHTTPSErrors: true,
|
||||
});
|
||||
} else {
|
||||
// Use regular Puppeteer for local development
|
||||
browser = await puppeteerInstance.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox']
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
|
|
@ -22,147 +47,131 @@ async function scrapeComicList(userId: string, limit: number = 10): Promise<stri
|
|||
timeout: 60000 // Longer timeout to ensure everything loads
|
||||
});
|
||||
|
||||
// Try different waiting strategies
|
||||
console.log('Waiting for page to load completely...');
|
||||
// Wait directly for the Next.js data script to load (the most reliable approach)
|
||||
console.log('Waiting for Next.js data to load...');
|
||||
|
||||
// First try waiting for specific elements that might be present
|
||||
const selectors = ['.book', '[data-testid="comic-item"]', '.comics-grid', '.comic-list', '.md-list'];
|
||||
let foundSelector = false;
|
||||
|
||||
for (const selector of selectors) {
|
||||
try {
|
||||
console.log(`Trying to wait for selector: ${selector}`);
|
||||
await page.waitForSelector(selector, { timeout: 5000 });
|
||||
console.log(`Found element with selector: ${selector}`);
|
||||
foundSelector = true;
|
||||
break;
|
||||
} catch (e) {
|
||||
console.log(`Selector ${selector} not found`);
|
||||
}
|
||||
}
|
||||
|
||||
// If no specific selector worked, wait for any links that might be comics
|
||||
if (!foundSelector) {
|
||||
console.log('Waiting for any comic links to appear...');
|
||||
try {
|
||||
await page.waitForSelector('a[href*="/comic/"]', { timeout: 5000 });
|
||||
console.log('Found comic links on the page');
|
||||
} catch (e) {
|
||||
console.log('No comic links found, continuing anyway');
|
||||
}
|
||||
}
|
||||
|
||||
// Interact with the page to ensure all content is loaded
|
||||
console.log('Interacting with page to ensure content is loaded...');
|
||||
|
||||
// Scroll down to trigger any lazy loading
|
||||
await page.evaluate(() => {
|
||||
// Scroll to the bottom of the page
|
||||
window.scrollTo(0, document.body.scrollHeight);
|
||||
});
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
|
||||
// Scroll back up
|
||||
await page.evaluate(() => {
|
||||
window.scrollTo(0, 0);
|
||||
});
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
|
||||
// Try clicking any buttons that might reveal the content
|
||||
try {
|
||||
const buttons = await page.$$('button, .button, [role="button"]');
|
||||
console.log(`Found ${buttons.length} buttons on the page`);
|
||||
|
||||
// Try clicking buttons that might be related to viewing the list
|
||||
for (const button of buttons.slice(0, 5)) { // Limit to first 5 buttons to avoid clicking too many
|
||||
const buttonText = await page.evaluate(el => el.textContent?.toLowerCase() || '', button);
|
||||
console.log(`Found button with text: ${buttonText}`);
|
||||
if (buttonText.includes('list') || buttonText.includes('view') ||
|
||||
buttonText.includes('show') || buttonText.includes('load')) {
|
||||
console.log(`Clicking button: ${buttonText}`);
|
||||
await button.click().catch(e => console.log(`Failed to click button: ${e.message}`));
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
await page.waitForFunction(
|
||||
() => document.getElementById('__NEXT_DATA__') !== null,
|
||||
{ timeout: 10000 }
|
||||
);
|
||||
console.log('Next.js data script found!');
|
||||
} catch (e) {
|
||||
console.log(`Error interacting with buttons: ${e.message}`);
|
||||
console.log('Next.js data script not found, page might not be fully loaded');
|
||||
}
|
||||
|
||||
// Give extra time for JavaScript to execute and everything to render
|
||||
console.log('Waiting additional time for page to stabilize...');
|
||||
// Minimal page interaction - Next.js will load the data we need
|
||||
console.log('Ensuring Next.js data is fully loaded...');
|
||||
|
||||
// Wait a bit for any dynamic data to populate
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
|
||||
// Verify Next.js data is available and complete
|
||||
const dataLoaded = await page.evaluate(() => {
|
||||
const script = document.getElementById('__NEXT_DATA__');
|
||||
if (!script) return false;
|
||||
|
||||
try {
|
||||
const data = JSON.parse(script.textContent || '{}');
|
||||
// Check if the follows data is present
|
||||
return !!data?.props?.pageProps?.follows;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
if (!dataLoaded) {
|
||||
console.log('Next.js data not fully loaded, waiting a bit longer...');
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
} else {
|
||||
console.log('Next.js data confirmed loaded!');
|
||||
}
|
||||
|
||||
console.log('Page loaded, extracting comic data...');
|
||||
|
||||
// Extract the HTML structure for debugging
|
||||
const pageHtml = await page.content();
|
||||
console.log('Page HTML length:', pageHtml.length);
|
||||
|
||||
// Extract comic slugs from the page with multiple selector attempts
|
||||
// Extract comic slugs using only Next.js data - no CSS selectors
|
||||
const slugs = await page.evaluate(() => {
|
||||
// Try multiple selectors to find comics
|
||||
const selectors = [
|
||||
// Try different selectors that might match comics
|
||||
'.book a[href^="/comic/"]',
|
||||
'a[href^="/comic/"]',
|
||||
'.md-comic',
|
||||
'.comics-grid a',
|
||||
'.comic-list a',
|
||||
'.md-list a',
|
||||
'[data-testid="comic-item"] a',
|
||||
// Generic link selector targeting comic URLs as fallback
|
||||
'a[href*="/comic/"]'
|
||||
];
|
||||
console.log('Extracting comics directly from Next.js data structure...');
|
||||
|
||||
// Try each selector and use the first one that returns results
|
||||
for (const selector of selectors) {
|
||||
console.log('Trying selector:', selector);
|
||||
const elements = document.querySelectorAll(selector);
|
||||
console.log(`Found ${elements.length} elements with selector: ${selector}`);
|
||||
try {
|
||||
// Find the __NEXT_DATA__ script which contains all the comic data
|
||||
const nextDataScript = document.getElementById('__NEXT_DATA__');
|
||||
if (!nextDataScript) {
|
||||
console.log('ERROR: __NEXT_DATA__ script not found!');
|
||||
return [];
|
||||
}
|
||||
|
||||
if (elements.length > 0) {
|
||||
// We found matching elements, extract the slugs
|
||||
const results = Array.from(elements).map(element => {
|
||||
const href = element.getAttribute('href');
|
||||
if (!href || !href.includes('/comic/')) return null;
|
||||
return href.replace('/comic/', '').split('/')[0]; // Get just the slug part
|
||||
console.log('Found Next.js data script!');
|
||||
const nextData = JSON.parse(nextDataScript.textContent || '{}');
|
||||
|
||||
// Navigate to the follows data in the structure
|
||||
const follows = nextData?.props?.pageProps?.follows || [];
|
||||
console.log(`Found ${follows.length} comics in Next.js data`);
|
||||
|
||||
if (follows.length > 0) {
|
||||
// Extract slugs from the follows array
|
||||
const comicSlugs = follows.map(follow => {
|
||||
const comic = follow.md_comics;
|
||||
if (comic && comic.slug) {
|
||||
console.log(`Found comic: ${comic.title} with slug: ${comic.slug}`);
|
||||
return comic.slug;
|
||||
}
|
||||
return null;
|
||||
}).filter(Boolean);
|
||||
|
||||
if (results.length > 0) {
|
||||
console.log(`Found ${results.length} comics with selector: ${selector}`);
|
||||
return results;
|
||||
console.log(`Successfully extracted ${comicSlugs.length} comics from Next.js data`);
|
||||
return comicSlugs;
|
||||
} else {
|
||||
console.log('No follows data found in Next.js data');
|
||||
|
||||
// Try to locate comics in a different location in the data structure
|
||||
const alternative = nextData?.props?.pageProps?.user_info?.user_comic_follow_count;
|
||||
if (alternative) {
|
||||
console.log(`Found user has ${alternative} comics according to user_info`);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error extracting from Next.js data:', e);
|
||||
return [];
|
||||
}
|
||||
|
||||
// If all selectors failed, try a generic approach to find any links to comics
|
||||
const allLinks = document.querySelectorAll('a');
|
||||
console.log(`Trying generic approach with ${allLinks.length} links`);
|
||||
|
||||
const comicLinks = Array.from(allLinks).filter(link => {
|
||||
const href = link.getAttribute('href');
|
||||
return href && href.includes('/comic/');
|
||||
});
|
||||
|
||||
console.log(`Found ${comicLinks.length} potential comic links using generic approach`);
|
||||
|
||||
return comicLinks.map(link => {
|
||||
const href = link.getAttribute('href');
|
||||
return href ? href.replace('/comic/', '').split('/')[0] : null;
|
||||
}).filter(Boolean);
|
||||
});
|
||||
|
||||
// Limit the number of comics to the specified limit
|
||||
const limitedSlugs = slugs.slice(0, limit);
|
||||
// Remove duplicates by using a Set
|
||||
const uniqueSlugs = Array.from(new Set(slugs));
|
||||
console.log(`Extracted ${slugs.length} comics from page (${uniqueSlugs.length} unique)`);
|
||||
|
||||
console.log(`Extracted ${slugs.length} comics from page, returning ${limitedSlugs.length} (limit: ${limit})`);
|
||||
// Limit the number of comics to the specified limit
|
||||
const limitedSlugs = uniqueSlugs.slice(0, limit);
|
||||
|
||||
console.log(`Returning ${limitedSlugs.length} comics (limit: ${limit})`);
|
||||
return limitedSlugs as string[];
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Define timeout for Vercel environment
|
||||
const VERCEL_TIMEOUT = 10000; // 10 seconds for Vercel functions
|
||||
const LOCAL_TIMEOUT = 60000; // 60 seconds for local development
|
||||
const FUNCTION_TIMEOUT = IS_VERCEL ? VERCEL_TIMEOUT : LOCAL_TIMEOUT;
|
||||
|
||||
export const GET: APIRoute = async ({ request }) => {
|
||||
// Ensure the function doesn't exceed Vercel's timeout limits
|
||||
let timeoutId: NodeJS.Timeout;
|
||||
|
||||
// Create a promise that rejects after the timeout
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeoutId = setTimeout(() => {
|
||||
reject(new Error(`Function timed out after ${FUNCTION_TIMEOUT}ms`));
|
||||
}, FUNCTION_TIMEOUT);
|
||||
});
|
||||
|
||||
try {
|
||||
const url = new URL(request.url);
|
||||
const userId = url.searchParams.get('userId');
|
||||
|
|
@ -213,25 +222,49 @@ export const GET: APIRoute = async ({ request }) => {
|
|||
|
||||
// Check if the list is empty
|
||||
if (totalComics === 0) {
|
||||
return new Response(JSON.stringify({
|
||||
comics: [],
|
||||
totalComics: 0,
|
||||
message: "No comics found in user's list"
|
||||
console.log('No comics found for user, returning empty list');
|
||||
const response = new Response(JSON.stringify({
|
||||
success: false,
|
||||
error: "No comics found in this user's list. Make sure the user ID is correct and the list is not empty.",
|
||||
data: {
|
||||
comics: [],
|
||||
total: 0,
|
||||
limit: validLimit,
|
||||
userId: cleanUserId
|
||||
}
|
||||
}), {
|
||||
status: 200,
|
||||
status: 404,
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
});
|
||||
|
||||
// Clear the timeout to prevent memory leaks
|
||||
clearTimeout(timeoutId);
|
||||
return response;
|
||||
}
|
||||
|
||||
console.log(`Returning ${totalComics} comics found for user ${cleanUserId}`);
|
||||
|
||||
return new Response(JSON.stringify({ comics, totalComics }), {
|
||||
const response = new Response(JSON.stringify({
|
||||
success: true,
|
||||
data: {
|
||||
comics,
|
||||
total: totalComics,
|
||||
limit: validLimit,
|
||||
userId: cleanUserId
|
||||
}
|
||||
}), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'max-age=3600' // Cache for 1 hour
|
||||
}
|
||||
});
|
||||
|
||||
// Clear the timeout to prevent memory leaks
|
||||
clearTimeout(timeoutId);
|
||||
return response;
|
||||
} catch (error) {
|
||||
console.error('Error fetching user comics:', error);
|
||||
|
||||
|
|
@ -252,8 +285,12 @@ export const GET: APIRoute = async ({ request }) => {
|
|||
}
|
||||
}
|
||||
|
||||
// Clear the timeout to prevent memory leaks
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
details: error instanceof Error ? error.message : String(error)
|
||||
}), {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue