import json import logging import sys import os import time from typing import List, Dict, Any, Optional from enum import Enum import cloudscraper import requests # Keep requests for RequestException if needed from bs4 import BeautifulSoup from fastapi import FastAPI, Query, HTTPException from fastapi.concurrency import run_in_threadpool import uvicorn # --- Logging Setup --- logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[logging.StreamHandler(sys.stdout)] ) logger = logging.getLogger(__name__) # Define Enum for sort order validation (can be used by both scraper and API) class SortOrder(str, Enum): asc = "asc" desc = "desc" # --- Scraper Function (Now handles sorting BEFORE limiting) --- def scrape_comic_list(user_id: str, limit: int = 10, sort_order: SortOrder = SortOrder.desc) -> List[Dict[str, Any]]: """ Scrape comic list from ComicK for a given user using cloudscraper. Sorts the comics based on the 'user_read_at' timestamp according to sort_order, then applies the limit. Returns a list of dictionaries, each containing comic slug, user's last read chapter data, the latest available chapter number for the comic, and the timestamp when the user last read a chapter of this comic. Raises exceptions on failure. """ url = f"https://comick.io/user/{user_id}/list" logger.info(f"Attempting to scrape comics for user: {user_id}. Sorting by 'read_at' {sort_order.value} before applying limit: {limit}") try: start_time = time.time() scraper = cloudscraper.create_scraper( browser={ 'browser': 'chrome', 'platform': 'windows', 'mobile': False }, delay=5 ) response = scraper.get(url, timeout=30) request_time = time.time() - start_time logger.info(f"Request URL: {response.url}") logger.info(f"Status Code: {response.status_code}") logger.info(f"Request Time: {request_time:.2f} seconds") if response.status_code != 200: logger.error(f"Comick request failed for user {user_id} with status code {response.status_code}: {response.text[:500]}...") response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') next_data_script = soup.find('script', {'id': '__NEXT_DATA__'}) if not next_data_script: logger.error(f"__NEXT_DATA__ script not found for user {user_id}. HTML structure might have changed or scraping was blocked.") raise ValueError("__NEXT_DATA__ script not found. Scraping likely blocked or page structure changed.") try: next_data = json.loads(next_data_script.string) except json.JSONDecodeError as e: logger.error(f"Error parsing JSON from __NEXT_DATA__ for user {user_id}: {e}") logger.debug(f"__NEXT_DATA__ content (start): {next_data_script.string[:500]}...") raise ValueError(f"Failed to parse internal page data: {e}") from e page_props = next_data.get('props', {}).get('pageProps', {}) # Extract the 'follows' list directly, as provided in the example JSON follows = page_props.get('follows') if not isinstance(follows, list): # Fallback: Try the structure found previously if 'follows' isn't directly under pageProps follows_data = page_props.get('data', {}) if isinstance(follows_data, dict): follows = follows_data.get('follows', []) else: follows = [] # Ensure follows is a list if data is not a dict if not isinstance(follows, list): logger.error(f"Expected 'follows' data to be a list for user {user_id}, but got: {type(follows)}. Data structure might be unexpected. Data: {str(follows)[:200]}") raise TypeError(f"'follows' data is not a list. Found type: {type(follows)}") # --- Extract ALL followed comics data first --- all_comics_data = [] for follow in follows: if isinstance(follow, dict): comic_info = follow.get('md_comics') user_last_read_chapter_details = follow.get('md_chapters') # 'read_at' is directly available in the 'follow' object read_at_timestamp = follow.get('read_at') # ISO 8601 string or None if isinstance(comic_info, dict) and comic_info.get('slug'): slug = comic_info['slug'] comic_latest_chapter_number = comic_info.get('last_chapter') all_comics_data.append({ "slug": slug, "user_last_read_chapter": user_last_read_chapter_details, "comic_latest_chapter": comic_latest_chapter_number, "user_read_at": read_at_timestamp }) else: logger.warning(f"Skipping follow item for user {user_id} due to missing 'md_comics' or 'slug'. Item: {str(follow)[:100]}...") else: logger.warning(f"Expected a dictionary in 'follows' list for user {user_id}, but got: {type(follow)}") logger.info(f"Extracted {len(all_comics_data)} followed comics entries for user {user_id}.") # --- Sort the ENTIRE list based on 'user_read_at' --- if all_comics_data: logger.debug(f"Sorting {len(all_comics_data)} comics for user {user_id} by 'user_read_at' in {sort_order.value} order.") reverse_order = (sort_order == SortOrder.desc) try: # Key function: Treat None as an empty string '' which sorts before ISO timestamps. # None comes first in 'asc', last in 'desc'. sorted_comics_data = sorted( all_comics_data, key=lambda item: item.get('user_read_at') or "", reverse=reverse_order ) logger.debug(f"Sorting complete for user {user_id}.") except TypeError as sort_error: logger.error(f"Error during sorting comics for user {user_id}: {sort_error}. Proceeding with unsorted data before limiting.") # Use the unsorted list if sorting fails, but still apply limit sorted_comics_data = all_comics_data else: sorted_comics_data = [] # Start with empty list if no data extracted # --- Apply limit AFTER sorting --- limited_comics_data = sorted_comics_data[:limit] logger.info(f"Returning {len(limited_comics_data)} comics for user {user_id} after sorting and applying limit {limit}.") return limited_comics_data # --- Exception Handling (no changes needed here) --- except requests.exceptions.HTTPError as e: raise e # Re-raise for API handler except requests.exceptions.RequestException as e: logger.error(f"Network or request failed for user {user_id}: {e}") if hasattr(e, 'response') and e.response is not None: logger.error(f"Response status: {e.response.status_code}") logger.error(f"Response text: {e.response.text[:500]}...") raise ConnectionError(f"Request failed: {e}") from e except (ValueError, TypeError) as e: # Catch JSON parsing or type errors from extraction/sorting raise e # Re-raise for API handler except Exception as e: logger.exception(f"An unexpected error occurred during scraping for user {user_id}: {e}") raise RuntimeError(f"An unexpected scraping error occurred: {e}") from e # --- FastAPI Application --- app = FastAPI( title="Comick User List Scraper API", description="API to scrape the public comic reading list for a given Comick user ID. Data is sorted by last read time *before* limiting.", version="1.5.0" # Bump version ) @app.get("/scrape/{user_id}") async def get_user_comics( user_id: str, limit: int = Query(10, ge=1, le=100, description="Maximum number of comic entries to return (1-100). Applied *after* sorting."), sort_order: SortOrder = Query(SortOrder.desc, description="Sort order for comics based on 'user_read_at' timestamp *before* limiting. 'asc' for oldest first, 'desc' for newest first. Comics never read ('user_read_at' is null) appear first in 'asc', last in 'desc'.") ): """ Scrapes the public comic list for the given Comick **user_id**. The list is **sorted** by the `user_read_at` timestamp according to the `sort_order` parameter *before* the `limit` is applied. Returns a list of comics, each containing: - **slug**: The unique identifier slug for the comic. - **user_last_read_chapter**: Information about the last chapter read *by the user* (can be `null` or an object). - **comic_latest_chapter**: The latest chapter number *available for the comic series* (can be a number or `null`). - **user_read_at**: The timestamp (ISO 8601 string) when the user last marked a chapter as read (can be `null`). Parameters: - **user_id**: The unique identifier for the user on comick.io. - **limit**: Optional query parameter to limit the number of results (applied *after* sorting). Defaults to 10, max 100. - **sort_order**: Optional query parameter to sort results by `user_read_at` before limiting. Defaults to `desc` (newest first). Use `asc` for oldest first. """ logger.info(f"API request received for user: {user_id}, limit: {limit}, sort_order: {sort_order.value}") try: # Run the synchronous scraper function in a thread pool # Pass the sort_order down to the scraper function comics_details = await run_in_threadpool( scrape_comic_list, user_id=user_id, limit=limit, sort_order=sort_order # Pass the enum value directly ) # The comics_details list is now already sorted and limited by the scraper function # Return success response return { "success": True, "data": { "user_id": user_id, "limit": limit, "sort_order": sort_order.value, "comics": comics_details # This list is already sorted and limited } } # --- Exception Handling (no changes needed here, covers errors from scraper) --- except requests.exceptions.HTTPError as e: status_code = 502 # Bad Gateway detail = f"Failed to retrieve data from Comick for user '{user_id}'." if e.response is not None: if e.response.status_code == 404: status_code = 404 detail = f"Comick user '{user_id}' not found." elif e.response.status_code >= 500: status_code = 502 detail += f" Upstream server error (Status: {e.response.status_code})." elif e.response.status_code >= 400: # e.g., 403 Forbidden status_code = 502 # Treat as Bad Gateway from API perspective detail += f" Upstream client error (Status: {e.response.status_code}). Check if scraping is blocked." else: detail += f" Upstream status: {e.response.status_code}" logger.warning(f"API Error ({status_code}) for user {user_id}: {detail} - {e}") raise HTTPException(status_code=status_code, detail=detail) from e except (ValueError, TypeError) as e: # Catch potential errors from JSON parsing, type mismatches during extraction or sorting logger.error(f"API Error (500) for user {user_id}: Data processing/sorting failed within scraper. Reason: {str(e)}") raise HTTPException(status_code=500, detail=f"Internal server error during scraping data processing or sorting: {str(e)}") from e except ConnectionError as e: logger.error(f"API Error (503) for user {user_id}: Network error connecting to Comick. Reason: {str(e)}") raise HTTPException(status_code=503, detail=f"Service Unavailable: Could not connect to Comick. {str(e)}") from e except RuntimeError as e: # Catch-all for unexpected scraper errors logger.error(f"API Error (500) for user {user_id}: Unexpected scraping error. Reason: {str(e)}") raise HTTPException(status_code=500, detail=f"Internal server error during scraping: {str(e)}") from e except Exception as e: # Catch-all for unexpected API level errors logger.exception(f"Unexpected API error for user {user_id}: {e}") raise HTTPException(status_code=500, detail="An unexpected internal server error occurred.") from e @app.get("/", include_in_schema=False) async def root(): return {"message": "Comick Scraper API is running. Use /docs for documentation."} # --- Server Runner (unchanged) --- if __name__ == "__main__": host = os.getenv("HOST", "0.0.0.0") port = int(os.getenv("PORT", "8000")) is_development = os.getenv("ENVIRONMENT", "production").lower() == "development" log_level = "debug" if is_development else "info" logging.getLogger().setLevel(log_level.upper()) logger.info(f"Log level set to: {log_level.upper()}") print(f"Starting server on {host}:{port} (Development Mode: {is_development})") uvicorn.run( "main:app", # Assuming your file is named main.py host=host, port=port, reload=is_development, log_level=log_level )