Upload files to "/"
This commit is contained in:
commit
940d472843
5 changed files with 635 additions and 0 deletions
275
main.py
Normal file
275
main.py
Normal file
|
|
@ -0,0 +1,275 @@
|
|||
import json
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
from typing import List, Dict, Any, Optional
|
||||
from enum import Enum
|
||||
|
||||
import cloudscraper
|
||||
import requests # Keep requests for RequestException if needed
|
||||
from bs4 import BeautifulSoup
|
||||
from fastapi import FastAPI, Query, HTTPException
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
import uvicorn
|
||||
|
||||
# --- Logging Setup ---
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[logging.StreamHandler(sys.stdout)]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Define Enum for sort order validation (can be used by both scraper and API)
|
||||
class SortOrder(str, Enum):
|
||||
asc = "asc"
|
||||
desc = "desc"
|
||||
|
||||
# --- Scraper Function (Now handles sorting BEFORE limiting) ---
|
||||
def scrape_comic_list(user_id: str, limit: int = 10, sort_order: SortOrder = SortOrder.desc) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Scrape comic list from ComicK for a given user using cloudscraper.
|
||||
Sorts the comics based on the 'user_read_at' timestamp according to sort_order,
|
||||
then applies the limit.
|
||||
Returns a list of dictionaries, each containing comic slug, user's last read
|
||||
chapter data, the latest available chapter number for the comic, and the
|
||||
timestamp when the user last read a chapter of this comic.
|
||||
Raises exceptions on failure.
|
||||
"""
|
||||
url = f"https://comick.io/user/{user_id}/list"
|
||||
logger.info(f"Attempting to scrape comics for user: {user_id}. Sorting by 'read_at' {sort_order.value} before applying limit: {limit}")
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
scraper = cloudscraper.create_scraper(
|
||||
browser={
|
||||
'browser': 'chrome',
|
||||
'platform': 'windows',
|
||||
'mobile': False
|
||||
},
|
||||
delay=5
|
||||
)
|
||||
|
||||
response = scraper.get(url, timeout=30)
|
||||
request_time = time.time() - start_time
|
||||
|
||||
logger.info(f"Request URL: {response.url}")
|
||||
logger.info(f"Status Code: {response.status_code}")
|
||||
logger.info(f"Request Time: {request_time:.2f} seconds")
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.error(f"Comick request failed for user {user_id} with status code {response.status_code}: {response.text[:500]}...")
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
next_data_script = soup.find('script', {'id': '__NEXT_DATA__'})
|
||||
if not next_data_script:
|
||||
logger.error(f"__NEXT_DATA__ script not found for user {user_id}. HTML structure might have changed or scraping was blocked.")
|
||||
raise ValueError("__NEXT_DATA__ script not found. Scraping likely blocked or page structure changed.")
|
||||
|
||||
try:
|
||||
next_data = json.loads(next_data_script.string)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Error parsing JSON from __NEXT_DATA__ for user {user_id}: {e}")
|
||||
logger.debug(f"__NEXT_DATA__ content (start): {next_data_script.string[:500]}...")
|
||||
raise ValueError(f"Failed to parse internal page data: {e}") from e
|
||||
|
||||
page_props = next_data.get('props', {}).get('pageProps', {})
|
||||
# Extract the 'follows' list directly, as provided in the example JSON
|
||||
follows = page_props.get('follows')
|
||||
|
||||
if not isinstance(follows, list):
|
||||
# Fallback: Try the structure found previously if 'follows' isn't directly under pageProps
|
||||
follows_data = page_props.get('data', {})
|
||||
if isinstance(follows_data, dict):
|
||||
follows = follows_data.get('follows', [])
|
||||
else:
|
||||
follows = [] # Ensure follows is a list if data is not a dict
|
||||
|
||||
if not isinstance(follows, list):
|
||||
logger.error(f"Expected 'follows' data to be a list for user {user_id}, but got: {type(follows)}. Data structure might be unexpected. Data: {str(follows)[:200]}")
|
||||
raise TypeError(f"'follows' data is not a list. Found type: {type(follows)}")
|
||||
|
||||
# --- Extract ALL followed comics data first ---
|
||||
all_comics_data = []
|
||||
for follow in follows:
|
||||
if isinstance(follow, dict):
|
||||
comic_info = follow.get('md_comics')
|
||||
user_last_read_chapter_details = follow.get('md_chapters')
|
||||
# 'read_at' is directly available in the 'follow' object
|
||||
read_at_timestamp = follow.get('read_at') # ISO 8601 string or None
|
||||
|
||||
if isinstance(comic_info, dict) and comic_info.get('slug'):
|
||||
slug = comic_info['slug']
|
||||
comic_latest_chapter_number = comic_info.get('last_chapter')
|
||||
|
||||
all_comics_data.append({
|
||||
"slug": slug,
|
||||
"user_last_read_chapter": user_last_read_chapter_details,
|
||||
"comic_latest_chapter": comic_latest_chapter_number,
|
||||
"user_read_at": read_at_timestamp
|
||||
})
|
||||
else:
|
||||
logger.warning(f"Skipping follow item for user {user_id} due to missing 'md_comics' or 'slug'. Item: {str(follow)[:100]}...")
|
||||
else:
|
||||
logger.warning(f"Expected a dictionary in 'follows' list for user {user_id}, but got: {type(follow)}")
|
||||
|
||||
logger.info(f"Extracted {len(all_comics_data)} followed comics entries for user {user_id}.")
|
||||
|
||||
# --- Sort the ENTIRE list based on 'user_read_at' ---
|
||||
if all_comics_data:
|
||||
logger.debug(f"Sorting {len(all_comics_data)} comics for user {user_id} by 'user_read_at' in {sort_order.value} order.")
|
||||
reverse_order = (sort_order == SortOrder.desc)
|
||||
try:
|
||||
# Key function: Treat None as an empty string '' which sorts before ISO timestamps.
|
||||
# None comes first in 'asc', last in 'desc'.
|
||||
sorted_comics_data = sorted(
|
||||
all_comics_data,
|
||||
key=lambda item: item.get('user_read_at') or "",
|
||||
reverse=reverse_order
|
||||
)
|
||||
logger.debug(f"Sorting complete for user {user_id}.")
|
||||
except TypeError as sort_error:
|
||||
logger.error(f"Error during sorting comics for user {user_id}: {sort_error}. Proceeding with unsorted data before limiting.")
|
||||
# Use the unsorted list if sorting fails, but still apply limit
|
||||
sorted_comics_data = all_comics_data
|
||||
else:
|
||||
sorted_comics_data = [] # Start with empty list if no data extracted
|
||||
|
||||
# --- Apply limit AFTER sorting ---
|
||||
limited_comics_data = sorted_comics_data[:limit]
|
||||
|
||||
logger.info(f"Returning {len(limited_comics_data)} comics for user {user_id} after sorting and applying limit {limit}.")
|
||||
return limited_comics_data
|
||||
|
||||
# --- Exception Handling (no changes needed here) ---
|
||||
except requests.exceptions.HTTPError as e:
|
||||
raise e # Re-raise for API handler
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Network or request failed for user {user_id}: {e}")
|
||||
if hasattr(e, 'response') and e.response is not None:
|
||||
logger.error(f"Response status: {e.response.status_code}")
|
||||
logger.error(f"Response text: {e.response.text[:500]}...")
|
||||
raise ConnectionError(f"Request failed: {e}") from e
|
||||
except (ValueError, TypeError) as e: # Catch JSON parsing or type errors from extraction/sorting
|
||||
raise e # Re-raise for API handler
|
||||
except Exception as e:
|
||||
logger.exception(f"An unexpected error occurred during scraping for user {user_id}: {e}")
|
||||
raise RuntimeError(f"An unexpected scraping error occurred: {e}") from e
|
||||
|
||||
|
||||
# --- FastAPI Application ---
|
||||
app = FastAPI(
|
||||
title="Comick User List Scraper API",
|
||||
description="API to scrape the public comic reading list for a given Comick user ID. Data is sorted by last read time *before* limiting.",
|
||||
version="1.5.0" # Bump version
|
||||
)
|
||||
|
||||
@app.get("/scrape/{user_id}")
|
||||
async def get_user_comics(
|
||||
user_id: str,
|
||||
limit: int = Query(10, ge=1, le=100, description="Maximum number of comic entries to return (1-100). Applied *after* sorting."),
|
||||
sort_order: SortOrder = Query(SortOrder.desc, description="Sort order for comics based on 'user_read_at' timestamp *before* limiting. 'asc' for oldest first, 'desc' for newest first. Comics never read ('user_read_at' is null) appear first in 'asc', last in 'desc'.")
|
||||
):
|
||||
"""
|
||||
Scrapes the public comic list for the given Comick **user_id**.
|
||||
|
||||
The list is **sorted** by the `user_read_at` timestamp according to the `sort_order` parameter *before* the `limit` is applied.
|
||||
|
||||
Returns a list of comics, each containing:
|
||||
- **slug**: The unique identifier slug for the comic.
|
||||
- **user_last_read_chapter**: Information about the last chapter read *by the user* (can be `null` or an object).
|
||||
- **comic_latest_chapter**: The latest chapter number *available for the comic series* (can be a number or `null`).
|
||||
- **user_read_at**: The timestamp (ISO 8601 string) when the user last marked a chapter as read (can be `null`).
|
||||
|
||||
Parameters:
|
||||
- **user_id**: The unique identifier for the user on comick.io.
|
||||
- **limit**: Optional query parameter to limit the number of results (applied *after* sorting). Defaults to 10, max 100.
|
||||
- **sort_order**: Optional query parameter to sort results by `user_read_at` before limiting. Defaults to `desc` (newest first). Use `asc` for oldest first.
|
||||
"""
|
||||
logger.info(f"API request received for user: {user_id}, limit: {limit}, sort_order: {sort_order.value}")
|
||||
try:
|
||||
# Run the synchronous scraper function in a thread pool
|
||||
# Pass the sort_order down to the scraper function
|
||||
comics_details = await run_in_threadpool(
|
||||
scrape_comic_list,
|
||||
user_id=user_id,
|
||||
limit=limit,
|
||||
sort_order=sort_order # Pass the enum value directly
|
||||
)
|
||||
|
||||
# The comics_details list is now already sorted and limited by the scraper function
|
||||
|
||||
# Return success response
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"user_id": user_id,
|
||||
"limit": limit,
|
||||
"sort_order": sort_order.value,
|
||||
"comics": comics_details # This list is already sorted and limited
|
||||
}
|
||||
}
|
||||
|
||||
# --- Exception Handling (no changes needed here, covers errors from scraper) ---
|
||||
except requests.exceptions.HTTPError as e:
|
||||
status_code = 502 # Bad Gateway
|
||||
detail = f"Failed to retrieve data from Comick for user '{user_id}'."
|
||||
if e.response is not None:
|
||||
if e.response.status_code == 404:
|
||||
status_code = 404
|
||||
detail = f"Comick user '{user_id}' not found."
|
||||
elif e.response.status_code >= 500:
|
||||
status_code = 502
|
||||
detail += f" Upstream server error (Status: {e.response.status_code})."
|
||||
elif e.response.status_code >= 400: # e.g., 403 Forbidden
|
||||
status_code = 502 # Treat as Bad Gateway from API perspective
|
||||
detail += f" Upstream client error (Status: {e.response.status_code}). Check if scraping is blocked."
|
||||
else:
|
||||
detail += f" Upstream status: {e.response.status_code}"
|
||||
|
||||
logger.warning(f"API Error ({status_code}) for user {user_id}: {detail} - {e}")
|
||||
raise HTTPException(status_code=status_code, detail=detail) from e
|
||||
|
||||
except (ValueError, TypeError) as e:
|
||||
# Catch potential errors from JSON parsing, type mismatches during extraction or sorting
|
||||
logger.error(f"API Error (500) for user {user_id}: Data processing/sorting failed within scraper. Reason: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Internal server error during scraping data processing or sorting: {str(e)}") from e
|
||||
|
||||
except ConnectionError as e:
|
||||
logger.error(f"API Error (503) for user {user_id}: Network error connecting to Comick. Reason: {str(e)}")
|
||||
raise HTTPException(status_code=503, detail=f"Service Unavailable: Could not connect to Comick. {str(e)}") from e
|
||||
|
||||
except RuntimeError as e: # Catch-all for unexpected scraper errors
|
||||
logger.error(f"API Error (500) for user {user_id}: Unexpected scraping error. Reason: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail=f"Internal server error during scraping: {str(e)}") from e
|
||||
|
||||
except Exception as e: # Catch-all for unexpected API level errors
|
||||
logger.exception(f"Unexpected API error for user {user_id}: {e}")
|
||||
raise HTTPException(status_code=500, detail="An unexpected internal server error occurred.") from e
|
||||
|
||||
|
||||
@app.get("/", include_in_schema=False)
|
||||
async def root():
|
||||
return {"message": "Comick Scraper API is running. Use /docs for documentation."}
|
||||
|
||||
|
||||
# --- Server Runner (unchanged) ---
|
||||
if __name__ == "__main__":
|
||||
host = os.getenv("HOST", "0.0.0.0")
|
||||
port = int(os.getenv("PORT", "8000"))
|
||||
is_development = os.getenv("ENVIRONMENT", "production").lower() == "development"
|
||||
log_level = "debug" if is_development else "info"
|
||||
|
||||
logging.getLogger().setLevel(log_level.upper())
|
||||
logger.info(f"Log level set to: {log_level.upper()}")
|
||||
|
||||
print(f"Starting server on {host}:{port} (Development Mode: {is_development})")
|
||||
uvicorn.run(
|
||||
"main:app", # Assuming your file is named main.py
|
||||
host=host,
|
||||
port=port,
|
||||
reload=is_development,
|
||||
log_level=log_level
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue