Compare commits

..

9 commits

Author SHA1 Message Date
ld3zian
89d09b45bb try edge 2025-03-04 18:41:46 -07:00
ld3zian
a5d10f26d5 idk anymore... 2025-03-04 18:37:47 -07:00
ld3zian
24eeb93f68 does this work? 2025-03-04 18:34:18 -07:00
ld3zian
8271bc4957 Revert "add cf maybe"
This reverts commit f79c45e5c3.
2025-03-04 18:23:52 -07:00
ld3zian
c51e6927dc Revert "revert changes"
This reverts commit 45a5b8776d.
2025-03-04 18:22:52 -07:00
ld3zian
43377803be Merge branch 'main' of https://github.com/ld3z/new-manga 2025-03-04 17:13:25 -07:00
ld3zian
45a5b8776d revert changes 2025-03-04 17:13:21 -07:00
ld3zian
f79c45e5c3 add cf maybe 2025-03-04 16:50:15 -07:00
ld3zian
80e7fe5c92 add more error logging for importing and vercel db improvements 2025-03-04 16:41:18 -07:00
7 changed files with 228 additions and 93 deletions

View file

@ -4,7 +4,11 @@ import tailwind from "@astrojs/tailwind";
export default defineConfig({ export default defineConfig({
output: "server", output: "server",
adapter: vercel(), adapter: vercel({
analytics: true,
maxDuration: 60,
runtime: 'edge',
}),
image: { image: {
remotePatterns: [{ protocol: "https" }], remotePatterns: [{ protocol: "https" }],
}, },
@ -14,7 +18,13 @@ export default defineConfig({
'process.env.PRIMARY_REDIS_URL': JSON.stringify(process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL), 'process.env.PRIMARY_REDIS_URL': JSON.stringify(process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL),
'process.env.REPLICA_REDIS_URL_1': JSON.stringify(process.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL), 'process.env.REPLICA_REDIS_URL_1': JSON.stringify(process.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL),
'process.env.REPLICA_REDIS_URL_2': JSON.stringify(process.env.REPLICA_REDIS_URL_2), 'process.env.REPLICA_REDIS_URL_2': JSON.stringify(process.env.REPLICA_REDIS_URL_2),
'process.env.REPLICA_REDIS_URL_3': JSON.stringify(process.env.REPLICA_REDIS_URL_3) 'process.env.REPLICA_REDIS_URL_3': JSON.stringify(process.env.REPLICA_REDIS_URL_3),
} 'process.env.VERCEL': JSON.stringify(process.env.VERCEL || false)
},
server: {
watch: {
usePolling: true,
},
},
} }
}); });

18
package-lock.json generated
View file

@ -14,7 +14,6 @@
"@floating-ui/dom": "^1.6.13", "@floating-ui/dom": "^1.6.13",
"@types/node": "^22.13.9", "@types/node": "^22.13.9",
"@vercel/analytics": "^1.5.0", "@vercel/analytics": "^1.5.0",
"@vercel/kv": "^3.0.0",
"astro": "^5.4.1", "astro": "^5.4.1",
"feed": "^4.2.2", "feed": "^4.2.2",
"ioredis": "^5.5.0", "ioredis": "^5.5.0",
@ -1486,6 +1485,8 @@
"version": "1.34.3", "version": "1.34.3",
"resolved": "https://registry.npmjs.org/@upstash/redis/-/redis-1.34.3.tgz", "resolved": "https://registry.npmjs.org/@upstash/redis/-/redis-1.34.3.tgz",
"integrity": "sha512-VT25TyODGy/8ljl7GADnJoMmtmJ1F8d84UXfGonRRF8fWYJz7+2J6GzW+a6ETGtk4OyuRTt7FRSvFG5GvrfSdQ==", "integrity": "sha512-VT25TyODGy/8ljl7GADnJoMmtmJ1F8d84UXfGonRRF8fWYJz7+2J6GzW+a6ETGtk4OyuRTt7FRSvFG5GvrfSdQ==",
"optional": true,
"peer": true,
"dependencies": { "dependencies": {
"crypto-js": "^4.2.0" "crypto-js": "^4.2.0"
} }
@ -1532,17 +1533,6 @@
"resolved": "https://registry.npmjs.org/@vercel/edge/-/edge-1.2.1.tgz", "resolved": "https://registry.npmjs.org/@vercel/edge/-/edge-1.2.1.tgz",
"integrity": "sha512-1++yncEyIAi68D3UEOlytYb1IUcIulMWdoSzX2h9LuSeeyR7JtaIgR8DcTQ6+DmYOQn+5MCh6LY+UmK6QBByNA==" "integrity": "sha512-1++yncEyIAi68D3UEOlytYb1IUcIulMWdoSzX2h9LuSeeyR7JtaIgR8DcTQ6+DmYOQn+5MCh6LY+UmK6QBByNA=="
}, },
"node_modules/@vercel/kv": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@vercel/kv/-/kv-3.0.0.tgz",
"integrity": "sha512-pKT8fRnfyYk2MgvyB6fn6ipJPCdfZwiKDdw7vB+HL50rjboEBHDVBEcnwfkEpVSp2AjNtoaOUH7zG+bVC/rvSg==",
"dependencies": {
"@upstash/redis": "^1.34.0"
},
"engines": {
"node": ">=14.6"
}
},
"node_modules/@vercel/nft": { "node_modules/@vercel/nft": {
"version": "0.29.1", "version": "0.29.1",
"resolved": "https://registry.npmjs.org/@vercel/nft/-/nft-0.29.1.tgz", "resolved": "https://registry.npmjs.org/@vercel/nft/-/nft-0.29.1.tgz",
@ -2383,7 +2373,9 @@
"node_modules/crypto-js": { "node_modules/crypto-js": {
"version": "4.2.0", "version": "4.2.0",
"resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz",
"integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==",
"optional": true,
"peer": true
}, },
"node_modules/cssesc": { "node_modules/cssesc": {
"version": "3.0.0", "version": "3.0.0",

View file

@ -17,7 +17,6 @@
"@floating-ui/dom": "^1.6.13", "@floating-ui/dom": "^1.6.13",
"@types/node": "^22.13.9", "@types/node": "^22.13.9",
"@vercel/analytics": "^1.5.0", "@vercel/analytics": "^1.5.0",
"@vercel/kv": "^3.0.0",
"astro": "^5.4.1", "astro": "^5.4.1",
"feed": "^4.2.2", "feed": "^4.2.2",
"ioredis": "^5.5.0", "ioredis": "^5.5.0",

View file

@ -479,7 +479,23 @@ const languages = [
if (!response.ok) { if (!response.ok) {
const errorText = await response.text(); const errorText = await response.text();
console.error(`Feed import failed with status ${response.status}: ${errorText}`); console.error(`Feed import failed with status ${response.status}: ${errorText}`);
throw new Error(`Failed to fetch feed: ${response.status} ${response.statusText}`);
// Display a more user-friendly error based on the status code
let userFriendlyError = 'Failed to import feed.';
if (response.status === 404) {
userFriendlyError = `Feed ID "${feedId}" not found. Please check the ID and try again.`;
} else if (response.status === 500) {
if (errorText.includes('ETIMEDOUT') || errorText.includes('timeout')) {
userFriendlyError = 'Connection to database timed out. This is common on the first request. Please try again in a few seconds.';
} else if (errorText.includes('ECONNREFUSED')) {
userFriendlyError = 'Database connection refused. Please try again later.';
} else {
userFriendlyError = 'Server error while importing feed. Please try again later.';
}
}
throw new Error(userFriendlyError);
} }
const data = await response.json(); const data = await response.json();

View file

@ -2,9 +2,21 @@ import Redis from 'ioredis';
import type { FeedMapping } from './types'; import type { FeedMapping } from './types';
import { getChaptersForSlugs } from './api'; import { getChaptersForSlugs } from './api';
// Primary Redis configuration // Extended Error interface for network errors
interface RedisNetworkError extends Error {
code?: string;
}
// Primary Redis configuration - with more robust fallbacks and logging
const PRIMARY_REDIS_URL = import.meta.env.PRIMARY_REDIS_URL || process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL; const PRIMARY_REDIS_URL = import.meta.env.PRIMARY_REDIS_URL || process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL;
// Log Redis URL status (without exposing sensitive data)
console.log(`Redis URLs status: Primary=${Boolean(PRIMARY_REDIS_URL)}`);
if (!PRIMARY_REDIS_URL) {
console.warn('WARNING: No Redis URL configured. Database functionality will not work properly.');
}
// Multiple replicas support // Multiple replicas support
const REPLICA_URLS = [ const REPLICA_URLS = [
import.meta.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL_1 || import.meta.env.REPLICA_REDIS_URL || process.env.REPLICA_REDIS_URL, import.meta.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL_1 || import.meta.env.REPLICA_REDIS_URL || process.env.REPLICA_REDIS_URL,
@ -12,6 +24,8 @@ const REPLICA_URLS = [
import.meta.env.REPLICA_REDIS_URL_3 || process.env.REPLICA_REDIS_URL_3, import.meta.env.REPLICA_REDIS_URL_3 || process.env.REPLICA_REDIS_URL_3,
].filter(Boolean) as string[]; ].filter(Boolean) as string[];
console.log(`Found ${REPLICA_URLS.length} Redis replica URLs`);
// Create a mapping of URLs to friendly names for logging // Create a mapping of URLs to friendly names for logging
const redisInstanceNames = new Map<string, string>(); const redisInstanceNames = new Map<string, string>();
@ -39,26 +53,43 @@ let replicaClients: Map<string, Redis> = new Map();
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
const MAX_RETRIES = 5; const MAX_RETRIES = 5;
const BASE_DELAY_MS = 100; const BASE_DELAY_MS = 100;
const IS_VERCEL = process.env.VERCEL || import.meta.env.VERCEL || false;
const VERCEL_CONNECTION_TIMEOUT = 25000; // 25 seconds for Vercel
const LOCAL_CONNECTION_TIMEOUT = 5000; // 5 seconds for local
// Improved connection handling with exponential backoff // Improved connection handling with exponential backoff
async function createRedisClient(url: string): Promise<Redis> { async function createRedisClient(url: string): Promise<Redis> {
if (!url) throw new Error('Redis URL is not set'); if (!url) throw new Error('Redis URL is not set');
const redisName = getRedisName(url); const redisName = getRedisName(url);
console.log(`Connecting to ${redisName} Redis instance in ${IS_VERCEL ? 'Vercel' : 'local'} environment`);
// Parse URL to extract authentication info // Parse URL to extract authentication info
const options = { const options = {
retryStrategy: (times) => { retryStrategy: (times) => {
if (times >= MAX_RETRIES) return null; if (times >= MAX_RETRIES) {
console.error(`${redisName} max retries (${MAX_RETRIES}) reached, giving up`);
return null;
}
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000); const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000);
console.log(`${redisName} retry attempt ${times}, next try in ${delay}ms`); console.log(`${redisName} retry attempt ${times}, next try in ${delay}ms`);
return delay + Math.random() * 200; // Add jitter return delay + Math.random() * 200; // Add jitter
}, },
maxRetriesPerRequest: null, // Allow retries for queued commands maxRetriesPerRequest: 3, // Limit retries for queued commands
enableOfflineQueue: true, // Enable command queuing enableOfflineQueue: true, // Enable command queuing
enableReadyCheck: true, enableReadyCheck: true,
connectTimeout: IS_VERCEL ? VERCEL_CONNECTION_TIMEOUT : LOCAL_CONNECTION_TIMEOUT,
disconnectTimeout: 10000, // Increase disconnect timeout to 10 seconds
keepAlive: 10000, // Send keepalive every 10 seconds
commandTimeout: 15000, // Command timeout
retryMaxDelay: 5000, // Maximum delay between retries
reconnectOnError: (err) => { reconnectOnError: (err) => {
return err.message.includes('READONLY'); const targetErrors = ['READONLY', 'ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND'];
const shouldReconnect = targetErrors.some(e => err.message.includes(e));
if (shouldReconnect) {
console.log(`Reconnecting due to error: ${err.message}`);
}
return shouldReconnect;
} }
}; };
@ -67,68 +98,65 @@ async function createRedisClient(url: string): Promise<Redis> {
// Get the best available Redis client // Get the best available Redis client
export async function getRedisClient(): Promise<Redis> { export async function getRedisClient(): Promise<Redis> {
// Try primary first // Special handling for Vercel environment
if (primaryClient?.status === 'ready') { if (IS_VERCEL) {
try { console.log('Running in Vercel environment - using optimized Redis connection strategy');
await primaryClient.ping(); }
return primaryClient;
} catch (error) { // Check if Redis is configured
console.warn('Primary Redis connection failed, will try replicas:', error); if (!PRIMARY_REDIS_URL && REPLICA_URLS.length === 0) {
// Fall through to try replicas throw new Error('No Redis URLs configured. Please check your environment variables in the Vercel dashboard.');
}
}
// Try replicas in sequence
for (const [url, client] of replicaClients.entries()) {
if (client.status === 'ready') {
try {
await client.ping();
console.log(`Using ${getRedisName(url)}`);
return client;
} catch (error) {
console.warn(`${getRedisName(url)} connection failed:`, error);
// Continue to next replica
}
}
} }
// Reconnect to primary
try { try {
console.log('Initializing new primary Redis connection...'); // Try primary first
primaryClient = await createRedisClient(PRIMARY_REDIS_URL!); if (!primaryClient) {
console.log('Creating new primary Redis client');
primaryClient if (!PRIMARY_REDIS_URL) {
.on('ready', () => console.log('Primary Redis connection ready')) throw new Error('Primary Redis URL is not set. Please check your environment variables.');
.on('error', (err) => console.error('Primary Redis error:', err)) }
.on('reconnecting', () => console.log('Primary Redis reconnecting')) primaryClient = await createRedisClient(PRIMARY_REDIS_URL);
.on('end', () => console.log('Primary Redis connection closed')); } else if (primaryClient.status !== 'ready') {
console.log('Primary Redis client not ready, reconnecting');
return primaryClient;
} catch (error) {
console.error('Failed to connect to primary Redis:', error);
// Try connecting to any replica that might be available
for (const url of REPLICA_URLS) {
try { try {
const redisName = getRedisName(url); await primaryClient.disconnect();
console.log(`Trying to connect to ${redisName}...`); } catch (e) {
const client = await createRedisClient(url); console.error('Error disconnecting primary client:', e);
}
client primaryClient = await createRedisClient(PRIMARY_REDIS_URL);
.on('ready', () => console.log(`${redisName} connection ready`)) }
.on('error', (err) => console.error(`${redisName} error:`, err))
.on('reconnecting', () => console.log(`${redisName} reconnecting`)) try {
.on('end', () => console.log(`${redisName} connection closed`)); // Add timeout to ping
const pingPromise = primaryClient.ping();
replicaClients.set(url, client); const timeoutPromise = new Promise((_, reject) =>
return client; setTimeout(() => reject(new Error('Ping timeout')), 5000)
} catch (replicaError) { );
console.error(`Failed to connect to ${getRedisName(url)}:`, replicaError);
// Continue to next replica await Promise.race([pingPromise, timeoutPromise]);
return primaryClient;
} catch (error) {
console.error('Primary Redis client ping failed:', error);
throw error;
}
} catch (primaryError) {
console.error('Error connecting to primary Redis:', primaryError);
// Special handling for Vercel ETIMEDOUT errors
if (IS_VERCEL && primaryError instanceof Error) {
const errorMsg = primaryError.message;
if (errorMsg.includes('ETIMEDOUT')) {
console.log('Encountered ETIMEDOUT in Vercel environment. This is a common issue with Vercel serverless functions and Redis.');
throw new Error('Redis connection timed out. Please ensure your Redis service allows connections from Vercel IP ranges. See Vercel documentation for IP ranges used by their serverless functions.');
} else if (errorMsg.includes('ECONNREFUSED')) {
throw new Error('Redis connection refused. Please check if your Redis service is running and accessible from Vercel.');
} else if (errorMsg.includes('ENOTFOUND')) {
throw new Error('Redis host not found. Please check your PRIMARY_REDIS_URL environment variable.');
} }
} }
throw new Error('All Redis connections failed'); throw primaryError;
} }
} }
@ -249,24 +277,57 @@ export async function warmChapterCache(slugs: string[], lang: string): Promise<v
} }
export async function storeFeedMapping(feedId: string, slugs: string[], lang: string): Promise<void> { export async function storeFeedMapping(feedId: string, slugs: string[], lang: string): Promise<void> {
if (!feedId) throw new Error('Feed ID is required');
if (!Array.isArray(slugs) || slugs.length === 0) throw new Error('Slugs array is required and cannot be empty');
if (!lang) throw new Error('Language is required');
const mapping = { const mapping = {
slugs, slugs,
lang, lang,
created_at: new Date().toISOString() created_at: new Date().toISOString()
}; };
console.log(`Attempting to store feed mapping with ID: ${feedId}, language: ${lang}, and ${slugs.length} slugs`);
// Check if Redis is configured
if (!PRIMARY_REDIS_URL && REPLICA_URLS.length === 0) {
console.error('No Redis URLs configured. Cannot store feed mapping.');
throw new Error('Database connection not configured. Please check your environment variables.');
}
try { try {
await writeToAllRedis(async (redis) => { await writeToAllRedis(async (redis) => {
await redis.set( console.log(`Writing feed mapping to Redis instance...`);
feedId,
JSON.stringify(mapping), try {
'EX', await redis.set(
DEFAULT_CACHE_TTL feedId,
); JSON.stringify(mapping),
'EX',
DEFAULT_CACHE_TTL
);
console.log(`Successfully stored feed mapping in Redis instance`);
} catch (redisError: any) {
console.error(`Redis set operation failed:`, redisError);
throw new Error(`Redis operation failed: ${redisError.message || 'Unknown Redis error'}`);
}
}); });
} catch (error) {
console.log(`Feed mapping successfully stored with ID: ${feedId}`);
} catch (error: any) {
console.error('Error storing feed mapping:', error); console.error('Error storing feed mapping:', error);
throw new Error('Failed to store feed mapping');
// Check for common connection issues
const errorMsg = error.message || '';
if (errorMsg.includes('ECONNREFUSED')) {
throw new Error('Unable to connect to the Redis database. Connection refused.');
} else if (errorMsg.includes('ETIMEDOUT')) {
throw new Error('Connection to Redis database timed out. This may be due to network issues or firewall settings.');
} else if (errorMsg.includes('ENOTFOUND')) {
throw new Error('Redis host not found. Please check your database URL configuration.');
} else {
throw new Error(`Failed to store feed mapping: ${errorMsg}`);
}
} }
} }

View file

@ -7,18 +7,52 @@ const feedMappings = new Map<string, { slugs: string[], lang: string }>();
export const POST: APIRoute = async ({ request }) => { export const POST: APIRoute = async ({ request }) => {
try { try {
const { slugs, lang } = await request.json(); console.log("Handling create-feed API request");
if (!request.body) {
console.error("Request has no body");
return new Response("Request body is required", { status: 400 });
}
let body;
try {
body = await request.json();
} catch (parseError) {
console.error("Failed to parse request JSON:", parseError);
return new Response("Invalid JSON in request body", { status: 400 });
}
const { slugs, lang } = body;
if (!Array.isArray(slugs) || slugs.length === 0) { if (!Array.isArray(slugs) || slugs.length === 0) {
console.error("Invalid slugs array:", slugs);
return new Response("Invalid slugs array", { status: 400 }); return new Response("Invalid slugs array", { status: 400 });
} }
if (!lang) {
console.error("Language not specified");
return new Response("Language is required", { status: 400 });
}
console.log(`Creating feed for ${slugs.length} slugs in language: ${lang}`);
// Create a unique hash based on slugs and language // Create a unique hash based on slugs and language
const dataString = `${slugs.sort().join(',')}:${lang}`; const dataString = `${slugs.sort().join(',')}:${lang}`;
const hash = createHash('md5').update(dataString).digest('hex').slice(0, 8); const hash = createHash('md5').update(dataString).digest('hex').slice(0, 8);
// Store the mapping in Vercel KV try {
await storeFeedMapping(hash, slugs, lang); // Store the mapping in Redis
await storeFeedMapping(hash, slugs, lang);
console.log(`Successfully created feed with ID: ${hash}`);
} catch (dbError: any) {
console.error("Database error while storing feed mapping:", dbError);
return new Response(`Database error: ${dbError.message || "Unknown error"}`, {
status: 500,
headers: {
'Content-Type': 'application/json'
}
});
}
return new Response(JSON.stringify({ feedId: hash }), { return new Response(JSON.stringify({ feedId: hash }), {
status: 200, status: 200,
@ -26,9 +60,18 @@ export const POST: APIRoute = async ({ request }) => {
'Content-Type': 'application/json' 'Content-Type': 'application/json'
} }
}); });
} catch (error) { } catch (error: any) {
console.error('Error creating feed:', error); console.error('Unexpected error creating feed:', error);
return new Response("Internal server error", { status: 500 }); return new Response(JSON.stringify({
error: "Internal server error",
message: error.message || "Unknown error",
stack: process.env.NODE_ENV === 'development' ? error.stack : undefined
}), {
status: 500,
headers: {
'Content-Type': 'application/json'
}
});
} }
}; };

View file

@ -45,7 +45,21 @@ export const GET: APIRoute = async ({ params }) => {
const errorMessage = error instanceof Error const errorMessage = error instanceof Error
? `Internal server error: ${error.message}` ? `Internal server error: ${error.message}`
: "Internal server error"; : "Internal server error";
// Add specific error codes for better client-side handling
let statusCode = 500;
let friendlyMessage = errorMessage;
if (error instanceof Error) {
if (error.message.includes('ETIMEDOUT')) {
friendlyMessage = `Database connection timeout. This is often temporary, especially after server idle periods. Please try again in a few seconds.`;
} else if (error.message.includes('ECONNREFUSED')) {
friendlyMessage = `Database connection refused. The service may be temporarily unavailable.`;
} else if (error.message.includes('ENOTFOUND')) {
friendlyMessage = `Database host not found. There may be a configuration issue.`;
}
}
return new Response(errorMessage, { status: 500 }); return new Response(friendlyMessage, { status: statusCode });
} }
}; };