From dd63aba90ef0b5e1baa2e0e09482278c8bda5ab1 Mon Sep 17 00:00:00 2001 From: ld3zian <42660576+ld3z@users.noreply.github.com> Date: Fri, 28 Feb 2025 17:03:48 -0700 Subject: [PATCH] add replicas --- astro.config.mjs | 4 +- src/lib/db.ts | 199 +++++++++++++++++++++++++++++++---------------- 2 files changed, 133 insertions(+), 70 deletions(-) diff --git a/astro.config.mjs b/astro.config.mjs index 5b10810..778d936 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -12,7 +12,9 @@ export default defineConfig({ vite: { define: { 'process.env.PRIMARY_REDIS_URL': JSON.stringify(process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL), - 'process.env.REPLICA_REDIS_URL': JSON.stringify(process.env.REPLICA_REDIS_URL) + 'process.env.REPLICA_REDIS_URL_1': JSON.stringify(process.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL), + 'process.env.REPLICA_REDIS_URL_2': JSON.stringify(process.env.REPLICA_REDIS_URL_2), + 'process.env.REPLICA_REDIS_URL_3': JSON.stringify(process.env.REPLICA_REDIS_URL_3) } } }); \ No newline at end of file diff --git a/src/lib/db.ts b/src/lib/db.ts index 8348c91..bb34a83 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -2,10 +2,40 @@ import Redis from 'ioredis'; import type { FeedMapping } from './types'; import { getChaptersForSlugs } from './api'; +// Primary Redis configuration const PRIMARY_REDIS_URL = import.meta.env.PRIMARY_REDIS_URL || process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL; -const REPLICA_REDIS_URL = import.meta.env.REPLICA_REDIS_URL || process.env.REPLICA_REDIS_URL; -let redisClient: Redis | null = null; -let replicaClient: Redis | null = null; + +// Multiple replicas support +const REPLICA_URLS = [ + import.meta.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL_1 || import.meta.env.REPLICA_REDIS_URL || process.env.REPLICA_REDIS_URL, + import.meta.env.REPLICA_REDIS_URL_2 || process.env.REPLICA_REDIS_URL_2, + import.meta.env.REPLICA_REDIS_URL_3 || process.env.REPLICA_REDIS_URL_3, +].filter(Boolean) as string[]; + +// Create a mapping of URLs to friendly names for logging +const redisInstanceNames = new Map(); + +// Initialize friendly names for each Redis instance +function initializeRedisNames() { + redisInstanceNames.set(PRIMARY_REDIS_URL!, 'Primary'); + + REPLICA_URLS.forEach((url, index) => { + redisInstanceNames.set(url, `Replica ${index + 1}`); + }); +} + +// Call this immediately +initializeRedisNames(); + +// Helper function to get a safe name for a Redis instance +function getRedisName(url: string): string { + return redisInstanceNames.get(url) || 'Unknown Redis instance'; +} + +// Client instances +let primaryClient: Redis | null = null; +let replicaClients: Map = new Map(); + const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days const MAX_RETRIES = 5; const BASE_DELAY_MS = 100; @@ -14,11 +44,14 @@ const BASE_DELAY_MS = 100; async function createRedisClient(url: string): Promise { if (!url) throw new Error('Redis URL is not set'); - return new Redis(url, { + const redisName = getRedisName(url); + + // Parse URL to extract authentication info + const options = { retryStrategy: (times) => { if (times >= MAX_RETRIES) return null; const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000); - console.log(`Redis retry attempt ${times}, next try in ${delay}ms`); + console.log(`${redisName} retry attempt ${times}, next try in ${delay}ms`); return delay + Math.random() * 200; // Add jitter }, maxRetriesPerRequest: null, // Allow retries for queued commands @@ -27,100 +60,150 @@ async function createRedisClient(url: string): Promise { reconnectOnError: (err) => { return err.message.includes('READONLY'); } - }); + }; + + return new Redis(url, options); } // Get the best available Redis client export async function getRedisClient(): Promise { // Try primary first - if (redisClient?.status === 'ready') { + if (primaryClient?.status === 'ready') { try { - await redisClient.ping(); - return redisClient; + await primaryClient.ping(); + return primaryClient; } catch (error) { - console.warn('Primary Redis connection failed, will try replica:', error); - // Fall through to try replica + console.warn('Primary Redis connection failed, will try replicas:', error); + // Fall through to try replicas } } - // Try replica if available - if (REPLICA_REDIS_URL && replicaClient?.status === 'ready') { - try { - await replicaClient.ping(); - return replicaClient; - } catch (error) { - console.warn('Replica Redis connection failed:', error); - // Fall through to reconnect + // Try replicas in sequence + for (const [url, client] of replicaClients.entries()) { + if (client.status === 'ready') { + try { + await client.ping(); + console.log(`Using ${getRedisName(url)}`); + return client; + } catch (error) { + console.warn(`${getRedisName(url)} connection failed:`, error); + // Continue to next replica + } } } // Reconnect to primary try { console.log('Initializing new primary Redis connection...'); - redisClient = await createRedisClient(PRIMARY_REDIS_URL!); + primaryClient = await createRedisClient(PRIMARY_REDIS_URL!); - redisClient + primaryClient .on('ready', () => console.log('Primary Redis connection ready')) .on('error', (err) => console.error('Primary Redis error:', err)) .on('reconnecting', () => console.log('Primary Redis reconnecting')) .on('end', () => console.log('Primary Redis connection closed')); - return redisClient; + return primaryClient; } catch (error) { console.error('Failed to connect to primary Redis:', error); - // If primary fails and we have a replica URL, try connecting to replica - if (REPLICA_REDIS_URL) { - console.log('Falling back to replica Redis...'); - replicaClient = await createRedisClient(REPLICA_REDIS_URL); - - replicaClient - .on('ready', () => console.log('Replica Redis connection ready')) - .on('error', (err) => console.error('Replica Redis error:', err)) - .on('reconnecting', () => console.log('Replica Redis reconnecting')) - .on('end', () => console.log('Replica Redis connection closed')); - - return replicaClient; + // Try connecting to any replica that might be available + for (const url of REPLICA_URLS) { + try { + const redisName = getRedisName(url); + console.log(`Trying to connect to ${redisName}...`); + const client = await createRedisClient(url); + + client + .on('ready', () => console.log(`${redisName} connection ready`)) + .on('error', (err) => console.error(`${redisName} error:`, err)) + .on('reconnecting', () => console.log(`${redisName} reconnecting`)) + .on('end', () => console.log(`${redisName} connection closed`)); + + replicaClients.set(url, client); + return client; + } catch (replicaError) { + console.error(`Failed to connect to ${getRedisName(url)}:`, replicaError); + // Continue to next replica + } } throw new Error('All Redis connections failed'); } } -// Write to both primary and replica for redundancy +// Write to primary and all replicas for redundancy export async function writeToAllRedis(operation: (redis: Redis) => Promise): Promise { const errors = []; + let successCount = 0; // Try primary try { - if (!redisClient || redisClient.status !== 'ready') { - redisClient = await createRedisClient(PRIMARY_REDIS_URL!); + if (!primaryClient || primaryClient.status !== 'ready') { + primaryClient = await createRedisClient(PRIMARY_REDIS_URL!); } - await operation(redisClient); + await operation(primaryClient); + successCount++; } catch (error) { - console.error('Failed to write to primary Redis:', error); + console.error('Failed to write to Primary Redis:', error); errors.push(error); } - // Try replica if available (for redundancy) - if (REPLICA_REDIS_URL) { + // Try all replicas + for (const url of REPLICA_URLS) { try { - if (!replicaClient || replicaClient.status !== 'ready') { - replicaClient = await createRedisClient(REPLICA_REDIS_URL); + let client = replicaClients.get(url); + if (!client || client.status !== 'ready') { + client = await createRedisClient(url); + replicaClients.set(url, client); } - await operation(replicaClient); + await operation(client); + successCount++; } catch (error) { - console.error('Failed to write to replica Redis:', error); + console.error(`Failed to write to ${getRedisName(url)}:`, error); errors.push(error); } } - // If both writes failed, throw an error - if (errors.length === (REPLICA_REDIS_URL ? 2 : 1)) { + // If all writes failed, throw an error + if (successCount === 0) { throw new Error('Failed to write to any Redis instance'); } + + // Log warning if some writes failed + if (errors.length > 0) { + console.warn(`${errors.length} Redis writes failed out of ${errors.length + successCount} attempts`); + } } +// Initialize all connections on startup +(async () => { + try { + // Initialize primary + if (PRIMARY_REDIS_URL) { + primaryClient = await createRedisClient(PRIMARY_REDIS_URL); + await primaryClient.ping(); + console.log('Primary Redis pre-connected successfully'); + } + + // Initialize all replicas + for (const url of REPLICA_URLS) { + try { + const client = await createRedisClient(url); + await client.ping(); + replicaClients.set(url, client); + console.log(`${getRedisName(url)} pre-connected successfully`); + } catch (error) { + console.error(`Failed to pre-connect to ${getRedisName(url)}:`, error); + } + } + + console.log(`Redis setup complete: 1 primary and ${replicaClients.size} replicas connected`); + } catch (error) { + console.error('Redis pre-connection failed:', error); + } +})(); + // Update existing functions to use the new methods export async function storeFeedMappings(mappings: Array<{ feedId: string, slugs: string[], lang: string }>): Promise { @@ -187,28 +270,6 @@ export async function storeFeedMapping(feedId: string, slugs: string[], lang: st } } -// Initialize both connections on startup -(async () => { - try { - // Initialize primary - if (PRIMARY_REDIS_URL) { - redisClient = await createRedisClient(PRIMARY_REDIS_URL); - await redisClient.ping(); - console.log('Primary Redis pre-connected successfully'); - } - - // Initialize replica if available - if (REPLICA_REDIS_URL) { - replicaClient = await createRedisClient(REPLICA_REDIS_URL); - await replicaClient.ping(); - console.log('Replica Redis pre-connected successfully'); - } - } catch (error) { - console.error('Redis pre-connection failed:', error); - } -})(); - -// Batch operations for better performance export async function checkCacheStatus(slugs: string[], lang: string): Promise { const redis = await getRedisClient(); const pipeline = redis.pipeline();