ldez/new-manga
1
0
Fork 0
mirror of https://github.com/ld3z/new-manga.git synced 2026-11-09 20:33:44 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

add replicas

Browse source
This commit is contained in:
ld3zian 2025-02-28 17:03:48 -07:00
parent 893dc60d0e
commit dd63aba90e
2 changed files with 133 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

4
astro.config.mjs
View file

@ -12,7 +12,9 @@ export default defineConfig({
vite: { vite: {
define: { define: {
'process.env.PRIMARY_REDIS_URL': JSON.stringify(process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL), 'process.env.PRIMARY_REDIS_URL': JSON.stringify(process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL),
'process.env.REPLICA_REDIS_URL': JSON.stringify(process.env.REPLICA_REDIS_URL) 'process.env.REPLICA_REDIS_URL_1': JSON.stringify(process.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL),
'process.env.REPLICA_REDIS_URL_2': JSON.stringify(process.env.REPLICA_REDIS_URL_2),
'process.env.REPLICA_REDIS_URL_3': JSON.stringify(process.env.REPLICA_REDIS_URL_3)
} }
} }
}); });

191
src/lib/db.ts
View file

@ -2,10 +2,40 @@ import Redis from 'ioredis';
import type { FeedMapping } from './types'; import type { FeedMapping } from './types';
import { getChaptersForSlugs } from './api'; import { getChaptersForSlugs } from './api';
// Primary Redis configuration
const PRIMARY_REDIS_URL = import.meta.env.PRIMARY_REDIS_URL || process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL; const PRIMARY_REDIS_URL = import.meta.env.PRIMARY_REDIS_URL || process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL;
const REPLICA_REDIS_URL = import.meta.env.REPLICA_REDIS_URL || process.env.REPLICA_REDIS_URL;
let redisClient: Redis | null = null; // Multiple replicas support
let replicaClient: Redis | null = null; const REPLICA_URLS = [
import.meta.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL_1 || import.meta.env.REPLICA_REDIS_URL || process.env.REPLICA_REDIS_URL,
import.meta.env.REPLICA_REDIS_URL_2 || process.env.REPLICA_REDIS_URL_2,
import.meta.env.REPLICA_REDIS_URL_3 || process.env.REPLICA_REDIS_URL_3,
].filter(Boolean) as string[];
// Create a mapping of URLs to friendly names for logging
const redisInstanceNames = new Map<string, string>();
// Initialize friendly names for each Redis instance
function initializeRedisNames() {
redisInstanceNames.set(PRIMARY_REDIS_URL!, 'Primary');
REPLICA_URLS.forEach((url, index) => {
redisInstanceNames.set(url, `Replica ${index + 1}`);
});
}
// Call this immediately
initializeRedisNames();
// Helper function to get a safe name for a Redis instance
function getRedisName(url: string): string {
return redisInstanceNames.get(url) || 'Unknown Redis instance';
}
// Client instances
let primaryClient: Redis | null = null;
let replicaClients: Map<string, Redis> = new Map();
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
const MAX_RETRIES = 5; const MAX_RETRIES = 5;
const BASE_DELAY_MS = 100; const BASE_DELAY_MS = 100;
@ -14,11 +44,14 @@ const BASE_DELAY_MS = 100;
async function createRedisClient(url: string): Promise<Redis> { async function createRedisClient(url: string): Promise<Redis> {
if (!url) throw new Error('Redis URL is not set'); if (!url) throw new Error('Redis URL is not set');
return new Redis(url, { const redisName = getRedisName(url);
// Parse URL to extract authentication info
const options = {
retryStrategy: (times) => { retryStrategy: (times) => {
if (times >= MAX_RETRIES) return null; if (times >= MAX_RETRIES) return null;
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000); const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000);
console.log(`Redis retry attempt ${times}, next try in ${delay}ms`); console.log(`${redisName} retry attempt ${times}, next try in ${delay}ms`);
return delay + Math.random() * 200; // Add jitter return delay + Math.random() * 200; // Add jitter
}, },
maxRetriesPerRequest: null, // Allow retries for queued commands maxRetriesPerRequest: null, // Allow retries for queued commands
@ -27,99 +60,149 @@ async function createRedisClient(url: string): Promise<Redis> {
reconnectOnError: (err) => { reconnectOnError: (err) => {
return err.message.includes('READONLY'); return err.message.includes('READONLY');
} }
}); };
return new Redis(url, options);
} }
// Get the best available Redis client // Get the best available Redis client
export async function getRedisClient(): Promise<Redis> { export async function getRedisClient(): Promise<Redis> {
// Try primary first // Try primary first
if (redisClient?.status === 'ready') { if (primaryClient?.status === 'ready') {
try { try {
await redisClient.ping(); await primaryClient.ping();
return redisClient; return primaryClient;
} catch (error) { } catch (error) {
console.warn('Primary Redis connection failed, will try replica:', error); console.warn('Primary Redis connection failed, will try replicas:', error);
// Fall through to try replica // Fall through to try replicas
} }
} }
// Try replica if available // Try replicas in sequence
if (REPLICA_REDIS_URL && replicaClient?.status === 'ready') { for (const [url, client] of replicaClients.entries()) {
if (client.status === 'ready') {
try { try {
await replicaClient.ping(); await client.ping();
return replicaClient; console.log(`Using ${getRedisName(url)}`);
return client;
} catch (error) { } catch (error) {
console.warn('Replica Redis connection failed:', error); console.warn(`${getRedisName(url)} connection failed:`, error);
// Fall through to reconnect // Continue to next replica
}
} }
} }
// Reconnect to primary // Reconnect to primary
try { try {
console.log('Initializing new primary Redis connection...'); console.log('Initializing new primary Redis connection...');
redisClient = await createRedisClient(PRIMARY_REDIS_URL!); primaryClient = await createRedisClient(PRIMARY_REDIS_URL!);
redisClient primaryClient
.on('ready', () => console.log('Primary Redis connection ready')) .on('ready', () => console.log('Primary Redis connection ready'))
.on('error', (err) => console.error('Primary Redis error:', err)) .on('error', (err) => console.error('Primary Redis error:', err))
.on('reconnecting', () => console.log('Primary Redis reconnecting')) .on('reconnecting', () => console.log('Primary Redis reconnecting'))
.on('end', () => console.log('Primary Redis connection closed')); .on('end', () => console.log('Primary Redis connection closed'));
return redisClient; return primaryClient;
} catch (error) { } catch (error) {
console.error('Failed to connect to primary Redis:', error); console.error('Failed to connect to primary Redis:', error);
// If primary fails and we have a replica URL, try connecting to replica // Try connecting to any replica that might be available
if (REPLICA_REDIS_URL) { for (const url of REPLICA_URLS) {
console.log('Falling back to replica Redis...'); try {
replicaClient = await createRedisClient(REPLICA_REDIS_URL); const redisName = getRedisName(url);
console.log(`Trying to connect to ${redisName}...`);
const client = await createRedisClient(url);
replicaClient client
.on('ready', () => console.log('Replica Redis connection ready')) .on('ready', () => console.log(`${redisName} connection ready`))
.on('error', (err) => console.error('Replica Redis error:', err)) .on('error', (err) => console.error(`${redisName} error:`, err))
.on('reconnecting', () => console.log('Replica Redis reconnecting')) .on('reconnecting', () => console.log(`${redisName} reconnecting`))
.on('end', () => console.log('Replica Redis connection closed')); .on('end', () => console.log(`${redisName} connection closed`));
return replicaClient; replicaClients.set(url, client);
return client;
} catch (replicaError) {
console.error(`Failed to connect to ${getRedisName(url)}:`, replicaError);
// Continue to next replica
}
} }
throw new Error('All Redis connections failed'); throw new Error('All Redis connections failed');
} }
} }
// Write to both primary and replica for redundancy // Write to primary and all replicas for redundancy
export async function writeToAllRedis(operation: (redis: Redis) => Promise<any>): Promise<void> { export async function writeToAllRedis(operation: (redis: Redis) => Promise<any>): Promise<void> {
const errors = []; const errors = [];
let successCount = 0;
// Try primary // Try primary
try { try {
if (!redisClient || redisClient.status !== 'ready') { if (!primaryClient || primaryClient.status !== 'ready') {
redisClient = await createRedisClient(PRIMARY_REDIS_URL!); primaryClient = await createRedisClient(PRIMARY_REDIS_URL!);
} }
await operation(redisClient); await operation(primaryClient);
successCount++;
} catch (error) { } catch (error) {
console.error('Failed to write to primary Redis:', error); console.error('Failed to write to Primary Redis:', error);
errors.push(error); errors.push(error);
} }
// Try replica if available (for redundancy) // Try all replicas
if (REPLICA_REDIS_URL) { for (const url of REPLICA_URLS) {
try { try {
if (!replicaClient || replicaClient.status !== 'ready') { let client = replicaClients.get(url);
replicaClient = await createRedisClient(REPLICA_REDIS_URL); if (!client || client.status !== 'ready') {
client = await createRedisClient(url);
replicaClients.set(url, client);
} }
await operation(replicaClient); await operation(client);
successCount++;
} catch (error) { } catch (error) {
console.error('Failed to write to replica Redis:', error); console.error(`Failed to write to ${getRedisName(url)}:`, error);
errors.push(error); errors.push(error);
} }
} }
// If both writes failed, throw an error // If all writes failed, throw an error
if (errors.length === (REPLICA_REDIS_URL ? 2 : 1)) { if (successCount === 0) {
throw new Error('Failed to write to any Redis instance'); throw new Error('Failed to write to any Redis instance');
} }
// Log warning if some writes failed
if (errors.length > 0) {
console.warn(`${errors.length} Redis writes failed out of ${errors.length + successCount} attempts`);
} }
}
// Initialize all connections on startup
(async () => {
try {
// Initialize primary
if (PRIMARY_REDIS_URL) {
primaryClient = await createRedisClient(PRIMARY_REDIS_URL);
await primaryClient.ping();
console.log('Primary Redis pre-connected successfully');
}
// Initialize all replicas
for (const url of REPLICA_URLS) {
try {
const client = await createRedisClient(url);
await client.ping();
replicaClients.set(url, client);
console.log(`${getRedisName(url)} pre-connected successfully`);
} catch (error) {
console.error(`Failed to pre-connect to ${getRedisName(url)}:`, error);
}
}
console.log(`Redis setup complete: 1 primary and ${replicaClients.size} replicas connected`);
} catch (error) {
console.error('Redis pre-connection failed:', error);
}
})();
// Update existing functions to use the new methods // Update existing functions to use the new methods
@ -187,28 +270,6 @@ export async function storeFeedMapping(feedId: string, slugs: string[], lang: st
} }
} }
// Initialize both connections on startup
(async () => {
try {
// Initialize primary
if (PRIMARY_REDIS_URL) {
redisClient = await createRedisClient(PRIMARY_REDIS_URL);
await redisClient.ping();
console.log('Primary Redis pre-connected successfully');
}
// Initialize replica if available
if (REPLICA_REDIS_URL) {
replicaClient = await createRedisClient(REPLICA_REDIS_URL);
await replicaClient.ping();
console.log('Replica Redis pre-connected successfully');
}
} catch (error) {
console.error('Redis pre-connection failed:', error);
}
})();
// Batch operations for better performance
export async function checkCacheStatus(slugs: string[], lang: string): Promise<string[]> { export async function checkCacheStatus(slugs: string[], lang: string): Promise<string[]> {
const redis = await getRedisClient(); const redis = await getRedisClient();
const pipeline = redis.pipeline(); const pipeline = redis.pipeline();