mirror of
https://github.com/ld3z/new-manga.git
synced 2026-11-09 19:44:02 -07:00
add replicas
This commit is contained in:
ld3zian
parent
893dc60d0e
commit
dd63aba90e
2 changed files with 133 additions and 70 deletions
199
src/lib/db.ts
199
src/lib/db.ts
|
|
@ -2,10 +2,40 @@ import Redis from 'ioredis';
|
|||
import type { FeedMapping } from './types';
|
||||
import { getChaptersForSlugs } from './api';
|
||||
|
||||
// Primary Redis configuration
|
||||
const PRIMARY_REDIS_URL = import.meta.env.PRIMARY_REDIS_URL || process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL;
|
||||
const REPLICA_REDIS_URL = import.meta.env.REPLICA_REDIS_URL || process.env.REPLICA_REDIS_URL;
|
||||
let redisClient: Redis | null = null;
|
||||
let replicaClient: Redis | null = null;
|
||||
|
||||
// Multiple replicas support
|
||||
const REPLICA_URLS = [
|
||||
import.meta.env.REPLICA_REDIS_URL_1 || process.env.REPLICA_REDIS_URL_1 || import.meta.env.REPLICA_REDIS_URL || process.env.REPLICA_REDIS_URL,
|
||||
import.meta.env.REPLICA_REDIS_URL_2 || process.env.REPLICA_REDIS_URL_2,
|
||||
import.meta.env.REPLICA_REDIS_URL_3 || process.env.REPLICA_REDIS_URL_3,
|
||||
].filter(Boolean) as string[];
|
||||
|
||||
// Create a mapping of URLs to friendly names for logging
|
||||
const redisInstanceNames = new Map<string, string>();
|
||||
|
||||
// Initialize friendly names for each Redis instance
|
||||
function initializeRedisNames() {
|
||||
redisInstanceNames.set(PRIMARY_REDIS_URL!, 'Primary');
|
||||
|
||||
REPLICA_URLS.forEach((url, index) => {
|
||||
redisInstanceNames.set(url, `Replica ${index + 1}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Call this immediately
|
||||
initializeRedisNames();
|
||||
|
||||
// Helper function to get a safe name for a Redis instance
|
||||
function getRedisName(url: string): string {
|
||||
return redisInstanceNames.get(url) || 'Unknown Redis instance';
|
||||
}
|
||||
|
||||
// Client instances
|
||||
let primaryClient: Redis | null = null;
|
||||
let replicaClients: Map<string, Redis> = new Map();
|
||||
|
||||
const DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30; // 30 days
|
||||
const MAX_RETRIES = 5;
|
||||
const BASE_DELAY_MS = 100;
|
||||
|
|
@ -14,11 +44,14 @@ const BASE_DELAY_MS = 100;
|
|||
async function createRedisClient(url: string): Promise<Redis> {
|
||||
if (!url) throw new Error('Redis URL is not set');
|
||||
|
||||
return new Redis(url, {
|
||||
const redisName = getRedisName(url);
|
||||
|
||||
// Parse URL to extract authentication info
|
||||
const options = {
|
||||
retryStrategy: (times) => {
|
||||
if (times >= MAX_RETRIES) return null;
|
||||
const delay = Math.min(BASE_DELAY_MS * Math.pow(2, times), 5000);
|
||||
console.log(`Redis retry attempt ${times}, next try in ${delay}ms`);
|
||||
console.log(`${redisName} retry attempt ${times}, next try in ${delay}ms`);
|
||||
return delay + Math.random() * 200; // Add jitter
|
||||
},
|
||||
maxRetriesPerRequest: null, // Allow retries for queued commands
|
||||
|
|
@ -27,100 +60,150 @@ async function createRedisClient(url: string): Promise<Redis> {
|
|||
reconnectOnError: (err) => {
|
||||
return err.message.includes('READONLY');
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
return new Redis(url, options);
|
||||
}
|
||||
|
||||
// Get the best available Redis client
|
||||
export async function getRedisClient(): Promise<Redis> {
|
||||
// Try primary first
|
||||
if (redisClient?.status === 'ready') {
|
||||
if (primaryClient?.status === 'ready') {
|
||||
try {
|
||||
await redisClient.ping();
|
||||
return redisClient;
|
||||
await primaryClient.ping();
|
||||
return primaryClient;
|
||||
} catch (error) {
|
||||
console.warn('Primary Redis connection failed, will try replica:', error);
|
||||
// Fall through to try replica
|
||||
console.warn('Primary Redis connection failed, will try replicas:', error);
|
||||
// Fall through to try replicas
|
||||
}
|
||||
}
|
||||
|
||||
// Try replica if available
|
||||
if (REPLICA_REDIS_URL && replicaClient?.status === 'ready') {
|
||||
try {
|
||||
await replicaClient.ping();
|
||||
return replicaClient;
|
||||
} catch (error) {
|
||||
console.warn('Replica Redis connection failed:', error);
|
||||
// Fall through to reconnect
|
||||
// Try replicas in sequence
|
||||
for (const [url, client] of replicaClients.entries()) {
|
||||
if (client.status === 'ready') {
|
||||
try {
|
||||
await client.ping();
|
||||
console.log(`Using ${getRedisName(url)}`);
|
||||
return client;
|
||||
} catch (error) {
|
||||
console.warn(`${getRedisName(url)} connection failed:`, error);
|
||||
// Continue to next replica
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reconnect to primary
|
||||
try {
|
||||
console.log('Initializing new primary Redis connection...');
|
||||
redisClient = await createRedisClient(PRIMARY_REDIS_URL!);
|
||||
primaryClient = await createRedisClient(PRIMARY_REDIS_URL!);
|
||||
|
||||
redisClient
|
||||
primaryClient
|
||||
.on('ready', () => console.log('Primary Redis connection ready'))
|
||||
.on('error', (err) => console.error('Primary Redis error:', err))
|
||||
.on('reconnecting', () => console.log('Primary Redis reconnecting'))
|
||||
.on('end', () => console.log('Primary Redis connection closed'));
|
||||
|
||||
return redisClient;
|
||||
return primaryClient;
|
||||
} catch (error) {
|
||||
console.error('Failed to connect to primary Redis:', error);
|
||||
|
||||
// If primary fails and we have a replica URL, try connecting to replica
|
||||
if (REPLICA_REDIS_URL) {
|
||||
console.log('Falling back to replica Redis...');
|
||||
replicaClient = await createRedisClient(REPLICA_REDIS_URL);
|
||||
|
||||
replicaClient
|
||||
.on('ready', () => console.log('Replica Redis connection ready'))
|
||||
.on('error', (err) => console.error('Replica Redis error:', err))
|
||||
.on('reconnecting', () => console.log('Replica Redis reconnecting'))
|
||||
.on('end', () => console.log('Replica Redis connection closed'));
|
||||
|
||||
return replicaClient;
|
||||
// Try connecting to any replica that might be available
|
||||
for (const url of REPLICA_URLS) {
|
||||
try {
|
||||
const redisName = getRedisName(url);
|
||||
console.log(`Trying to connect to ${redisName}...`);
|
||||
const client = await createRedisClient(url);
|
||||
|
||||
client
|
||||
.on('ready', () => console.log(`${redisName} connection ready`))
|
||||
.on('error', (err) => console.error(`${redisName} error:`, err))
|
||||
.on('reconnecting', () => console.log(`${redisName} reconnecting`))
|
||||
.on('end', () => console.log(`${redisName} connection closed`));
|
||||
|
||||
replicaClients.set(url, client);
|
||||
return client;
|
||||
} catch (replicaError) {
|
||||
console.error(`Failed to connect to ${getRedisName(url)}:`, replicaError);
|
||||
// Continue to next replica
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('All Redis connections failed');
|
||||
}
|
||||
}
|
||||
|
||||
// Write to both primary and replica for redundancy
|
||||
// Write to primary and all replicas for redundancy
|
||||
export async function writeToAllRedis(operation: (redis: Redis) => Promise<any>): Promise<void> {
|
||||
const errors = [];
|
||||
let successCount = 0;
|
||||
|
||||
// Try primary
|
||||
try {
|
||||
if (!redisClient || redisClient.status !== 'ready') {
|
||||
redisClient = await createRedisClient(PRIMARY_REDIS_URL!);
|
||||
if (!primaryClient || primaryClient.status !== 'ready') {
|
||||
primaryClient = await createRedisClient(PRIMARY_REDIS_URL!);
|
||||
}
|
||||
await operation(redisClient);
|
||||
await operation(primaryClient);
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
console.error('Failed to write to primary Redis:', error);
|
||||
console.error('Failed to write to Primary Redis:', error);
|
||||
errors.push(error);
|
||||
}
|
||||
|
||||
// Try replica if available (for redundancy)
|
||||
if (REPLICA_REDIS_URL) {
|
||||
// Try all replicas
|
||||
for (const url of REPLICA_URLS) {
|
||||
try {
|
||||
if (!replicaClient || replicaClient.status !== 'ready') {
|
||||
replicaClient = await createRedisClient(REPLICA_REDIS_URL);
|
||||
let client = replicaClients.get(url);
|
||||
if (!client || client.status !== 'ready') {
|
||||
client = await createRedisClient(url);
|
||||
replicaClients.set(url, client);
|
||||
}
|
||||
await operation(replicaClient);
|
||||
await operation(client);
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
console.error('Failed to write to replica Redis:', error);
|
||||
console.error(`Failed to write to ${getRedisName(url)}:`, error);
|
||||
errors.push(error);
|
||||
}
|
||||
}
|
||||
|
||||
// If both writes failed, throw an error
|
||||
if (errors.length === (REPLICA_REDIS_URL ? 2 : 1)) {
|
||||
// If all writes failed, throw an error
|
||||
if (successCount === 0) {
|
||||
throw new Error('Failed to write to any Redis instance');
|
||||
}
|
||||
|
||||
// Log warning if some writes failed
|
||||
if (errors.length > 0) {
|
||||
console.warn(`${errors.length} Redis writes failed out of ${errors.length + successCount} attempts`);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize all connections on startup
|
||||
(async () => {
|
||||
try {
|
||||
// Initialize primary
|
||||
if (PRIMARY_REDIS_URL) {
|
||||
primaryClient = await createRedisClient(PRIMARY_REDIS_URL);
|
||||
await primaryClient.ping();
|
||||
console.log('Primary Redis pre-connected successfully');
|
||||
}
|
||||
|
||||
// Initialize all replicas
|
||||
for (const url of REPLICA_URLS) {
|
||||
try {
|
||||
const client = await createRedisClient(url);
|
||||
await client.ping();
|
||||
replicaClients.set(url, client);
|
||||
console.log(`${getRedisName(url)} pre-connected successfully`);
|
||||
} catch (error) {
|
||||
console.error(`Failed to pre-connect to ${getRedisName(url)}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Redis setup complete: 1 primary and ${replicaClients.size} replicas connected`);
|
||||
} catch (error) {
|
||||
console.error('Redis pre-connection failed:', error);
|
||||
}
|
||||
})();
|
||||
|
||||
// Update existing functions to use the new methods
|
||||
|
||||
export async function storeFeedMappings(mappings: Array<{ feedId: string, slugs: string[], lang: string }>): Promise<void> {
|
||||
|
|
@ -187,28 +270,6 @@ export async function storeFeedMapping(feedId: string, slugs: string[], lang: st
|
|||
}
|
||||
}
|
||||
|
||||
// Initialize both connections on startup
|
||||
(async () => {
|
||||
try {
|
||||
// Initialize primary
|
||||
if (PRIMARY_REDIS_URL) {
|
||||
redisClient = await createRedisClient(PRIMARY_REDIS_URL);
|
||||
await redisClient.ping();
|
||||
console.log('Primary Redis pre-connected successfully');
|
||||
}
|
||||
|
||||
// Initialize replica if available
|
||||
if (REPLICA_REDIS_URL) {
|
||||
replicaClient = await createRedisClient(REPLICA_REDIS_URL);
|
||||
await replicaClient.ping();
|
||||
console.log('Replica Redis pre-connected successfully');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Redis pre-connection failed:', error);
|
||||
}
|
||||
})();
|
||||
|
||||
// Batch operations for better performance
|
||||
export async function checkCacheStatus(slugs: string[], lang: string): Promise<string[]> {
|
||||
const redis = await getRedisClient();
|
||||
const pipeline = redis.pipeline();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue