mirror of
https://github.com/ld3z/new-manga.git
synced 2026-11-11 06:03:44 -07:00
Compare commits
No commits in common. "6d783ad0d04499a9cf0aecdffe2f0a254614319a" and "90f0e2417e04c198acd572f303d5233e3147297e" have entirely different histories.
6d783ad0d0
...
90f0e2417e
6 changed files with 67 additions and 412 deletions
Binary file not shown.
|
Before Width: | Height: | Size: 629 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 147 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 817 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 324 KiB |
|
|
@ -33,7 +33,7 @@ export const backgrounds: BackgroundImage[] = [
|
||||||
source: "https://danbooru.donmai.us/posts/6900673",
|
source: "https://danbooru.donmai.us/posts/6900673",
|
||||||
socials: [
|
socials: [
|
||||||
{
|
{
|
||||||
name: "Artist's Twitter",
|
name: "Twitter",
|
||||||
link: "https://twitter.com/ayanagi0319",
|
link: "https://twitter.com/ayanagi0319",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
|
@ -45,7 +45,7 @@ export const backgrounds: BackgroundImage[] = [
|
||||||
source: "https://danbooru.donmai.us/posts/8743208",
|
source: "https://danbooru.donmai.us/posts/8743208",
|
||||||
socials: [
|
socials: [
|
||||||
{
|
{
|
||||||
name: "Artist's Twitter",
|
name: "Twitter",
|
||||||
link: "https://x.com/kamepan44231",
|
link: "https://x.com/kamepan44231",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
|
@ -57,7 +57,7 @@ export const backgrounds: BackgroundImage[] = [
|
||||||
source: "https://danbooru.donmai.us/posts/8074444",
|
source: "https://danbooru.donmai.us/posts/8074444",
|
||||||
socials: [
|
socials: [
|
||||||
{
|
{
|
||||||
name: "Artist's Twitter",
|
name: "Twitter",
|
||||||
link: "https://x.com/Ten_0123_/",
|
link: "https://x.com/Ten_0123_/",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
|
@ -69,8 +69,8 @@ export const backgrounds: BackgroundImage[] = [
|
||||||
source: "https://www.pixiv.net/en/artworks/124222053",
|
source: "https://www.pixiv.net/en/artworks/124222053",
|
||||||
socials: [
|
socials: [
|
||||||
{
|
{
|
||||||
name: "Artist's Twitter",
|
name: "Twitter",
|
||||||
link: "https://x.com/greatodoggo",
|
link: "https://twitter.com/greatodoggo",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
|
@ -81,7 +81,7 @@ export const backgrounds: BackgroundImage[] = [
|
||||||
source: "https://safebooru.org/index.php?page=post&s=view&id=5418793",
|
source: "https://safebooru.org/index.php?page=post&s=view&id=5418793",
|
||||||
socials: [
|
socials: [
|
||||||
{
|
{
|
||||||
name: "Artist's Twitter",
|
name: "Twitter",
|
||||||
link: "https://x.com/soaphisoap",
|
link: "https://x.com/soaphisoap",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
|
@ -93,7 +93,7 @@ export const backgrounds: BackgroundImage[] = [
|
||||||
source: "https://safebooru.org/index.php?page=post&s=view&id=5531630",
|
source: "https://safebooru.org/index.php?page=post&s=view&id=5531630",
|
||||||
socials: [
|
socials: [
|
||||||
{
|
{
|
||||||
name: "Artist's Twitter",
|
name: "Twitter",
|
||||||
link: "https://x.com/charlieinoni",
|
link: "https://x.com/charlieinoni",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
|
@ -106,59 +106,11 @@ export const backgrounds: BackgroundImage[] = [
|
||||||
"https://i.pximg.net/img-original/img/2025/03/03/15/13/24/127822472_p0.png",
|
"https://i.pximg.net/img-original/img/2025/03/03/15/13/24/127822472_p0.png",
|
||||||
socials: [
|
socials: [
|
||||||
{
|
{
|
||||||
name: "Artist's Twitter",
|
name: "Twitter",
|
||||||
link: "https://x.com/",
|
link: "https://x.com/",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
|
||||||
url: "/cynthia1.webp",
|
|
||||||
title: "Cynthia (Pokemon)",
|
|
||||||
artist: "???",
|
|
||||||
source: "https://safebooru.org/index.php?page=post&s=view&id=5547740",
|
|
||||||
socials: [
|
|
||||||
{
|
|
||||||
name: "Artist's Twitter",
|
|
||||||
link: "https://x.com/",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
url: "/darling.webp",
|
|
||||||
title: "Zero Two",
|
|
||||||
artist: "AWZ",
|
|
||||||
source: "https://www.pixiv.net/en/artworks/123110068",
|
|
||||||
socials: [
|
|
||||||
{
|
|
||||||
name: "Pixiv Profile",
|
|
||||||
link: "https://www.pixiv.net/en/users/67355371",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
url: "/granny.webp",
|
|
||||||
title: "Turbo Granny (Dandadan)",
|
|
||||||
artist: "MajimartBCN",
|
|
||||||
source: "https://twitter.com/MajimartBCN/status/1864741369383694841",
|
|
||||||
socials: [
|
|
||||||
{
|
|
||||||
name: "Artist's Twitter",
|
|
||||||
link: "https://x.com/MajimartBCN",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
url: "/numi_lily.webp",
|
|
||||||
title: "Nihmune & Shylily",
|
|
||||||
artist: "Gryever_AW",
|
|
||||||
source: "https://x.com/gryever_aw/status/1890772686084686118",
|
|
||||||
socials: [
|
|
||||||
{
|
|
||||||
name: "Artist's Twitter",
|
|
||||||
link: "https://x.com/Gryever_AW",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
// Add more backgrounds here!
|
// Add more backgrounds here!
|
||||||
// {
|
// {
|
||||||
// url: 'your-image-url',
|
// url: 'your-image-url',
|
||||||
|
|
|
||||||
407
src/lib/db.ts
407
src/lib/db.ts
|
|
@ -7,10 +7,6 @@ export const CHAPTER_CACHE_TTL = 60 * 60; // 1 hour
|
||||||
const MAX_RETRIES = 3;
|
const MAX_RETRIES = 3;
|
||||||
const BASE_DELAY_MS = 50;
|
const BASE_DELAY_MS = 50;
|
||||||
const IS_VERCEL = process.env.VERCEL || false;
|
const IS_VERCEL = process.env.VERCEL || false;
|
||||||
const HEALTH_CHECK_INTERVAL = 60 * 1000; // Check primary health every minute
|
|
||||||
const REPLICA_PROMOTION_THRESHOLD = 5; // After 5 failed primary health checks, promote a replica to primary
|
|
||||||
const SYNC_BUFFER_SIZE = 100; // Maximum number of operations to buffer before syncing
|
|
||||||
const SYNC_INTERVAL = 10 * 1000; // Sync replicas every 10 seconds
|
|
||||||
|
|
||||||
// Redis URLs from environment variables
|
// Redis URLs from environment variables
|
||||||
const PRIMARY_REDIS_URL =
|
const PRIMARY_REDIS_URL =
|
||||||
|
|
@ -22,35 +18,13 @@ const REPLICA_URLS = [
|
||||||
import.meta.env.REPLICA_REDIS_URL_1 ||
|
import.meta.env.REPLICA_REDIS_URL_1 ||
|
||||||
process.env.REPLICA_REDIS_URL_1 ||
|
process.env.REPLICA_REDIS_URL_1 ||
|
||||||
import.meta.env.REPLICA_REDIS_URL,
|
import.meta.env.REPLICA_REDIS_URL,
|
||||||
import.meta.env.REPLICA_REDIS_URL_2 || process.env.REPLICA_REPLICA_URL_2 || process.env.REPLICA_REDIS_URL_2,
|
import.meta.env.REPLICA_REDIS_URL_2 || process.env.REPLICA_REDIS_URL_2,
|
||||||
import.meta.env.REPLICA_REDIS_URL_3 || process.env.REPLICA_REDIS_URL_3,
|
import.meta.env.REPLICA_REDIS_URL_3 || process.env.REPLICA_REDIS_URL_3,
|
||||||
].filter(Boolean) as string[];
|
].filter(Boolean) as string[];
|
||||||
|
|
||||||
// Log available Redis instances
|
// Singleton clients
|
||||||
console.log(`Primary Redis URL: ${PRIMARY_REDIS_URL ? 'Available' : 'Not configured'}`);
|
|
||||||
console.log(`Replica URLs available: ${REPLICA_URLS.length}`);
|
|
||||||
|
|
||||||
// Singleton clients with health status tracking
|
|
||||||
let primaryClient: Redis | null = null;
|
let primaryClient: Redis | null = null;
|
||||||
let primaryHealthy: boolean = true;
|
|
||||||
let primaryFailedChecks = 0;
|
|
||||||
let replicaClients: Map<string, Redis> = new Map();
|
let replicaClients: Map<string, Redis> = new Map();
|
||||||
let activeReplicaUrl: string | null = null;
|
|
||||||
|
|
||||||
// Track which replica was used when primary was down
|
|
||||||
let lastUsedReplica: string | null = null;
|
|
||||||
|
|
||||||
// Operation buffer for sync
|
|
||||||
interface RedisOperation {
|
|
||||||
key: string;
|
|
||||||
value: string;
|
|
||||||
ttl?: number;
|
|
||||||
type: 'set' | 'del';
|
|
||||||
timestamp: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Buffer to store operations that need to be synced
|
|
||||||
const operationBuffer: RedisOperation[] = [];
|
|
||||||
|
|
||||||
// Redis instance naming for logging
|
// Redis instance naming for logging
|
||||||
const redisInstanceNames = new Map<string, string>();
|
const redisInstanceNames = new Map<string, string>();
|
||||||
|
|
@ -83,7 +57,6 @@ async function createRedisClient(url: string): Promise<Redis> {
|
||||||
throw new Error("Redis URL not provided");
|
throw new Error("Redis URL not provided");
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`Creating Redis client for ${getRedisName(url)}: ${url.slice(0, 20)}...`);
|
|
||||||
const redisName = getRedisName(url);
|
const redisName = getRedisName(url);
|
||||||
|
|
||||||
const client = new Redis(url, {
|
const client = new Redis(url, {
|
||||||
|
|
@ -109,66 +82,14 @@ async function createRedisClient(url: string): Promise<Redis> {
|
||||||
|
|
||||||
// Setup event listeners
|
// Setup event listeners
|
||||||
client
|
client
|
||||||
.on("ready", () => {
|
.on("ready", () => console.log(`${redisName} ready`))
|
||||||
console.log(`${redisName} ready`);
|
.on("error", (err) => console.error(`${redisName} error:`, err))
|
||||||
|
|
||||||
// If primary recovers and we had used a replica, initiate sync from replica to primary
|
|
||||||
if (url === PRIMARY_REDIS_URL && lastUsedReplica) {
|
|
||||||
syncFromReplicaToPrimary(lastUsedReplica).catch(err =>
|
|
||||||
console.error(`Failed to sync from replica to primary: ${err}`));
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.on("error", (err) => {
|
|
||||||
console.error(`${redisName} error:`, err);
|
|
||||||
if (url === PRIMARY_REDIS_URL) {
|
|
||||||
primaryHealthy = false;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.on("reconnecting", () => console.log(`${redisName} reconnecting`))
|
.on("reconnecting", () => console.log(`${redisName} reconnecting`))
|
||||||
.on("end", () => {
|
.on("end", () => console.log(`${redisName} closed`));
|
||||||
console.log(`${redisName} closed`);
|
|
||||||
if (url === PRIMARY_REDIS_URL) {
|
|
||||||
primaryHealthy = false;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
return client;
|
return client;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize all replicas to establish connections early
|
|
||||||
*/
|
|
||||||
async function initializeAllClients(): Promise<void> {
|
|
||||||
try {
|
|
||||||
// Initialize primary
|
|
||||||
if (PRIMARY_REDIS_URL && (!primaryClient || primaryClient.status !== "ready")) {
|
|
||||||
primaryClient = await createRedisClient(PRIMARY_REDIS_URL);
|
|
||||||
primaryHealthy = await checkConnection(primaryClient, "Primary");
|
|
||||||
console.log(`Primary Redis initialized, healthy: ${primaryHealthy}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize all replicas
|
|
||||||
for (const url of REPLICA_URLS) {
|
|
||||||
if (!replicaClients.has(url)) {
|
|
||||||
try {
|
|
||||||
const client = await createRedisClient(url);
|
|
||||||
const isHealthy = await checkConnection(client, getRedisName(url));
|
|
||||||
if (isHealthy) {
|
|
||||||
replicaClients.set(url, client);
|
|
||||||
console.log(`Replica ${getRedisName(url)} initialized successfully`);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Failed to initialize replica ${getRedisName(url)}:`, error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Initialized ${replicaClients.size} replica connections`);
|
|
||||||
} catch (error) {
|
|
||||||
console.error("Error during client initialization:", error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if a Redis client is connected and responsive
|
* Check if a Redis client is connected and responsive
|
||||||
*/
|
*/
|
||||||
|
|
@ -189,257 +110,69 @@ async function checkConnection(client: Redis, name: string): Promise<boolean> {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get an active Redis client from the pool
|
* Get an active Redis client from the pool
|
||||||
* Prioritizes primary, falls back to replicas
|
|
||||||
*/
|
*/
|
||||||
export async function getRedisClient(): Promise<Redis> {
|
export async function getRedisClient(): Promise<Redis> {
|
||||||
// Try primary client first if it's considered healthy
|
// Try primary client
|
||||||
if (primaryHealthy && (primaryClient === null || primaryClient.status !== "ready")) {
|
if (!primaryClient || primaryClient.status !== "ready") {
|
||||||
try {
|
primaryClient = await createRedisClient(PRIMARY_REDIS_URL!);
|
||||||
primaryClient = await createRedisClient(PRIMARY_REDIS_URL!);
|
}
|
||||||
const isPrimaryConnected = await checkConnection(primaryClient, "Primary");
|
|
||||||
primaryHealthy = isPrimaryConnected;
|
if (await checkConnection(primaryClient, "Primary")) {
|
||||||
if (isPrimaryConnected) {
|
|
||||||
primaryFailedChecks = 0;
|
|
||||||
console.log("Initialized primary Redis connection");
|
|
||||||
return primaryClient;
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.error("Failed to initialize primary Redis:", err);
|
|
||||||
primaryHealthy = false;
|
|
||||||
primaryFailedChecks++;
|
|
||||||
}
|
|
||||||
} else if (primaryHealthy && primaryClient && primaryClient.status === "ready") {
|
|
||||||
return primaryClient;
|
return primaryClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Primary is unhealthy, try to use the active replica if we have one
|
// Try replicas
|
||||||
if (activeReplicaUrl && replicaClients.has(activeReplicaUrl)) {
|
|
||||||
const activeReplica = replicaClients.get(activeReplicaUrl)!;
|
|
||||||
if (activeReplica.status === "ready" && await checkConnection(activeReplica, getRedisName(activeReplicaUrl))) {
|
|
||||||
console.log(`Using active replica: ${getRedisName(activeReplicaUrl)}`);
|
|
||||||
lastUsedReplica = activeReplicaUrl; // Track that we used this replica
|
|
||||||
return activeReplica;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// No active replica, try replicas in order
|
|
||||||
for (const url of REPLICA_URLS) {
|
for (const url of REPLICA_URLS) {
|
||||||
try {
|
let client = replicaClients.get(url);
|
||||||
let client = replicaClients.get(url);
|
if (!client || client.status !== "ready") {
|
||||||
if (!client || client.status !== "ready") {
|
client = await createRedisClient(url);
|
||||||
client = await createRedisClient(url);
|
replicaClients.set(url, client);
|
||||||
replicaClients.set(url, client);
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if (await checkConnection(client, getRedisName(url))) {
|
if (await checkConnection(client, getRedisName(url))) {
|
||||||
console.log(`Switching to replica: ${getRedisName(url)}`);
|
return client;
|
||||||
activeReplicaUrl = url;
|
|
||||||
lastUsedReplica = url; // Track that we used this replica
|
|
||||||
|
|
||||||
// If primary has been down for a while, promote this replica to be the new primary
|
|
||||||
if (primaryFailedChecks >= REPLICA_PROMOTION_THRESHOLD) {
|
|
||||||
console.log(`Promoting ${getRedisName(url)} to primary after ${primaryFailedChecks} failed checks`);
|
|
||||||
primaryClient = client;
|
|
||||||
primaryHealthy = true;
|
|
||||||
primaryFailedChecks = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
return client;
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.error(`Failed to connect to replica ${getRedisName(url)}:`, err);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// If all else fails, try one more time with primary
|
// Fallback: Force new primary connection
|
||||||
console.warn("All Redis connections failed, making last attempt with primary");
|
|
||||||
primaryClient = await createRedisClient(PRIMARY_REDIS_URL!);
|
primaryClient = await createRedisClient(PRIMARY_REDIS_URL!);
|
||||||
return primaryClient;
|
return primaryClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get all available replicas that are not the active client
|
* Write data to all available Redis instances
|
||||||
*/
|
*/
|
||||||
async function getAvailableReplicas(activeClientUrl: string): Promise<Redis[]> {
|
async function writeToAllRedis(
|
||||||
const availableReplicas: Redis[] = [];
|
operation: (redis: Redis) => Promise<any>
|
||||||
|
|
||||||
// Initialize clients if not already done
|
|
||||||
await initializeAllClients();
|
|
||||||
|
|
||||||
// Check each replica
|
|
||||||
for (const [url, client] of replicaClients.entries()) {
|
|
||||||
if (url !== activeClientUrl && client.status === "ready") {
|
|
||||||
try {
|
|
||||||
const isHealthy = await checkConnection(client, getRedisName(url));
|
|
||||||
if (isHealthy) {
|
|
||||||
availableReplicas.push(client);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Error checking replica ${getRedisName(url)}:`, error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Found ${availableReplicas.length} available replicas for replication`);
|
|
||||||
return availableReplicas;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Immediately replicate a Redis operation to all available replicas
|
|
||||||
*/
|
|
||||||
async function replicateOperation(
|
|
||||||
activeClientUrl: string,
|
|
||||||
key: string,
|
|
||||||
value: string,
|
|
||||||
ttl?: number
|
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
try {
|
const clients = [
|
||||||
// Get available replicas that are not the active client
|
primaryClient,
|
||||||
const replicas = await getAvailableReplicas(activeClientUrl);
|
...Array.from(replicaClients.values()),
|
||||||
|
].filter(Boolean) as Redis[];
|
||||||
|
|
||||||
if (replicas.length === 0) {
|
const errors: Error[] = [];
|
||||||
console.log('No available replicas for replication');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Replicating operation for key ${key} to ${replicas.length} replicas`);
|
for (const client of clients) {
|
||||||
|
try {
|
||||||
// Replicate to each available replica
|
if (client.status === "ready") {
|
||||||
const replicationPromises = replicas.map(async (replica) => {
|
await operation(client);
|
||||||
try {
|
|
||||||
if (ttl) {
|
|
||||||
await replica.set(key, value, 'EX', ttl);
|
|
||||||
} else {
|
|
||||||
await replica.set(key, value);
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Replication failed for key ${key}:`, error);
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
});
|
} catch (err) {
|
||||||
|
errors.push(err as Error);
|
||||||
|
console.error(
|
||||||
|
`Write failed for ${getRedisName(client.options.host || "")}:`,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const results = await Promise.allSettled(replicationPromises);
|
if (errors.length === clients.length && clients.length > 0) {
|
||||||
const successful = results.filter(r => r.status === 'fulfilled' && r.value === true).length;
|
throw new Error("All Redis writes failed");
|
||||||
|
|
||||||
console.log(`Successfully replicated to ${successful}/${replicas.length} replicas`);
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error during replication:', error);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Synchronize data from a replica to the primary
|
* Store a feed mapping in Redis
|
||||||
* Used when primary recovers after being down
|
|
||||||
*/
|
|
||||||
async function syncFromReplicaToPrimary(replicaUrl: string): Promise<void> {
|
|
||||||
if (!primaryClient || primaryClient.status !== "ready" || !replicaClients.has(replicaUrl)) {
|
|
||||||
console.log("Cannot sync: primary or replica not available");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const replicaClient = replicaClients.get(replicaUrl)!;
|
|
||||||
console.log(`Starting sync from ${getRedisName(replicaUrl)} to Primary...`);
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Get all feed keys using scan to prevent blocking
|
|
||||||
const feedKeys: string[] = [];
|
|
||||||
let cursor = '0';
|
|
||||||
|
|
||||||
do {
|
|
||||||
const [nextCursor, keys] = await replicaClient.scan(
|
|
||||||
cursor,
|
|
||||||
'MATCH',
|
|
||||||
'[0-9a-f]*', // Match feed IDs (hex format)
|
|
||||||
'COUNT',
|
|
||||||
'100'
|
|
||||||
);
|
|
||||||
|
|
||||||
cursor = nextCursor;
|
|
||||||
feedKeys.push(...keys);
|
|
||||||
} while (cursor !== '0');
|
|
||||||
|
|
||||||
// Also try scanning for any key pattern that might be a feed ID
|
|
||||||
cursor = '0';
|
|
||||||
const additionalKeys: string[] = [];
|
|
||||||
|
|
||||||
do {
|
|
||||||
const [nextCursor, keys] = await replicaClient.scan(
|
|
||||||
cursor,
|
|
||||||
'MATCH',
|
|
||||||
'feed:*', // Try another pattern
|
|
||||||
'COUNT',
|
|
||||||
'100'
|
|
||||||
);
|
|
||||||
|
|
||||||
cursor = nextCursor;
|
|
||||||
additionalKeys.push(...keys);
|
|
||||||
} while (cursor !== '0');
|
|
||||||
|
|
||||||
feedKeys.push(...additionalKeys);
|
|
||||||
|
|
||||||
// Get chapter cache keys
|
|
||||||
const chapterKeys: string[] = [];
|
|
||||||
cursor = '0';
|
|
||||||
|
|
||||||
do {
|
|
||||||
const [nextCursor, keys] = await replicaClient.scan(
|
|
||||||
cursor,
|
|
||||||
'MATCH',
|
|
||||||
'chapters:*',
|
|
||||||
'COUNT',
|
|
||||||
'100'
|
|
||||||
);
|
|
||||||
|
|
||||||
cursor = nextCursor;
|
|
||||||
chapterKeys.push(...keys);
|
|
||||||
} while (cursor !== '0');
|
|
||||||
|
|
||||||
console.log(`Found ${feedKeys.length} feed keys and ${chapterKeys.length} chapter keys to sync`);
|
|
||||||
|
|
||||||
// Sync feed mappings
|
|
||||||
if (feedKeys.length > 0) {
|
|
||||||
const feedPipeline = primaryClient.pipeline();
|
|
||||||
|
|
||||||
for (const key of feedKeys) {
|
|
||||||
const value = await replicaClient.get(key);
|
|
||||||
const ttl = await replicaClient.ttl(key);
|
|
||||||
|
|
||||||
if (value && ttl > 0) {
|
|
||||||
feedPipeline.set(key, value, 'EX', ttl);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
await feedPipeline.exec();
|
|
||||||
console.log(`Synced ${feedKeys.length} feed mappings from replica to primary`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sync chapter cache
|
|
||||||
if (chapterKeys.length > 0) {
|
|
||||||
const chapterPipeline = primaryClient.pipeline();
|
|
||||||
|
|
||||||
for (const key of chapterKeys) {
|
|
||||||
const value = await replicaClient.get(key);
|
|
||||||
const ttl = await replicaClient.ttl(key);
|
|
||||||
|
|
||||||
if (value && ttl > 0) {
|
|
||||||
chapterPipeline.set(key, value, 'EX', ttl);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
await chapterPipeline.exec();
|
|
||||||
console.log(`Synced ${chapterKeys.length} chapter caches from replica to primary`);
|
|
||||||
}
|
|
||||||
|
|
||||||
lastUsedReplica = null; // Reset after successful sync
|
|
||||||
console.log("Replica to primary sync completed successfully");
|
|
||||||
} catch (error) {
|
|
||||||
console.error("Error during replica to primary sync:", error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Store a feed mapping in Redis with immediate replication
|
|
||||||
*/
|
*/
|
||||||
export async function storeFeedMapping(
|
export async function storeFeedMapping(
|
||||||
feedId: string,
|
feedId: string,
|
||||||
|
|
@ -452,54 +185,31 @@ export async function storeFeedMapping(
|
||||||
created_at: new Date().toISOString()
|
created_at: new Date().toISOString()
|
||||||
};
|
};
|
||||||
|
|
||||||
const stringValue = JSON.stringify(mapping);
|
await writeToAllRedis(async (redis) =>
|
||||||
|
redis.set(feedId, JSON.stringify(mapping), "EX", DEFAULT_CACHE_TTL)
|
||||||
try {
|
);
|
||||||
// First write to primary/active client
|
|
||||||
const redis = await getRedisClient(); // This already handles failover
|
|
||||||
const activeUrl = redis.options.host || '';
|
|
||||||
|
|
||||||
await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL);
|
|
||||||
console.log(`Successfully stored mapping for feed ${feedId}`);
|
|
||||||
|
|
||||||
// Immediately replicate to other instances
|
|
||||||
await replicateOperation(activeUrl, feedId, stringValue, DEFAULT_CACHE_TTL);
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Error storing feed mapping for ${feedId}:`, error);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Retrieve a feed mapping from Redis with automatic failover
|
* Retrieve a feed mapping from Redis
|
||||||
*/
|
*/
|
||||||
export async function getFeedMapping(
|
export async function getFeedMapping(
|
||||||
feedId: string
|
feedId: string
|
||||||
): Promise<FeedMapping | null> {
|
): Promise<FeedMapping | null> {
|
||||||
try {
|
try {
|
||||||
const redis = await getRedisClient(); // This already handles failover
|
const redis = await getRedisClient();
|
||||||
const mapping = await redis.get(feedId);
|
const mapping = await redis.get(feedId);
|
||||||
|
|
||||||
if (!mapping) return null;
|
if (!mapping) return null;
|
||||||
|
|
||||||
// Reset expiration on access
|
// Reset expiration on access
|
||||||
const activeUrl = redis.options.host || '';
|
|
||||||
await redis.expire(feedId, DEFAULT_CACHE_TTL);
|
await redis.expire(feedId, DEFAULT_CACHE_TTL);
|
||||||
|
|
||||||
// Also reset TTL on other replicas
|
const data = JSON.parse(mapping);
|
||||||
replicateOperation(activeUrl, feedId, mapping, DEFAULT_CACHE_TTL)
|
return {
|
||||||
.catch(err => console.error(`Failed to reset TTL on replicas for ${feedId}:`, err));
|
slugs: data.slugs,
|
||||||
|
lang: data.lang
|
||||||
try {
|
};
|
||||||
const data = JSON.parse(mapping);
|
|
||||||
return {
|
|
||||||
slugs: data.slugs,
|
|
||||||
lang: data.lang
|
|
||||||
};
|
|
||||||
} catch (e) {
|
|
||||||
console.error(`Error parsing feed mapping for ${feedId}:`, e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error retrieving feed mapping for ${feedId}:`, error);
|
console.error(`Error retrieving feed mapping for ${feedId}:`, error);
|
||||||
return null;
|
return null;
|
||||||
|
|
@ -517,7 +227,7 @@ export async function warmChapterCache(
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
try {
|
try {
|
||||||
console.log(`Warming chapter cache for ${slugs.length} comics in ${lang}`);
|
console.log(`Warming chapter cache for ${slugs.length} comics in ${lang}`);
|
||||||
const redis = await getRedisClient(); // This already handles failover
|
const redis = await getRedisClient();
|
||||||
|
|
||||||
// Validate slugs
|
// Validate slugs
|
||||||
const validSlugs = slugs.filter(slug =>
|
const validSlugs = slugs.filter(slug =>
|
||||||
|
|
@ -571,12 +281,5 @@ export async function warmChapterCache(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize connections eagerly to establish connections as soon as possible
|
// Initialize connections lazily
|
||||||
if (!import.meta.env.SSR) {
|
|
||||||
initializeAllClients()
|
|
||||||
.then(() => console.log("Redis clients initialized"))
|
|
||||||
.catch(err => console.error("Failed to initialize Redis clients:", err));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize connections lazily as backup
|
|
||||||
console.log("Redis will connect on first use");
|
console.log("Redis will connect on first use");
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue