diff --git a/astro.config.mjs b/astro.config.mjs index 814a9a1..778d936 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -1,17 +1,14 @@ import { defineConfig } from "astro/config"; +import vercel from "@astrojs/vercel"; import tailwind from "@astrojs/tailwind"; -import netlify from "@astrojs/netlify"; - export default defineConfig({ output: "server", - + adapter: vercel(), image: { remotePatterns: [{ protocol: "https" }], }, - integrations: [tailwind()], - vite: { define: { 'process.env.PRIMARY_REDIS_URL': JSON.stringify(process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL), @@ -19,7 +16,5 @@ export default defineConfig({ 'process.env.REPLICA_REDIS_URL_2': JSON.stringify(process.env.REPLICA_REDIS_URL_2), 'process.env.REPLICA_REDIS_URL_3': JSON.stringify(process.env.REPLICA_REDIS_URL_3) } - }, - - adapter: netlify() + } }); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 5896c4d..109fad9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,18 +1,19 @@ { "name": "@ldez/manga", - "version": "1.0.1", + "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@ldez/manga", - "version": "1.0.1", + "version": "1.0.0", "dependencies": { - "@astrojs/netlify": "^6.2.3", "@astrojs/rss": "^4.0.11", "@astrojs/tailwind": "^6.0.1", + "@astrojs/vercel": "^8.1.3", "@floating-ui/dom": "^1.6.13", "@types/node": "^22.13.11", + "@vercel/analytics": "^1.5.0", "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", @@ -70,24 +71,6 @@ "vfile": "^6.0.3" } }, - "node_modules/@astrojs/netlify": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/@astrojs/netlify/-/netlify-6.2.3.tgz", - "integrity": "sha512-QWSvoECD1ctC8N376TICRrc6F6XyyRiW3L9G1Z37B6o5BmMBXlkwsk9cMboDLbLqhcxR3RHBHXJYZ4rgjQDvQA==", - "dependencies": { - "@astrojs/internal-helpers": "0.6.1", - "@astrojs/underscore-redirects": "0.6.0", - "@netlify/blobs": "^8.1.1", - "@netlify/functions": "^3.0.0", - "@vercel/nft": "^0.29.2", - "esbuild": "^0.25.0", - "tinyglobby": "^0.2.12", - "vite": "^6.2.0" - }, - "peerDependencies": { - "astro": "^5.3.0" - } - }, "node_modules/@astrojs/prism": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@astrojs/prism/-/prism-3.2.0.tgz", @@ -139,10 +122,22 @@ "node": "^18.17.1 || ^20.3.0 || >=22.0.0" } }, - "node_modules/@astrojs/underscore-redirects": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/@astrojs/underscore-redirects/-/underscore-redirects-0.6.0.tgz", - "integrity": "sha512-dnJgFpaM955IFNIkEEmMaaIdWXRdeZs1ID6mlGBqdjh6NiCXfKmOdq7L4fd9Dd/tr18fkLrOJ25IUJSxRAEhjQ==" + "node_modules/@astrojs/vercel": { + "version": "8.1.3", + "resolved": "https://registry.npmjs.org/@astrojs/vercel/-/vercel-8.1.3.tgz", + "integrity": "sha512-mHO28cc0FQbA6ncFteW5Hqf2l0gSthPNF8/dpM1sQAOD92Mbyi93/4EhTYj07Xhi3t37FU4tPtzyQoCAn4P4Kg==", + "dependencies": { + "@astrojs/internal-helpers": "0.6.1", + "@vercel/analytics": "^1.5.0", + "@vercel/edge": "^1.2.1", + "@vercel/nft": "^0.29.2", + "@vercel/routing-utils": "^5.0.4", + "esbuild": "^0.25.0", + "tinyglobby": "^0.2.12" + }, + "peerDependencies": { + "astro": "^5.0.0" + } }, "node_modules/@babel/helper-string-parser": { "version": "7.25.9", @@ -1066,33 +1061,6 @@ "node": ">=18" } }, - "node_modules/@netlify/blobs": { - "version": "8.1.2", - "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-8.1.2.tgz", - "integrity": "sha512-coQlePCMpgyMxfeCvxa6qPHlahECin0lSRtg8UOn2rzXRWdvJk+yUhhUstW4HLa9ynvAXFAGTEZoVt4BTESNbw==", - "engines": { - "node": "^14.16.0 || >=16.0.0" - } - }, - "node_modules/@netlify/functions": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@netlify/functions/-/functions-3.0.4.tgz", - "integrity": "sha512-Ox8+ABI+nsLK+c4/oC5dpquXuEIjzfTlJrdQKgQijCsDQoje7inXFAtKDLvvaGvuvE+PVpMLwQcIUL6P9Ob1hQ==", - "dependencies": { - "@netlify/serverless-functions-api": "1.36.0" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@netlify/serverless-functions-api": { - "version": "1.36.0", - "resolved": "https://registry.npmjs.org/@netlify/serverless-functions-api/-/serverless-functions-api-1.36.0.tgz", - "integrity": "sha512-z6okREyK8in0486a22Oro0k+YsuyEjDXJt46FpgeOgXqKJ9ElM8QPll0iuLBkpbH33ENiNbIPLd1cuClRQnhiw==", - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1518,6 +1486,48 @@ "crypto-js": "^4.2.0" } }, + "node_modules/@vercel/analytics": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-1.5.0.tgz", + "integrity": "sha512-MYsBzfPki4gthY5HnYN7jgInhAZ7Ac1cYDoRWFomwGHWEX7odTEzbtg9kf/QSo7XEsEAqlQugA6gJ2WS2DEa3g==", + "peerDependencies": { + "@remix-run/react": "^2", + "@sveltejs/kit": "^1 || ^2", + "next": ">= 13", + "react": "^18 || ^19 || ^19.0.0-rc", + "svelte": ">= 4", + "vue": "^3", + "vue-router": "^4" + }, + "peerDependenciesMeta": { + "@remix-run/react": { + "optional": true + }, + "@sveltejs/kit": { + "optional": true + }, + "next": { + "optional": true + }, + "react": { + "optional": true + }, + "svelte": { + "optional": true + }, + "vue": { + "optional": true + }, + "vue-router": { + "optional": true + } + } + }, + "node_modules/@vercel/edge": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@vercel/edge/-/edge-1.2.1.tgz", + "integrity": "sha512-1++yncEyIAi68D3UEOlytYb1IUcIulMWdoSzX2h9LuSeeyR7JtaIgR8DcTQ6+DmYOQn+5MCh6LY+UmK6QBByNA==" + }, "node_modules/@vercel/nft": { "version": "0.29.2", "resolved": "https://registry.npmjs.org/@vercel/nft/-/nft-0.29.2.tgz", @@ -1543,6 +1553,18 @@ "node": ">=18" } }, + "node_modules/@vercel/routing-utils": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@vercel/routing-utils/-/routing-utils-5.0.4.tgz", + "integrity": "sha512-4ke67zkXVi2fRZdoYckABcsSkRC9CnrdadOGxoS/Bk22+ObHjGQWvUHExRSXh339anwu9YY7ZacNSGH4gUnTQA==", + "dependencies": { + "path-to-regexp": "6.1.0", + "path-to-regexp-updated": "npm:path-to-regexp@6.3.0" + }, + "optionalDependencies": { + "ajv": "^6.12.3" + } + }, "node_modules/abbrev": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.0.tgz", @@ -1578,6 +1600,22 @@ "node": ">= 14" } }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "optional": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/ansi-align": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", @@ -2304,9 +2342,9 @@ "integrity": "sha512-L3sHRo1pXXEqX8VU28kfgUY+YGsk09hPqZiZmLacNib6XNTCM8ubYeT7ryXQw8asB1sKgcU5lkB7ONug08aB8w==" }, "node_modules/consola": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", - "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.0.tgz", + "integrity": "sha512-EiPU8G6dQG0GFHNR8ljnZFki/8a+cQwEQ+7wpxdChl02Q8HXlwEZWD5lqAF8vC2sEC3Tehr8hy7vErz88LHyUA==", "engines": { "node": "^14.18.0 || >=16.10.0" } @@ -2588,6 +2626,12 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "optional": true + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -2603,6 +2647,12 @@ "node": ">=8.6.0" } }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "optional": true + }, "node_modules/fast-xml-parser": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.1.tgz", @@ -3213,6 +3263,12 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "optional": true + }, "node_modules/kleur": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", @@ -4407,6 +4463,17 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/path-to-regexp": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.1.0.tgz", + "integrity": "sha512-h9DqehX3zZZDCEm+xbfU0ZmwCGFCAAraPJWMXJ4+v32NjZJilVg3k1TcKsRgIb8IQ/izZSaydDc1OhJCZvs2Dw==" + }, + "node_modules/path-to-regexp-updated": { + "name": "path-to-regexp", + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -4612,6 +4679,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "optional": true, + "engines": { + "node": ">=6" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -5692,6 +5768,15 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "optional": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index 8719059..1350193 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@ldez/manga", "type": "module", - "version": "1.0.1", + "version": "1.0.0", "private": true, "scripts": { "dev": "astro dev", @@ -11,11 +11,12 @@ "astro": "astro" }, "dependencies": { - "@astrojs/netlify": "^6.2.3", "@astrojs/rss": "^4.0.11", "@astrojs/tailwind": "^6.0.1", + "@astrojs/vercel": "^8.1.3", "@floating-ui/dom": "^1.6.13", "@types/node": "^22.13.11", + "@vercel/analytics": "^1.5.0", "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", diff --git a/src/lib/api.ts b/src/lib/api.ts index c0329e2..b77f572 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -291,7 +291,7 @@ export async function getChaptersForSlugs( lang: string = 'en' ): Promise { console.log(`Fetching chapters for ${slugs.length} comics`); - const redis = await getRedisClient(); + const { client: redis } = await getRedisClient(); const uniqueSlugs = [...new Set(slugs)]; const pipeline = redis.pipeline(); diff --git a/src/lib/db.ts b/src/lib/db.ts index f1fe845..4094f66 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -39,6 +39,7 @@ let primaryHealthy: boolean = true; let primaryFailedChecks = 0; let replicaClients: Map = new Map(); let activeReplicaUrl: string | null = null; +let initializationPromise: Promise | null = null; // Promise to track initialization // Track which replica was used when primary was down let lastUsedReplica: string | null = null; @@ -157,34 +158,92 @@ async function createRedisClient(url: string): Promise { * Initialize all replicas to establish connections early */ async function initializeAllClients(): Promise { - try { - // Initialize primary - if (PRIMARY_REDIS_URL && (!primaryClient || primaryClient.status !== "ready")) { - primaryClient = await createRedisClient(PRIMARY_REDIS_URL); - primaryHealthy = await checkConnection(primaryClient, "Primary"); - console.log(`Primary Redis initialized, healthy: ${primaryHealthy}`); - } - - // Initialize all replicas - for (const url of REPLICA_URLS) { - if (!replicaClients.has(url)) { - try { - const client = await createRedisClient(url); - const isHealthy = await checkConnection(client, getRedisName(url)); - if (isHealthy) { - replicaClients.set(url, client); - console.log(`Replica ${getRedisName(url)} initialized successfully`); - } - } catch (error) { - console.error(`Failed to initialize replica ${getRedisName(url)}:`, error); - } - } - } - - console.log(`Initialized ${replicaClients.size} replica connections`); - } catch (error) { - console.error("Error during client initialization:", error); + // Prevent multiple concurrent initializations + if (initializationPromise) { + console.log("Initialization already in progress, awaiting existing promise."); + return initializationPromise; } + + console.log("Starting Redis client initialization..."); + const initStartTime = Date.now(); + + // Create a promise that resolves when initialization is complete + initializationPromise = (async () => { + try { + // Initialize primary + if (PRIMARY_REDIS_URL && (!primaryClient || primaryClient.status !== "ready")) { + console.log("Initializing primary client..."); + primaryClient = await createRedisClient(PRIMARY_REDIS_URL); + primaryHealthy = await checkConnection(primaryClient, "Primary"); + console.log(`Primary Redis initialized, healthy: ${primaryHealthy}`); + } else if (primaryClient) { + console.log(`Primary client already exists (Status: ${primaryClient.status}), checking health...`); + primaryHealthy = await checkConnection(primaryClient, "Primary"); + console.log(`Primary health check result: ${primaryHealthy}`); + } else { + console.log("Primary Redis URL not configured."); + } + + // Initialize all replicas + console.log(`Found ${REPLICA_URLS.length} replica URLs to initialize.`); + const replicaInitPromises: Promise[] = []; + + for (const url of REPLICA_URLS) { + // Add promise to array to run in parallel + replicaInitPromises.push( + (async () => { + // Check if already initialized and healthy + if (replicaClients.has(url)) { + const existingClient = replicaClients.get(url)!; + console.log(`Replica ${getRedisName(url)} already in map (Status: ${existingClient.status}), checking health...`); + if (await checkConnection(existingClient, getRedisName(url))) { + console.log(`Replica ${getRedisName(url)} is healthy.`); + return; // Skip re-initialization if healthy + } else { + console.log(`Replica ${getRedisName(url)} found but unhealthy, attempting re-initialization.`); + // Optionally disconnect the old client first + existingClient.disconnect(); + } + } + + try { + console.log(`Initializing replica ${getRedisName(url)}...`); + const client = await createRedisClient(url); + const isHealthy = await checkConnection(client, getRedisName(url)); + if (isHealthy) { + replicaClients.set(url, client); // Add to map *only if healthy* + console.log(`Replica ${getRedisName(url)} initialized successfully and added to map.`); + } else { + console.warn(`Replica ${getRedisName(url)} initialized but failed health check.`); + // Optionally disconnect if unhealthy after creation + client.disconnect(); + } + } catch (error) { + console.error(`Failed to initialize replica ${getRedisName(url)}:`, error); + } + })() + ); + } + + // Wait for all replica initializations to complete + await Promise.allSettled(replicaInitPromises); + + console.log(`Initialization attempt finished. Current replica map size: ${replicaClients.size}. Time taken: ${Date.now() - initStartTime}ms`); + + } catch (error) { + console.error("CRITICAL Error during client initialization process:", error); + // Reset promise on critical failure to allow retry + initializationPromise = null; + throw error; // Re-throw if needed + } finally { + // Keep the promise reference after success, but maybe reset after some time? + // For now, keep it to prevent re-running unnecessarily. + // If you suspect clients disconnect often, you might want to clear this promise + // periodically or on errors to force re-initialization attempts. + } + })(); // Immediately invoke the async IIFE + + return initializationPromise; } /** @@ -208,71 +267,133 @@ async function checkConnection(client: Redis, name: string): Promise { /** * Get an active Redis client from the pool * Prioritizes primary, falls back to replicas + * Returns the client and its connection URL. */ -export async function getRedisClient(): Promise { - // Try primary client first if it's considered healthy - if (primaryHealthy && (primaryClient === null || primaryClient.status !== "ready")) { - try { - primaryClient = await createRedisClient(PRIMARY_REDIS_URL!); - const isPrimaryConnected = await checkConnection(primaryClient, "Primary"); - primaryHealthy = isPrimaryConnected; - if (isPrimaryConnected) { - primaryFailedChecks = 0; - console.log("Initialized primary Redis connection"); - return primaryClient; - } - } catch (err) { - console.error("Failed to initialize primary Redis:", err); - primaryHealthy = false; - primaryFailedChecks++; - } - } else if (primaryHealthy && primaryClient && primaryClient.status === "ready") { - return primaryClient; +export async function getRedisClient(): Promise<{ client: Redis; url: string }> { + // Ensure initialization has been attempted at least once + if (!initializationPromise) { + console.log("[getRedisClient] Initialization promise not found, triggering initialization..."); + await initializeAllClients(); // Await the first initialization attempt + } else { + // Optional: Await if already running, prevents race conditions on first load + // await initializationPromise; } + + // Try primary client first if it's considered healthy + if (primaryHealthy && primaryClient && primaryClient.status === "ready") { + // Check connection just in case before returning + if (await checkConnection(primaryClient, "Primary")) { + return { client: primaryClient, url: PRIMARY_REDIS_URL! }; // <-- Return URL + } else { + console.warn("Primary client status was ready but ping failed. Marking unhealthy."); + primaryHealthy = false; + primaryFailedChecks++; + // Fall through to replica logic + } + } + + // --- Replica Logic --- + console.log(`[getRedisClient] Primary unhealthy or unavailable. Checking replicas. Current map size: ${replicaClients.size}`); + + // If the map is empty, maybe initialization failed or hasn't run? Try again. + // This adds robustness but might hide underlying init issues if called repeatedly. + if (replicaClients.size === 0 && REPLICA_URLS.length > 0) { + console.warn("[getRedisClient] Replica map is empty, attempting re-initialization..."); + await initializeAllClients(); // Await re-initialization attempt + console.log(`[getRedisClient] Re-initialization finished. New map size: ${replicaClients.size}`); + } + + // Primary is unhealthy, try to use the active replica if we have one if (activeReplicaUrl && replicaClients.has(activeReplicaUrl)) { const activeReplica = replicaClients.get(activeReplicaUrl)!; if (activeReplica.status === "ready" && await checkConnection(activeReplica, getRedisName(activeReplicaUrl))) { console.log(`Using active replica: ${getRedisName(activeReplicaUrl)}`); lastUsedReplica = activeReplicaUrl; // Track that we used this replica - return activeReplica; + return { client: activeReplica, url: activeReplicaUrl }; // <-- Return URL } } - // No active replica, try replicas in order + // No active replica, try replicas in order (using the potentially updated map) + console.log(`[getRedisClient] No active replica. Iterating through ${replicaClients.size} clients in map...`); + // Iterate through the *map* instead of the URL list first + for (const [url, client] of replicaClients.entries()) { + const clientName = getRedisName(url); + console.log(`[getRedisClient] Checking client from map: ${clientName} (Status: ${client.status})`); + if (client.status === "ready" && await checkConnection(client, clientName)) { + console.log(`[getRedisClient] Using healthy client from map: ${clientName}`); + activeReplicaUrl = url; + lastUsedReplica = url; + // Promotion logic remains the same + if (primaryFailedChecks >= REPLICA_PROMOTION_THRESHOLD) { + console.log(`Promoting ${getRedisName(url)} to primary after ${primaryFailedChecks} failed checks`); + primaryClient = client; // The client instance is now considered primary + primaryHealthy = true; + primaryFailedChecks = 0; + // Note: PRIMARY_REDIS_URL still points to the original primary config. + // The 'url' returned here is the replica's URL, which is correct for identifying the *current* primary instance. + } + return { client: client, url: url }; + } else { + console.log(`[getRedisClient] Client ${clientName} from map is not ready or failed check.`); + } + } + + // If no client in the map worked, fall back to trying URLs again (covers cases where a client wasn't added to map) + console.log(`[getRedisClient] No suitable client found in map. Falling back to iterating through REPLICA_URLS (${REPLICA_URLS.length})...`); for (const url of REPLICA_URLS) { + const clientName = getRedisName(url); try { let client = replicaClients.get(url); + // If client exists but wasn't ready above, try creating a new one if (!client || client.status !== "ready") { + console.log(`[getRedisClient] Client ${clientName} not in map or not ready. Attempting connection...`); client = await createRedisClient(url); + // Add to map immediately upon successful creation attempt, check health next replicaClients.set(url, client); + console.log(`[getRedisClient] Added/updated client ${clientName} in map.`); } - - if (await checkConnection(client, getRedisName(url))) { - console.log(`Switching to replica: ${getRedisName(url)}`); + + if (await checkConnection(client, clientName)) { + console.log(`[getRedisClient] Successfully connected/verified replica: ${clientName}`); activeReplicaUrl = url; - lastUsedReplica = url; // Track that we used this replica - - // If primary has been down for a while, promote this replica to be the new primary + lastUsedReplica = url; + // Promotion logic remains the same if (primaryFailedChecks >= REPLICA_PROMOTION_THRESHOLD) { - console.log(`Promoting ${getRedisName(url)} to primary after ${primaryFailedChecks} failed checks`); - primaryClient = client; - primaryHealthy = true; - primaryFailedChecks = 0; + console.log(`Promoting ${getRedisName(url)} to primary after ${primaryFailedChecks} failed checks`); + primaryClient = client; // The client instance is now considered primary + primaryHealthy = true; + primaryFailedChecks = 0; + // Note: PRIMARY_REDIS_URL still points to the original primary config. + // The 'url' returned here is the replica's URL, which is correct for identifying the *current* primary instance. } - - return client; + return { client: client, url: url }; + } else { + console.log(`[getRedisClient] Health check failed for ${clientName} after connection attempt.`); + // Optionally remove from map if health check fails immediately after creation? + // replicaClients.delete(url); } } catch (err) { - console.error(`Failed to connect to replica ${getRedisName(url)}:`, err); + console.error(`[getRedisClient] Failed to connect to replica ${clientName}:`, err); + // Remove from map if connection failed + replicaClients.delete(url); } } // If all else fails, try one more time with primary console.warn("All Redis connections failed, making last attempt with primary"); - primaryClient = await createRedisClient(PRIMARY_REDIS_URL!); - return primaryClient; + // Ensure primaryClient is re-initialized if it failed earlier + if (!primaryClient || primaryClient.status !== 'connecting' && primaryClient.status !== 'ready') { + try { + primaryClient = await createRedisClient(PRIMARY_REDIS_URL!); + } catch (err) { + console.error("Fatal: Failed to re-initialize primary Redis on last attempt:", err); + throw new Error("Could not establish any Redis connection."); // Throw if even last attempt fails + } + } + // Even if creation succeeded, we return it, subsequent operations might fail but we need to return *something* + return { client: primaryClient, url: PRIMARY_REDIS_URL! }; // <-- Return URL } /** @@ -303,46 +424,561 @@ async function getAvailableReplicas(activeClientUrl: string): Promise { } /** - * Immediately replicate a Redis operation to all available replicas + * Store a feed mapping in Redis with immediate replication */ -async function replicateOperation( - activeClientUrl: string, - key: string, - value: string, - ttl?: number +export async function storeFeedMapping( + feedId: string, + slugs: string[], + lang: string +): Promise { + const mapping = { + slugs, + lang, + created_at: new Date().toISOString() + }; + + const stringValue = JSON.stringify(mapping); + const regularKey = feedId; // Key for the non-compressed version + const compressedKey = `${feedId}:compressed`; // Key for the compressed version + + try { + const { client: redis, url: activeUrl } = await getRedisClient(); + const clientName = getRedisName(activeUrl); + + // --- 1. Store the non-compressed version ALWAYS --- + console.log(`[storeFeedMapping] Attempting initial SET for regular key ${regularKey} to ${clientName}`); + await redis.set(regularKey, stringValue, "EX", DEFAULT_CACHE_TTL); + console.log(`[storeFeedMapping] SUCCESS: Stored regular mapping for feed ${feedId} to ${clientName}`); + + // Replicate the non-compressed version + console.log(`[storeFeedMapping] Calling ensurePrimaryAndReplicate for regular key ${regularKey} (source: ${clientName})`); + await ensurePrimaryAndReplicate(activeUrl, regularKey, stringValue, DEFAULT_CACHE_TTL); + + + // --- 2. Store the compressed version IF needed --- + if (shouldCompress(stringValue)) { + console.log(`[storeFeedMapping] Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`); + const compressed = await gzip(stringValue); + const base64Compressed = compressed.toString('base64'); + + console.log(`[storeFeedMapping] Attempting initial SET for compressed key ${compressedKey} to ${clientName}`); + await redis.set(compressedKey, base64Compressed, "EX", DEFAULT_CACHE_TTL); + console.log(`[storeFeedMapping] SUCCESS: Stored compressed mapping (${base64Compressed.length} bytes) to ${clientName}`); + + // Replicate the compressed version + console.log(`[storeFeedMapping] Calling ensurePrimaryAndReplicate for compressed key ${compressedKey} (source: ${clientName})`); + await ensurePrimaryAndReplicate(activeUrl, compressedKey, base64Compressed, DEFAULT_CACHE_TTL); + + } else { + console.log(`[storeFeedMapping] Skipping compressed storage for ${feedId} (data size below threshold).`); + // Optional: If a compressed version might exist from a previous larger save, delete it + // console.log(`[storeFeedMapping] Attempting to delete potentially stale compressed key ${compressedKey} on ${clientName}`); + // await redis.del(compressedKey).catch(err => console.warn(`Failed to delete stale compressed key ${compressedKey}:`, err)); + // // Also replicate the deletion + // console.log(`[storeFeedMapping] Calling ensurePrimaryAndReplicate for DELETE of compressed key ${compressedKey}`); + // await ensurePrimaryAndReplicate(activeUrl, compressedKey, '', 0); // Use TTL 0 or a specific marker for deletion if needed + } + + console.log(`[storeFeedMapping] Successfully processed storage and replication for feed ${feedId} (both versions if applicable)`); + + } catch (error) { + // Log the specific error during the initial write attempt + // Note: Error could happen during either set operation or replication call + console.error(`[storeFeedMapping] CRITICAL ERROR storing feed mapping for ${feedId}:`, error); + throw error; // Re-throw the error after logging + } +} + +/** + * Retrieve a feed mapping from Redis with automatic failover + */ +export async function getFeedMapping( + feedId: string +): Promise { + console.log(`[getFeedMapping] Attempting to get mapping for feedId: ${feedId}`); + try { + // Try primary first + const { client, url: clientUrl } = await getRedisClient(); + const clientName = getRedisName(clientUrl); + console.log(`[getFeedMapping] Using client: ${clientName} (${clientUrl})`); + + // Check for compressed version first + const compressedKey = `${feedId}:compressed`; + console.log(`[getFeedMapping] Checking for compressed key: ${compressedKey} on ${clientName}`); + const compressedData = await client.get(compressedKey); // This is the base64 string + + if (compressedData) { + console.log(`[getFeedMapping] SUCCESS: Found compressed data for ${feedId} on ${clientName}`); + try { + // Decompress and parse + const buffer = Buffer.from(compressedData, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + // --- Add back direct expire on the source client --- + console.log(`[getFeedMapping] Resetting TTL directly on source ${clientName} for compressed key ${compressedKey}`); + client.expire(compressedKey, DEFAULT_CACHE_TTL) + .catch(err => console.warn(`[getFeedMapping] Failed direct TTL reset on ${clientName} for ${compressedKey}:`, err)); + // --- End Add back --- + + // Replicate the COMPRESSED data using ensurePrimaryAndReplicate + console.log(`[getFeedMapping] Triggering DATA replication for compressed key ${compressedKey}`); + ensurePrimaryAndReplicate(clientUrl, compressedKey, compressedData, DEFAULT_CACHE_TTL) + .catch(err => console.error(`[getFeedMapping] Error during background DATA replication for ${compressedKey}:`, err)); + + return { + slugs: data.slugs, + lang: data.lang + }; + } catch (error) { + console.error(`[getFeedMapping] Error decompressing data for ${feedId} from ${clientName}:`, error); + // Fall through to try uncompressed version + } + } else { + console.log(`[getFeedMapping] No compressed key found for ${feedId} on ${clientName}.`); + } + + // If no compressed version, try regular key + const regularKey = feedId; + console.log(`[getFeedMapping] Checking for regular key: ${regularKey} on ${clientName}`); + let jsonString = await client.get(regularKey); // This is the JSON string + + // Variables to track source if found on replica + let foundOnReplica = false; + let replicaUrlSource: string | null = null; + let replicaClientSource: Redis | null = null; // Store the replica client instance + let valueFromReplica: string | null = null; // Store value found on replica + let isValueFromReplicaCompressed = false; // Track if replica value was compressed + + if (jsonString) { + console.log(`[getFeedMapping] SUCCESS: Found regular key ${regularKey} on ${clientName}`); + } + + // If not found in the current client AND the client was the primary, check replicas + if (!jsonString && REPLICA_URLS.length > 0 && client === primaryClient) { + console.log(`[getFeedMapping] Key ${regularKey} not found on PRIMARY, checking replicas...`); + + // Try each replica + for (const [replicaUrl, replicaClient] of replicaClients.entries()) { + const replicaName = getRedisName(replicaUrl); + if (replicaClient.status === "ready") { + try { + // Check compressed key on replica first + const replicaCompressedData = await replicaClient.get(compressedKey); // Base64 string + if (replicaCompressedData) { + console.log(`[getFeedMapping] SUCCESS: Found COMPRESSED key ${compressedKey} on REPLICA ${replicaName}`); + try { + // Decompress for return value + const buffer = Buffer.from(replicaCompressedData, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + // --- Add back direct expire on the source replica client --- + console.log(`[getFeedMapping] Resetting TTL directly on source ${replicaName} for compressed key ${compressedKey}`); + replicaClient.expire(compressedKey, DEFAULT_CACHE_TTL) + .catch(err => console.warn(`[getFeedMapping] Failed direct TTL reset on ${replicaName} for ${compressedKey}:`, err)); + // --- End Add back --- + + // Replicate the COMPRESSED data found on replica + console.log(`[getFeedMapping] Triggering DATA replication for compressed key ${compressedKey} (found on replica ${replicaName})`); + ensurePrimaryAndReplicate(replicaUrl, compressedKey, replicaCompressedData, DEFAULT_CACHE_TTL) + .catch(err => console.error(`[getFeedMapping] Error during background DATA replication for ${compressedKey}:`, err)); + + return { slugs: data.slugs, lang: data.lang }; + } catch (error) { + console.error(`[getFeedMapping] Error decompressing data for ${feedId} from REPLICA ${replicaName}:`, error); + } + } else { + // Check regular key on replica + const replicaJsonString = await replicaClient.get(regularKey); // JSON string + if (replicaJsonString) { + console.log(`[getFeedMapping] SUCCESS: Found REGULAR key ${regularKey} on REPLICA ${replicaName}`); + foundOnReplica = true; + replicaUrlSource = replicaUrl; + replicaClientSource = replicaClient; // Store the client instance + valueFromReplica = replicaJsonString; // Store the value found + isValueFromReplicaCompressed = false; // Mark as not compressed + jsonString = replicaJsonString; // Use this value going forward + break; // Found on this replica, stop checking others + } else { + console.log(`[getFeedMapping] No regular key found on replica ${replicaName}.`); + } + } + + } catch (error) { + console.error(`[getFeedMapping] Error checking replica ${replicaName} for ${feedId}:`, error); + } + } else { + console.log(`[getFeedMapping] Skipping replica ${replicaName} (status: ${replicaClient.status})`); + } + } + } + + // If jsonString is still null/empty after checks, the key wasn't found + if (!jsonString) { + console.log(`[getFeedMapping] Key ${feedId} (and compressed variant) not found on any checked instance.`); + return null; + } + + // --- Add back direct expire on the source client (primary or replica) --- + const sourceClientForExpire = foundOnReplica ? replicaClientSource! : client; + const sourceClientNameForExpire = foundOnReplica ? getRedisName(replicaUrlSource!) : clientName; + console.log(`[getFeedMapping] Resetting TTL directly on source ${sourceClientNameForExpire} for regular key ${regularKey}`); + sourceClientForExpire.expire(regularKey, DEFAULT_CACHE_TTL) + .catch(err => console.warn(`[getFeedMapping] Failed direct TTL reset on ${sourceClientNameForExpire} for ${regularKey}:`, err)); + // --- End Add back --- + + // Replicate the REGULAR data using ensurePrimaryAndReplicate + console.log(`[getFeedMapping] Triggering DATA replication for regular key ${regularKey}`); + const sourceUrlForReplication = foundOnReplica ? replicaUrlSource! : clientUrl; + ensurePrimaryAndReplicate(sourceUrlForReplication, regularKey, jsonString, DEFAULT_CACHE_TTL) + .catch(err => console.error(`[getFeedMapping] Error during background DATA replication for ${regularKey}:`, err)); + + // Parse the result + try { + const data = JSON.parse(jsonString); + return { + slugs: data.slugs, + lang: data.lang + }; + } catch (parseError) { + console.error(`[getFeedMapping] CRITICAL ERROR parsing feed mapping JSON for ${feedId}:`, parseError, `JSON String: ${jsonString}`); + return null; // Return null if parsing fails + } + + } catch (error) { + console.error(`[getFeedMapping] CRITICAL ERROR retrieving feed mapping for ${feedId}:`, error); + return null; + } +} + +/** + * Pre-warm the chapter cache for better performance + * This function checks cache status and uses the getChaptersForSlugs function from api.ts + * to fetch and cache data in the background + */ +export async function warmChapterCache( + slugs: string[], + lang: string ): Promise { try { - // Get available replicas that are not the active client - const replicas = await getAvailableReplicas(activeClientUrl); + console.log(`Warming chapter cache for ${slugs.length} comics in ${lang}`); + const { client } = await getRedisClient(); // This already handles failover - if (replicas.length === 0) { - console.log('No available replicas for replication'); - return; + // Validate slugs + const validSlugs = slugs.filter(slug => + slug && typeof slug === 'string' && slug.match(/^[a-z0-9-]+$/)); + + if (validSlugs.length !== slugs.length) { + console.warn(`Filtered out ${slugs.length - validSlugs.length} invalid slugs`); } - console.log(`Replicating operation for key ${key} to ${replicas.length} replicas`); + // Check which slugs are already in cache + const pipeline = client.pipeline(); + const cacheKeys = validSlugs.map(slug => `chapters:${slug}:${lang}`); - // Replicate to each available replica - const replicationPromises = replicas.map(async (replica) => { - try { - if (ttl) { - await replica.set(key, value, 'EX', ttl); - } else { - await replica.set(key, value); - } - return true; - } catch (error) { - console.error(`Replication failed for key ${key}:`, error); - return false; + cacheKeys.forEach(key => { + pipeline.exists(key); + }); + + const results = await pipeline.exec(); + const slugsToFetch: string[] = []; + + results?.forEach(([err, exists], index) => { + if (err) { + console.error(`Error checking cache for ${cacheKeys[index]}:`, err); + slugsToFetch.push(validSlugs[index]); + } else if (!exists) { + slugsToFetch.push(validSlugs[index]); } }); - const results = await Promise.allSettled(replicationPromises); - const successful = results.filter(r => r.status === 'fulfilled' && r.value === true).length; + if (slugsToFetch.length === 0) { + console.log('All chapters already in cache, no warming needed'); + return; + } - console.log(`Successfully replicated to ${successful}/${replicas.length} replicas`); + console.log(`Cache miss for ${slugsToFetch.length} comics - starting background fetch`); + + // To avoid circular dependencies, we'll import getChaptersForSlugs dynamically + const { getChaptersForSlugs } = await import('./api'); + + // Start fetching in the background without waiting for the result + getChaptersForSlugs(slugsToFetch, lang) + .then(chapters => { + console.log(`Successfully warmed cache with ${chapters.length} chapters`); + }) + .catch(error => { + console.error('Error warming chapter cache:', error); + }); + } catch (error) { - console.error('Error during replication:', error); + console.error('Error in warmChapterCache:', error); + } +} + +// Add a new function for handling large feed mappings with compression +export async function storeLargeFeedMapping( + feedId: string, + slugs: string[], + lang: string +): Promise { + // This function now essentially calls storeFeedMapping which handles compression logic. + // We can simplify this or keep it as a specific entry point. + // For consistency, let's just call the main function. + console.log(`Using storeLargeFeedMapping for ${feedId}, deferring to storeFeedMapping.`); + await storeFeedMapping(feedId, slugs, lang); + + /* // Original logic updated with ensurePrimaryAndReplicate: + const mapping = { + slugs, + lang, + created_at: new Date().toISOString() + }; + + const jsonString = JSON.stringify(mapping); + let keyToStore = feedId; + let valueToStore = jsonString; + let ttl = DEFAULT_CACHE_TTL; + + try { + const redis = await getRedisClient(); + const activeUrl = redis.options.url || `${redis.options.host}:${redis.options.port}`; + + // Use compression for large mappings + if (shouldCompress(jsonString)) { + const compressed = await gzip(jsonString); + const base64Compressed = compressed.toString('base64'); + + keyToStore = `${feedId}:compressed`; + valueToStore = base64Compressed; + + // Store with compression flag + await redis.set(keyToStore, valueToStore, "EX", ttl); + console.log(`Stored compressed mapping for feed ${feedId} to ${getRedisName(activeUrl)} (${jsonString.length} → ${valueToStore.length} bytes)`); + + } else { + // Use normal storage for small mappings + await redis.set(keyToStore, jsonString, "EX", ttl); + console.log(`Stored mapping for feed ${feedId} to ${getRedisName(activeUrl)}`); + } + + // Ensure data is on primary and replicate to others + await ensurePrimaryAndReplicate(activeUrl, keyToStore, valueToStore, ttl); + + console.log(`Successfully processed storage and replication for large feed ${feedId}`); + + } catch (error) { + console.error(`Error storing large feed mapping for ${feedId}:`, error); + throw error; + } + */ +} + +// Batch get multiple feed mappings at once +export async function getBatchFeedMappings( + feedIds: string[] +): Promise> { + if (!feedIds.length) return {}; + + try { + const { client } = await getRedisClient(); + const pipeline = client.pipeline(); + + // Check regular keys first + feedIds.forEach(id => { + pipeline.get(id); + }); + + const results = await pipeline.exec(); + if (!results) return {}; + + // Process results + const mappings: Record = {}; + const ttlUpdates = client.pipeline(); + const compressedChecks = client.pipeline(); + const compressedIds: string[] = []; + + for (let i = 0; i < feedIds.length; i++) { + const id = feedIds[i]; + const result = results[i]; + + if (result[0]) { + mappings[id] = null; + continue; + } + + if (result[1]) { + // Process regular mapping + try { + const data = JSON.parse(result[1] as string); + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL + ttlUpdates.expire(id, DEFAULT_CACHE_TTL); + } catch (e) { + mappings[id] = null; + } + } else { + // Check for compressed version + compressedChecks.exists(`${id}:compressed`); + compressedIds.push(id); + } + } + + // Execute TTL updates in background + ttlUpdates.exec().catch(err => console.error('TTL update error:', err)); + + // Check compressed versions if needed + if (compressedIds.length > 0) { + const compressedResults = await compressedChecks.exec(); + + if (compressedResults) { + const compressedFetches = client.pipeline(); + const compressedToFetch: string[] = []; + + compressedResults.forEach((result, index) => { + if (!result[0] && result[1]) { + const id = compressedIds[index]; + compressedFetches.get(`${id}:compressed`); + compressedToFetch.push(id); + } + }); + + if (compressedToFetch.length > 0) { + const fetchResults = await compressedFetches.exec(); + + if (fetchResults) { + for (let i = 0; i < compressedToFetch.length; i++) { + const id = compressedToFetch[i]; + const [err, compressedData] = fetchResults[i]; + + if (!err && compressedData) { + try { + const buffer = Buffer.from(compressedData as string, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL in background + client.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {}); + } catch (e) { + mappings[id] = null; + } + } else { + mappings[id] = null; + } + } + } + } + } + } + + return mappings; + } catch (error) { + console.error(`Error retrieving batch feed mappings:`, error); + return {}; + } +} + +/** + * Ensures data is written to the primary (if not the source) and replicated. + * Handles SET operations. For DEL, a separate function or modification might be needed. + */ +async function ensurePrimaryAndReplicate( + sourceUrl: string, + key: string, + value: string, // Value to set + ttl: number // TTL for SET, use > 0 +): Promise { + const replicationTasks: Promise[] = []; + const sourceName = getRedisName(sourceUrl); + const primaryUrl = PRIMARY_REDIS_URL; + const primaryName = getRedisName(primaryUrl!); + + // Check if operation is a delete based on TTL (simple approach) + const isDelete = ttl <= 0; + const operationType = isDelete ? 'DELETE' : 'SET'; + + console.log(`[ensurePrimaryAndReplicate] Starting sync for ${operationType} on key ${key}. Source: ${sourceName}`); + + // 1. Sync to Primary (if source wasn't primary and primary exists/is healthy) + if (primaryClient && sourceUrl !== primaryUrl) { + const primaryStatus = primaryClient.status; + console.log(`[ensurePrimaryAndReplicate] Checking primary (${primaryName}) status: ${primaryStatus}`); + let primaryIsReady = primaryStatus === 'ready'; + + if (!primaryIsReady) { + try { + await primaryClient.ping(); + primaryIsReady = true; + console.log(`[ensurePrimaryAndReplicate] Primary (${primaryName}) ping successful after status check.`); + } catch (pingErr) { + console.warn(`[ensurePrimaryAndReplicate] Primary (${primaryName}) ping failed, skipping sync attempt. Status: ${primaryStatus}`, pingErr); + } + } + + if (primaryIsReady) { + console.log(`[ensurePrimaryAndReplicate] Attempting sync ${operationType} to PRIMARY (${primaryName}) for key ${key}`); + const task = isDelete + ? primaryClient.del(key) + : primaryClient.set(key, value, "EX", ttl); + + replicationTasks.push( + task + .then(() => console.log(`[ensurePrimaryAndReplicate] SUCCESS: Synced ${operationType} key ${key} to PRIMARY (${primaryName})`)) + .catch(err => console.error(`[ensurePrimaryAndReplicate] FAILED: Sync ${operationType} key ${key} to PRIMARY (${primaryName})`, err)) + ); + } else { + console.log(`[ensurePrimaryAndReplicate] SKIPPED: Sync ${operationType} to PRIMARY (${primaryName}) because it's not ready (Status: ${primaryStatus}).`); + } + } else if (sourceUrl === primaryUrl) { + console.log(`[ensurePrimaryAndReplicate] Source was PRIMARY (${primaryName}), no primary sync needed for ${operationType} on key ${key}.`); + } else { + console.log(`[ensurePrimaryAndReplicate] Primary client not available, cannot sync ${operationType} key ${key} to primary.`); + } + + // 2. Replicate to other available replicas + console.log(`[ensurePrimaryAndReplicate] Current replicaClients map size: ${replicaClients.size}, Keys: ${Array.from(replicaClients.keys())}`); + console.log(`[ensurePrimaryAndReplicate] Checking ${replicaClients.size} replicas for ${operationType} on key ${key}`); + for (const [replicaUrl, replicaClient] of replicaClients.entries()) { + const replicaName = getRedisName(replicaUrl); + // Don't replicate back to the source or to the primary (if primary sync was attempted above) + if (replicaUrl !== sourceUrl && replicaUrl !== primaryUrl) { + const replicaStatus = replicaClient.status; + console.log(`[ensurePrimaryAndReplicate] Checking replica ${replicaName} status: ${replicaStatus}`); + if (replicaStatus === 'ready') { + console.log(`[ensurePrimaryAndReplicate] Attempting replication ${operationType} to REPLICA (${replicaName}) for key ${key}`); + const task = isDelete + ? replicaClient.del(key) + : replicaClient.set(key, value, "EX", ttl); + replicationTasks.push( + task + .then(() => console.log(`[ensurePrimaryAndReplicate] SUCCESS: Replicated ${operationType} key ${key} to REPLICA (${replicaName})`)) + .catch(err => console.error(`[ensurePrimaryAndReplicate] FAILED: Replicate ${operationType} key ${key} to REPLICA (${replicaName})`, err)) + ); + } else { + console.log(`[ensurePrimaryAndReplicate] SKIPPED: Replication ${operationType} to REPLICA (${replicaName}) because it's not ready (Status: ${replicaStatus}).`); + } + } else { + console.log(`[ensurePrimaryAndReplicate] SKIPPED: Replication ${operationType} to ${replicaName} (Source or Primary) for key ${key}.`); + } + } + + // Wait for all sync/replication attempts to settle + if (replicationTasks.length > 0) { + console.log(`[ensurePrimaryAndReplicate] Waiting for ${replicationTasks.length} sync/replication tasks for ${operationType} on key ${key}...`); + const results = await Promise.allSettled(replicationTasks); + console.log(`[ensurePrimaryAndReplicate] Sync/replication tasks settled for ${operationType} on key ${key}. Results:`, results.map(r => r.status)); + } else { + console.log(`[ensurePrimaryAndReplicate] No sync/replication tasks needed for ${operationType} on key ${key}.`); } } @@ -456,368 +1092,13 @@ async function syncFromReplicaToPrimary(replicaUrl: string): Promise { } } -/** - * Store a feed mapping in Redis with immediate replication - */ -export async function storeFeedMapping( - feedId: string, - slugs: string[], - lang: string -): Promise { - const mapping = { - slugs, - lang, - created_at: new Date().toISOString() - }; - - const stringValue = JSON.stringify(mapping); - - try { - // Get Redis client - const redis = await getRedisClient(); - const activeUrl = redis.options.host || ''; - - // Check if compression should be used - if (shouldCompress(stringValue)) { - console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`); - const compressed = await gzip(stringValue); - const base64Compressed = compressed.toString('base64'); - - // Store with compression flag - await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); - console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`); - - // Replicate to other instances - const replicas = await getAvailableReplicas(activeUrl); - for (const replica of replicas) { - await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); - } - } else { - // Use standard set operations without compression - await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); - - // Replicate to other instances - const replicas = await getAvailableReplicas(activeUrl); - for (const replica of replicas) { - await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); - } - } - - console.log(`Successfully stored mapping for feed ${feedId}`); - } catch (error) { - console.error(`Error storing feed mapping for ${feedId}:`, error); - throw error; - } -} - -/** - * Retrieve a feed mapping from Redis with automatic failover - */ -export async function getFeedMapping( - feedId: string -): Promise { - try { - // Try primary first - const redis = await getRedisClient(); - - // Check for compressed version first - const compressedKey = `${feedId}:compressed`; - const compressedData = await redis.get(compressedKey); - - if (compressedData) { - console.log(`Found compressed data for ${feedId}`); - try { - // Decompress and parse - const buffer = Buffer.from(compressedData, 'base64'); - const decompressed = await gunzip(buffer); - const data = JSON.parse(decompressed.toString()); - - // Reset expiration - await redis.expire(compressedKey, DEFAULT_CACHE_TTL); - - return { - slugs: data.slugs, - lang: data.lang - }; - } catch (error) { - console.error(`Error decompressing data for ${feedId}:`, error); - // Fall through to try uncompressed version - } - } - - // If no compressed version, try regular key - let jsonString = await redis.get(feedId); - - // If not found in primary but we have replicas, check them - if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) { - console.log(`Key ${feedId} not found in primary, checking replicas...`); - - // Try each replica - for (const [url, replicaClient] of replicaClients.entries()) { - if (replicaClient.status === "ready") { - try { - jsonString = await replicaClient.get(feedId); - if (jsonString) { - console.log(`Found key ${feedId} in replica ${getRedisName(url)}`); - - // Sync this key back to primary for future requests - if (primaryClient && primaryClient.status === "ready") { - primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL) - .catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err)); - } - - break; - } - } catch (error) { - console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error); - } - } - } - } - - if (!jsonString) return null; - - // Reset expiration on access - await redis.expire(feedId, DEFAULT_CACHE_TTL); - - // Parse the result - const data = JSON.parse(jsonString as string); - return { - slugs: data.slugs, - lang: data.lang - }; - } catch (error) { - console.error(`Error retrieving feed mapping for ${feedId}:`, error); - return null; - } -} - -/** - * Pre-warm the chapter cache for better performance - * This function checks cache status and uses the getChaptersForSlugs function from api.ts - * to fetch and cache data in the background - */ -export async function warmChapterCache( - slugs: string[], - lang: string -): Promise { - try { - console.log(`Warming chapter cache for ${slugs.length} comics in ${lang}`); - const redis = await getRedisClient(); // This already handles failover - - // Validate slugs - const validSlugs = slugs.filter(slug => - slug && typeof slug === 'string' && slug.match(/^[a-z0-9-]+$/)); - - if (validSlugs.length !== slugs.length) { - console.warn(`Filtered out ${slugs.length - validSlugs.length} invalid slugs`); - } - - // Check which slugs are already in cache - const pipeline = redis.pipeline(); - const cacheKeys = validSlugs.map(slug => `chapters:${slug}:${lang}`); - - cacheKeys.forEach(key => { - pipeline.exists(key); - }); - - const results = await pipeline.exec(); - const slugsToFetch: string[] = []; - - results?.forEach(([err, exists], index) => { - if (err) { - console.error(`Error checking cache for ${cacheKeys[index]}:`, err); - slugsToFetch.push(validSlugs[index]); - } else if (!exists) { - slugsToFetch.push(validSlugs[index]); - } - }); - - if (slugsToFetch.length === 0) { - console.log('All chapters already in cache, no warming needed'); - return; - } - - console.log(`Cache miss for ${slugsToFetch.length} comics - starting background fetch`); - - // To avoid circular dependencies, we'll import getChaptersForSlugs dynamically - const { getChaptersForSlugs } = await import('./api'); - - // Start fetching in the background without waiting for the result - getChaptersForSlugs(slugsToFetch, lang) - .then(chapters => { - console.log(`Successfully warmed cache with ${chapters.length} chapters`); - }) - .catch(error => { - console.error('Error warming chapter cache:', error); - }); - - } catch (error) { - console.error('Error in warmChapterCache:', error); - } -} - -// Add a new function for handling large feed mappings with compression -export async function storeLargeFeedMapping( - feedId: string, - slugs: string[], - lang: string -): Promise { - const mapping = { - slugs, - lang, - created_at: new Date().toISOString() - }; - - const jsonString = JSON.stringify(mapping); - - try { - const redis = await getRedisClient(); - const activeUrl = redis.options.host || ''; - - // Use compression for large mappings - if (shouldCompress(jsonString)) { - const compressed = await gzip(jsonString); - const base64Compressed = compressed.toString('base64'); - - // Store with compression flag - await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); - - // Replicate to other instances - await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL); - - console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`); - } else { - // Use normal storage for small mappings - await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL); - await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL); - console.log(`Successfully stored mapping for feed ${feedId}`); - } - } catch (error) { - console.error(`Error storing feed mapping for ${feedId}:`, error); - throw error; - } -} - -// Batch get multiple feed mappings at once -export async function getBatchFeedMappings( - feedIds: string[] -): Promise> { - if (!feedIds.length) return {}; - - try { - const redis = await getRedisClient(); - const pipeline = redis.pipeline(); - - // Check regular keys first - feedIds.forEach(id => { - pipeline.get(id); - }); - - const results = await pipeline.exec(); - if (!results) return {}; - - // Process results - const mappings: Record = {}; - const ttlUpdates = redis.pipeline(); - const compressedChecks = redis.pipeline(); - const compressedIds: string[] = []; - - for (let i = 0; i < feedIds.length; i++) { - const id = feedIds[i]; - const result = results[i]; - - if (result[0]) { - mappings[id] = null; - continue; - } - - if (result[1]) { - // Process regular mapping - try { - const data = JSON.parse(result[1] as string); - mappings[id] = { - slugs: data.slugs, - lang: data.lang - }; - - // Reset TTL - ttlUpdates.expire(id, DEFAULT_CACHE_TTL); - } catch (e) { - mappings[id] = null; - } - } else { - // Check for compressed version - compressedChecks.exists(`${id}:compressed`); - compressedIds.push(id); - } - } - - // Execute TTL updates in background - ttlUpdates.exec().catch(err => console.error('TTL update error:', err)); - - // Check compressed versions if needed - if (compressedIds.length > 0) { - const compressedResults = await compressedChecks.exec(); - - if (compressedResults) { - const compressedFetches = redis.pipeline(); - const compressedToFetch: string[] = []; - - compressedResults.forEach((result, index) => { - if (!result[0] && result[1]) { - const id = compressedIds[index]; - compressedFetches.get(`${id}:compressed`); - compressedToFetch.push(id); - } - }); - - if (compressedToFetch.length > 0) { - const fetchResults = await compressedFetches.exec(); - - if (fetchResults) { - for (let i = 0; i < compressedToFetch.length; i++) { - const id = compressedToFetch[i]; - const [err, compressedData] = fetchResults[i]; - - if (!err && compressedData) { - try { - const buffer = Buffer.from(compressedData as string, 'base64'); - const decompressed = await gunzip(buffer); - const data = JSON.parse(decompressed.toString()); - - mappings[id] = { - slugs: data.slugs, - lang: data.lang - }; - - // Reset TTL in background - redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {}); - } catch (e) { - mappings[id] = null; - } - } else { - mappings[id] = null; - } - } - } - } - } - } - - return mappings; - } catch (error) { - console.error(`Error retrieving batch feed mappings:`, error); - return {}; - } -} - // Initialize connections eagerly to establish connections as soon as possible if (!import.meta.env.SSR) { + // Trigger initialization but don't block module loading initializeAllClients() - .then(() => console.log("Redis clients initialized")) - .catch(err => console.error("Failed to initialize Redis clients:", err)); + .then(() => console.log("Initial background Redis client initialization attempt complete.")) + .catch(err => console.error("Initial background Redis client initialization failed:", err)); } // Initialize connections lazily as backup -console.log("Redis will connect on first use"); +console.log("Redis will connect on first use (or relies on background initialization).");