diff --git a/astro.config.mjs b/astro.config.mjs index 778d936..814a9a1 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -1,14 +1,17 @@ import { defineConfig } from "astro/config"; -import vercel from "@astrojs/vercel"; import tailwind from "@astrojs/tailwind"; +import netlify from "@astrojs/netlify"; + export default defineConfig({ output: "server", - adapter: vercel(), + image: { remotePatterns: [{ protocol: "https" }], }, + integrations: [tailwind()], + vite: { define: { 'process.env.PRIMARY_REDIS_URL': JSON.stringify(process.env.PRIMARY_REDIS_URL || process.env.REDIS_URL), @@ -16,5 +19,7 @@ export default defineConfig({ 'process.env.REPLICA_REDIS_URL_2': JSON.stringify(process.env.REPLICA_REDIS_URL_2), 'process.env.REPLICA_REDIS_URL_3': JSON.stringify(process.env.REPLICA_REDIS_URL_3) } - } + }, + + adapter: netlify() }); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 109fad9..5896c4d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,19 +1,18 @@ { "name": "@ldez/manga", - "version": "1.0.0", + "version": "1.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@ldez/manga", - "version": "1.0.0", + "version": "1.0.1", "dependencies": { + "@astrojs/netlify": "^6.2.3", "@astrojs/rss": "^4.0.11", "@astrojs/tailwind": "^6.0.1", - "@astrojs/vercel": "^8.1.3", "@floating-ui/dom": "^1.6.13", "@types/node": "^22.13.11", - "@vercel/analytics": "^1.5.0", "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", @@ -71,6 +70,24 @@ "vfile": "^6.0.3" } }, + "node_modules/@astrojs/netlify": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/@astrojs/netlify/-/netlify-6.2.3.tgz", + "integrity": "sha512-QWSvoECD1ctC8N376TICRrc6F6XyyRiW3L9G1Z37B6o5BmMBXlkwsk9cMboDLbLqhcxR3RHBHXJYZ4rgjQDvQA==", + "dependencies": { + "@astrojs/internal-helpers": "0.6.1", + "@astrojs/underscore-redirects": "0.6.0", + "@netlify/blobs": "^8.1.1", + "@netlify/functions": "^3.0.0", + "@vercel/nft": "^0.29.2", + "esbuild": "^0.25.0", + "tinyglobby": "^0.2.12", + "vite": "^6.2.0" + }, + "peerDependencies": { + "astro": "^5.3.0" + } + }, "node_modules/@astrojs/prism": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@astrojs/prism/-/prism-3.2.0.tgz", @@ -122,22 +139,10 @@ "node": "^18.17.1 || ^20.3.0 || >=22.0.0" } }, - "node_modules/@astrojs/vercel": { - "version": "8.1.3", - "resolved": "https://registry.npmjs.org/@astrojs/vercel/-/vercel-8.1.3.tgz", - "integrity": "sha512-mHO28cc0FQbA6ncFteW5Hqf2l0gSthPNF8/dpM1sQAOD92Mbyi93/4EhTYj07Xhi3t37FU4tPtzyQoCAn4P4Kg==", - "dependencies": { - "@astrojs/internal-helpers": "0.6.1", - "@vercel/analytics": "^1.5.0", - "@vercel/edge": "^1.2.1", - "@vercel/nft": "^0.29.2", - "@vercel/routing-utils": "^5.0.4", - "esbuild": "^0.25.0", - "tinyglobby": "^0.2.12" - }, - "peerDependencies": { - "astro": "^5.0.0" - } + "node_modules/@astrojs/underscore-redirects": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@astrojs/underscore-redirects/-/underscore-redirects-0.6.0.tgz", + "integrity": "sha512-dnJgFpaM955IFNIkEEmMaaIdWXRdeZs1ID6mlGBqdjh6NiCXfKmOdq7L4fd9Dd/tr18fkLrOJ25IUJSxRAEhjQ==" }, "node_modules/@babel/helper-string-parser": { "version": "7.25.9", @@ -1061,6 +1066,33 @@ "node": ">=18" } }, + "node_modules/@netlify/blobs": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-8.1.2.tgz", + "integrity": "sha512-coQlePCMpgyMxfeCvxa6qPHlahECin0lSRtg8UOn2rzXRWdvJk+yUhhUstW4HLa9ynvAXFAGTEZoVt4BTESNbw==", + "engines": { + "node": "^14.16.0 || >=16.0.0" + } + }, + "node_modules/@netlify/functions": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@netlify/functions/-/functions-3.0.4.tgz", + "integrity": "sha512-Ox8+ABI+nsLK+c4/oC5dpquXuEIjzfTlJrdQKgQijCsDQoje7inXFAtKDLvvaGvuvE+PVpMLwQcIUL6P9Ob1hQ==", + "dependencies": { + "@netlify/serverless-functions-api": "1.36.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@netlify/serverless-functions-api": { + "version": "1.36.0", + "resolved": "https://registry.npmjs.org/@netlify/serverless-functions-api/-/serverless-functions-api-1.36.0.tgz", + "integrity": "sha512-z6okREyK8in0486a22Oro0k+YsuyEjDXJt46FpgeOgXqKJ9ElM8QPll0iuLBkpbH33ENiNbIPLd1cuClRQnhiw==", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1486,48 +1518,6 @@ "crypto-js": "^4.2.0" } }, - "node_modules/@vercel/analytics": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-1.5.0.tgz", - "integrity": "sha512-MYsBzfPki4gthY5HnYN7jgInhAZ7Ac1cYDoRWFomwGHWEX7odTEzbtg9kf/QSo7XEsEAqlQugA6gJ2WS2DEa3g==", - "peerDependencies": { - "@remix-run/react": "^2", - "@sveltejs/kit": "^1 || ^2", - "next": ">= 13", - "react": "^18 || ^19 || ^19.0.0-rc", - "svelte": ">= 4", - "vue": "^3", - "vue-router": "^4" - }, - "peerDependenciesMeta": { - "@remix-run/react": { - "optional": true - }, - "@sveltejs/kit": { - "optional": true - }, - "next": { - "optional": true - }, - "react": { - "optional": true - }, - "svelte": { - "optional": true - }, - "vue": { - "optional": true - }, - "vue-router": { - "optional": true - } - } - }, - "node_modules/@vercel/edge": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@vercel/edge/-/edge-1.2.1.tgz", - "integrity": "sha512-1++yncEyIAi68D3UEOlytYb1IUcIulMWdoSzX2h9LuSeeyR7JtaIgR8DcTQ6+DmYOQn+5MCh6LY+UmK6QBByNA==" - }, "node_modules/@vercel/nft": { "version": "0.29.2", "resolved": "https://registry.npmjs.org/@vercel/nft/-/nft-0.29.2.tgz", @@ -1553,18 +1543,6 @@ "node": ">=18" } }, - "node_modules/@vercel/routing-utils": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/@vercel/routing-utils/-/routing-utils-5.0.4.tgz", - "integrity": "sha512-4ke67zkXVi2fRZdoYckABcsSkRC9CnrdadOGxoS/Bk22+ObHjGQWvUHExRSXh339anwu9YY7ZacNSGH4gUnTQA==", - "dependencies": { - "path-to-regexp": "6.1.0", - "path-to-regexp-updated": "npm:path-to-regexp@6.3.0" - }, - "optionalDependencies": { - "ajv": "^6.12.3" - } - }, "node_modules/abbrev": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.0.tgz", @@ -1600,22 +1578,6 @@ "node": ">= 14" } }, - "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "optional": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, "node_modules/ansi-align": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", @@ -2342,9 +2304,9 @@ "integrity": "sha512-L3sHRo1pXXEqX8VU28kfgUY+YGsk09hPqZiZmLacNib6XNTCM8ubYeT7ryXQw8asB1sKgcU5lkB7ONug08aB8w==" }, "node_modules/consola": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.0.tgz", - "integrity": "sha512-EiPU8G6dQG0GFHNR8ljnZFki/8a+cQwEQ+7wpxdChl02Q8HXlwEZWD5lqAF8vC2sEC3Tehr8hy7vErz88LHyUA==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", + "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==", "engines": { "node": "^14.18.0 || >=16.10.0" } @@ -2626,12 +2588,6 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "optional": true - }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -2647,12 +2603,6 @@ "node": ">=8.6.0" } }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "optional": true - }, "node_modules/fast-xml-parser": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.1.tgz", @@ -3263,12 +3213,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "optional": true - }, "node_modules/kleur": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", @@ -4463,17 +4407,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/path-to-regexp": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.1.0.tgz", - "integrity": "sha512-h9DqehX3zZZDCEm+xbfU0ZmwCGFCAAraPJWMXJ4+v32NjZJilVg3k1TcKsRgIb8IQ/izZSaydDc1OhJCZvs2Dw==" - }, - "node_modules/path-to-regexp-updated": { - "name": "path-to-regexp", - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", - "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==" - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -4679,15 +4612,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "optional": true, - "engines": { - "node": ">=6" - } - }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -5768,15 +5692,6 @@ "browserslist": ">= 4.21.0" } }, - "node_modules/uri-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "optional": true, - "dependencies": { - "punycode": "^2.1.0" - } - }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index 1350193..8719059 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@ldez/manga", "type": "module", - "version": "1.0.0", + "version": "1.0.1", "private": true, "scripts": { "dev": "astro dev", @@ -11,12 +11,11 @@ "astro": "astro" }, "dependencies": { + "@astrojs/netlify": "^6.2.3", "@astrojs/rss": "^4.0.11", "@astrojs/tailwind": "^6.0.1", - "@astrojs/vercel": "^8.1.3", "@floating-ui/dom": "^1.6.13", "@types/node": "^22.13.11", - "@vercel/analytics": "^1.5.0", "astro": "^5.5.4", "feed": "^4.2.2", "ioredis": "^5.6.0", diff --git a/src/lib/api.ts b/src/lib/api.ts index b77f572..c0329e2 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -291,7 +291,7 @@ export async function getChaptersForSlugs( lang: string = 'en' ): Promise { console.log(`Fetching chapters for ${slugs.length} comics`); - const { client: redis } = await getRedisClient(); + const redis = await getRedisClient(); const uniqueSlugs = [...new Set(slugs)]; const pipeline = redis.pipeline(); diff --git a/src/lib/db.ts b/src/lib/db.ts index 4094f66..f1fe845 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -39,7 +39,6 @@ let primaryHealthy: boolean = true; let primaryFailedChecks = 0; let replicaClients: Map = new Map(); let activeReplicaUrl: string | null = null; -let initializationPromise: Promise | null = null; // Promise to track initialization // Track which replica was used when primary was down let lastUsedReplica: string | null = null; @@ -158,92 +157,34 @@ async function createRedisClient(url: string): Promise { * Initialize all replicas to establish connections early */ async function initializeAllClients(): Promise { - // Prevent multiple concurrent initializations - if (initializationPromise) { - console.log("Initialization already in progress, awaiting existing promise."); - return initializationPromise; - } - - console.log("Starting Redis client initialization..."); - const initStartTime = Date.now(); - - // Create a promise that resolves when initialization is complete - initializationPromise = (async () => { - try { - // Initialize primary - if (PRIMARY_REDIS_URL && (!primaryClient || primaryClient.status !== "ready")) { - console.log("Initializing primary client..."); - primaryClient = await createRedisClient(PRIMARY_REDIS_URL); - primaryHealthy = await checkConnection(primaryClient, "Primary"); - console.log(`Primary Redis initialized, healthy: ${primaryHealthy}`); - } else if (primaryClient) { - console.log(`Primary client already exists (Status: ${primaryClient.status}), checking health...`); - primaryHealthy = await checkConnection(primaryClient, "Primary"); - console.log(`Primary health check result: ${primaryHealthy}`); - } else { - console.log("Primary Redis URL not configured."); - } - - // Initialize all replicas - console.log(`Found ${REPLICA_URLS.length} replica URLs to initialize.`); - const replicaInitPromises: Promise[] = []; - - for (const url of REPLICA_URLS) { - // Add promise to array to run in parallel - replicaInitPromises.push( - (async () => { - // Check if already initialized and healthy - if (replicaClients.has(url)) { - const existingClient = replicaClients.get(url)!; - console.log(`Replica ${getRedisName(url)} already in map (Status: ${existingClient.status}), checking health...`); - if (await checkConnection(existingClient, getRedisName(url))) { - console.log(`Replica ${getRedisName(url)} is healthy.`); - return; // Skip re-initialization if healthy - } else { - console.log(`Replica ${getRedisName(url)} found but unhealthy, attempting re-initialization.`); - // Optionally disconnect the old client first - existingClient.disconnect(); - } - } - - try { - console.log(`Initializing replica ${getRedisName(url)}...`); - const client = await createRedisClient(url); - const isHealthy = await checkConnection(client, getRedisName(url)); - if (isHealthy) { - replicaClients.set(url, client); // Add to map *only if healthy* - console.log(`Replica ${getRedisName(url)} initialized successfully and added to map.`); - } else { - console.warn(`Replica ${getRedisName(url)} initialized but failed health check.`); - // Optionally disconnect if unhealthy after creation - client.disconnect(); - } - } catch (error) { - console.error(`Failed to initialize replica ${getRedisName(url)}:`, error); - } - })() - ); - } - - // Wait for all replica initializations to complete - await Promise.allSettled(replicaInitPromises); - - console.log(`Initialization attempt finished. Current replica map size: ${replicaClients.size}. Time taken: ${Date.now() - initStartTime}ms`); - - } catch (error) { - console.error("CRITICAL Error during client initialization process:", error); - // Reset promise on critical failure to allow retry - initializationPromise = null; - throw error; // Re-throw if needed - } finally { - // Keep the promise reference after success, but maybe reset after some time? - // For now, keep it to prevent re-running unnecessarily. - // If you suspect clients disconnect often, you might want to clear this promise - // periodically or on errors to force re-initialization attempts. + try { + // Initialize primary + if (PRIMARY_REDIS_URL && (!primaryClient || primaryClient.status !== "ready")) { + primaryClient = await createRedisClient(PRIMARY_REDIS_URL); + primaryHealthy = await checkConnection(primaryClient, "Primary"); + console.log(`Primary Redis initialized, healthy: ${primaryHealthy}`); } - })(); // Immediately invoke the async IIFE - - return initializationPromise; + + // Initialize all replicas + for (const url of REPLICA_URLS) { + if (!replicaClients.has(url)) { + try { + const client = await createRedisClient(url); + const isHealthy = await checkConnection(client, getRedisName(url)); + if (isHealthy) { + replicaClients.set(url, client); + console.log(`Replica ${getRedisName(url)} initialized successfully`); + } + } catch (error) { + console.error(`Failed to initialize replica ${getRedisName(url)}:`, error); + } + } + } + + console.log(`Initialized ${replicaClients.size} replica connections`); + } catch (error) { + console.error("Error during client initialization:", error); + } } /** @@ -267,133 +208,71 @@ async function checkConnection(client: Redis, name: string): Promise { /** * Get an active Redis client from the pool * Prioritizes primary, falls back to replicas - * Returns the client and its connection URL. */ -export async function getRedisClient(): Promise<{ client: Redis; url: string }> { - // Ensure initialization has been attempted at least once - if (!initializationPromise) { - console.log("[getRedisClient] Initialization promise not found, triggering initialization..."); - await initializeAllClients(); // Await the first initialization attempt - } else { - // Optional: Await if already running, prevents race conditions on first load - // await initializationPromise; - } - - +export async function getRedisClient(): Promise { // Try primary client first if it's considered healthy - if (primaryHealthy && primaryClient && primaryClient.status === "ready") { - // Check connection just in case before returning - if (await checkConnection(primaryClient, "Primary")) { - return { client: primaryClient, url: PRIMARY_REDIS_URL! }; // <-- Return URL - } else { - console.warn("Primary client status was ready but ping failed. Marking unhealthy."); - primaryHealthy = false; - primaryFailedChecks++; - // Fall through to replica logic + if (primaryHealthy && (primaryClient === null || primaryClient.status !== "ready")) { + try { + primaryClient = await createRedisClient(PRIMARY_REDIS_URL!); + const isPrimaryConnected = await checkConnection(primaryClient, "Primary"); + primaryHealthy = isPrimaryConnected; + if (isPrimaryConnected) { + primaryFailedChecks = 0; + console.log("Initialized primary Redis connection"); + return primaryClient; + } + } catch (err) { + console.error("Failed to initialize primary Redis:", err); + primaryHealthy = false; + primaryFailedChecks++; } + } else if (primaryHealthy && primaryClient && primaryClient.status === "ready") { + return primaryClient; } - // --- Replica Logic --- - console.log(`[getRedisClient] Primary unhealthy or unavailable. Checking replicas. Current map size: ${replicaClients.size}`); - - // If the map is empty, maybe initialization failed or hasn't run? Try again. - // This adds robustness but might hide underlying init issues if called repeatedly. - if (replicaClients.size === 0 && REPLICA_URLS.length > 0) { - console.warn("[getRedisClient] Replica map is empty, attempting re-initialization..."); - await initializeAllClients(); // Await re-initialization attempt - console.log(`[getRedisClient] Re-initialization finished. New map size: ${replicaClients.size}`); - } - - // Primary is unhealthy, try to use the active replica if we have one if (activeReplicaUrl && replicaClients.has(activeReplicaUrl)) { const activeReplica = replicaClients.get(activeReplicaUrl)!; if (activeReplica.status === "ready" && await checkConnection(activeReplica, getRedisName(activeReplicaUrl))) { console.log(`Using active replica: ${getRedisName(activeReplicaUrl)}`); lastUsedReplica = activeReplicaUrl; // Track that we used this replica - return { client: activeReplica, url: activeReplicaUrl }; // <-- Return URL + return activeReplica; } } - // No active replica, try replicas in order (using the potentially updated map) - console.log(`[getRedisClient] No active replica. Iterating through ${replicaClients.size} clients in map...`); - // Iterate through the *map* instead of the URL list first - for (const [url, client] of replicaClients.entries()) { - const clientName = getRedisName(url); - console.log(`[getRedisClient] Checking client from map: ${clientName} (Status: ${client.status})`); - if (client.status === "ready" && await checkConnection(client, clientName)) { - console.log(`[getRedisClient] Using healthy client from map: ${clientName}`); - activeReplicaUrl = url; - lastUsedReplica = url; - // Promotion logic remains the same - if (primaryFailedChecks >= REPLICA_PROMOTION_THRESHOLD) { - console.log(`Promoting ${getRedisName(url)} to primary after ${primaryFailedChecks} failed checks`); - primaryClient = client; // The client instance is now considered primary - primaryHealthy = true; - primaryFailedChecks = 0; - // Note: PRIMARY_REDIS_URL still points to the original primary config. - // The 'url' returned here is the replica's URL, which is correct for identifying the *current* primary instance. - } - return { client: client, url: url }; - } else { - console.log(`[getRedisClient] Client ${clientName} from map is not ready or failed check.`); - } - } - - // If no client in the map worked, fall back to trying URLs again (covers cases where a client wasn't added to map) - console.log(`[getRedisClient] No suitable client found in map. Falling back to iterating through REPLICA_URLS (${REPLICA_URLS.length})...`); + // No active replica, try replicas in order for (const url of REPLICA_URLS) { - const clientName = getRedisName(url); try { let client = replicaClients.get(url); - // If client exists but wasn't ready above, try creating a new one if (!client || client.status !== "ready") { - console.log(`[getRedisClient] Client ${clientName} not in map or not ready. Attempting connection...`); client = await createRedisClient(url); - // Add to map immediately upon successful creation attempt, check health next replicaClients.set(url, client); - console.log(`[getRedisClient] Added/updated client ${clientName} in map.`); } - - if (await checkConnection(client, clientName)) { - console.log(`[getRedisClient] Successfully connected/verified replica: ${clientName}`); + + if (await checkConnection(client, getRedisName(url))) { + console.log(`Switching to replica: ${getRedisName(url)}`); activeReplicaUrl = url; - lastUsedReplica = url; - // Promotion logic remains the same + lastUsedReplica = url; // Track that we used this replica + + // If primary has been down for a while, promote this replica to be the new primary if (primaryFailedChecks >= REPLICA_PROMOTION_THRESHOLD) { - console.log(`Promoting ${getRedisName(url)} to primary after ${primaryFailedChecks} failed checks`); - primaryClient = client; // The client instance is now considered primary - primaryHealthy = true; - primaryFailedChecks = 0; - // Note: PRIMARY_REDIS_URL still points to the original primary config. - // The 'url' returned here is the replica's URL, which is correct for identifying the *current* primary instance. + console.log(`Promoting ${getRedisName(url)} to primary after ${primaryFailedChecks} failed checks`); + primaryClient = client; + primaryHealthy = true; + primaryFailedChecks = 0; } - return { client: client, url: url }; - } else { - console.log(`[getRedisClient] Health check failed for ${clientName} after connection attempt.`); - // Optionally remove from map if health check fails immediately after creation? - // replicaClients.delete(url); + + return client; } } catch (err) { - console.error(`[getRedisClient] Failed to connect to replica ${clientName}:`, err); - // Remove from map if connection failed - replicaClients.delete(url); + console.error(`Failed to connect to replica ${getRedisName(url)}:`, err); } } // If all else fails, try one more time with primary console.warn("All Redis connections failed, making last attempt with primary"); - // Ensure primaryClient is re-initialized if it failed earlier - if (!primaryClient || primaryClient.status !== 'connecting' && primaryClient.status !== 'ready') { - try { - primaryClient = await createRedisClient(PRIMARY_REDIS_URL!); - } catch (err) { - console.error("Fatal: Failed to re-initialize primary Redis on last attempt:", err); - throw new Error("Could not establish any Redis connection."); // Throw if even last attempt fails - } - } - // Even if creation succeeded, we return it, subsequent operations might fail but we need to return *something* - return { client: primaryClient, url: PRIMARY_REDIS_URL! }; // <-- Return URL + primaryClient = await createRedisClient(PRIMARY_REDIS_URL!); + return primaryClient; } /** @@ -424,561 +303,46 @@ async function getAvailableReplicas(activeClientUrl: string): Promise { } /** - * Store a feed mapping in Redis with immediate replication + * Immediately replicate a Redis operation to all available replicas */ -export async function storeFeedMapping( - feedId: string, - slugs: string[], - lang: string -): Promise { - const mapping = { - slugs, - lang, - created_at: new Date().toISOString() - }; - - const stringValue = JSON.stringify(mapping); - const regularKey = feedId; // Key for the non-compressed version - const compressedKey = `${feedId}:compressed`; // Key for the compressed version - - try { - const { client: redis, url: activeUrl } = await getRedisClient(); - const clientName = getRedisName(activeUrl); - - // --- 1. Store the non-compressed version ALWAYS --- - console.log(`[storeFeedMapping] Attempting initial SET for regular key ${regularKey} to ${clientName}`); - await redis.set(regularKey, stringValue, "EX", DEFAULT_CACHE_TTL); - console.log(`[storeFeedMapping] SUCCESS: Stored regular mapping for feed ${feedId} to ${clientName}`); - - // Replicate the non-compressed version - console.log(`[storeFeedMapping] Calling ensurePrimaryAndReplicate for regular key ${regularKey} (source: ${clientName})`); - await ensurePrimaryAndReplicate(activeUrl, regularKey, stringValue, DEFAULT_CACHE_TTL); - - - // --- 2. Store the compressed version IF needed --- - if (shouldCompress(stringValue)) { - console.log(`[storeFeedMapping] Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`); - const compressed = await gzip(stringValue); - const base64Compressed = compressed.toString('base64'); - - console.log(`[storeFeedMapping] Attempting initial SET for compressed key ${compressedKey} to ${clientName}`); - await redis.set(compressedKey, base64Compressed, "EX", DEFAULT_CACHE_TTL); - console.log(`[storeFeedMapping] SUCCESS: Stored compressed mapping (${base64Compressed.length} bytes) to ${clientName}`); - - // Replicate the compressed version - console.log(`[storeFeedMapping] Calling ensurePrimaryAndReplicate for compressed key ${compressedKey} (source: ${clientName})`); - await ensurePrimaryAndReplicate(activeUrl, compressedKey, base64Compressed, DEFAULT_CACHE_TTL); - - } else { - console.log(`[storeFeedMapping] Skipping compressed storage for ${feedId} (data size below threshold).`); - // Optional: If a compressed version might exist from a previous larger save, delete it - // console.log(`[storeFeedMapping] Attempting to delete potentially stale compressed key ${compressedKey} on ${clientName}`); - // await redis.del(compressedKey).catch(err => console.warn(`Failed to delete stale compressed key ${compressedKey}:`, err)); - // // Also replicate the deletion - // console.log(`[storeFeedMapping] Calling ensurePrimaryAndReplicate for DELETE of compressed key ${compressedKey}`); - // await ensurePrimaryAndReplicate(activeUrl, compressedKey, '', 0); // Use TTL 0 or a specific marker for deletion if needed - } - - console.log(`[storeFeedMapping] Successfully processed storage and replication for feed ${feedId} (both versions if applicable)`); - - } catch (error) { - // Log the specific error during the initial write attempt - // Note: Error could happen during either set operation or replication call - console.error(`[storeFeedMapping] CRITICAL ERROR storing feed mapping for ${feedId}:`, error); - throw error; // Re-throw the error after logging - } -} - -/** - * Retrieve a feed mapping from Redis with automatic failover - */ -export async function getFeedMapping( - feedId: string -): Promise { - console.log(`[getFeedMapping] Attempting to get mapping for feedId: ${feedId}`); - try { - // Try primary first - const { client, url: clientUrl } = await getRedisClient(); - const clientName = getRedisName(clientUrl); - console.log(`[getFeedMapping] Using client: ${clientName} (${clientUrl})`); - - // Check for compressed version first - const compressedKey = `${feedId}:compressed`; - console.log(`[getFeedMapping] Checking for compressed key: ${compressedKey} on ${clientName}`); - const compressedData = await client.get(compressedKey); // This is the base64 string - - if (compressedData) { - console.log(`[getFeedMapping] SUCCESS: Found compressed data for ${feedId} on ${clientName}`); - try { - // Decompress and parse - const buffer = Buffer.from(compressedData, 'base64'); - const decompressed = await gunzip(buffer); - const data = JSON.parse(decompressed.toString()); - - // --- Add back direct expire on the source client --- - console.log(`[getFeedMapping] Resetting TTL directly on source ${clientName} for compressed key ${compressedKey}`); - client.expire(compressedKey, DEFAULT_CACHE_TTL) - .catch(err => console.warn(`[getFeedMapping] Failed direct TTL reset on ${clientName} for ${compressedKey}:`, err)); - // --- End Add back --- - - // Replicate the COMPRESSED data using ensurePrimaryAndReplicate - console.log(`[getFeedMapping] Triggering DATA replication for compressed key ${compressedKey}`); - ensurePrimaryAndReplicate(clientUrl, compressedKey, compressedData, DEFAULT_CACHE_TTL) - .catch(err => console.error(`[getFeedMapping] Error during background DATA replication for ${compressedKey}:`, err)); - - return { - slugs: data.slugs, - lang: data.lang - }; - } catch (error) { - console.error(`[getFeedMapping] Error decompressing data for ${feedId} from ${clientName}:`, error); - // Fall through to try uncompressed version - } - } else { - console.log(`[getFeedMapping] No compressed key found for ${feedId} on ${clientName}.`); - } - - // If no compressed version, try regular key - const regularKey = feedId; - console.log(`[getFeedMapping] Checking for regular key: ${regularKey} on ${clientName}`); - let jsonString = await client.get(regularKey); // This is the JSON string - - // Variables to track source if found on replica - let foundOnReplica = false; - let replicaUrlSource: string | null = null; - let replicaClientSource: Redis | null = null; // Store the replica client instance - let valueFromReplica: string | null = null; // Store value found on replica - let isValueFromReplicaCompressed = false; // Track if replica value was compressed - - if (jsonString) { - console.log(`[getFeedMapping] SUCCESS: Found regular key ${regularKey} on ${clientName}`); - } - - // If not found in the current client AND the client was the primary, check replicas - if (!jsonString && REPLICA_URLS.length > 0 && client === primaryClient) { - console.log(`[getFeedMapping] Key ${regularKey} not found on PRIMARY, checking replicas...`); - - // Try each replica - for (const [replicaUrl, replicaClient] of replicaClients.entries()) { - const replicaName = getRedisName(replicaUrl); - if (replicaClient.status === "ready") { - try { - // Check compressed key on replica first - const replicaCompressedData = await replicaClient.get(compressedKey); // Base64 string - if (replicaCompressedData) { - console.log(`[getFeedMapping] SUCCESS: Found COMPRESSED key ${compressedKey} on REPLICA ${replicaName}`); - try { - // Decompress for return value - const buffer = Buffer.from(replicaCompressedData, 'base64'); - const decompressed = await gunzip(buffer); - const data = JSON.parse(decompressed.toString()); - - // --- Add back direct expire on the source replica client --- - console.log(`[getFeedMapping] Resetting TTL directly on source ${replicaName} for compressed key ${compressedKey}`); - replicaClient.expire(compressedKey, DEFAULT_CACHE_TTL) - .catch(err => console.warn(`[getFeedMapping] Failed direct TTL reset on ${replicaName} for ${compressedKey}:`, err)); - // --- End Add back --- - - // Replicate the COMPRESSED data found on replica - console.log(`[getFeedMapping] Triggering DATA replication for compressed key ${compressedKey} (found on replica ${replicaName})`); - ensurePrimaryAndReplicate(replicaUrl, compressedKey, replicaCompressedData, DEFAULT_CACHE_TTL) - .catch(err => console.error(`[getFeedMapping] Error during background DATA replication for ${compressedKey}:`, err)); - - return { slugs: data.slugs, lang: data.lang }; - } catch (error) { - console.error(`[getFeedMapping] Error decompressing data for ${feedId} from REPLICA ${replicaName}:`, error); - } - } else { - // Check regular key on replica - const replicaJsonString = await replicaClient.get(regularKey); // JSON string - if (replicaJsonString) { - console.log(`[getFeedMapping] SUCCESS: Found REGULAR key ${regularKey} on REPLICA ${replicaName}`); - foundOnReplica = true; - replicaUrlSource = replicaUrl; - replicaClientSource = replicaClient; // Store the client instance - valueFromReplica = replicaJsonString; // Store the value found - isValueFromReplicaCompressed = false; // Mark as not compressed - jsonString = replicaJsonString; // Use this value going forward - break; // Found on this replica, stop checking others - } else { - console.log(`[getFeedMapping] No regular key found on replica ${replicaName}.`); - } - } - - } catch (error) { - console.error(`[getFeedMapping] Error checking replica ${replicaName} for ${feedId}:`, error); - } - } else { - console.log(`[getFeedMapping] Skipping replica ${replicaName} (status: ${replicaClient.status})`); - } - } - } - - // If jsonString is still null/empty after checks, the key wasn't found - if (!jsonString) { - console.log(`[getFeedMapping] Key ${feedId} (and compressed variant) not found on any checked instance.`); - return null; - } - - // --- Add back direct expire on the source client (primary or replica) --- - const sourceClientForExpire = foundOnReplica ? replicaClientSource! : client; - const sourceClientNameForExpire = foundOnReplica ? getRedisName(replicaUrlSource!) : clientName; - console.log(`[getFeedMapping] Resetting TTL directly on source ${sourceClientNameForExpire} for regular key ${regularKey}`); - sourceClientForExpire.expire(regularKey, DEFAULT_CACHE_TTL) - .catch(err => console.warn(`[getFeedMapping] Failed direct TTL reset on ${sourceClientNameForExpire} for ${regularKey}:`, err)); - // --- End Add back --- - - // Replicate the REGULAR data using ensurePrimaryAndReplicate - console.log(`[getFeedMapping] Triggering DATA replication for regular key ${regularKey}`); - const sourceUrlForReplication = foundOnReplica ? replicaUrlSource! : clientUrl; - ensurePrimaryAndReplicate(sourceUrlForReplication, regularKey, jsonString, DEFAULT_CACHE_TTL) - .catch(err => console.error(`[getFeedMapping] Error during background DATA replication for ${regularKey}:`, err)); - - // Parse the result - try { - const data = JSON.parse(jsonString); - return { - slugs: data.slugs, - lang: data.lang - }; - } catch (parseError) { - console.error(`[getFeedMapping] CRITICAL ERROR parsing feed mapping JSON for ${feedId}:`, parseError, `JSON String: ${jsonString}`); - return null; // Return null if parsing fails - } - - } catch (error) { - console.error(`[getFeedMapping] CRITICAL ERROR retrieving feed mapping for ${feedId}:`, error); - return null; - } -} - -/** - * Pre-warm the chapter cache for better performance - * This function checks cache status and uses the getChaptersForSlugs function from api.ts - * to fetch and cache data in the background - */ -export async function warmChapterCache( - slugs: string[], - lang: string +async function replicateOperation( + activeClientUrl: string, + key: string, + value: string, + ttl?: number ): Promise { try { - console.log(`Warming chapter cache for ${slugs.length} comics in ${lang}`); - const { client } = await getRedisClient(); // This already handles failover + // Get available replicas that are not the active client + const replicas = await getAvailableReplicas(activeClientUrl); - // Validate slugs - const validSlugs = slugs.filter(slug => - slug && typeof slug === 'string' && slug.match(/^[a-z0-9-]+$/)); - - if (validSlugs.length !== slugs.length) { - console.warn(`Filtered out ${slugs.length - validSlugs.length} invalid slugs`); - } - - // Check which slugs are already in cache - const pipeline = client.pipeline(); - const cacheKeys = validSlugs.map(slug => `chapters:${slug}:${lang}`); - - cacheKeys.forEach(key => { - pipeline.exists(key); - }); - - const results = await pipeline.exec(); - const slugsToFetch: string[] = []; - - results?.forEach(([err, exists], index) => { - if (err) { - console.error(`Error checking cache for ${cacheKeys[index]}:`, err); - slugsToFetch.push(validSlugs[index]); - } else if (!exists) { - slugsToFetch.push(validSlugs[index]); - } - }); - - if (slugsToFetch.length === 0) { - console.log('All chapters already in cache, no warming needed'); + if (replicas.length === 0) { + console.log('No available replicas for replication'); return; } - console.log(`Cache miss for ${slugsToFetch.length} comics - starting background fetch`); + console.log(`Replicating operation for key ${key} to ${replicas.length} replicas`); - // To avoid circular dependencies, we'll import getChaptersForSlugs dynamically - const { getChaptersForSlugs } = await import('./api'); - - // Start fetching in the background without waiting for the result - getChaptersForSlugs(slugsToFetch, lang) - .then(chapters => { - console.log(`Successfully warmed cache with ${chapters.length} chapters`); - }) - .catch(error => { - console.error('Error warming chapter cache:', error); - }); - - } catch (error) { - console.error('Error in warmChapterCache:', error); - } -} - -// Add a new function for handling large feed mappings with compression -export async function storeLargeFeedMapping( - feedId: string, - slugs: string[], - lang: string -): Promise { - // This function now essentially calls storeFeedMapping which handles compression logic. - // We can simplify this or keep it as a specific entry point. - // For consistency, let's just call the main function. - console.log(`Using storeLargeFeedMapping for ${feedId}, deferring to storeFeedMapping.`); - await storeFeedMapping(feedId, slugs, lang); - - /* // Original logic updated with ensurePrimaryAndReplicate: - const mapping = { - slugs, - lang, - created_at: new Date().toISOString() - }; - - const jsonString = JSON.stringify(mapping); - let keyToStore = feedId; - let valueToStore = jsonString; - let ttl = DEFAULT_CACHE_TTL; - - try { - const redis = await getRedisClient(); - const activeUrl = redis.options.url || `${redis.options.host}:${redis.options.port}`; - - // Use compression for large mappings - if (shouldCompress(jsonString)) { - const compressed = await gzip(jsonString); - const base64Compressed = compressed.toString('base64'); - - keyToStore = `${feedId}:compressed`; - valueToStore = base64Compressed; - - // Store with compression flag - await redis.set(keyToStore, valueToStore, "EX", ttl); - console.log(`Stored compressed mapping for feed ${feedId} to ${getRedisName(activeUrl)} (${jsonString.length} → ${valueToStore.length} bytes)`); - - } else { - // Use normal storage for small mappings - await redis.set(keyToStore, jsonString, "EX", ttl); - console.log(`Stored mapping for feed ${feedId} to ${getRedisName(activeUrl)}`); - } - - // Ensure data is on primary and replicate to others - await ensurePrimaryAndReplicate(activeUrl, keyToStore, valueToStore, ttl); - - console.log(`Successfully processed storage and replication for large feed ${feedId}`); - - } catch (error) { - console.error(`Error storing large feed mapping for ${feedId}:`, error); - throw error; - } - */ -} - -// Batch get multiple feed mappings at once -export async function getBatchFeedMappings( - feedIds: string[] -): Promise> { - if (!feedIds.length) return {}; - - try { - const { client } = await getRedisClient(); - const pipeline = client.pipeline(); - - // Check regular keys first - feedIds.forEach(id => { - pipeline.get(id); + // Replicate to each available replica + const replicationPromises = replicas.map(async (replica) => { + try { + if (ttl) { + await replica.set(key, value, 'EX', ttl); + } else { + await replica.set(key, value); + } + return true; + } catch (error) { + console.error(`Replication failed for key ${key}:`, error); + return false; + } }); - const results = await pipeline.exec(); - if (!results) return {}; + const results = await Promise.allSettled(replicationPromises); + const successful = results.filter(r => r.status === 'fulfilled' && r.value === true).length; - // Process results - const mappings: Record = {}; - const ttlUpdates = client.pipeline(); - const compressedChecks = client.pipeline(); - const compressedIds: string[] = []; - - for (let i = 0; i < feedIds.length; i++) { - const id = feedIds[i]; - const result = results[i]; - - if (result[0]) { - mappings[id] = null; - continue; - } - - if (result[1]) { - // Process regular mapping - try { - const data = JSON.parse(result[1] as string); - mappings[id] = { - slugs: data.slugs, - lang: data.lang - }; - - // Reset TTL - ttlUpdates.expire(id, DEFAULT_CACHE_TTL); - } catch (e) { - mappings[id] = null; - } - } else { - // Check for compressed version - compressedChecks.exists(`${id}:compressed`); - compressedIds.push(id); - } - } - - // Execute TTL updates in background - ttlUpdates.exec().catch(err => console.error('TTL update error:', err)); - - // Check compressed versions if needed - if (compressedIds.length > 0) { - const compressedResults = await compressedChecks.exec(); - - if (compressedResults) { - const compressedFetches = client.pipeline(); - const compressedToFetch: string[] = []; - - compressedResults.forEach((result, index) => { - if (!result[0] && result[1]) { - const id = compressedIds[index]; - compressedFetches.get(`${id}:compressed`); - compressedToFetch.push(id); - } - }); - - if (compressedToFetch.length > 0) { - const fetchResults = await compressedFetches.exec(); - - if (fetchResults) { - for (let i = 0; i < compressedToFetch.length; i++) { - const id = compressedToFetch[i]; - const [err, compressedData] = fetchResults[i]; - - if (!err && compressedData) { - try { - const buffer = Buffer.from(compressedData as string, 'base64'); - const decompressed = await gunzip(buffer); - const data = JSON.parse(decompressed.toString()); - - mappings[id] = { - slugs: data.slugs, - lang: data.lang - }; - - // Reset TTL in background - client.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {}); - } catch (e) { - mappings[id] = null; - } - } else { - mappings[id] = null; - } - } - } - } - } - } - - return mappings; + console.log(`Successfully replicated to ${successful}/${replicas.length} replicas`); } catch (error) { - console.error(`Error retrieving batch feed mappings:`, error); - return {}; - } -} - -/** - * Ensures data is written to the primary (if not the source) and replicated. - * Handles SET operations. For DEL, a separate function or modification might be needed. - */ -async function ensurePrimaryAndReplicate( - sourceUrl: string, - key: string, - value: string, // Value to set - ttl: number // TTL for SET, use > 0 -): Promise { - const replicationTasks: Promise[] = []; - const sourceName = getRedisName(sourceUrl); - const primaryUrl = PRIMARY_REDIS_URL; - const primaryName = getRedisName(primaryUrl!); - - // Check if operation is a delete based on TTL (simple approach) - const isDelete = ttl <= 0; - const operationType = isDelete ? 'DELETE' : 'SET'; - - console.log(`[ensurePrimaryAndReplicate] Starting sync for ${operationType} on key ${key}. Source: ${sourceName}`); - - // 1. Sync to Primary (if source wasn't primary and primary exists/is healthy) - if (primaryClient && sourceUrl !== primaryUrl) { - const primaryStatus = primaryClient.status; - console.log(`[ensurePrimaryAndReplicate] Checking primary (${primaryName}) status: ${primaryStatus}`); - let primaryIsReady = primaryStatus === 'ready'; - - if (!primaryIsReady) { - try { - await primaryClient.ping(); - primaryIsReady = true; - console.log(`[ensurePrimaryAndReplicate] Primary (${primaryName}) ping successful after status check.`); - } catch (pingErr) { - console.warn(`[ensurePrimaryAndReplicate] Primary (${primaryName}) ping failed, skipping sync attempt. Status: ${primaryStatus}`, pingErr); - } - } - - if (primaryIsReady) { - console.log(`[ensurePrimaryAndReplicate] Attempting sync ${operationType} to PRIMARY (${primaryName}) for key ${key}`); - const task = isDelete - ? primaryClient.del(key) - : primaryClient.set(key, value, "EX", ttl); - - replicationTasks.push( - task - .then(() => console.log(`[ensurePrimaryAndReplicate] SUCCESS: Synced ${operationType} key ${key} to PRIMARY (${primaryName})`)) - .catch(err => console.error(`[ensurePrimaryAndReplicate] FAILED: Sync ${operationType} key ${key} to PRIMARY (${primaryName})`, err)) - ); - } else { - console.log(`[ensurePrimaryAndReplicate] SKIPPED: Sync ${operationType} to PRIMARY (${primaryName}) because it's not ready (Status: ${primaryStatus}).`); - } - } else if (sourceUrl === primaryUrl) { - console.log(`[ensurePrimaryAndReplicate] Source was PRIMARY (${primaryName}), no primary sync needed for ${operationType} on key ${key}.`); - } else { - console.log(`[ensurePrimaryAndReplicate] Primary client not available, cannot sync ${operationType} key ${key} to primary.`); - } - - // 2. Replicate to other available replicas - console.log(`[ensurePrimaryAndReplicate] Current replicaClients map size: ${replicaClients.size}, Keys: ${Array.from(replicaClients.keys())}`); - console.log(`[ensurePrimaryAndReplicate] Checking ${replicaClients.size} replicas for ${operationType} on key ${key}`); - for (const [replicaUrl, replicaClient] of replicaClients.entries()) { - const replicaName = getRedisName(replicaUrl); - // Don't replicate back to the source or to the primary (if primary sync was attempted above) - if (replicaUrl !== sourceUrl && replicaUrl !== primaryUrl) { - const replicaStatus = replicaClient.status; - console.log(`[ensurePrimaryAndReplicate] Checking replica ${replicaName} status: ${replicaStatus}`); - if (replicaStatus === 'ready') { - console.log(`[ensurePrimaryAndReplicate] Attempting replication ${operationType} to REPLICA (${replicaName}) for key ${key}`); - const task = isDelete - ? replicaClient.del(key) - : replicaClient.set(key, value, "EX", ttl); - replicationTasks.push( - task - .then(() => console.log(`[ensurePrimaryAndReplicate] SUCCESS: Replicated ${operationType} key ${key} to REPLICA (${replicaName})`)) - .catch(err => console.error(`[ensurePrimaryAndReplicate] FAILED: Replicate ${operationType} key ${key} to REPLICA (${replicaName})`, err)) - ); - } else { - console.log(`[ensurePrimaryAndReplicate] SKIPPED: Replication ${operationType} to REPLICA (${replicaName}) because it's not ready (Status: ${replicaStatus}).`); - } - } else { - console.log(`[ensurePrimaryAndReplicate] SKIPPED: Replication ${operationType} to ${replicaName} (Source or Primary) for key ${key}.`); - } - } - - // Wait for all sync/replication attempts to settle - if (replicationTasks.length > 0) { - console.log(`[ensurePrimaryAndReplicate] Waiting for ${replicationTasks.length} sync/replication tasks for ${operationType} on key ${key}...`); - const results = await Promise.allSettled(replicationTasks); - console.log(`[ensurePrimaryAndReplicate] Sync/replication tasks settled for ${operationType} on key ${key}. Results:`, results.map(r => r.status)); - } else { - console.log(`[ensurePrimaryAndReplicate] No sync/replication tasks needed for ${operationType} on key ${key}.`); + console.error('Error during replication:', error); } } @@ -1092,13 +456,368 @@ async function syncFromReplicaToPrimary(replicaUrl: string): Promise { } } +/** + * Store a feed mapping in Redis with immediate replication + */ +export async function storeFeedMapping( + feedId: string, + slugs: string[], + lang: string +): Promise { + const mapping = { + slugs, + lang, + created_at: new Date().toISOString() + }; + + const stringValue = JSON.stringify(mapping); + + try { + // Get Redis client + const redis = await getRedisClient(); + const activeUrl = redis.options.host || ''; + + // Check if compression should be used + if (shouldCompress(stringValue)) { + console.log(`Compressing large feed mapping for ${feedId} (${Buffer.byteLength(stringValue, 'utf8')} bytes)`); + const compressed = await gzip(stringValue); + const base64Compressed = compressed.toString('base64'); + + // Store with compression flag + await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + console.log(`Stored compressed mapping (${base64Compressed.length} bytes)`); + + // Replicate to other instances + const replicas = await getAvailableReplicas(activeUrl); + for (const replica of replicas) { + await replica.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + } + } else { + // Use standard set operations without compression + await redis.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); + + // Replicate to other instances + const replicas = await getAvailableReplicas(activeUrl); + for (const replica of replicas) { + await replica.set(feedId, stringValue, "EX", DEFAULT_CACHE_TTL); + } + } + + console.log(`Successfully stored mapping for feed ${feedId}`); + } catch (error) { + console.error(`Error storing feed mapping for ${feedId}:`, error); + throw error; + } +} + +/** + * Retrieve a feed mapping from Redis with automatic failover + */ +export async function getFeedMapping( + feedId: string +): Promise { + try { + // Try primary first + const redis = await getRedisClient(); + + // Check for compressed version first + const compressedKey = `${feedId}:compressed`; + const compressedData = await redis.get(compressedKey); + + if (compressedData) { + console.log(`Found compressed data for ${feedId}`); + try { + // Decompress and parse + const buffer = Buffer.from(compressedData, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + // Reset expiration + await redis.expire(compressedKey, DEFAULT_CACHE_TTL); + + return { + slugs: data.slugs, + lang: data.lang + }; + } catch (error) { + console.error(`Error decompressing data for ${feedId}:`, error); + // Fall through to try uncompressed version + } + } + + // If no compressed version, try regular key + let jsonString = await redis.get(feedId); + + // If not found in primary but we have replicas, check them + if (!jsonString && REPLICA_URLS.length > 0 && redis === primaryClient) { + console.log(`Key ${feedId} not found in primary, checking replicas...`); + + // Try each replica + for (const [url, replicaClient] of replicaClients.entries()) { + if (replicaClient.status === "ready") { + try { + jsonString = await replicaClient.get(feedId); + if (jsonString) { + console.log(`Found key ${feedId} in replica ${getRedisName(url)}`); + + // Sync this key back to primary for future requests + if (primaryClient && primaryClient.status === "ready") { + primaryClient.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL) + .catch(err => console.error(`Failed to sync key ${feedId} to primary:`, err)); + } + + break; + } + } catch (error) { + console.error(`Error checking replica ${getRedisName(url)} for ${feedId}:`, error); + } + } + } + } + + if (!jsonString) return null; + + // Reset expiration on access + await redis.expire(feedId, DEFAULT_CACHE_TTL); + + // Parse the result + const data = JSON.parse(jsonString as string); + return { + slugs: data.slugs, + lang: data.lang + }; + } catch (error) { + console.error(`Error retrieving feed mapping for ${feedId}:`, error); + return null; + } +} + +/** + * Pre-warm the chapter cache for better performance + * This function checks cache status and uses the getChaptersForSlugs function from api.ts + * to fetch and cache data in the background + */ +export async function warmChapterCache( + slugs: string[], + lang: string +): Promise { + try { + console.log(`Warming chapter cache for ${slugs.length} comics in ${lang}`); + const redis = await getRedisClient(); // This already handles failover + + // Validate slugs + const validSlugs = slugs.filter(slug => + slug && typeof slug === 'string' && slug.match(/^[a-z0-9-]+$/)); + + if (validSlugs.length !== slugs.length) { + console.warn(`Filtered out ${slugs.length - validSlugs.length} invalid slugs`); + } + + // Check which slugs are already in cache + const pipeline = redis.pipeline(); + const cacheKeys = validSlugs.map(slug => `chapters:${slug}:${lang}`); + + cacheKeys.forEach(key => { + pipeline.exists(key); + }); + + const results = await pipeline.exec(); + const slugsToFetch: string[] = []; + + results?.forEach(([err, exists], index) => { + if (err) { + console.error(`Error checking cache for ${cacheKeys[index]}:`, err); + slugsToFetch.push(validSlugs[index]); + } else if (!exists) { + slugsToFetch.push(validSlugs[index]); + } + }); + + if (slugsToFetch.length === 0) { + console.log('All chapters already in cache, no warming needed'); + return; + } + + console.log(`Cache miss for ${slugsToFetch.length} comics - starting background fetch`); + + // To avoid circular dependencies, we'll import getChaptersForSlugs dynamically + const { getChaptersForSlugs } = await import('./api'); + + // Start fetching in the background without waiting for the result + getChaptersForSlugs(slugsToFetch, lang) + .then(chapters => { + console.log(`Successfully warmed cache with ${chapters.length} chapters`); + }) + .catch(error => { + console.error('Error warming chapter cache:', error); + }); + + } catch (error) { + console.error('Error in warmChapterCache:', error); + } +} + +// Add a new function for handling large feed mappings with compression +export async function storeLargeFeedMapping( + feedId: string, + slugs: string[], + lang: string +): Promise { + const mapping = { + slugs, + lang, + created_at: new Date().toISOString() + }; + + const jsonString = JSON.stringify(mapping); + + try { + const redis = await getRedisClient(); + const activeUrl = redis.options.host || ''; + + // Use compression for large mappings + if (shouldCompress(jsonString)) { + const compressed = await gzip(jsonString); + const base64Compressed = compressed.toString('base64'); + + // Store with compression flag + await redis.set(`${feedId}:compressed`, base64Compressed, "EX", DEFAULT_CACHE_TTL); + + // Replicate to other instances + await replicateOperation(activeUrl, `${feedId}:compressed`, base64Compressed, DEFAULT_CACHE_TTL); + + console.log(`Successfully stored compressed mapping for feed ${feedId} (${jsonString.length} → ${base64Compressed.length} bytes)`); + } else { + // Use normal storage for small mappings + await redis.set(feedId, jsonString, "EX", DEFAULT_CACHE_TTL); + await replicateOperation(activeUrl, feedId, jsonString, DEFAULT_CACHE_TTL); + console.log(`Successfully stored mapping for feed ${feedId}`); + } + } catch (error) { + console.error(`Error storing feed mapping for ${feedId}:`, error); + throw error; + } +} + +// Batch get multiple feed mappings at once +export async function getBatchFeedMappings( + feedIds: string[] +): Promise> { + if (!feedIds.length) return {}; + + try { + const redis = await getRedisClient(); + const pipeline = redis.pipeline(); + + // Check regular keys first + feedIds.forEach(id => { + pipeline.get(id); + }); + + const results = await pipeline.exec(); + if (!results) return {}; + + // Process results + const mappings: Record = {}; + const ttlUpdates = redis.pipeline(); + const compressedChecks = redis.pipeline(); + const compressedIds: string[] = []; + + for (let i = 0; i < feedIds.length; i++) { + const id = feedIds[i]; + const result = results[i]; + + if (result[0]) { + mappings[id] = null; + continue; + } + + if (result[1]) { + // Process regular mapping + try { + const data = JSON.parse(result[1] as string); + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL + ttlUpdates.expire(id, DEFAULT_CACHE_TTL); + } catch (e) { + mappings[id] = null; + } + } else { + // Check for compressed version + compressedChecks.exists(`${id}:compressed`); + compressedIds.push(id); + } + } + + // Execute TTL updates in background + ttlUpdates.exec().catch(err => console.error('TTL update error:', err)); + + // Check compressed versions if needed + if (compressedIds.length > 0) { + const compressedResults = await compressedChecks.exec(); + + if (compressedResults) { + const compressedFetches = redis.pipeline(); + const compressedToFetch: string[] = []; + + compressedResults.forEach((result, index) => { + if (!result[0] && result[1]) { + const id = compressedIds[index]; + compressedFetches.get(`${id}:compressed`); + compressedToFetch.push(id); + } + }); + + if (compressedToFetch.length > 0) { + const fetchResults = await compressedFetches.exec(); + + if (fetchResults) { + for (let i = 0; i < compressedToFetch.length; i++) { + const id = compressedToFetch[i]; + const [err, compressedData] = fetchResults[i]; + + if (!err && compressedData) { + try { + const buffer = Buffer.from(compressedData as string, 'base64'); + const decompressed = await gunzip(buffer); + const data = JSON.parse(decompressed.toString()); + + mappings[id] = { + slugs: data.slugs, + lang: data.lang + }; + + // Reset TTL in background + redis.expire(`${id}:compressed`, DEFAULT_CACHE_TTL).catch(() => {}); + } catch (e) { + mappings[id] = null; + } + } else { + mappings[id] = null; + } + } + } + } + } + } + + return mappings; + } catch (error) { + console.error(`Error retrieving batch feed mappings:`, error); + return {}; + } +} + // Initialize connections eagerly to establish connections as soon as possible if (!import.meta.env.SSR) { - // Trigger initialization but don't block module loading initializeAllClients() - .then(() => console.log("Initial background Redis client initialization attempt complete.")) - .catch(err => console.error("Initial background Redis client initialization failed:", err)); + .then(() => console.log("Redis clients initialized")) + .catch(err => console.error("Failed to initialize Redis clients:", err)); } // Initialize connections lazily as backup -console.log("Redis will connect on first use (or relies on background initialization)."); +console.log("Redis will connect on first use");