diff --git a/package-lock.json b/package-lock.json index dbcdf00..3017436 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,10 +15,12 @@ "@types/node": "^22.15.3", "@vercel/analytics": "^1.5.0", "astro": "^5.7.10", + "chrome-aws-lambda": "^10.1.0", "feed": "^4.2.2", "ioredis": "^5.6.1", "p-throttle": "^7.0.0", "puppeteer": "^24.8.0", + "puppeteer-core": "^10.1.0", "redis-modules-sdk": "^1.4.0", "tailwindcss": "^3.4.1" } @@ -2242,6 +2244,16 @@ "file-uri-to-path": "1.0.0" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/blob-to-buffer": { "version": "1.2.9", "resolved": "https://registry.npmjs.org/blob-to-buffer/-/blob-to-buffer-1.2.9.tgz", @@ -2340,6 +2352,29 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -2472,6 +2507,20 @@ "node": ">=18" } }, + "node_modules/chrome-aws-lambda": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/chrome-aws-lambda/-/chrome-aws-lambda-10.1.0.tgz", + "integrity": "sha512-NZQVf+J4kqG4sVhRm3WNmOfzY0OtTSm+S8rg77pwePa9RCYHzhnzRs8YvNI6L9tALIW6RpmefWiPURt3vURXcw==", + "dependencies": { + "lambdafs": "^2.0.3" + }, + "engines": { + "node": ">= 10.16" + }, + "peerDependencies": { + "puppeteer-core": "^10.1.0" + } + }, "node_modules/chromium-bidi": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-4.1.1.tgz", @@ -2674,6 +2723,11 @@ "resolved": "https://registry.npmjs.org/common-ancestor-path/-/common-ancestor-path-1.0.1.tgz", "integrity": "sha512-L3sHRo1pXXEqX8VU28kfgUY+YGsk09hPqZiZmLacNib6XNTCM8ubYeT7ryXQw8asB1sKgcU5lkB7ONug08aB8w==" }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, "node_modules/consola": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.0.tgz", @@ -3239,6 +3293,18 @@ "node": ">=8" } }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/flattie": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/flattie/-/flattie-1.1.1.tgz", @@ -3290,6 +3356,16 @@ "url": "https://github.com/sponsors/rawify" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -3643,6 +3719,25 @@ "node": ">= 14" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", @@ -3675,6 +3770,21 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "node_modules/ioredis": { "version": "5.6.1", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz", @@ -3907,6 +4017,214 @@ "node": ">=6" } }, + "node_modules/lambdafs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/lambdafs/-/lambdafs-2.1.1.tgz", + "integrity": "sha512-x5k8JcoJWkWLvCVBzrl4pzvkEHSgSBqFjg3Dpsc4AcTMq7oUMym4cL/gRTZ6VM4mUMY+M0dIbQ+V1c1tsqqanQ==", + "bundleDependencies": [ + "tar-fs" + ], + "dependencies": { + "tar-fs": "*" + }, + "bin": { + "lambdafs": "bin/brotli.js" + }, + "engines": { + "node": ">= 10.16" + } + }, + "node_modules/lambdafs/node_modules/base64-js": { + "version": "1.5.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "inBundle": true, + "license": "MIT" + }, + "node_modules/lambdafs/node_modules/bl": { + "version": "4.1.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/lambdafs/node_modules/bl/node_modules/readable-stream": { + "version": "3.6.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/lambdafs/node_modules/buffer": { + "version": "5.7.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "inBundle": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/lambdafs/node_modules/chownr": { + "version": "1.1.4", + "inBundle": true, + "license": "ISC" + }, + "node_modules/lambdafs/node_modules/end-of-stream": { + "version": "1.4.4", + "inBundle": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/lambdafs/node_modules/fs-constants": { + "version": "1.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/lambdafs/node_modules/ieee754": { + "version": "1.2.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "inBundle": true, + "license": "BSD-3-Clause" + }, + "node_modules/lambdafs/node_modules/inherits": { + "version": "2.0.4", + "inBundle": true, + "license": "ISC" + }, + "node_modules/lambdafs/node_modules/mkdirp-classic": { + "version": "0.5.3", + "inBundle": true, + "license": "MIT" + }, + "node_modules/lambdafs/node_modules/once": { + "version": "1.4.0", + "inBundle": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/lambdafs/node_modules/pump": { + "version": "3.0.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/lambdafs/node_modules/string_decoder": { + "version": "1.1.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/lambdafs/node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/lambdafs/node_modules/tar-fs": { + "version": "2.1.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/lambdafs/node_modules/tar-stream": { + "version": "2.2.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lambdafs/node_modules/tar-stream/node_modules/readable-stream": { + "version": "3.6.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/lambdafs/node_modules/util-deprecate": { + "version": "1.0.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/lambdafs/node_modules/wrappy": { + "version": "1.0.2", + "inBundle": true, + "license": "ISC" + }, "node_modules/lilconfig": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", @@ -3923,6 +4241,17 @@ "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==" }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -4771,6 +5100,14 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -5021,6 +5358,31 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-queue": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.1.0.tgz", @@ -5058,6 +5420,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/pac-proxy-agent": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", @@ -5159,6 +5529,22 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -5235,6 +5621,17 @@ "node": ">= 6" } }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/postcss": { "version": "8.5.3", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.3.tgz", @@ -5486,6 +5883,190 @@ } }, "node_modules/puppeteer-core": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-10.1.0.tgz", + "integrity": "sha512-x2yDSJI/PRiWhDqAt1jd4rhTotxwjwKzHLIIqD2MlJ+TmzGJfBY9snAGIVXJwkWfKJg+Ef5xupdK0EbHDqBpFw==", + "dependencies": { + "debug": "4.3.1", + "devtools-protocol": "0.0.883894", + "extract-zip": "2.0.1", + "https-proxy-agent": "5.0.0", + "node-fetch": "2.6.1", + "pkg-dir": "4.2.0", + "progress": "2.0.1", + "proxy-from-env": "1.1.0", + "rimraf": "3.0.2", + "tar-fs": "2.0.0", + "unbzip2-stream": "1.3.3", + "ws": "7.4.6" + }, + "engines": { + "node": ">=10.18.1" + } + }, + "node_modules/puppeteer-core/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/puppeteer-core/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/puppeteer-core/node_modules/debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/puppeteer-core/node_modules/devtools-protocol": { + "version": "0.0.883894", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.883894.tgz", + "integrity": "sha512-33idhm54QJzf3Q7QofMgCvIVSd2o9H3kQPWaKT/fhoZh+digc+WSiMhbkeG3iN79WY4Hwr9G05NpbhEVrsOYAg==" + }, + "node_modules/puppeteer-core/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/puppeteer-core/node_modules/https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/puppeteer-core/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/puppeteer-core/node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/puppeteer-core/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/puppeteer-core/node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "engines": { + "node": "4.x || >=6.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/progress": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.1.tgz", + "integrity": "sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/puppeteer-core/node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/puppeteer-core/node_modules/tar-fs": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.0.tgz", + "integrity": "sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp": "^0.5.1", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + } + }, + "node_modules/puppeteer-core/node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/puppeteer/node_modules/puppeteer-core": { "version": "24.8.0", "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.8.0.tgz", "integrity": "sha512-tDf2YKIo5kM5r0vOzT52+PTgN0bBZOA4OFgQaqYyfarrcXLLJ92wi/lSMe44hd+F+gk0gw9QsAzyRW8v6ra93w==", @@ -5501,6 +6082,26 @@ "node": ">=18" } }, + "node_modules/puppeteer/node_modules/ws": { + "version": "8.18.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", + "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -5533,6 +6134,19 @@ "pify": "^2.3.0" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -5939,6 +6553,25 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/sax": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", @@ -6147,6 +6780,14 @@ "bare-events": "^2.2.0" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-width": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", @@ -6398,6 +7039,11 @@ "node": ">=0.8" } }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, "node_modules/tiny-inflate": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", @@ -6525,6 +7171,15 @@ "resolved": "https://registry.npmjs.org/ultrahtml/-/ultrahtml-1.6.0.tgz", "integrity": "sha512-R9fBn90VTJrqqLDwyMph+HGne8eqY1iPfYhPzZrvKpIfwkWZbcYlfpsb8B9dTvBfpy1/hqAD7Wi8EKfP9e8zdw==" }, + "node_modules/unbzip2-stream": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz", + "integrity": "sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg==", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, "node_modules/uncrypto": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz", @@ -7015,15 +7670,15 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/ws": { - "version": "8.18.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", - "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", + "version": "7.4.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", + "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==", "engines": { - "node": ">=10.0.0" + "node": ">=8.3.0" }, "peerDependencies": { "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" + "utf-8-validate": "^5.0.2" }, "peerDependenciesMeta": { "bufferutil": { diff --git a/package.json b/package.json index 4c68713..e205560 100644 --- a/package.json +++ b/package.json @@ -18,10 +18,12 @@ "@types/node": "^22.15.3", "@vercel/analytics": "^1.5.0", "astro": "^5.7.10", + "chrome-aws-lambda": "^10.1.0", "feed": "^4.2.2", "ioredis": "^5.6.1", "p-throttle": "^7.0.0", "puppeteer": "^24.8.0", + "puppeteer-core": "^10.1.0", "redis-modules-sdk": "^1.4.0", "tailwindcss": "^3.4.1" } diff --git a/src/pages/api/user-comics.ts b/src/pages/api/user-comics.ts index 2a48c80..d4d7fc1 100644 --- a/src/pages/api/user-comics.ts +++ b/src/pages/api/user-comics.ts @@ -1,12 +1,37 @@ import type { APIRoute } from "astro"; +import chromium from 'chrome-aws-lambda'; +import puppeteerCore from 'puppeteer-core'; + +// Check if running on Vercel +const IS_VERCEL = process.env.VERCEL === '1' || process.env.VERCEL === 'true'; + +// Import puppeteer dynamically for ESM compatibility import puppeteer from 'puppeteer'; +// Decide which Puppeteer to use based on environment +const puppeteerInstance = IS_VERCEL ? puppeteerCore : puppeteer; + // Helper function to extract comic slugs from a ComicK user list page async function scrapeComicList(userId: string, limit: number = 10): Promise { - const browser = await puppeteer.launch({ - headless: true, - args: ['--no-sandbox', '--disable-setuid-sandbox'] - }); + let browser; + + if (IS_VERCEL) { + // Use chrome-aws-lambda for Vercel environment + console.log('Running on Vercel - using chrome-aws-lambda'); + browser = await puppeteerCore.launch({ + args: chromium.args, + defaultViewport: chromium.defaultViewport, + executablePath: await chromium.executablePath, + headless: chromium.headless, + ignoreHTTPSErrors: true, + }); + } else { + // Use regular Puppeteer for local development + browser = await puppeteerInstance.launch({ + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox'] + }); + } try { const page = await browser.newPage(); @@ -22,147 +47,131 @@ async function scrapeComicList(userId: string, limit: number = 10): Promise { - // Scroll to the bottom of the page - window.scrollTo(0, document.body.scrollHeight); - }); - await new Promise(resolve => setTimeout(resolve, 2000)); - - // Scroll back up - await page.evaluate(() => { - window.scrollTo(0, 0); - }); - await new Promise(resolve => setTimeout(resolve, 1000)); - - // Try clicking any buttons that might reveal the content try { - const buttons = await page.$$('button, .button, [role="button"]'); - console.log(`Found ${buttons.length} buttons on the page`); - - // Try clicking buttons that might be related to viewing the list - for (const button of buttons.slice(0, 5)) { // Limit to first 5 buttons to avoid clicking too many - const buttonText = await page.evaluate(el => el.textContent?.toLowerCase() || '', button); - console.log(`Found button with text: ${buttonText}`); - if (buttonText.includes('list') || buttonText.includes('view') || - buttonText.includes('show') || buttonText.includes('load')) { - console.log(`Clicking button: ${buttonText}`); - await button.click().catch(e => console.log(`Failed to click button: ${e.message}`)); - await new Promise(resolve => setTimeout(resolve, 1000)); - } - } + await page.waitForFunction( + () => document.getElementById('__NEXT_DATA__') !== null, + { timeout: 10000 } + ); + console.log('Next.js data script found!'); } catch (e) { - console.log(`Error interacting with buttons: ${e.message}`); + console.log('Next.js data script not found, page might not be fully loaded'); } - // Give extra time for JavaScript to execute and everything to render - console.log('Waiting additional time for page to stabilize...'); + // Minimal page interaction - Next.js will load the data we need + console.log('Ensuring Next.js data is fully loaded...'); + + // Wait a bit for any dynamic data to populate await new Promise(resolve => setTimeout(resolve, 3000)); + // Verify Next.js data is available and complete + const dataLoaded = await page.evaluate(() => { + const script = document.getElementById('__NEXT_DATA__'); + if (!script) return false; + + try { + const data = JSON.parse(script.textContent || '{}'); + // Check if the follows data is present + return !!data?.props?.pageProps?.follows; + } catch (e) { + return false; + } + }); + + if (!dataLoaded) { + console.log('Next.js data not fully loaded, waiting a bit longer...'); + await new Promise(resolve => setTimeout(resolve, 3000)); + } else { + console.log('Next.js data confirmed loaded!'); + } + console.log('Page loaded, extracting comic data...'); // Extract the HTML structure for debugging const pageHtml = await page.content(); console.log('Page HTML length:', pageHtml.length); - // Extract comic slugs from the page with multiple selector attempts + // Extract comic slugs using only Next.js data - no CSS selectors const slugs = await page.evaluate(() => { - // Try multiple selectors to find comics - const selectors = [ - // Try different selectors that might match comics - '.book a[href^="/comic/"]', - 'a[href^="/comic/"]', - '.md-comic', - '.comics-grid a', - '.comic-list a', - '.md-list a', - '[data-testid="comic-item"] a', - // Generic link selector targeting comic URLs as fallback - 'a[href*="/comic/"]' - ]; + console.log('Extracting comics directly from Next.js data structure...'); - // Try each selector and use the first one that returns results - for (const selector of selectors) { - console.log('Trying selector:', selector); - const elements = document.querySelectorAll(selector); - console.log(`Found ${elements.length} elements with selector: ${selector}`); + try { + // Find the __NEXT_DATA__ script which contains all the comic data + const nextDataScript = document.getElementById('__NEXT_DATA__'); + if (!nextDataScript) { + console.log('ERROR: __NEXT_DATA__ script not found!'); + return []; + } - if (elements.length > 0) { - // We found matching elements, extract the slugs - const results = Array.from(elements).map(element => { - const href = element.getAttribute('href'); - if (!href || !href.includes('/comic/')) return null; - return href.replace('/comic/', '').split('/')[0]; // Get just the slug part + console.log('Found Next.js data script!'); + const nextData = JSON.parse(nextDataScript.textContent || '{}'); + + // Navigate to the follows data in the structure + const follows = nextData?.props?.pageProps?.follows || []; + console.log(`Found ${follows.length} comics in Next.js data`); + + if (follows.length > 0) { + // Extract slugs from the follows array + const comicSlugs = follows.map(follow => { + const comic = follow.md_comics; + if (comic && comic.slug) { + console.log(`Found comic: ${comic.title} with slug: ${comic.slug}`); + return comic.slug; + } + return null; }).filter(Boolean); - if (results.length > 0) { - console.log(`Found ${results.length} comics with selector: ${selector}`); - return results; + console.log(`Successfully extracted ${comicSlugs.length} comics from Next.js data`); + return comicSlugs; + } else { + console.log('No follows data found in Next.js data'); + + // Try to locate comics in a different location in the data structure + const alternative = nextData?.props?.pageProps?.user_info?.user_comic_follow_count; + if (alternative) { + console.log(`Found user has ${alternative} comics according to user_info`); } + + return []; } + } catch (e) { + console.log('Error extracting from Next.js data:', e); + return []; } - - // If all selectors failed, try a generic approach to find any links to comics - const allLinks = document.querySelectorAll('a'); - console.log(`Trying generic approach with ${allLinks.length} links`); - - const comicLinks = Array.from(allLinks).filter(link => { - const href = link.getAttribute('href'); - return href && href.includes('/comic/'); - }); - - console.log(`Found ${comicLinks.length} potential comic links using generic approach`); - - return comicLinks.map(link => { - const href = link.getAttribute('href'); - return href ? href.replace('/comic/', '').split('/')[0] : null; - }).filter(Boolean); }); - // Limit the number of comics to the specified limit - const limitedSlugs = slugs.slice(0, limit); + // Remove duplicates by using a Set + const uniqueSlugs = Array.from(new Set(slugs)); + console.log(`Extracted ${slugs.length} comics from page (${uniqueSlugs.length} unique)`); - console.log(`Extracted ${slugs.length} comics from page, returning ${limitedSlugs.length} (limit: ${limit})`); + // Limit the number of comics to the specified limit + const limitedSlugs = uniqueSlugs.slice(0, limit); + + console.log(`Returning ${limitedSlugs.length} comics (limit: ${limit})`); return limitedSlugs as string[]; } finally { await browser.close(); } } +// Define timeout for Vercel environment +const VERCEL_TIMEOUT = 10000; // 10 seconds for Vercel functions +const LOCAL_TIMEOUT = 60000; // 60 seconds for local development +const FUNCTION_TIMEOUT = IS_VERCEL ? VERCEL_TIMEOUT : LOCAL_TIMEOUT; + export const GET: APIRoute = async ({ request }) => { + // Ensure the function doesn't exceed Vercel's timeout limits + let timeoutId: NodeJS.Timeout; + + // Create a promise that rejects after the timeout + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject(new Error(`Function timed out after ${FUNCTION_TIMEOUT}ms`)); + }, FUNCTION_TIMEOUT); + }); + try { const url = new URL(request.url); const userId = url.searchParams.get('userId'); @@ -213,25 +222,49 @@ export const GET: APIRoute = async ({ request }) => { // Check if the list is empty if (totalComics === 0) { - return new Response(JSON.stringify({ - comics: [], - totalComics: 0, - message: "No comics found in user's list" + console.log('No comics found for user, returning empty list'); + const response = new Response(JSON.stringify({ + success: false, + error: "No comics found in this user's list. Make sure the user ID is correct and the list is not empty.", + data: { + comics: [], + total: 0, + limit: validLimit, + userId: cleanUserId + } }), { - status: 200, + status: 404, headers: { 'Content-Type': 'application/json' } }); + + // Clear the timeout to prevent memory leaks + clearTimeout(timeoutId); + return response; } + + console.log(`Returning ${totalComics} comics found for user ${cleanUserId}`); - return new Response(JSON.stringify({ comics, totalComics }), { + const response = new Response(JSON.stringify({ + success: true, + data: { + comics, + total: totalComics, + limit: validLimit, + userId: cleanUserId + } + }), { status: 200, headers: { 'Content-Type': 'application/json', 'Cache-Control': 'max-age=3600' // Cache for 1 hour } }); + + // Clear the timeout to prevent memory leaks + clearTimeout(timeoutId); + return response; } catch (error) { console.error('Error fetching user comics:', error); @@ -252,8 +285,12 @@ export const GET: APIRoute = async ({ request }) => { } } + // Clear the timeout to prevent memory leaks + clearTimeout(timeoutId); + return new Response( JSON.stringify({ + success: false, error: errorMessage, details: error instanceof Error ? error.message : String(error) }), {