From e5866e83899d724ad7981e81aeebdf53b044ec17 Mon Sep 17 00:00:00 2001 From: Jack Stenglein Date: Sun, 30 Jun 2024 16:54:06 -0500 Subject: [PATCH 1/4] feat(#568): automatically extract games from TWIC --- backend/package-lock.json | 612 ++++++++++++++++++++++++-- backend/package.json | 3 +- backend/serverless-compose.yml | 4 + backend/twicService/requirements.txt | 4 + backend/twicService/scrape_twic.py | 535 ++++++++++++++++++++++ backend/twicService/serverless.yml | 22 + backend/twicService/time_controls.csv | 159 +++++++ scripts/upload_twic_games.py | 29 ++ 8 files changed, 1335 insertions(+), 33 deletions(-) create mode 100644 backend/twicService/requirements.txt create mode 100644 backend/twicService/scrape_twic.py create mode 100644 backend/twicService/serverless.yml create mode 100644 backend/twicService/time_controls.csv create mode 100644 scripts/upload_twic_games.py diff --git a/backend/package-lock.json b/backend/package-lock.json index 096029035..9bc7a0575 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -10,7 +10,8 @@ "devDependencies": { "@serverless/compose": "^1.3.0", "serverless-go-plugin": "^2.4.0", - "serverless-plugin-custom-roles": "^3.1.1" + "serverless-plugin-custom-roles": "^3.1.1", + "serverless-python-requirements": "^6.1.0" } }, "node_modules/@aws-crypto/crc32": { @@ -996,6 +997,12 @@ "node": ">=14.0.0" } }, + "node_modules/@iarna/toml": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.5.tgz", + "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==", + "dev": true + }, "node_modules/@kwsites/file-exists": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", @@ -2253,6 +2260,13 @@ "node": ">= 8" } }, + "node_modules/appdirectory": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/appdirectory/-/appdirectory-0.1.0.tgz", + "integrity": "sha512-DJ5DV8vZXBbusyiyPlH28xppwS8eAMRuuyMo88xeEcf4bV64lbLtbxRxqixZuJBXsZzLtXFmA13GwVjJc7vdQw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "dev": true + }, "node_modules/archive-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/archive-type/-/archive-type-4.0.0.tgz", @@ -2670,6 +2684,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -2902,6 +2925,17 @@ "node": ">= 10" } }, + "node_modules/cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, "node_modules/clone": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", @@ -3083,6 +3117,15 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/decompress": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/decompress/-/decompress-4.2.1.tgz", @@ -3723,6 +3766,19 @@ "find-requires": "bin/find-requires.js" } }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/flat": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", @@ -3868,6 +3924,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-intrinsic": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", @@ -3927,6 +3992,19 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/glob-all": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/glob-all/-/glob-all-3.3.1.tgz", + "integrity": "sha512-Y+ESjdI7ZgMwfzanHZYQ87C59jOO0i+Hd+QYtVt9PhLi6d8wlOpzQnfBxWUlaTuAoR3TkybLqqbIoWveU4Ji7Q==", + "dev": true, + "dependencies": { + "glob": "^7.2.3", + "yargs": "^15.3.1" + }, + "bin": { + "glob-all": "bin/glob-all" + } + }, "node_modules/glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -4420,6 +4498,27 @@ "node": ">=0.10.0" } }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-primitive": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/is-primitive/-/is-primitive-3.0.1.tgz", + "integrity": "sha512-GljRxhWvlCNRfZyORiH77FwdFwGcMO620o37EOYC0ORWdq+WYNVqW0w2Juzew4M+L81l6/QS3t5gkkihyRqv9w==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-promise": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", @@ -4479,6 +4578,15 @@ "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/isomorphic-ws": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz", @@ -4690,6 +4798,18 @@ "immediate": "~3.0.5" } }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -4737,6 +4857,18 @@ "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==" }, + "node_modules/lodash.uniqby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz", + "integrity": "sha512-e/zcLx6CSbmaEgFHCA7BnoQKyCtKMxnuWrJygbwPs/AIn+IMKl66L8/s+wBUn5LRw2pZx3bUHibiV1b6aTWIww==", + "dev": true + }, + "node_modules/lodash.values": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.values/-/lodash.values-4.3.0.tgz", + "integrity": "sha512-r0RwvdCv8id9TUblb/O7rYPwVy6lerCbcawrfdo9iC/1t1wsNMJknO79WNBgwkH0hIeJ08jmvvESbFpNb4jH0Q==", + "dev": true + }, "node_modules/log": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/log/-/log-6.3.1.tgz", @@ -4881,17 +5013,6 @@ "node": ">=8" } }, - "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/lru-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", @@ -5372,6 +5493,33 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-map": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", @@ -5395,11 +5543,29 @@ "node": ">=8" } }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -5705,6 +5871,15 @@ "node": ">=8.10.0" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -5713,6 +5888,12 @@ "node": ">=0.10.0" } }, + "node_modules/require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, "node_modules/resolve-alpn": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", @@ -5750,6 +5931,22 @@ "node": ">=0.10.0" } }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/run-async": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", @@ -5852,12 +6049,9 @@ } }, "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", + "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", "bin": { "semver": "bin/semver.js" }, @@ -5962,6 +6156,36 @@ "node": ">=10" } }, + "node_modules/serverless-python-requirements": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/serverless-python-requirements/-/serverless-python-requirements-6.1.0.tgz", + "integrity": "sha512-+cgHLXT7AnjAiOXF/lIuw9RWz7GrgTmgpq8FijzKuahEMzcK5J1p+8odiX0V3zMm4CWpFBxQrcCcGaq2zmw8dA==", + "dev": true, + "dependencies": { + "@iarna/toml": "^2.2.5", + "appdirectory": "^0.1.0", + "bluebird": "^3.7.2", + "child-process-ext": "^2.1.1", + "fs-extra": "^10.1.0", + "glob-all": "^3.3.1", + "is-wsl": "^2.2.0", + "jszip": "^3.10.1", + "lodash.get": "^4.4.2", + "lodash.uniqby": "^4.7.0", + "lodash.values": "^4.3.0", + "rimraf": "^3.0.2", + "semver": "^7.6.0", + "set-value": "^4.1.0", + "sha256-file": "1.0.0", + "shell-quote": "^1.8.1" + }, + "engines": { + "node": ">=12.0" + }, + "peerDependencies": { + "serverless": "^2.32 || 3" + } + }, "node_modules/serverless/node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -6052,6 +6276,12 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "dev": true + }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -6068,11 +6298,35 @@ "node": ">= 0.4" } }, + "node_modules/set-value": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-4.1.0.tgz", + "integrity": "sha512-zTEg4HL0RwVrqcWs3ztF+x1vkxfm0lP+MQQFPiMJTKVceBwEV0A569Ou8l9IYQG8jOZdMVI1hGsc0tmeD2o/Lw==", + "dev": true, + "funding": [ + "https://github.com/sponsors/jonschlinkert", + "https://paypal.me/jonathanschlinkert", + "https://jonschlinkert.dev/sponsor" + ], + "dependencies": { + "is-plain-object": "^2.0.4", + "is-primitive": "^3.0.1" + }, + "engines": { + "node": ">=11.0" + } + }, "node_modules/setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" }, + "node_modules/sha256-file": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/sha256-file/-/sha256-file-1.0.0.tgz", + "integrity": "sha512-nqf+g0veqgQAkDx0U2y2Tn2KWyADuuludZTw9A7J3D+61rKlIIl9V5TS4mfnwKuXZOH9B7fQyjYJ9pKRHIsAyg==", + "dev": true + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -6092,6 +6346,15 @@ "node": ">=8" } }, + "node_modules/shell-quote": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", + "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/side-channel": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", @@ -6614,6 +6877,12 @@ "node": ">= 8" } }, + "node_modules/which-module": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.1.tgz", + "integrity": "sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==", + "dev": true + }, "node_modules/which-typed-array": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.15.tgz", @@ -6740,6 +7009,12 @@ "node": ">=0.4" } }, + "node_modules/y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==", + "dev": true + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -6771,6 +7046,41 @@ "sprintf-js": "~1.0.2" } }, + "node_modules/yargs": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", + "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "dev": true, + "dependencies": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dev": true, + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", @@ -7689,6 +7999,12 @@ "tslib": "^2.5.0" } }, + "@iarna/toml": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.5.tgz", + "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==", + "dev": true + }, "@kwsites/file-exists": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", @@ -8699,6 +9015,12 @@ "picomatch": "^2.0.4" } }, + "appdirectory": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/appdirectory/-/appdirectory-0.1.0.tgz", + "integrity": "sha512-DJ5DV8vZXBbusyiyPlH28xppwS8eAMRuuyMo88xeEcf4bV64lbLtbxRxqixZuJBXsZzLtXFmA13GwVjJc7vdQw==", + "dev": true + }, "archive-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/archive-type/-/archive-type-4.0.0.tgz", @@ -9021,6 +9343,12 @@ "set-function-length": "^1.2.1" } }, + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -9189,6 +9517,17 @@ "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==" }, + "cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, "clone": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", @@ -9333,6 +9672,12 @@ } } }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "dev": true + }, "decompress": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/decompress/-/decompress-4.2.1.tgz", @@ -9821,6 +10166,16 @@ "esniff": "^1.1.0" } }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, "flat": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", @@ -9923,6 +10278,12 @@ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, "get-intrinsic": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", @@ -9958,6 +10319,16 @@ "path-is-absolute": "^1.0.0" } }, + "glob-all": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/glob-all/-/glob-all-3.3.1.tgz", + "integrity": "sha512-Y+ESjdI7ZgMwfzanHZYQ87C59jOO0i+Hd+QYtVt9PhLi6d8wlOpzQnfBxWUlaTuAoR3TkybLqqbIoWveU4Ji7Q==", + "dev": true, + "requires": { + "glob": "^7.2.3", + "yargs": "^15.3.1" + } + }, "glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -10291,6 +10662,21 @@ "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==" }, + "is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "requires": { + "isobject": "^3.0.1" + } + }, + "is-primitive": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/is-primitive/-/is-primitive-3.0.1.tgz", + "integrity": "sha512-GljRxhWvlCNRfZyORiH77FwdFwGcMO620o37EOYC0ORWdq+WYNVqW0w2Juzew4M+L81l6/QS3t5gkkihyRqv9w==", + "dev": true + }, "is-promise": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", @@ -10332,6 +10718,12 @@ "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true + }, "isomorphic-ws": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz", @@ -10521,6 +10913,15 @@ "immediate": "~3.0.5" } }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -10568,6 +10969,18 @@ "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==" }, + "lodash.uniqby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz", + "integrity": "sha512-e/zcLx6CSbmaEgFHCA7BnoQKyCtKMxnuWrJygbwPs/AIn+IMKl66L8/s+wBUn5LRw2pZx3bUHibiV1b6aTWIww==", + "dev": true + }, + "lodash.values": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.values/-/lodash.values-4.3.0.tgz", + "integrity": "sha512-r0RwvdCv8id9TUblb/O7rYPwVy6lerCbcawrfdo9iC/1t1wsNMJknO79WNBgwkH0hIeJ08jmvvESbFpNb4jH0Q==", + "dev": true + }, "log": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/log/-/log-6.3.1.tgz", @@ -10671,14 +11084,6 @@ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==" }, - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "requires": { - "yallist": "^4.0.0" - } - }, "lru-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", @@ -11014,6 +11419,26 @@ "yocto-queue": "^0.1.0" } }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + }, + "dependencies": { + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + } + } + }, "p-map": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", @@ -11031,11 +11456,23 @@ "p-finally": "^1.0.0" } }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, "pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -11241,11 +11678,23 @@ "picomatch": "^2.2.1" } }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true + }, "require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, "resolve-alpn": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", @@ -11273,6 +11722,15 @@ "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==" }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, "run-async": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", @@ -11326,12 +11784,9 @@ } }, "semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "requires": { - "lru-cache": "^6.0.0" - } + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", + "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==" }, "serverless": { "version": "3.38.0", @@ -11479,6 +11934,36 @@ "semver": "^7.3.5" } }, + "serverless-python-requirements": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/serverless-python-requirements/-/serverless-python-requirements-6.1.0.tgz", + "integrity": "sha512-+cgHLXT7AnjAiOXF/lIuw9RWz7GrgTmgpq8FijzKuahEMzcK5J1p+8odiX0V3zMm4CWpFBxQrcCcGaq2zmw8dA==", + "dev": true, + "requires": { + "@iarna/toml": "^2.2.5", + "appdirectory": "^0.1.0", + "bluebird": "^3.7.2", + "child-process-ext": "^2.1.1", + "fs-extra": "^10.1.0", + "glob-all": "^3.3.1", + "is-wsl": "^2.2.0", + "jszip": "^3.10.1", + "lodash.get": "^4.4.2", + "lodash.uniqby": "^4.7.0", + "lodash.values": "^4.3.0", + "rimraf": "^3.0.2", + "semver": "^7.6.0", + "set-value": "^4.1.0", + "sha256-file": "1.0.0", + "shell-quote": "^1.8.1" + } + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "dev": true + }, "set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -11492,11 +11977,27 @@ "has-property-descriptors": "^1.0.2" } }, + "set-value": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-4.1.0.tgz", + "integrity": "sha512-zTEg4HL0RwVrqcWs3ztF+x1vkxfm0lP+MQQFPiMJTKVceBwEV0A569Ou8l9IYQG8jOZdMVI1hGsc0tmeD2o/Lw==", + "dev": true, + "requires": { + "is-plain-object": "^2.0.4", + "is-primitive": "^3.0.1" + } + }, "setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" }, + "sha256-file": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/sha256-file/-/sha256-file-1.0.0.tgz", + "integrity": "sha512-nqf+g0veqgQAkDx0U2y2Tn2KWyADuuludZTw9A7J3D+61rKlIIl9V5TS4mfnwKuXZOH9B7fQyjYJ9pKRHIsAyg==", + "dev": true + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -11510,6 +12011,12 @@ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" }, + "shell-quote": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", + "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "dev": true + }, "side-channel": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", @@ -11944,6 +12451,12 @@ "isexe": "^2.0.0" } }, + "which-module": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.1.tgz", + "integrity": "sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==", + "dev": true + }, "which-typed-array": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.15.tgz", @@ -12028,6 +12541,12 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, + "y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==", + "dev": true + }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -12057,6 +12576,35 @@ } } }, + "yargs": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", + "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "dev": true, + "requires": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.2" + } + }, + "yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dev": true, + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, "yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", diff --git a/backend/package.json b/backend/package.json index 52010c577..fc91b5695 100644 --- a/backend/package.json +++ b/backend/package.json @@ -2,7 +2,8 @@ "devDependencies": { "@serverless/compose": "^1.3.0", "serverless-go-plugin": "^2.4.0", - "serverless-plugin-custom-roles": "^3.1.1" + "serverless-plugin-custom-roles": "^3.1.1", + "serverless-python-requirements": "^6.1.0" }, "dependencies": { "serverless": "^3.38.0" diff --git a/backend/serverless-compose.yml b/backend/serverless-compose.yml index cbf6876c9..7695cb65e 100644 --- a/backend/serverless-compose.yml +++ b/backend/serverless-compose.yml @@ -150,3 +150,7 @@ services: TimelineTableArn: ${chess-dojo-scheduler.TimelineTableArn} ExamsTableArn: ${examService.ExamsTableArn} ExamsTableStreamArn: ${examService.ExamsTableStreamArn} + + twicService: + path: twicService + \ No newline at end of file diff --git a/backend/twicService/requirements.txt b/backend/twicService/requirements.txt new file mode 100644 index 000000000..33d924a5a --- /dev/null +++ b/backend/twicService/requirements.txt @@ -0,0 +1,4 @@ +chess +beautifulsoup4 +requests +dynamodb-json diff --git a/backend/twicService/scrape_twic.py b/backend/twicService/scrape_twic.py new file mode 100644 index 000000000..2ef7160a8 --- /dev/null +++ b/backend/twicService/scrape_twic.py @@ -0,0 +1,535 @@ +import csv +import chess.pgn +import uuid +import datetime +import io +import boto3 +import traceback +import re +import requests +from bs4 import BeautifulSoup +from io import BytesIO +from zipfile import ZipFile +from urllib.request import urlopen, Request +from dynamodb_json import json_util as json +import os +import ssl + +ssl._create_default_https_context = ssl._create_unverified_context + +time_control_re = re.compile('\[TimeControl \"(.*)\"\]') +event_re = re.compile('\[Event \"(.*)\"]') +site_re = re.compile('\[Site \"(.*)\"]') +twic_header_re = re.compile(r'^\d+\) ') + +stage = os.environ['stage'] + +# db = boto3.resource('dynamodb') +# table = db.Table(f'{stage}-games') + + +def handler(event, context): + print('Event: ', event) + + archive_num = 1549 + + try: + twic_info = fetch_twic_info(archive_num) + if len(twic_info) == 0: + handle_error(archive_num, 'Empty twic_info') + + print(f'INFO {archive_num}: Got TWIC Info: ', twic_info) + validate_twic_info(archive_num, twic_info) + + pgns = fetch_twic_pgns(archive_num) + print(f'INFO {archive_num}: Got {len(pgns)} PGNs') + + upload_pgns(archive_num, pgns, twic_info) + except Exception as e: + handle_error(archive_num, e) + + +def handle_error(archive_num, msg): + """ + Logs the given error and exits. + @param archive_num The TWIC archive number that generated the error. + @param msg The error message to log. + """ + print(f'ERROR {archive_num}: {msg}') + print(traceback.format_exc()) + exit() + + +def fetch_twic_pgns(archive_num): + """ + Returns a list of the PGNs in the given TWIC archive. + @param archive_num The TWIC archive number to fetch. + """ + resp = urlopen(Request(f'https://theweekinchess.com/zips/twic{archive_num}g.zip', data=None, headers={'User-Agent': 'curl/8.4.0'})) + zip = ZipFile(BytesIO(resp.read())) + f = zip.open(f'twic{archive_num}.pgn') + pgns = [] + + while pgn := read_zip_pgn(f, archive_num): + pgns.append(pgn) + return pgns + + +def read_zip_pgn(file, archive_num): + """ + Reads a PGN from the given zip file data. + @param file The zip file data. + @param archive_num The TWIC archive number the file is from. + """ + + while True: + pgn = read_zip_line(file, archive_num) + if len(pgn) == 0: + return '' + pgn = pgn.lstrip() + if len(pgn) > 0: + break + + if not pgn.startswith('[Event'): + raise Exception(f'PGN first line does not start with Event header: `{pgn}`') + + foundMoves = False + while line := read_zip_line(file, archive_num): + if foundMoves and is_pgn_end(line, pgn): + break + elif line == '\n' or line == '\r\n': + foundMoves = True + + pgn += line + + return pgn + + +def is_pgn_end(line, pgn): + """ + Returns true if the line is the end of the PGN. + @param line The line to check. + @param pgn The PGN being read. + """ + if line == '\n' or line == '\r\n': + return True + + line = line.strip() + if line.endswith('1-0') or line.endswith('0-1') or line.endswith('1/2-1/2') or line.endswith('*'): + pgn += line + return True + + return False + + +def read_zip_line(file, archive_num): + """ + Reads a line from the given zip file data. + @param file The zip file data. + @param archive_num The TWIC archive number the file is from. + """ + line = file.readline() + try: + return line.decode('windows-1251') + except UnicodeDecodeError: + print(f'WARNING {archive_num}: failed to decode with windows-1251. Attempting utf-8.') + return line.decode('utf-8') + + +def fetch_twic_info(archive_num): + """ + Fetches the TWIC HTML page for the given archive number, and extracts the info from each event. + The event infos are returned as a list in the order of the TWIC page. Only events with games are included. + @param archive_num The TWIC archive number to fetch + """ + res = requests.get(f'https://theweekinchess.com/html/twic{archive_num}.html', headers={'User-Agent': 'curl/8.4.0'}) + if res.status_code != 200: + print(f'ERROR {archive_num}: TWIC status code != 200: ', repr(res)) + return [] + + soup = BeautifulSoup(res.text, 'html.parser') + results = [] + + events = soup.find_all('h2') + for event in events: + twic_event = get_twic_event(event) + if twic_event is not None: + results.append(twic_event) + + return results + + +def get_twic_event(soup): + """ + Gets a TWIC event from the given soup. If the soup does not represent a TWIC event + or the event has no sections, None is returned. + @param soup The BeautifulSoup object of the TWIC event name. + """ + event_name = soup.get_text().strip() + event_name = twic_header_re.sub('', event_name) + + if event_name == "Forthcoming Events and Links": + return None + + event = { + 'event': event_name, + 'sections': [], + } + + curr = soup.next_sibling + while curr is not None: + if curr.name == 'h2': + break + + if curr.name == 'ul' and curr.has_attr('class') and curr['class'][0] == 'tourn_details': + event['sections'].append(get_twic_section(curr)) + + curr = curr.next_sibling + + if len(event['sections']) == 0: + return None + + return event + + +def get_twic_section(tourn_details): + """ + Converts the given tournament details object into a dictionary of the section data. + @param tourn_details The BeautifulSoup object of the tournament details. + """ + event = tourn_details.find('li', class_='Event') + start = tourn_details.find('li', class_='StartDate') + end = tourn_details.find('li', class_='EndDate') + place = tourn_details.find('li', class_='Place') + nat = tourn_details.find('li', class_='NAT') + time_control = tourn_details.find('li', class_='TimeControl') + + site = '' + if place is not None: + site += place.get_text() + if nat is not None: + site += ' ' + nat.get_text() + + return { + 'event': '' if event is None else event.get_text(), + 'start': '' if start is None else start.get_text(), + 'end': '' if end is None else end.get_text(), + 'place': 'UNKNOWN' if place is None else place.get_text(), + 'nat': 'UNKNOWN' if nat is None else nat.get_text(), + 'site': 'UNKNOWN' if site == '' else site, + 'time_control': 'Unknown' if time_control is None else time_control.get_text().removeprefix('Time Control: ') + } + + +def validate_twic_info(archive_num, twic_info): + """ + Performs sanity checks on the given TWIC info. An exception is raised if something is wrong with the info. + @param archive_num The TWIC archive number from which the info was extracted. + @param twic_info The TWIC information to validate. + """ + no_sections_count = 0 + for info in twic_info: + if len(info['sections']) == 0: + no_sections_count += 1 + + if no_sections_count >= 3: + raise Exception(f'{archive_num}: TWIC info has {no_sections_count} events with no sections') + + +def upload_pgns(archive_num, pgns, twic_info): + time_control_info = load_time_control_info('time_controls.csv') + + twic_index = 0 + success = 0 + failed = 0 + + pgns_per_event = {} + + for i, pgn in enumerate(pgns): + site_changed = i > 0 and get_site(pgns[i-1]) != get_site(pgn) + time_headers, used_twic_index = get_time_headers(archive_num, pgn, site_changed, twic_index, twic_info, time_control_info) + + twic_index = used_twic_index + pgns_per_event[twic_info[twic_index]['event']] = pgns_per_event.get(twic_info[twic_index]['event'], 0) + 1 + + game = chess.pgn.read_game(io.StringIO(pgn)) + if game is None: + record_failure(archive_num, pgn) + failed += 1 + continue + + game = convert_game(game, time_headers, archive_num) + with open(f'twic_games_{archive_num}.json', 'a') as f: + f.write('{"Item":') + f.write(json.dumps(game)) + f.write('}\n') + + success += 1 + + print(f'INFO {archive_num} Success: {success}') + print(f'INFO {archive_num} Failed: {failed}') + print(f'INFO {archive_num} Total: ', success+failed) + print(f'INFO {archive_num} PGNs per event: ', pgns_per_event) + + +def matches_site(site, info): + """ + Returns true if the given PGN site matches any of the sections in the given TWIC info. + @param site The PGN site string to check. + @param info The TWIC info to check. + """ + return any(site == section['site'] or section['place'] in site or section['nat'] in site for section in info['sections']) + + +def record_failure(archive_num, pgn): + """ + Writes the failed PGN to the logs. + @param archive_num The TWIC archive number of the failed PGN. + @param pgn The pgn to write. + """ + print(f'ERROR {archive_num} Failed PGN: ', pgn) + + +def load_time_control_info(filename): + """ + Reads the mapping file from TWIC time controls to PGN standard. + The result is returned as a dictionary where the key is the TWIC time control and the + value is the information on the time control. + @param filename The CSV file containing the time control data. + """ + result = {} + with open(filename, 'r') as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + result[row['twic']] = row + return result + + +def convert_game(game, time_headers, archive_num): + """ + Converts the given chess.pgn game into a record for the DynamoDB games table. + @param game The chess.pgn game instance. + @param time_headers The PGN time control information. + @param archive_num The TWIC archive number this game comes from. + """ + time_class = time_headers['time_class'] + game.headers['TimeClass'] = time_class + if time_headers['pgn']: + game.headers['TimeControl'] = time_headers['pgn'] + if time_headers['white_clock']: + game.headers['WhiteClock'] = time_headers['white_clock'] + if time_headers['black_clock']: + game.headers['BlackClock'] = time_headers['black_clock'] + + ply_count = len(list(game.mainline_moves())) + game.headers['PlyCount'] = f'{ply_count}' + game.headers['TwicArchive'] = f'{archive_num}' + + headers = dict(game.headers) + + white = headers['White'].lower() + black = headers['Black'].lower() + date = headers['Date'] + if not date: + date = '????.??.??' + + createdAt = datetime.datetime.utcnow().isoformat() + + result = { + 'cohort': 'masters', + 'id': date + '_' + str(uuid.uuid4()), + 'white': white.lower(), + 'black': black.lower(), + 'date': date, + 'createdAt': createdAt, + 'updatedAt': createdAt, + 'publishedAt': createdAt, + 'owner': 'masters', + 'ownerDisplayName': 'Masters DB', + 'headers': headers, + 'pgn': str(game), + 'positionComments': {}, + 'timeClass': time_headers['time_class'] + } + return result + + +def get_time_headers(archive_num, pgn, site_changed, twic_index, twic_infos, time_control_info): + """ + Returns the time control headers for the given PGN and the used TWIC index. + @param archive_num The archive number the PGN is from. + @param pgn The PGN string to get the time control info for. + @param site_changed Whether the current PGN site changed from the previous one. + @param twic_index The index of the current TWIC info. + @param twic_infos The list of TWIC infos. + @param time_control_info The mapping from TWIC time controls to PGN time controls. + """ + used_twic_index = get_matching_twic_index(archive_num, pgn, site_changed, twic_index, twic_infos) + twic_info = twic_infos[used_twic_index] + + event = get_event(pgn) + matched_section = None + for section in twic_info['sections']: + if section['event'] == event: + matched_section = section + + possible_time_controls = [] + if matched_section: + possible_time_controls.append(matched_section['time_control']) + else: + possible_time_controls.extend([section['time_control'] for section in twic_info['sections'] if section['time_control']]) + + possible_time_classes = set() + for tc in possible_time_controls: + tc = tc.strip() + if tc in time_control_info: + possible_time_classes.add(time_control_info[tc]['time_class']) + elif tc and tc != "Unknown": + print(f'ERROR {archive_num}: unknown TWIC time control: {tc}') + + if len(possible_time_classes) == 0: + return get_unknown_time_control_headers(pgn), used_twic_index + + if len(possible_time_classes) == 1: + tc = possible_time_controls[0].strip() + if tc in time_control_info: + return time_control_info[tc], used_twic_index + return get_unknown_time_control_headers(pgn), used_twic_index + + preferred_time_class = 'Blitz' + if 'Standard' in possible_time_classes: + preferred_time_class = 'Standard' + elif 'Rapid' in possible_time_classes: + preferred_time_class = 'Rapid' + + for tc in possible_time_controls: + tc = tc.strip() + if tc in time_control_info and time_control_info[tc]['time_class'] == preferred_time_class: + return time_control_info[tc], used_twic_index + + raise Exception(f'Reached end of get_time_control_headers for PGN: {pgn}') + + +def get_matching_twic_index(archive_num, pgn, site_changed, curr_index, twic_infos): + """ + Returns the TWIC index that best matches the given PGN based on the event location. If no good match + is found, curr_index is returned. + @param archive_num The archive number of the TWIC info. + @param pgn The PGN to get the TWIC info index of. + @param site_changed Whether the current PGN site changed from the previous one. + @param curr_index The current TWIC index to use as a fallback. + @param twic_infos The TWIC info list. + """ + site = get_site(pgn) + + if not site_changed and matches_site(site, twic_infos[curr_index]): + return curr_index + + site_found, used_twic_index = matches_future_twic(archive_num, site, curr_index, twic_infos) + if site_found: + return used_twic_index + + if site_changed and matches_site(site, twic_infos[curr_index]): + print(f'INFO {archive_num}: site `{site}` matches current TWIC info {curr_index}: `{twic_infos[curr_index]["event"]}`. Continuing to use that info.') + return curr_index + + site_found, used_twic_index = matches_previous_twic(archive_num, site, curr_index, twic_infos) + if site_found: + return used_twic_index + + print(f'WARNING {archive_num}: site `{site}` not found for PGN (could indicate mismatched event): {pgn}\r\t', twic_infos[curr_index]) + + +def matches_future_twic(archive_num, site, curr_index, twic_infos): + """ + Returns true if the given site matches a future TWIC info, as well as the matched TWIC index. If no + match is found, the current TWIC index is returned. + @param archive_num The archive number of the TWIC info. + @param site The site name to check. + @param curr_index The current TWIC index to start searching from. + @param twic_infos The TWIC information list. + """ + i = curr_index + 1 + while i < len(twic_infos): + if matches_site(site, twic_infos[i]): + print(f'INFO {archive_num}: site `{site}` matches TWIC info {i+1}: `{twic_infos[i]["event"]}`. Jumping to that info.') + return True, i + i += 1 + return False, curr_index + + +def matches_previous_twic(archive_num, site, curr_index, twic_infos): + """ + Returns true if the given site matches a previous TWIC info, as well as the matched TWIC index. If no + match is found, the current TWIC index is returned. + @param archive_num The archive number of the TWIC info. + @param site The site name to check. + @param curr_index The current TWIC index to start searching from. + @param twic_infos The TWIC information list. + """ + i = curr_index - 1 + while i >= 0: + if matches_site(site, twic_infos[i]): + print(f'INFO {archive_num}: site `{site}` matches TWIC info {i+1}: `{twic_infos[i]["event"]}`. Jumping to that info.') + return True, i + i -= 1 + return False, curr_index + + +def get_event(pgn): + """ + Returns the value of the event header or ? if it doesnt not exist. + @param pgn The PGN to get the event header for. + """ + event = event_re.search(pgn) + if event is None: + return '?' + return event.group(1) + + +def get_site(pgn): + """ + Returns the value of the site header or ? if it does not exist. + @param pgn The PGN to get the site header for. + """ + site = site_re.search(pgn) + if site is None: + return '?' + return site.group(1) + + +def get_unknown_time_control_headers(pgn): + """ + Returns the time control headers for the given PGN, assuming the PGN + has an otherwise unknown time control. + @param pgn The PGN string to get the time control headers for. + """ + event = get_event(pgn).lower() + + if 'titled tue' in event: + return { + 'pgn': '180+1', + 'time_class': 'Blitz', + 'white_clock': '', + 'black_clock': '', + } + + time_class = 'Unknown' + + if 'classical' in event: + time_class = 'Standard' + elif 'rapid' in event or 'quick' in event: + time_class = 'Rapid' + elif 'blitz' in event or 'bullet' in event or 'armageddon' in event: + time_class = 'Blitz' + + return { + 'pgn': '', + 'time_class': time_class, + 'white_clock': '', + 'black_clock': '' + } + + +if __name__ == '__main__': + handler(None, None) diff --git a/backend/twicService/serverless.yml b/backend/twicService/serverless.yml new file mode 100644 index 000000000..aa458899c --- /dev/null +++ b/backend/twicService/serverless.yml @@ -0,0 +1,22 @@ +service: chess-dojo-twic +frameworkVersion: '3' + +plugins: + - serverless-python-requirements + +provider: + name: aws + runtime: python3.9 + region: us-east-1 + logRetentionInDays: 14 + environment: + stage: ${sls:stage} + deploymentMethod: direct + +custom: + pythonRequirements: + dockerizePip: true + +functions: + scrapeTwic: + handler: scrape_twic.handler diff --git a/backend/twicService/time_controls.csv b/backend/twicService/time_controls.csv new file mode 100644 index 000000000..025bc3254 --- /dev/null +++ b/backend/twicService/time_controls.csv @@ -0,0 +1,159 @@ +twic,pgn,time_class,white_clock,black_clock +120m:60m,40/7200:3600,Standard,, +10m/7m 3spm(61)Armageddon,60/600:0+3,Rapid,0:10:00,0:07:00 +120m:30m+30spm(41),40/7200:1800+30,Standard,, +70m+30spm(1),4200+30,Standard,, +13m+2spm,780+2,Rapid,, +105m+30spm(1),6300+30,Standard,, +105m(40)+30m+30spm,40/6300+30:1800+30,Standard,, +120m:25m+5spm(41),40/7200:1500+5,Standard,, +30m+20spm,1800+20,Rapid,, +60m+30spm,3600+30,Standard,, +90m+15spm,5400+15,Standard,, +5m:vs:4m,300,Blitz,0:05:00,0:04:00 +120m+10spm(40),40/7200:0+10,Standard,, +100m(40)+60m+30sec Delay,40/6000+30d:3600+30d,Standard,, +100m:40m+30smp(1),40/6000+30:2400+30,Standard,, +70m+5spm(1),4200+5,Standard,, +25m,1500,Rapid,, +60m,3600,Standard,, +30m,1800,Rapid,, +75m+5spm(1),4500+5,Standard,, +120m,7200,Standard,, +120m:60m+5spm(41),40/7200:3600+5,Standard,, +90m:40m+30spm(1),40/5400+30:2400+30,Standard,, +3m+2spm,180+2,Blitz,, +7m+3spm,420+3,Blitz,, +8m+3spm,480+3,Rapid,, +120m(40)+45m+10spm,40/7200+10:2700+10,Standard,, +40m,2400,Rapid,, +110m(40)+30m,40/6600:1800,Standard,, +110m+5spm,6600+5,Standard,, +70m+10spm(1),4200+10,Standard,, +10m+5spm,600+5,Rapid,, +90m+35spm(1),5400+35,Standard,, +60m+7spm,3600+7,Standard,, +12m,720,Rapid,, +105m+10m+5spm(41),40/6300:600+5,Standard,, +80m(40):20m+30spm,40/4800:1200+30,Standard,, +15m+12spm,900+12,Rapid,, +40m+5spm,2400+5,Rapid,, +120m+30spm(40),40/7200:0+30,Standard,, +12m+2spm,720+2,Rapid,, +120m+30m,40/7200:1800,Standard,, +25m+2spm,1500+2,Rapid,, +3m,180,Blitz,, +100m:50m:15m+30spm(1),40/6000+30:20/4500+30:900+30,Standard,, +90m+45spm(1),5400+45,Standard,, +45m+30spm,2700+30,Standard,, +5m+2spm,300+2,Blitz,, +30m+15spm,1800+15,Rapid,, +25m+10spm,1500+10,Rapid,, +75m+30spm(1),4500+30,Standard,, +20m+10spm,1200+10,Rapid,, +75m+10spm(1),4500+10,Standard,, +75m:15m:30spm(1),40/4500+30:900+30,Standard,, +25m+15spm,1500+15,Rapid,, +5m+3spm,300+3,Blitz,, +40m+30spm,2400+30,Standard,, +120m+5spm,7200+5,Standard,, +15m+6spm,900+6,Rapid,, +120m:30m+30spm,40/7200+30:1800+30,Standard,, +70m+5spm,4200+5,Standard,, +100m+30spm,6000+30,Standard,, +15m+3spm,900+3,Rapid,, +10m+10spm,600+10,Rapid,, +85m:20m+30spm(1),40/5100+30:1200+30,Standard,, +90m:15m:10spm(1),40/5400+10:900+10,Standard,, +60m+5sd,3600+5d,Standard,, +90m:20m+30spm(1),40/5400+30:1200+30,Standard,, +90m+5spm(1),5400+5,Standard,, +60m(40)+15m,40/3600:900,Standard,, +120m+15spm,7200+15,Standard,, +3m+3spm,180+3,Blitz,, +120m(40)+60m+30spm,40/7200+30:3600+30,Standard,, +8m+4spm,540+4,Rapid,, +75m(40)+15m,40/4500:900,Standard,, +150m(40)+adj,9000,Standard,, +5vs4Arm,300,Blitz,0:05:00,0:04:00 +110m+10spm(1),6600+10,Standard,, +100m:50m:10m+30spm(1),40/6600+30:20/3000+30:600+30,Standard,, +45m+15spm,2700+15,Standard,, +10m/7m 1spm(41)Armageddon,40/600:0+1,Rapid,0:10:00,0:07:00 +5mvs3m Armageddon,300,Blitz,0:05:00,0:03:00 +10m,600,Blitz,, +105m+15m,40/6300:900,Standard,, +25m+5spm,1500+5,Rapid,, +60m+15spm,3600+15,Standard,, +90m:30m,40/5400:1800,Standard,, +120m+10spm,7200+10,Standard,, +6m+5spm,360+5,Rapid,, +1m,60,Blitz,, +90m+30spm(40)+120m,40/5400:7200+30,Standard,, +7m+2spm(1),420+2,Blitz,, +100m+10spm,6000+10,Standard,, +45m+10spm,2700+10,Rapid,, +40m+10spm,2400+10,Rapid,, +100m:30m+30spm(1),40/6000+30:1800+30,Standard,, +60m(30)+30m,30/3600:1800,Standard,, +120m+30spm,7200+30,Standard,, +20m,1200,Rapid,, +15m+10spm,900+10,Rapid,, +4m+2spm,240+2,Blitz,, +90m:60m+10spm(41),40/5400:3600+10,Standard,, +180m+15spm,10800+15,Standard,, +20m+10spm(1),1200+10,Rapid,, +130m+30spm,7800+30,Standard,, +115m+5spm,6900+5,Standard,, +100m:30m,40/6000:1800,Standard,, +120:60:15+30spm(61),40/7200:20/3600:900+30,Standard,, +7m+7spm,420+7,Rapid,, +30m+5spm,1800+5,Rapid,, +3m+1spm,180+1,Blitz,, +85m:25m,40/5100:1500,Standard,, +100:50:15+30spm(1),40/6000+30:20/3000+30:900+30,Standard,, +90m:15m:30spm(1),40/5400+30:900+30,Standard,, +80m+10spm,4800+10,Standard,, +30m+10spm,1800+10,Rapid,, +25m:5m+5spm(31),30/1500:300+5,Rapid,, +90m+30spm,5400+30,Standard,, +120m:30m+30spm(41)),40/7200:1800+30,Standard,, +10m+3spm,600+3,Rapid,, +4m+3spm,240+3,Blitz,, +50m+10spm,3000+10,Standard,, +90m:60m+30spm,40/5400+30:3600+30,Standard,, +130m + 30sec delay,7800+30d,Standard,, +90m+10spm(1),5400+10,Standard,, +12m+3spm,720+3,Rapid,, +5m,300,Blitz,, +25m+30spm,1500+30,Rapid,, +25m+20spm,1500+20,Rapid,, +20m+5spm,1200+5,Rapid,, +60m+10spm,3600+10,Standard,, +5m+1spm,300+1,Blitz,, +90m+30spm(1),5400+30,Standard,, +15m+10spm(1),1350+10,Rapid,, +35m+30spm,2100+30,Standard,, +80m(40):10m+30spm,40/4800+30:600+30,Standard,, +8m+2spm,480+2,Blitz,, +110m+10spm,6600+10,Standard,, +90m:30m+30spm(1),40/5400+30:1800+30,Standard,, +60m(30)+30m+30spm,30/3600+30:1800+30,Standard,, +5m+5spm,300+5,Blitz,, +120m:60m:15m+30spm(61),40/7200:20/3600:900+30,Standard,, +60m+5spm,3600+5,Standard,, +120m(40):30m+10spm(1),40/7200+10:1800+10,Standard,, +100m:20m,40/6000:1200,Standard,, +15m+2spm,900+2,Rapid,, +15m+5spm,900+5,Rapid,, +1m+1spm,60+1,Blitz,, +15m,900,Rapid,, +120m(40)+15m,40/7200:900,Standard,, +100m:50m:15m+30spm(61),40/6000:20/3000:900+30,Standard,, +12m+5spm,720+5,Rapid,, +120m:60m:30m,40/7200:20/3600:1800,Standard,, +45m,2700,Rapid,, +10m+2spm,600+2,Rapid,, +4m+3smp,240+3,Blitz,, +100m:50+30smp(1),40/6000+30:3000+30,Standard,, +60m+30m,40/3600:1800,Standard,, \ No newline at end of file diff --git a/scripts/upload_twic_games.py b/scripts/upload_twic_games.py new file mode 100644 index 000000000..a502547a0 --- /dev/null +++ b/scripts/upload_twic_games.py @@ -0,0 +1,29 @@ +import boto3 +import json +from boto3.dynamodb.types import TypeDeserializer + +db = boto3.resource('dynamodb') +table = db.Table('dev-games') +td = TypeDeserializer() + +def main(): + success = 0 + + with table.batch_writer() as batch: + for i in range(1538, 1550): + if i == 1548: + continue + + print(f'Uploading file {i}') + with open(f'twic_games_{i}.json', 'r') as file: + for line in file: + game = line.strip() + game = json.loads(game) + game = td.deserialize({'M': game['Item']}) + batch.put_item(Item=game) + success += 1 + + print('Successful Uploads: ', success) + +if __name__ == '__main__': + main() From 955f7c58789cba0d5c39a3f0f53cd3388717fb83 Mon Sep 17 00:00:00 2001 From: Jack Stenglein Date: Mon, 22 Jul 2024 13:02:33 -0500 Subject: [PATCH 2/4] feat(#568): upload TWIC games to dynamo --- backend/serverless-compose.yml | 2 + backend/twicService/scrape_twic.py | 157 +++++++++++++++++++++++------ backend/twicService/serverless.yml | 35 +++++++ scripts/upload_caissabase.py | 2 +- 4 files changed, 162 insertions(+), 34 deletions(-) diff --git a/backend/serverless-compose.yml b/backend/serverless-compose.yml index 7695cb65e..09925548a 100644 --- a/backend/serverless-compose.yml +++ b/backend/serverless-compose.yml @@ -153,4 +153,6 @@ services: twicService: path: twicService + params: + GamesTableArn: ${chess-dojo-scheduler.GamesTableArn} \ No newline at end of file diff --git a/backend/twicService/scrape_twic.py b/backend/twicService/scrape_twic.py index 2ef7160a8..00a23642c 100644 --- a/backend/twicService/scrape_twic.py +++ b/backend/twicService/scrape_twic.py @@ -13,9 +13,6 @@ from urllib.request import urlopen, Request from dynamodb_json import json_util as json import os -import ssl - -ssl._create_default_https_context = ssl._create_unverified_context time_control_re = re.compile('\[TimeControl \"(.*)\"\]') event_re = re.compile('\[Event \"(.*)\"]') @@ -24,19 +21,36 @@ stage = os.environ['stage'] -# db = boto3.resource('dynamodb') -# table = db.Table(f'{stage}-games') +db = boto3.resource('dynamodb') +twic_table = db.Table(f'{stage}-twic-metadata') +games_table = db.Table(f'{stage}-games') +ses = boto3.client('ses') + +unknown_time_controls = set() def handler(event, context): + """ + Handles a Lambda CloudWatch event by fetching the last known successful TWIC archive number, + extracting/uploading the games from the next TWIC archive number and then saving that as the + new last known successful number. Failure notifications are sent via email, as are notifications + about unknown TWIC time controls. + """ print('Event: ', event) - - archive_num = 1549 + archive_num = 0 + metadata = None try: + metadata = twic_table.get_item(Key={'id': 'METADATA'})['Item'] + print(f'INFO: Current TWIC metadata: ', metadata) + + last_archive_num = metadata['lastSuccess']['archiveNum'] + archive_num = last_archive_num + 1 + print(f'INFO: last successful archive was {last_archive_num}. Fetching {archive_num}') + twic_info = fetch_twic_info(archive_num) if len(twic_info) == 0: - handle_error(archive_num, 'Empty twic_info') + handle_error(archive_num, 'Empty twic_info', metadata) print(f'INFO {archive_num}: Got TWIC Info: ', twic_info) validate_twic_info(archive_num, twic_info) @@ -45,19 +59,92 @@ def handler(event, context): print(f'INFO {archive_num}: Got {len(pgns)} PGNs') upload_pgns(archive_num, pgns, twic_info) + save_success(archive_num, metadata) + notify_unknown_time_controls(archive_num) except Exception as e: - handle_error(archive_num, e) + handle_error(archive_num, e, metadata) -def handle_error(archive_num, msg): +def handle_error(archive_num, msg, metadata): """ - Logs the given error and exits. + Logs the given error and sends a notification email about the failure. @param archive_num The TWIC archive number that generated the error. @param msg The error message to log. + @param metadata The TWIC metadata before the failure. """ + trace = traceback.format_exc() print(f'ERROR {archive_num}: {msg}') - print(traceback.format_exc()) - exit() + print(trace) + + if metadata != None: + metadata['lastFailure'] = { + 'archiveNum': archive_num, + 'updatedAt': datetime.datetime.utcnow().isoformat(), + } + twic_table.put_item(Item=metadata) + + ses.send_email( + Source='ChessDojo TWIC ', + Destination={ + 'ToAddresses': ['jackstenglein@gmail.com'], + }, + Message={ + 'Subject': { + 'Data': f'TWIC Failure - Archive {archive_num}', + 'Charset': 'UTF-8', + }, + 'Body': { + 'Text': { + 'Data': f'ERROR {archive_num}: {msg}\n\n{trace}', + 'Charset': 'UTF-8', + } + } + } + ) + + +def save_success(archive_num, metadata): + """ + Records that the archive number was successful in the TWIC metadata table. + @param archive_num The successful archive number. + @param metadata The TWIC metadata before the success. + """ + metadata['lastSuccess'] = { + 'archiveNum': archive_num, + 'updatedAt': datetime.datetime.utcnow().isoformat(), + } + twic_table.put_item(Item=metadata) + + +def notify_unknown_time_controls(archive_num): + """ + If necessary, sends an email notifying about unknown time controls in the given + TWIC archive. + @param archive_num The archive num with the unknown time controls. + """ + if len(unknown_time_controls) == 0: + return + + msg = ', '.join(unknown_time_controls) + ses.send_email( + Source='ChessDojo TWIC ', + Destination={ + 'ToAddresses': ['jackstenglein@gmail.com'], + }, + Message={ + 'Subject': { + 'Data': f'TWIC Unknown Time Controls - Archive {archive_num}', + 'Charset': 'UTF-8', + }, + 'Body': { + 'Text': { + 'Data': f'Unknown time controls: {msg}', + 'Charset': 'UTF-8', + } + } + } + ) + unknown_time_controls.clear() def fetch_twic_pgns(archive_num): @@ -237,6 +324,12 @@ def validate_twic_info(archive_num, twic_info): def upload_pgns(archive_num, pgns, twic_info): + """ + Uploads the given PGNs to the games database table. + @param archive_num The TWIC archive number of the PGNs. + @param pgns The list of PGNs to upload. + @param twic_info The list of TWIC info objects. + """ time_control_info = load_time_control_info('time_controls.csv') twic_index = 0 @@ -245,26 +338,23 @@ def upload_pgns(archive_num, pgns, twic_info): pgns_per_event = {} - for i, pgn in enumerate(pgns): - site_changed = i > 0 and get_site(pgns[i-1]) != get_site(pgn) - time_headers, used_twic_index = get_time_headers(archive_num, pgn, site_changed, twic_index, twic_info, time_control_info) - - twic_index = used_twic_index - pgns_per_event[twic_info[twic_index]['event']] = pgns_per_event.get(twic_info[twic_index]['event'], 0) + 1 - - game = chess.pgn.read_game(io.StringIO(pgn)) - if game is None: - record_failure(archive_num, pgn) - failed += 1 - continue - - game = convert_game(game, time_headers, archive_num) - with open(f'twic_games_{archive_num}.json', 'a') as f: - f.write('{"Item":') - f.write(json.dumps(game)) - f.write('}\n') - - success += 1 + with games_table.batch_writer() as batch: + for i, pgn in enumerate(pgns): + site_changed = i > 0 and get_site(pgns[i-1]) != get_site(pgn) + time_headers, used_twic_index = get_time_headers(archive_num, pgn, site_changed, twic_index, twic_info, time_control_info) + + twic_index = used_twic_index + pgns_per_event[twic_info[twic_index]['event']] = pgns_per_event.get(twic_info[twic_index]['event'], 0) + 1 + + game = chess.pgn.read_game(io.StringIO(pgn)) + if game is None: + record_failure(archive_num, pgn) + failed += 1 + continue + + game = convert_game(game, time_headers, archive_num) + batch.put_item(Item=game) + success += 1 print(f'INFO {archive_num} Success: {success}') print(f'INFO {archive_num} Failed: {failed}') @@ -386,6 +476,7 @@ def get_time_headers(archive_num, pgn, site_changed, twic_index, twic_infos, tim possible_time_classes.add(time_control_info[tc]['time_class']) elif tc and tc != "Unknown": print(f'ERROR {archive_num}: unknown TWIC time control: {tc}') + unknown_time_controls.add(tc) if len(possible_time_classes) == 0: return get_unknown_time_control_headers(pgn), used_twic_index diff --git a/backend/twicService/serverless.yml b/backend/twicService/serverless.yml index aa458899c..f4f116cdc 100644 --- a/backend/twicService/serverless.yml +++ b/backend/twicService/serverless.yml @@ -3,6 +3,7 @@ frameworkVersion: '3' plugins: - serverless-python-requirements + - serverless-plugin-custom-roles provider: name: aws @@ -20,3 +21,37 @@ custom: functions: scrapeTwic: handler: scrape_twic.handler + timeout: 900 + iamRoleStatements: + - Effect: Allow + Action: + - dynamodb:PutItem + - dynamodb:GetItem + Resource: !GetAtt TwicTable.Arn + - Effect: Allow + Action: + - dynamodb:BatchWriteItem + Resource: ${param:GamesTableArn} + - Effect: Allow + Action: + - ses:SendEmail + Resource: + - arn:aws:ses:${aws:region}:${aws:accountId}:identity/chessdojo.club + +resources: + Conditions: + IsProd: !Equals ['${sls:stage}', 'prod'] + + Resources: + TwicTable: + Type: AWS::DynamoDB::Table + DeletionPolicy: Retain + Properties: + TableName: ${sls:stage}-twic-metadata + BillingMode: PAY_PER_REQUEST + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH diff --git a/scripts/upload_caissabase.py b/scripts/upload_caissabase.py index a3e708621..97c23488f 100644 --- a/scripts/upload_caissabase.py +++ b/scripts/upload_caissabase.py @@ -9,7 +9,7 @@ db = boto3.resource('dynamodb') -table = db.Table('prod-games') +table = db.Table('dev-games') def main(): From 731a13428514e6b6947fc8ac7e02dbd509ba5157 Mon Sep 17 00:00:00 2001 From: Jack Stenglein Date: Mon, 22 Jul 2024 18:00:12 -0500 Subject: [PATCH 3/4] fix(#568): skip TWIC variant games --- backend/twicService/scrape_twic.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/twicService/scrape_twic.py b/backend/twicService/scrape_twic.py index 00a23642c..9511d9c46 100644 --- a/backend/twicService/scrape_twic.py +++ b/backend/twicService/scrape_twic.py @@ -334,6 +334,7 @@ def upload_pgns(archive_num, pgns, twic_info): twic_index = 0 success = 0 + skipped = 0 failed = 0 pgns_per_event = {} @@ -353,15 +354,30 @@ def upload_pgns(archive_num, pgns, twic_info): continue game = convert_game(game, time_headers, archive_num) + if is_variant(game): + print(f'INFO {archive_num} Skipping variant PGN: ', pgn) + skipped += 1 + continue + batch.put_item(Item=game) success += 1 print(f'INFO {archive_num} Success: {success}') print(f'INFO {archive_num} Failed: {failed}') - print(f'INFO {archive_num} Total: ', success+failed) + print(f'INFO {archive_num} Skipped: {skipped}') + print(f'INFO {archive_num} Total: ', success+failed+skipped) print(f'INFO {archive_num} PGNs per event: ', pgns_per_event) +def is_variant(game): + """ + Returns true if the game is a chess variant like 960 or atomic chess, + based on the game's Variant header. + """ + variant = game['headers'].get('Variant', 'Standard') + return variant == 'Standard' or variant == 'From Position' + + def matches_site(site, info): """ Returns true if the given PGN site matches any of the sections in the given TWIC info. From 545c429a23ce16c9a08ca32194dfa491dc58db33 Mon Sep 17 00:00:00 2001 From: Jack Stenglein Date: Mon, 22 Jul 2024 18:00:41 -0500 Subject: [PATCH 4/4] fix(#568): improve processGame performance --- backend/pgnService/explorer/processGame.ts | 295 +++++++++++++++++++-- backend/pgnService/serverless.yml | 2 +- 2 files changed, 274 insertions(+), 23 deletions(-) diff --git a/backend/pgnService/explorer/processGame.ts b/backend/pgnService/explorer/processGame.ts index 80b5d91e4..08b5d0562 100644 --- a/backend/pgnService/explorer/processGame.ts +++ b/backend/pgnService/explorer/processGame.ts @@ -26,6 +26,8 @@ const STARTING_POSITION_FEN = 'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQk const dynamo = new DynamoDBClient({ region: 'us-east-1' }); const explorerTable = process.env.stage + '-explorer'; +const mastersTable = + process.env.stage === 'dev' ? 'dev-explorer' : 'prod-masters-explorer'; /** An ExplorerPosition extracted from a specific game. */ interface ExplorerPositionExtraction { @@ -74,23 +76,50 @@ interface ExplorerMoveUpdate { newResult?: keyof ExplorerResult; } +interface KnownPositionsCache { + masters: Map; + dojo: Map; +} + +const knownPositions: KnownPositionsCache = { + masters: new Map(), + dojo: new Map(), +}; + /** * Extracts the positions from a list of Games and saves them to the explorer table. * @param event The DynamoDB stream event that triggered this Lambda. It contains the Game table objects. */ export const handler: DynamoDBStreamHandler = async (event) => { - const promises: Promise[] = []; + console.log( + 'Size of known positions cache at start: ', + knownPositions.masters.size, + knownPositions.dojo.size, + ); + + const positionUpdates = new Map(); + for (const record of event.Records) { - promises.push(processRecord(record)); + await processRecord(record, positionUpdates, knownPositions); } - await Promise.all(promises); + + console.log( + `Applying ${positionUpdates.size} positions updates for ${event.Records.length} records`, + ); + await applyBatchUpdates(positionUpdates); }; /** * Extracts the positions from a single Game and saves or removes them as necessary. * @param record A single DynamoDB stream record to extract positions from. + * @param positionUpdates The cache of position updates across all records in the stream's batch. + * @param knownPositions The cache of known explorer positions in the database. */ -async function processRecord(record: DynamoDBRecord) { +async function processRecord( + record: DynamoDBRecord, + positionUpdates: Map, + knownPositions: KnownPositionsCache, +) { try { const oldGame = record.dynamodb?.OldImage ? (unmarshall( @@ -118,14 +147,16 @@ async function processRecord(record: DynamoDBRecord) { const newExplorerPositions = extractPositions(newGame); const updates = getUpdates(oldExplorerPositions, newExplorerPositions); - console.log('Length of updates: ', updates.length); - const promises: Promise[] = []; for (const update of Object.values(updates)) { - promises.push(writeExplorerPosition(game, update)); + promises.push( + writeExplorerPosition(game, update, positionUpdates, knownPositions), + ); } - const results = await Promise.all(promises); - console.log('Finished with %d results: %j', results.length, results); + await Promise.all(promises); + console.log( + `Successfully applied ${updates.length} updates for game ${game.cohort}/${game.id}`, + ); } catch (err) { console.log('ERROR: Failed to process record %j: ', record, err); } @@ -311,41 +342,65 @@ function getExplorerMoveResult(result?: string): keyof ExplorerResult { * for the same FEN is not overwritten. If this is a new ExplorerPosition, then it is created in a * way that allows for future updates. * @param game The game that generated the ExplorerPosition. - * @param chess The Chess.ts instance to use when generating initial ExplorerPosition moves. * @param update The update to apply to the ExplorerPosition. + * @param positionUpdates The cache of position updates across all records in the stream's batch. + * @param knownPositions The cache of known explorer positions in the database. */ async function writeExplorerPosition( game: Game, update: ExplorerPositionUpdate, + positionUpdates: Map, + knownPositions: KnownPositionsCache, ): Promise { if (!update.newResult && !update.oldResult) { throw new Error('ERROR: update does not contain newResult nor oldResult'); } let success = false; + const cohort = getExplorerCohort(game); - const position = await fetchExplorerPosition(update.normalizedFen); - if (!position && !update.oldResult) { + const position = await fetchExplorerPosition( + cohort, + update.normalizedFen, + knownPositions, + ); + if (!position) { success = await setOrUpdateExplorerPosition(game, update); + } else if (!position.results[cohort]) { + success = await updateExplorerPosition(cohort, update, position); } else { - success = await updateExplorerPosition(getExplorerCohort(game), update, position); + mergeUpdate(positionUpdates, update, cohort); + success = true; } if (success) { - updateExplorerGame(game, update); + await updateExplorerGame(game, update); } return success; } /** * Fetches the explorer position with the given FEN from the database. - * If it does not exist, undefined is returned. + * If it does not exist, undefined is returned. If the position is already + * in the knownPositions cache, it is returned unchanged without fetching + * from the database. + * @param cohort The cohort that needs to be fetched. * @param normalizedFen The normalized FEN to fetch. + * @param knownPositions The cache of known explorer positions in the database. * @returns The explorer position with the normalized FEN. */ async function fetchExplorerPosition( + cohort: string, normalizedFen: string, + knownPositions: KnownPositionsCache, ): Promise { + const isMastersTable = cohort.startsWith('masters'); + const cache = isMastersTable ? knownPositions.masters : knownPositions.dojo; + + if (cache.has(normalizedFen)) { + return cache.get(normalizedFen); + } + const input = new GetItemCommand({ Key: { normalizedFen: { @@ -355,7 +410,7 @@ async function fetchExplorerPosition( S: 'POSITION', }, }, - TableName: explorerTable, + TableName: isMastersTable ? mastersTable : explorerTable, }); const output = await dynamo.send(input); @@ -363,7 +418,9 @@ async function fetchExplorerPosition( return undefined; } - return unmarshall(output.Item) as ExplorerPosition; + const position = unmarshall(output.Item) as ExplorerPosition; + cache.set(normalizedFen, position); + return position; } /** @@ -385,6 +442,7 @@ async function setOrUpdateExplorerPosition( } const cohort = getExplorerCohort(game); + const isMastersTable = cohort.startsWith('masters'); try { const initialExplorerPosition = getInitialExplorerPosition(update, cohort); @@ -392,7 +450,7 @@ async function setOrUpdateExplorerPosition( new PutItemCommand({ Item: marshall(initialExplorerPosition), ConditionExpression: 'attribute_not_exists(normalizedFen)', - TableName: explorerTable, + TableName: isMastersTable ? mastersTable : explorerTable, }), ); return true; @@ -529,6 +587,7 @@ async function updateExplorerPosition( updateExpression.length - ', '.length, ); + const isMastersTable = cohort.startsWith('masters'); const input = new UpdateItemCommand({ Key: { normalizedFen: { @@ -541,7 +600,7 @@ async function updateExplorerPosition( UpdateExpression: updateExpression, ExpressionAttributeNames: expressionAttrNames, ExpressionAttributeValues: expressionAttrValues, - TableName: explorerTable, + TableName: isMastersTable ? mastersTable : explorerTable, ReturnValues: 'NONE', }); @@ -613,6 +672,7 @@ async function setExplorerPositionCohort( updateExpression.length - ', '.length, ); + const isMastersTable = cohort.startsWith('masters'); const input = new UpdateItemCommand({ Key: { normalizedFen: { @@ -626,7 +686,7 @@ async function setExplorerPositionCohort( ConditionExpression: 'attribute_not_exists(results.#cohort)', ExpressionAttributeNames: exprAttrNames, ExpressionAttributeValues: exprAttrValues, - TableName: explorerTable, + TableName: isMastersTable ? mastersTable : explorerTable, ReturnValues: 'NONE', }); @@ -645,6 +705,195 @@ async function setExplorerPositionCohort( } } +/** + * Merges the given update into the cache of batch position updates. + * @param positionUpdates The cache of batch position updates. + * @param update The update to apply to the cache. + * @param cohort The cohort the update applies to. + */ +function mergeUpdate( + positionUpdates: Map, + update: ExplorerPositionUpdate, + cohort: string, +) { + if (!positionUpdates.has(update.normalizedFen)) { + positionUpdates.set(update.normalizedFen, { + normalizedFen: update.normalizedFen, + id: '', + results: {}, + moves: {}, + }); + } + + const currentUpdates = positionUpdates.get(update.normalizedFen); + if (!currentUpdates) { + throw new Error('currentUpdates is undefined'); + } + + if (update.oldResult) { + if (!currentUpdates.results[cohort]) { + currentUpdates.results[cohort] = {}; + } + currentUpdates.results[cohort][update.oldResult] = + (currentUpdates.results[cohort][update.oldResult] ?? 0) - 1; + } + + if (update.newResult) { + if (!currentUpdates.results[cohort]) { + currentUpdates.results[cohort] = {}; + } + currentUpdates.results[cohort][update.newResult] = + (currentUpdates.results[cohort][update.newResult] ?? 0) + 1; + } + + update.moves.forEach((move) => { + if (!currentUpdates.moves[move.san]) { + currentUpdates.moves[move.san] = { + san: move.san, + results: {}, + }; + } + const updateMove = currentUpdates.moves[move.san]; + + if (move.oldResult) { + if (!updateMove.results[cohort]) { + updateMove.results[cohort] = {}; + } + updateMove.results[cohort][move.oldResult] = + (updateMove.results[cohort][move.oldResult] ?? 0) - 1; + } + + if (move.newResult) { + if (!updateMove.results[cohort]) { + updateMove.results[cohort] = {}; + } + updateMove.results[cohort][move.newResult] = + (updateMove.results[cohort][move.newResult] ?? 0) + 1; + } + }); +} + +/** + * Applies the given batch updates to the explorer database. + * @param updates The updates to apply. + */ +async function applyBatchUpdates(updates: Map) { + for (const update of updates.values()) { + await applyBatchUpdate(update); + } +} + +/** + * Applies a batch update to an ExplorerPosition. The update is applied separately + * for the masters and non-masters cohorts. + * @param update The update to apply. + */ +async function applyBatchUpdate(update: ExplorerPosition) { + await Promise.all([ + applyBatchUpdateTable(update, mastersTable, (cohort) => + cohort.startsWith('masters'), + ), + applyBatchUpdateTable( + update, + explorerTable, + (cohort) => !cohort.startsWith('masters'), + ), + ]); +} + +/** + * Applies a batch update to the given table using the given cohortPredicate function. + * If the cohortPredicate returns false for a given cohort, then that cohort's results + * are not included in the update. + * @param update The update to apply. + * @param table The table to apply the update to. + * @param cohortPredicate The cohort predicate function to check updates against. + */ +async function applyBatchUpdateTable( + update: ExplorerPosition, + table: string, + cohortPredicate: (cohort: string) => boolean, +) { + let updateExpression = 'ADD '; + const expressionAttrValues: Record = {}; + const expressionAttrNames: Record = {}; + + let nameIdx = 0; + let valIdx = 0; + + for (const [cohort, results] of Object.entries(update.results)) { + if (!cohortPredicate(cohort)) { + continue; + } + + for (const [resultName, count] of Object.entries(results)) { + updateExpression += `results.#n${nameIdx}.${resultName} :v${valIdx}, `; + expressionAttrNames[`#n${nameIdx}`] = cohort; + expressionAttrValues[`:v${valIdx}`] = { + N: `${count}`, + }; + nameIdx++; + valIdx++; + } + } + + Object.values(update.moves).forEach((move, moveIdx) => { + for (const [cohort, results] of Object.entries(move.results)) { + if (!cohortPredicate(cohort)) { + continue; + } + + for (const [resultName, count] of Object.entries(results)) { + updateExpression += `moves.#san${moveIdx}.results.#n${nameIdx}.${resultName} :v${valIdx}, `; + expressionAttrNames[`#san${moveIdx}`] = move.san; + expressionAttrNames[`#n${nameIdx}`] = cohort; + expressionAttrValues[`:v${valIdx}`] = { + N: `${count}`, + }; + nameIdx++; + valIdx++; + } + } + }); + + if (nameIdx === 0) { + return; + } + + updateExpression = updateExpression.substring( + 0, + updateExpression.length - ', '.length, + ); + + const input = new UpdateItemCommand({ + Key: { + normalizedFen: { + S: update.normalizedFen, + }, + id: { + S: 'POSITION', + }, + }, + UpdateExpression: updateExpression, + ExpressionAttributeNames: expressionAttrNames, + ExpressionAttributeValues: expressionAttrValues, + TableName: table, + ReturnValues: 'NONE', + }); + + try { + await dynamo.send(input); + } catch (err) { + console.log( + 'ERROR: Failed to update explorer position %j with input %j: ', + update, + input, + err, + ); + throw err; + } +} + /** * Sets or removes the ExplorerGame associated with this game and update as necessary. * @param game The game associated with the ExplorerPosition. @@ -700,10 +949,11 @@ async function putExplorerGame(game: Game, update: ExplorerPositionUpdate) { }; try { + const isMastersTable = game.cohort.startsWith('masters'); await dynamo.send( new PutItemCommand({ Item: marshall(explorerGame, { removeUndefinedValues: true }), - TableName: explorerTable, + TableName: isMastersTable ? mastersTable : explorerTable, }), ); } catch (err) { @@ -724,13 +974,14 @@ async function removeExplorerGame(game: Game, update: ExplorerPositionUpdate) { try { const id = `GAME#${getExplorerCohort(game)}#${game.id}`; + const isMastersTable = game.cohort.startsWith('masters'); await dynamo.send( new DeleteItemCommand({ Key: { normalizedFen: { S: update.normalizedFen }, id: { S: id }, }, - TableName: explorerTable, + TableName: isMastersTable ? mastersTable : explorerTable, }), ); } catch (err) { diff --git a/backend/pgnService/serverless.yml b/backend/pgnService/serverless.yml index 7aa931765..5c6a7734a 100644 --- a/backend/pgnService/serverless.yml +++ b/backend/pgnService/serverless.yml @@ -32,7 +32,7 @@ functions: type: dynamodb arn: ${param:GamesTableStreamArn} batchWindow: 20 - batchSize: 300 + batchSize: 1000 maximumRetryAttempts: 2 parallelizationFactor: 10 functionResponseType: ReportBatchItemFailures