From c2548c8f8752c2a0d9816530e2135307c2ec2046 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sat, 3 Sep 2022 19:46:22 -0400 Subject: [PATCH] Add script to update local BOSS content --- package-lock.json | 349 ++++++++++++++++++++++++++++++++++++++++++++-- package.json | 6 +- scrape.js | 112 +++++++++++++++ 3 files changed, 456 insertions(+), 11 deletions(-) create mode 100644 scrape.js diff --git a/package-lock.json b/package-lock.json index 77fc0d5..c83c701 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "BOSS", "version": "1.0.0", "license": "ISC", "dependencies": { @@ -13,10 +14,68 @@ "dotenv": "^10.0.0", "express": "^4.17.1", "express-subdomain": "^1.0.5", - "fs-extra": "^10.0.0", "morgan": "^1.10.0" + }, + "devDependencies": { + "axios": "^0.27.2", + "fs-extra": "^10.1.0", + "xmlbuilder2": "^3.0.2" } }, + "node_modules/@oozcitak/dom": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@oozcitak/dom/-/dom-1.15.10.tgz", + "integrity": "sha512-0JT29/LaxVgRcGKvHmSrUTEvZ8BXvZhGl2LASRUgHqDTC1M5g1pLmVv56IYNyt3bG2CUjDkc67wnyZC14pbQrQ==", + "dev": true, + "dependencies": { + "@oozcitak/infra": "1.0.8", + "@oozcitak/url": "1.0.4", + "@oozcitak/util": "8.3.8" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/@oozcitak/infra": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@oozcitak/infra/-/infra-1.0.8.tgz", + "integrity": "sha512-JRAUc9VR6IGHOL7OGF+yrvs0LO8SlqGnPAMqyzOuFZPSZSXI7Xf2O9+awQPSMXgIWGtgUf/dA6Hs6X6ySEaWTg==", + "dev": true, + "dependencies": { + "@oozcitak/util": "8.3.8" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/@oozcitak/url": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@oozcitak/url/-/url-1.0.4.tgz", + "integrity": "sha512-kDcD8y+y3FCSOvnBI6HJgl00viO/nGbQoCINmQ0h98OhnGITrWR3bOGfwYCthgcrV8AnTJz8MzslTQbC3SOAmw==", + "dev": true, + "dependencies": { + "@oozcitak/infra": "1.0.8", + "@oozcitak/util": "8.3.8" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/@oozcitak/util": { + "version": "8.3.8", + "resolved": "https://registry.npmjs.org/@oozcitak/util/-/util-8.3.8.tgz", + "integrity": "sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==", + "dev": true, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/@types/node": { + "version": "18.7.14", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.14.tgz", + "integrity": "sha512-6bbDaETVi8oyIARulOE9qF1/Qdi/23z6emrUh0fNJRUmjznqrixD4MpGDdgOFk5Xb0m2H6Xu42JGdvAxaJR/wA==", + "dev": true + }, "node_modules/accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -29,11 +88,36 @@ "node": ">= 0.6" } }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true + }, + "node_modules/axios": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", + "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "dev": true, + "dependencies": { + "follow-redirects": "^1.14.9", + "form-data": "^4.0.0" + } + }, "node_modules/basic-auth": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", @@ -86,6 +170,18 @@ "node": ">=0.1.90" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -126,6 +222,15 @@ "ms": "2.0.0" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -165,6 +270,19 @@ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -235,6 +353,40 @@ "node": ">= 0.8" } }, + "node_modules/follow-redirects": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", + "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -252,9 +404,10 @@ } }, "node_modules/fs-extra": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", - "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "dev": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -267,7 +420,8 @@ "node_modules/graceful-fs": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", - "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==" + "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==", + "dev": true }, "node_modules/http-errors": { "version": "1.7.2", @@ -308,10 +462,24 @@ "node": ">= 0.10" } }, + "node_modules/js-yaml": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, "dependencies": { "universalify": "^2.0.0" }, @@ -537,6 +705,12 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true + }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -569,6 +743,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true, "engines": { "node": ">= 10.0.0" } @@ -596,9 +771,67 @@ "engines": { "node": ">= 0.8" } + }, + "node_modules/xmlbuilder2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/xmlbuilder2/-/xmlbuilder2-3.0.2.tgz", + "integrity": "sha512-h4MUawGY21CTdhV4xm3DG9dgsqyhDkZvVJBx88beqX8wJs3VgyGQgAn5VreHuae6unTQxh115aMK5InCVmOIKw==", + "dev": true, + "dependencies": { + "@oozcitak/dom": "1.15.10", + "@oozcitak/infra": "1.0.8", + "@oozcitak/util": "8.3.8", + "@types/node": "*", + "js-yaml": "3.14.0" + }, + "engines": { + "node": ">=12.0" + } } }, "dependencies": { + "@oozcitak/dom": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@oozcitak/dom/-/dom-1.15.10.tgz", + "integrity": "sha512-0JT29/LaxVgRcGKvHmSrUTEvZ8BXvZhGl2LASRUgHqDTC1M5g1pLmVv56IYNyt3bG2CUjDkc67wnyZC14pbQrQ==", + "dev": true, + "requires": { + "@oozcitak/infra": "1.0.8", + "@oozcitak/url": "1.0.4", + "@oozcitak/util": "8.3.8" + } + }, + "@oozcitak/infra": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@oozcitak/infra/-/infra-1.0.8.tgz", + "integrity": "sha512-JRAUc9VR6IGHOL7OGF+yrvs0LO8SlqGnPAMqyzOuFZPSZSXI7Xf2O9+awQPSMXgIWGtgUf/dA6Hs6X6ySEaWTg==", + "dev": true, + "requires": { + "@oozcitak/util": "8.3.8" + } + }, + "@oozcitak/url": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@oozcitak/url/-/url-1.0.4.tgz", + "integrity": "sha512-kDcD8y+y3FCSOvnBI6HJgl00viO/nGbQoCINmQ0h98OhnGITrWR3bOGfwYCthgcrV8AnTJz8MzslTQbC3SOAmw==", + "dev": true, + "requires": { + "@oozcitak/infra": "1.0.8", + "@oozcitak/util": "8.3.8" + } + }, + "@oozcitak/util": { + "version": "8.3.8", + "resolved": "https://registry.npmjs.org/@oozcitak/util/-/util-8.3.8.tgz", + "integrity": "sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==", + "dev": true + }, + "@types/node": { + "version": "18.7.14", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.14.tgz", + "integrity": "sha512-6bbDaETVi8oyIARulOE9qF1/Qdi/23z6emrUh0fNJRUmjznqrixD4MpGDdgOFk5Xb0m2H6Xu42JGdvAxaJR/wA==", + "dev": true + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -608,11 +841,36 @@ "negotiator": "0.6.2" } }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true + }, + "axios": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", + "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "dev": true, + "requires": { + "follow-redirects": "^1.14.9", + "form-data": "^4.0.0" + } + }, "basic-auth": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", @@ -652,6 +910,15 @@ "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "requires": { + "delayed-stream": "~1.0.0" + } + }, "content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -683,6 +950,12 @@ "ms": "2.0.0" } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -713,6 +986,12 @@ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -774,6 +1053,23 @@ "unpipe": "~1.0.0" } }, + "follow-redirects": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", + "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==", + "dev": true + }, + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -785,9 +1081,10 @@ "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, "fs-extra": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", - "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "dev": true, "requires": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -797,7 +1094,8 @@ "graceful-fs": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", - "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==" + "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==", + "dev": true }, "http-errors": { "version": "1.7.2", @@ -829,10 +1127,21 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, + "js-yaml": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, "requires": { "graceful-fs": "^4.1.6", "universalify": "^2.0.0" @@ -1006,6 +1315,12 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true + }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -1028,7 +1343,8 @@ "universalify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", - "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==" + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true }, "unpipe": { "version": "1.0.0", @@ -1044,6 +1360,19 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "xmlbuilder2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/xmlbuilder2/-/xmlbuilder2-3.0.2.tgz", + "integrity": "sha512-h4MUawGY21CTdhV4xm3DG9dgsqyhDkZvVJBx88beqX8wJs3VgyGQgAn5VreHuae6unTQxh115aMK5InCVmOIKw==", + "dev": true, + "requires": { + "@oozcitak/dom": "1.15.10", + "@oozcitak/infra": "1.0.8", + "@oozcitak/util": "8.3.8", + "@types/node": "*", + "js-yaml": "3.14.0" + } } } } diff --git a/package.json b/package.json index 3d69953..0ba94ae 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,11 @@ "dotenv": "^10.0.0", "express": "^4.17.1", "express-subdomain": "^1.0.5", - "fs-extra": "^10.0.0", "morgan": "^1.10.0" + }, + "devDependencies": { + "axios": "^0.27.2", + "fs-extra": "^10.1.0", + "xmlbuilder2": "^3.0.2" } } diff --git a/scrape.js b/scrape.js new file mode 100644 index 0000000..d70930c --- /dev/null +++ b/scrape.js @@ -0,0 +1,112 @@ +const fs = require('fs-extra'); +const https = require('https'); +const axios = require('axios'); +const { convert: xmlParser } = require('xmlbuilder2'); + +const { get } = axios.create({ + httpsAgent: new https.Agent({ + rejectUnauthorized: false + }) +}); + +const TASKSHEET_BASE = 'https://npts.app.nintendo.net/p01/tasksheet/1'; +const TASKSHEET_BASE_NEW = 'https://npts.app.pretendo.cc/p01/tasksheet/1'; + +const TASKSHEETS = [ + { + id: '8UsM86l8xgkjFk8z', + files: ['wood1', 'woodBGM'] + }, + { + id: 'bb6tOEckvgZ50ciH', + files: ['optdat2', 'schdat2'] + }, + { + id: 'bieC9ACJlisFg5xS', + files: ['solv'] + }, + { + id: 'IeUc4hQsKKe9rJHB', + files: ['CHARA'] + }, + { + id: 'IXmFUqR2qenXfF61', + files: ['promo1', 'promo2', 'promo3', 'push'] + }, + { + id: 'LRmanFo4Tx3kEGDp', + files: ['sysmsg1', 'sysmsg2'] + }, + { + id: 'rjVlM7hUXPxmYQJh', + files: ['optdat2', 'schdat2'] + }, + { + id: 'tOaQcoBLtPTgVN3Y', + files: ['solv'] + }, + { + id: 'TZr27FE8wzKiEaTO', + files: ['sysmsg1', 'sysmsg2'] + }, + { + id: 'v1cqzWykBKUg0rHQ', + files: ['solv'] + }, + { + id: 'vGwChBW1ExOoHDsm', + files: ['CHARA', 'CHARA/Boss002.pack', 'CHARA/BossStatic002.pack'] + }, + { + id: 'zvGSM4kOrXpkKnpT', + files: ['optdat2', 'schdat2'] + }, +]; + +for (const tasksheet of TASKSHEETS) { + const tasksheetId = tasksheet.id; + + fs.ensureDirSync(`cdn/tasksheet/1/${tasksheetId}`); + fs.ensureDirSync(`cdn/content/encrypted/${tasksheetId}`); + + for (const tasksheetFileName of tasksheet.files) { + if (tasksheetFileName.includes('/')) { + fs.ensureDirSync(`cdn/tasksheet/1/${tasksheetId}/_subfolder`); + } + + get(`${TASKSHEET_BASE}/${tasksheetId}/${tasksheetFileName}?c=US&l=en`) + .then(response => { + const replacedUrls = response.data.replaceAll('nintendo.net', 'pretendo.cc'); + + if (tasksheetFileName.includes('/')) { + fs.writeFileSync(`cdn/tasksheet/1/${tasksheetId}/_subfolder/${tasksheetFileName}`, replacedUrls); + } else { + fs.writeFileSync(`cdn/tasksheet/1/${tasksheetId}/${tasksheetFileName}`, replacedUrls); + } + + const { TaskSheet: xml } = xmlParser(response.data, { format: "object" }); + const files = xml.Files.File || []; + + if (files instanceof Array) { + for (const contentFile of files) { + const fileName = contentFile.Url.split('/').pop(); + + get(contentFile.Url, { + responseType: 'arraybuffer', + }).then(response => { + fs.writeFileSync(`cdn/content/encrypted/${tasksheetId}/${fileName}`, response.data); + }); + } + } else { + const contentFile = files; // Only one file + const fileName = contentFile.Url.split('/').pop(); + + get(contentFile.Url, { + responseType: 'arraybuffer', + }).then(response => { + fs.writeFileSync(`cdn/content/encrypted/${tasksheetId}/${fileName}`, response.data); + }); + } + }); + } +} \ No newline at end of file