From cd26860c46bcc8c15cd4a9a09f1a13f472d1ffc4 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sat, 28 Aug 2021 10:46:22 -0400 Subject: [PATCH] moved to DigitalOcean Spaces CDN for Miis --- package-lock.json | 220 ++++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/example.config.json | 6 ++ src/models/pnid.js | 14 ++- src/util.js | 23 ++++- 5 files changed, 255 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3d191b0..5b47995 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "aws-sdk": "^2.978.0", "bcrypt": "^5.0.0", "colors": "^1.4.0", "express": "^4.17.1", @@ -319,6 +320,40 @@ "resolved": "https://registry.npmjs.org/atob-lite/-/atob-lite-2.0.0.tgz", "integrity": "sha1-D+9a1G8b16hQLGVyfwNn1e5D1pY=" }, + "node_modules/aws-sdk": { + "version": "2.978.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.978.0.tgz", + "integrity": "sha512-UO1HP2b9FFtPhc5KbzEJ9oMBdzCmVv9nQTb+yIEh6SajcaErmK/ryleLOuPq9EsirE6wgb6mtf2qodRFucUA3g==", + "hasInstallScript": true, + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/aws-sdk/node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "node_modules/aws-sdk/node_modules/uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, "node_modules/aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -337,6 +372,25 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/basic-auth": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", @@ -424,6 +478,16 @@ "node": ">=0.6.19" } }, + "node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, "node_modules/buffer-to-uint8array": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/buffer-to-uint8array/-/buffer-to-uint8array-1.1.0.tgz", @@ -757,6 +821,14 @@ "node": ">= 0.6" } }, + "node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/express": { "version": "4.17.1", "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", @@ -1109,6 +1181,11 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "node_modules/ignore-walk": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", @@ -1314,6 +1391,14 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "node_modules/jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/jpeg-js": { "version": "0.3.7", "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.3.7.tgz", @@ -2084,6 +2169,15 @@ "node": ">=0.6" } }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -2594,6 +2688,20 @@ "punycode": "^2.1.0" } }, + "node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/url/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, "node_modules/utif": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/utif/-/utif-2.0.1.tgz", @@ -2841,6 +2949,23 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, + "node_modules/xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "node_modules/xml2js/node_modules/xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", + "engines": { + "node": ">=4.0" + } + }, "node_modules/xmlbuilder": { "version": "13.0.2", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-13.0.2.tgz", @@ -3109,6 +3234,34 @@ "resolved": "https://registry.npmjs.org/atob-lite/-/atob-lite-2.0.0.tgz", "integrity": "sha1-D+9a1G8b16hQLGVyfwNn1e5D1pY=" }, + "aws-sdk": { + "version": "2.978.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.978.0.tgz", + "integrity": "sha512-UO1HP2b9FFtPhc5KbzEJ9oMBdzCmVv9nQTb+yIEh6SajcaErmK/ryleLOuPq9EsirE6wgb6mtf2qodRFucUA3g==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -3124,6 +3277,11 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, "basic-auth": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", @@ -3199,6 +3357,16 @@ "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==" }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, "buffer-to-uint8array": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/buffer-to-uint8array/-/buffer-to-uint8array-1.1.0.tgz", @@ -3459,6 +3627,11 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "express": { "version": "4.17.1", "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", @@ -3743,6 +3916,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore-walk": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", @@ -3907,6 +4085,11 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "jpeg-js": { "version": "0.3.7", "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.3.7.tgz", @@ -4507,6 +4690,11 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -4916,6 +5104,22 @@ "punycode": "^2.1.0" } }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, "utif": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/utif/-/utif-2.0.1.tgz", @@ -5140,6 +5344,22 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + }, + "dependencies": { + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } + }, "xmlbuilder": { "version": "13.0.2", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-13.0.2.tgz", diff --git a/package.json b/package.json index 9a721f5..2c5d0f0 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ }, "homepage": "https://github.com/PretendoNetwork/account#readme", "dependencies": { + "aws-sdk": "^2.978.0", "bcrypt": "^5.0.0", "colors": "^1.4.0", "express": "^4.17.1", diff --git a/src/example.config.json b/src/example.config.json index b3a7e36..64e548b 100644 --- a/src/example.config.json +++ b/src/example.config.json @@ -15,5 +15,11 @@ "user": "email@address.com", "pass": "password" } + }, + "aws": { + "spaces": { + "key": "ACCESS_KEY", + "secret": "ACCESS_SECRET" + } } } \ No newline at end of file diff --git a/src/models/pnid.js b/src/models/pnid.js index 798c2c1..8e05d5d 100644 --- a/src/models/pnid.js +++ b/src/models/pnid.js @@ -2,8 +2,6 @@ const { Schema, model } = require('mongoose'); const uniqueValidator = require('mongoose-unique-validator'); const bcrypt = require('bcrypt'); const crypto = require('crypto'); -const fs = require('fs-extra'); -const path = require('path'); const imagePixels = require('image-pixels'); const TGA = require('tga'); const got = require('got'); @@ -215,21 +213,21 @@ PNIDSchema.methods.generateMiiImages = async function() { const pngData = await imagePixels(miiStudioNormalFaceImageData); const tga = TGA.createTgaBuffer(pngData.width, pngData.height, pngData.data); - const userMiiPath = path.normalize(`${__dirname}/../../cdn/${this.get('pid')}/miis`); - fs.ensureDirSync(userMiiPath); - fs.writeFileSync(`${userMiiPath}/standard.tga`, tga); - fs.writeFileSync(`${userMiiPath}/normal_face.png`, miiStudioNormalFaceImageData); + const userMiiKey = `mii/${this.get('pid')}`; + + await util.uploadCDNAsset(`${userMiiKey}/standard.tga`, tga); + await util.uploadCDNAsset(`${userMiiKey}/normal_face.png`, miiStudioNormalFaceImageData); const expressions = ['frustrated', 'smile_open_mouth', 'wink_left', 'sorrow', 'surprise_open_mouth']; for (const expression of expressions) { const miiStudioExpressionUrl = `https://studio.mii.nintendo.com/miis/image.png?data=${encodedStudioMiiData}&type=face&expression=${expression}&width=128&instanceCount=1`; const miiStudioExpressionImageData = await got(miiStudioExpressionUrl).buffer(); - fs.writeFileSync(`${userMiiPath}/${expression}.png`, miiStudioExpressionImageData); + await util.uploadCDNAsset(`${userMiiKey}/${expression}.png`, miiStudioExpressionImageData); } const miiStudioBodyUrl = `https://studio.mii.nintendo.com/miis/image.png?data=${encodedStudioMiiData}&type=all_body&width=270&instanceCount=1`; const miiStudioBodyImageData = await got(miiStudioBodyUrl).buffer(); - fs.writeFileSync(`${userMiiPath}/body.png`, miiStudioBodyImageData); + await util.uploadCDNAsset(`${userMiiKey}/body.png`, miiStudioBodyImageData); }; PNIDSchema.pre('save', async function(next) { diff --git a/src/util.js b/src/util.js index 10766bf..9b069e2 100644 --- a/src/util.js +++ b/src/util.js @@ -1,6 +1,15 @@ const crypto = require('crypto'); const NodeRSA = require('node-rsa'); const fs = require('fs-extra'); +const aws = require('aws-sdk'); +const config = require('./config.json'); + +const spacesEndpoint = new aws.Endpoint('nyc3.digitaloceanspaces.com'); +const s3 = new aws.S3({ + endpoint: spacesEndpoint, + accessKeyId: config.aws.spaces.key, + secretAccessKey: config.aws.spaces.secret +}); function nintendoPasswordHash(password, pid) { const pidBuffer = Buffer.alloc(4); @@ -167,10 +176,22 @@ function fullUrl(request) { return `${protocol}://${host}${path}`; } +async function uploadCDNAsset(key, data) { + const awsPutParams = { + Body: data, + Key: key, + Bucket: 'pn-cdn', + ACL: 'public-read' + }; + + await s3.putObject(awsPutParams).promise(); +} + module.exports = { nintendoPasswordHash, generateToken, decryptToken, unpackToken, - fullUrl + fullUrl, + uploadCDNAsset }; \ No newline at end of file