From 8ab51d7f96496ef585c7d4a81c856e5fc52f01a5 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 9 Oct 2022 15:24:15 -0400 Subject: [PATCH] Made s3 optional --- example.env | 3 +- src/cache.js | 22 +++++++ src/config-manager.js | 89 ++++++++++++++++++++------ src/server.js | 2 + src/services/local-cdn/index.js | 30 +++++++++ src/services/local-cdn/routes/get.js | 16 +++++ src/services/local-cdn/routes/index.js | 3 + src/util.js | 49 +++++++++----- 8 files changed, 177 insertions(+), 37 deletions(-) create mode 100644 src/services/local-cdn/index.js create mode 100644 src/services/local-cdn/routes/get.js create mode 100644 src/services/local-cdn/routes/index.js diff --git a/example.env b/example.env index 5c34138..a5c1007 100644 --- a/example.env +++ b/example.env @@ -14,5 +14,6 @@ PN_ACT_CONFIG_EMAIL_FROM=Company Name PN_ACT_CONFIG_S3_ACCESS_KEY=ACCESS_KEY PN_ACT_CONFIG_S3_ACCESS_SECRET=ACCESS_SECRET PN_ACT_CONFIG_HCAPTCHA_SECRET=0x0000000000000000000000000000000000000000 -PN_ACT_CONFIG_CDN_BASE=https://example.com +PN_ACT_CONFIG_CDN_BASE=https://local-cdn.example.com +PN_ACT_CONFIG_CDN_SUBDOMAIN=local-cdn PN_ACT_CONFIG_WEBSITE_BASE=https://example.com \ No newline at end of file diff --git a/src/cache.js b/src/cache.js index f20a8fe..e889483 100644 --- a/src/cache.js +++ b/src/cache.js @@ -7,6 +7,7 @@ const memoryCache = {}; const SERVICE_CERTS_BASE = `${__dirname}/../certs/service`; const NEX_CERTS_BASE = `${__dirname}/../certs/nex`; +const LOCAL_CDN_BASE = `${__dirname}/../cdn`; async function connect() { if (!disabledFeatures.redis) { @@ -169,6 +170,25 @@ async function setServiceAESKey(name, value) { await setCachedFile(`service:${name}:aes_key`, value); } +// Local CDN cache functions + +async function getLocalCDNFile(name, encoding) { + let file = await getCachedFile(`local_cdn:${name}`, encoding); + + if (file === null) { + if (await fs.pathExists(`${LOCAL_CDN_BASE}/${name}`)) { + file = await fs.readFile(`${LOCAL_CDN_BASE}/${name}`, { encoding }); + await setLocalCDNFile(name, file); + } + } + + return file; +} + +async function setLocalCDNFile(name, value) { + await setCachedFile(`local_cdn:${name}`, value); +} + module.exports = { connect, getNEXPublicKey, @@ -187,4 +207,6 @@ module.exports = { setServicePrivateKey, setServiceSecretKey, setServiceAESKey, + getLocalCDNFile, + setLocalCDNFile }; \ No newline at end of file diff --git a/src/config-manager.js b/src/config-manager.js index 0a8733c..12615ac 100644 --- a/src/config-manager.js +++ b/src/config-manager.js @@ -13,23 +13,24 @@ require('dotenv').config(); * @property {string} mongoose.uri URI Mongoose will connect to * @property {string} mongoose.database MongoDB database name * @property {object} mongoose.options MongoDB connection options - * @property {object} redis redis settings - * @property {string} redis.client redis client settings - * @property {string} redis.client.url redis server URL - * @property {object} email node-mailer client settings - * @property {string} email.host SMTP server address - * @property {number} email.port SMTP server port - * @property {boolean} email.secure Secure SMTP - * @property {string} email.from Email 'from' name/address - * @property {object} email.auth Email authentication settings - * @property {string} email.auth.user Email username - * @property {string} email.auth.pass Email password - * @property {object} aws s3 client settings - * @property {object} aws.spaces Digital Ocean Spaces settings - * @property {string} aws.spaces.key s3 access key - * @property {string} aws.spaces.secret s3 access secret - * @property {object} hcaptcha hCaptcha settings - * @property {string} hcaptcha.secret hCaptcha secret + * @property {object} [redis] redis settings + * @property {string} [redis.client] redis client settings + * @property {string} [redis.client.url] redis server URL + * @property {object} [email] node-mailer client settings + * @property {string} [email.host] SMTP server address + * @property {number} [email.port] SMTP server port + * @property {boolean} [email.secure] Secure SMTP + * @property {string} [email.from] Email 'from' name/address + * @property {object} [email.auth] Email authentication settings + * @property {string} [email.auth.user] Email username + * @property {string} [email.auth.pass] Email password + * @property {object} [aws] s3 client settings + * @property {object} [aws.spaces] Digital Ocean Spaces settings + * @property {string} [aws.spaces.key] s3 access key + * @property {string} [aws.spaces.secret] s3 access secret + * @property {object} [hcaptcha] hCaptcha settings + * @property {string} [hcaptcha.secret] hCaptcha secret + * @property {string} [cdn_subdomain] Subdomain used for serving CDN contents when s3 is disabled * @property {string} cdn_base Base URL for CDN location * @property {string} website_base Base URL for service website (used with emails) */ @@ -45,6 +46,7 @@ let config = {}; * @property {boolean} redis true if redis is disabled * @property {boolean} email true if email sending is disabled * @property {boolean} captcha true if captcha verification is disabled + * @property {boolean} s3 true if s3 services is disabled */ /** @@ -53,15 +55,14 @@ let config = {}; const disabledFeatures = { redis: false, email: false, - captcha: false + captcha: false, + s3: false }; const requiredFields = [ ['http.port', 'PN_ACT_CONFIG_HTTP_PORT', Number], ['mongoose.uri', 'PN_ACT_CONFIG_MONGO_URI'], ['mongoose.database', 'PN_ACT_CONFIG_MONGO_DB_NAME'], - ['aws.spaces.key', 'PN_ACT_CONFIG_S3_ACCESS_KEY'], - ['aws.spaces.secret', 'PN_ACT_CONFIG_S3_ACCESS_SECRET'], ['cdn_base', 'PN_ACT_CONFIG_CDN_BASE'], ['website_base', 'PN_ACT_CONFIG_WEBSITE_BASE'], ]; @@ -268,6 +269,54 @@ function configure() { } } + const s3AccessKeyConfigValue = get(config, 'aws.spaces.key'); + const s3AccessKeyEnvValue = get(process.env, 'PN_ACT_CONFIG_S3_ACCESS_KEY'); + + if (!s3AccessKeyConfigValue || s3AccessKeyConfigValue.trim() === '') { + if (!s3AccessKeyEnvValue || s3AccessKeyEnvValue.trim() === '') { + logger.warn('Failed to find s3 access key config. Disabling feature'); + + disabledFeatures.s3 = true; + } else { + logger.info('aws.spaces.key not found in config, using environment variable PN_ACT_CONFIG_S3_ACCESS_KEY'); + + set(config, 'aws.spaces.key', s3AccessKeyEnvValue); + } + } + + const s3SecretKeyConfigValue = get(config, 'aws.spaces.secret'); + const s3SecretKeyEnvValue = get(process.env, 'PN_ACT_CONFIG_S3_ACCESS_SECRET'); + + if (!s3SecretKeyConfigValue || s3SecretKeyConfigValue.trim() === '') { + if (!s3SecretKeyEnvValue || s3SecretKeyEnvValue.trim() === '') { + logger.warn('Failed to find s3 secret key config. Disabling feature'); + + disabledFeatures.s3 = true; + } else { + logger.info('aws.spaces.secret not found in config, using environment variable PN_ACT_CONFIG_S3_ACCESS_SECRET'); + + set(config, 'aws.spaces.secret', s3AccessKeyEnvValue); + } + } + + if (disabledFeatures.s3) { + const cdnSubdomainConfigValue = get(config, 'cdn_subdomain'); + const cdnSubdomainEnvValue = get(process.env, 'PN_ACT_CONFIG_CDN_SUBDOMAIN'); + + if (!cdnSubdomainConfigValue || cdnSubdomainConfigValue.trim() === '') { + if (!cdnSubdomainEnvValue || cdnSubdomainEnvValue.trim() === '') { + logger.error('s3 file storage is disabled and no CDN subdomain was set. Set cdn_subdomain in config.json or the PN_ACT_CONFIG_CDN_SUBDOMAIN environment variable'); + process.exit(0); + } else { + logger.info('cdn_subdomain not found in config, using environment variable PN_ACT_CONFIG_CDN_SUBDOMAIN'); + + set(config, 'cdn_subdomain', cdnSubdomainEnvValue); + } + } + + logger.warn(`s3 file storage disabled. Using disk-based file storage. Please ensure cdn_base config or PN_ACT_CONFIG_CDN_BASE env variable is set to point to this server with the subdomain being ${config.cdn_subdomain}`); + } + module.exports.config = config; } diff --git a/src/server.js b/src/server.js index b3202c6..c4dcf6b 100644 --- a/src/server.js +++ b/src/server.js @@ -22,6 +22,7 @@ const nnid = require('./services/nnid'); const nasc = require('./services/nasc'); const datastore = require('./services/datastore'); const api = require('./services/api'); +const localcdn = require('./services/local-cdn'); // START APPLICATION @@ -40,6 +41,7 @@ app.use(nnid); app.use(nasc); app.use(datastore); app.use(api); +app.use(localcdn); // 404 handler logger.info('Creating 404 status handler'); diff --git a/src/services/local-cdn/index.js b/src/services/local-cdn/index.js new file mode 100644 index 0000000..1a0a9dc --- /dev/null +++ b/src/services/local-cdn/index.js @@ -0,0 +1,30 @@ +const express = require('express'); +const subdomain = require('express-subdomain'); +const logger = require('../../../logger'); +const { config, disabledFeatures } = require('../../config-manager'); + +if (!disabledFeatures.s3) { + // * s3 enabled, no need for this + + module.exports = express.Router(); + + return; +} + +const routes = require('./routes'); + +// Router to handle the subdomain +const localcdn = express.Router(); + +// Setup routes +logger.info('[LOCAL-CDN] Applying imported routes'); +localcdn.use(routes.GET); + +// Main router for endpoints +const router = express.Router(); + +// Create subdomains +logger.info(`[LOCAL-CDN] Creating '${config.cdn_subdomain}' subdomain`); +router.use(subdomain(config.cdn_subdomain, localcdn)); + +module.exports = router; \ No newline at end of file diff --git a/src/services/local-cdn/routes/get.js b/src/services/local-cdn/routes/get.js new file mode 100644 index 0000000..7ae69b9 --- /dev/null +++ b/src/services/local-cdn/routes/get.js @@ -0,0 +1,16 @@ +const router = require('express').Router(); +const cache = require('../../../cache'); + +router.get('/*', async (request, response) => { + const filePath = request.params[0]; + + const file = await cache.getLocalCDNFile(filePath); + + if (file) { + response.send(file); + } else { + response.sendStatus(404); + } +}); + +module.exports = router; \ No newline at end of file diff --git a/src/services/local-cdn/routes/index.js b/src/services/local-cdn/routes/index.js new file mode 100644 index 0000000..033fca1 --- /dev/null +++ b/src/services/local-cdn/routes/index.js @@ -0,0 +1,3 @@ +module.exports = { + GET: require('./get.js'), +}; \ No newline at end of file diff --git a/src/util.js b/src/util.js index c7073f6..1f298d5 100644 --- a/src/util.js +++ b/src/util.js @@ -1,16 +1,21 @@ const crypto = require('crypto'); +const path = require('path'); const NodeRSA = require('node-rsa'); const aws = require('aws-sdk'); +const fs = require('fs-extra'); const mailer = require('./mailer'); const cache = require('./cache'); -const { config } = require('./config-manager'); +const { config, disabledFeatures } = require('./config-manager'); -const spacesEndpoint = new aws.Endpoint('nyc3.digitaloceanspaces.com'); -const s3 = new aws.S3({ - endpoint: spacesEndpoint, - accessKeyId: config.aws.spaces.key, - secretAccessKey: config.aws.spaces.secret -}); +let s3; + +if (!disabledFeatures.s3) { + s3 = new aws.S3({ + endpoint: new aws.Endpoint('nyc3.digitaloceanspaces.com'), + accessKeyId: config.aws.spaces.key, + secretAccessKey: config.aws.spaces.secret + }); +} function nintendoPasswordHash(password, pid) { const pidBuffer = Buffer.alloc(4); @@ -203,20 +208,32 @@ function unpackToken(token) { function fullUrl(request) { const protocol = request.protocol; const host = request.host; - const path = request.originalUrl; + const opath = request.originalUrl; - return `${protocol}://${host}${path}`; + return `${protocol}://${host}${opath}`; } async function uploadCDNAsset(bucket, key, data, acl) { - const awsPutParams = { - Body: data, - Key: key, - Bucket: bucket, - ACL: acl - }; + if (disabledFeatures.s3) { + await writeLocalCDNFile(key, data); + } else { + const awsPutParams = { + Body: data, + Key: key, + Bucket: bucket, + ACL: acl + }; - await s3.putObject(awsPutParams).promise(); + await s3.putObject(awsPutParams).promise(); + } +} + +async function writeLocalCDNFile(key, data) { + const filePath = `${__dirname}/../cdn/${key}`; + const folder = path.dirname(filePath); + + await fs.ensureDir(folder); + await fs.writeFile(filePath, data); } function nascError(errorCode) {