Made s3 optional

example.env

@@ -14,5 +14,6 @@ PN_ACT_CONFIG_EMAIL_FROM=Company Name <user@company.net>
 PN_ACT_CONFIG_S3_ACCESS_KEY=ACCESS_KEY
 PN_ACT_CONFIG_S3_ACCESS_SECRET=ACCESS_SECRET
 PN_ACT_CONFIG_HCAPTCHA_SECRET=0x0000000000000000000000000000000000000000
-PN_ACT_CONFIG_CDN_BASE=https://example.com
+PN_ACT_CONFIG_CDN_BASE=https://local-cdn.example.com
+PN_ACT_CONFIG_CDN_SUBDOMAIN=local-cdn
 PN_ACT_CONFIG_WEBSITE_BASE=https://example.com

src/cache.js

@@ -7,6 +7,7 @@ const memoryCache = {};
 
 const SERVICE_CERTS_BASE = `${__dirname}/../certs/service`;
 const NEX_CERTS_BASE = `${__dirname}/../certs/nex`;
+const LOCAL_CDN_BASE = `${__dirname}/../cdn`;
 
 async function connect() {
 	if (!disabledFeatures.redis) {
@@ -169,6 +170,25 @@ async function setServiceAESKey(name, value) {
 	await setCachedFile(`service:${name}:aes_key`, value);
 }
 
+// Local CDN cache functions
+
+async function getLocalCDNFile(name, encoding) {
+	let file = await getCachedFile(`local_cdn:${name}`, encoding);
+
+	if (file === null) {
+		if (await fs.pathExists(`${LOCAL_CDN_BASE}/${name}`)) {
+			file = await fs.readFile(`${LOCAL_CDN_BASE}/${name}`, { encoding });
+			await setLocalCDNFile(name, file);
+		}
+	}
+
+	return file;
+}
+
+async function setLocalCDNFile(name, value) {
+	await setCachedFile(`local_cdn:${name}`, value);
+}
+
 module.exports = {
 	connect,
 	getNEXPublicKey,
@@ -187,4 +207,6 @@ module.exports = {
 	setServicePrivateKey,
 	setServiceSecretKey,
 	setServiceAESKey,
+	getLocalCDNFile,
+	setLocalCDNFile
 };

src/config-manager.js

@@ -13,23 +13,24 @@ require('dotenv').config();
  * @property {string} mongoose.uri URI Mongoose will connect to
  * @property {string} mongoose.database MongoDB database name
  * @property {object} mongoose.options MongoDB connection options
- * @property {object} redis redis settings
- * @property {string} redis.client redis client settings
- * @property {string} redis.client.url redis server URL
- * @property {object} email node-mailer client settings
- * @property {string} email.host SMTP server address
- * @property {number} email.port SMTP server port
- * @property {boolean} email.secure Secure SMTP
- * @property {string} email.from Email 'from' name/address
- * @property {object} email.auth Email authentication settings
- * @property {string} email.auth.user Email username
- * @property {string} email.auth.pass Email password
- * @property {object} aws s3 client settings
- * @property {object} aws.spaces Digital Ocean Spaces settings
- * @property {string} aws.spaces.key s3 access key
- * @property {string} aws.spaces.secret s3 access secret
- * @property {object} hcaptcha hCaptcha settings
- * @property {string} hcaptcha.secret hCaptcha secret
+ * @property {object} [redis] redis settings
+ * @property {string} [redis.client] redis client settings
+ * @property {string} [redis.client.url] redis server URL
+ * @property {object} [email] node-mailer client settings
+ * @property {string} [email.host] SMTP server address
+ * @property {number} [email.port] SMTP server port
+ * @property {boolean} [email.secure] Secure SMTP
+ * @property {string} [email.from] Email 'from' name/address
+ * @property {object} [email.auth] Email authentication settings
+ * @property {string} [email.auth.user] Email username
+ * @property {string} [email.auth.pass] Email password
+ * @property {object} [aws] s3 client settings
+ * @property {object} [aws.spaces] Digital Ocean Spaces settings
+ * @property {string} [aws.spaces.key] s3 access key
+ * @property {string} [aws.spaces.secret] s3 access secret
+ * @property {object} [hcaptcha] hCaptcha settings
+ * @property {string} [hcaptcha.secret] hCaptcha secret
+ * @property {string} [cdn_subdomain] Subdomain used for serving CDN contents when s3 is disabled
  * @property {string} cdn_base Base URL for CDN location
  * @property {string} website_base Base URL for service website (used with emails)
  */
@@ -45,6 +46,7 @@ let config = {};
  * @property {boolean} redis true if redis is disabled
  * @property {boolean} email true if email sending is disabled
  * @property {boolean} captcha true if captcha verification is disabled
+ * @property {boolean} s3 true if s3 services is disabled
  */
 
 /**
@@ -53,15 +55,14 @@ let config = {};
 const disabledFeatures = {
 	redis: false,
 	email: false,
-	captcha: false
+	captcha: false,
+	s3: false
 };
 
 const requiredFields = [
 	['http.port', 'PN_ACT_CONFIG_HTTP_PORT', Number],
 	['mongoose.uri', 'PN_ACT_CONFIG_MONGO_URI'],
 	['mongoose.database', 'PN_ACT_CONFIG_MONGO_DB_NAME'],
-	['aws.spaces.key', 'PN_ACT_CONFIG_S3_ACCESS_KEY'],
-	['aws.spaces.secret', 'PN_ACT_CONFIG_S3_ACCESS_SECRET'],
 	['cdn_base', 'PN_ACT_CONFIG_CDN_BASE'],
 	['website_base', 'PN_ACT_CONFIG_WEBSITE_BASE'],
 ];
@@ -268,6 +269,54 @@ function configure() {
 		}
 	}
 
+	const s3AccessKeyConfigValue = get(config, 'aws.spaces.key');
+	const s3AccessKeyEnvValue = get(process.env, 'PN_ACT_CONFIG_S3_ACCESS_KEY');
+
+	if (!s3AccessKeyConfigValue || s3AccessKeyConfigValue.trim() === '') {
+		if (!s3AccessKeyEnvValue || s3AccessKeyEnvValue.trim() === '') {
+			logger.warn('Failed to find s3 access key config. Disabling feature');
+
+			disabledFeatures.s3 = true;
+		} else {
+			logger.info('aws.spaces.key not found in config, using environment variable PN_ACT_CONFIG_S3_ACCESS_KEY');
+
+			set(config, 'aws.spaces.key', s3AccessKeyEnvValue);
+		}
+	}
+
+	const s3SecretKeyConfigValue = get(config, 'aws.spaces.secret');
+	const s3SecretKeyEnvValue = get(process.env, 'PN_ACT_CONFIG_S3_ACCESS_SECRET');
+
+	if (!s3SecretKeyConfigValue || s3SecretKeyConfigValue.trim() === '') {
+		if (!s3SecretKeyEnvValue || s3SecretKeyEnvValue.trim() === '') {
+			logger.warn('Failed to find s3 secret key config. Disabling feature');
+
+			disabledFeatures.s3 = true;
+		} else {
+			logger.info('aws.spaces.secret not found in config, using environment variable PN_ACT_CONFIG_S3_ACCESS_SECRET');
+
+			set(config, 'aws.spaces.secret', s3AccessKeyEnvValue);
+		}
+	}
+
+	if (disabledFeatures.s3) {
+		const cdnSubdomainConfigValue = get(config, 'cdn_subdomain');
+		const cdnSubdomainEnvValue = get(process.env, 'PN_ACT_CONFIG_CDN_SUBDOMAIN');
+
+		if (!cdnSubdomainConfigValue || cdnSubdomainConfigValue.trim() === '') {
+			if (!cdnSubdomainEnvValue || cdnSubdomainEnvValue.trim() === '') {
+				logger.error('s3 file storage is disabled and no CDN subdomain was set. Set cdn_subdomain in config.json or the PN_ACT_CONFIG_CDN_SUBDOMAIN environment variable');
+				process.exit(0);
+			} else {
+				logger.info('cdn_subdomain not found in config, using environment variable PN_ACT_CONFIG_CDN_SUBDOMAIN');
+
+				set(config, 'cdn_subdomain', cdnSubdomainEnvValue);
+			}
+		}
+
+		logger.warn(`s3 file storage disabled. Using disk-based file storage. Please ensure cdn_base config or PN_ACT_CONFIG_CDN_BASE env variable is set to point to this server with the subdomain being ${config.cdn_subdomain}`);
+	}
+
 	module.exports.config = config;
 }
 

src/server.js

@@ -22,6 +22,7 @@ const nnid = require('./services/nnid');
 const nasc = require('./services/nasc');
 const datastore = require('./services/datastore');
 const api = require('./services/api');
+const localcdn = require('./services/local-cdn');
 
 // START APPLICATION
 
@@ -40,6 +41,7 @@ app.use(nnid);
 app.use(nasc);
 app.use(datastore);
 app.use(api);
+app.use(localcdn);
 
 // 404 handler
 logger.info('Creating 404 status handler');

src/services/local-cdn/index.js

@@ -0,0 +1,30 @@
+const express = require('express');
+const subdomain = require('express-subdomain');
+const logger = require('../../../logger');
+const { config, disabledFeatures } = require('../../config-manager');
+
+if (!disabledFeatures.s3) {
+	// * s3 enabled, no need for this
+
+	module.exports = express.Router();
+
+	return;
+}
+
+const routes = require('./routes');
+
+// Router to handle the subdomain
+const localcdn = express.Router();
+
+// Setup routes
+logger.info('[LOCAL-CDN] Applying imported routes');
+localcdn.use(routes.GET);
+
+// Main router for endpoints
+const router = express.Router();
+
+// Create subdomains
+logger.info(`[LOCAL-CDN] Creating '${config.cdn_subdomain}' subdomain`);
+router.use(subdomain(config.cdn_subdomain, localcdn));
+
+module.exports = router;

src/services/local-cdn/routes/get.js

@@ -0,0 +1,16 @@
+const router = require('express').Router();
+const cache = require('../../../cache');
+
+router.get('/*', async (request, response) => {
+	const filePath = request.params[0];
+
+	const file = await cache.getLocalCDNFile(filePath);
+
+	if (file) {
+		response.send(file);
+	} else {
+		response.sendStatus(404);
+	}
+});
+
+module.exports = router;

src/services/local-cdn/routes/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+	GET: require('./get.js'),
+};

src/util.js

@@ -1,16 +1,21 @@
 const crypto = require('crypto');
+const path = require('path');
 const NodeRSA = require('node-rsa');
 const aws = require('aws-sdk');
+const fs = require('fs-extra');
 const mailer = require('./mailer');
 const cache = require('./cache');
-const { config } = require('./config-manager');
+const { config, disabledFeatures } = require('./config-manager');
 
-const spacesEndpoint = new aws.Endpoint('nyc3.digitaloceanspaces.com');
-const s3 = new aws.S3({
-	endpoint: spacesEndpoint,
-	accessKeyId: config.aws.spaces.key,
-	secretAccessKey: config.aws.spaces.secret
-});
+let s3;
+
+if (!disabledFeatures.s3) {
+	s3 = new aws.S3({
+		endpoint: new aws.Endpoint('nyc3.digitaloceanspaces.com'),
+		accessKeyId: config.aws.spaces.key,
+		secretAccessKey: config.aws.spaces.secret
+	});
+}
 
 function nintendoPasswordHash(password, pid) {
 	const pidBuffer = Buffer.alloc(4);
@@ -203,20 +208,32 @@ function unpackToken(token) {
 function fullUrl(request) {
 	const protocol = request.protocol;
 	const host = request.host;
-	const path = request.originalUrl;
+	const opath = request.originalUrl;
 
-	return `${protocol}://${host}${path}`;
+	return `${protocol}://${host}${opath}`;
 }
 
 async function uploadCDNAsset(bucket, key, data, acl) {
-	const awsPutParams = {
-		Body: data,
-		Key: key,
-		Bucket: bucket,
-		ACL: acl
-	};
+	if (disabledFeatures.s3) {
+		await writeLocalCDNFile(key, data);
+	} else {
+		const awsPutParams = {
+			Body: data,
+			Key: key,
+			Bucket: bucket,
+			ACL: acl
+		};
 
-	await s3.putObject(awsPutParams).promise();
+		await s3.putObject(awsPutParams).promise();
+	}
+}
+
+async function writeLocalCDNFile(key, data) {
+	const filePath = `${__dirname}/../cdn/${key}`;
+	const folder = path.dirname(filePath);
+
+	await fs.ensureDir(folder);
+	await fs.writeFile(filePath, data);
 }
 
 function nascError(errorCode) {