Merge pull request #116 from binaryoverload/dev

This commit is contained in:
Jonathan Barrow
2024-09-24 11:44:42 -04:00
committed by GitHub
17 changed files with 3052 additions and 354 deletions

3127
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -4,15 +4,13 @@
"description": "",
"main": "./dist/server.js",
"scripts": {
"lint": "npx eslint .",
"lint": "eslint .",
"lint:fix": "eslint --fix .",
"build": "npm run lint && npm run clean && npx tsc && npx tsc-alias && npm run copy-static",
"clean": "rimraf ./dist",
"copy-static": "npm run copy-assets && npm run copy-timezones && npm run copy-views",
"copy-assets": "cp -r ./src/assets ./dist/assets",
"copy-views": "cp -r ./src/views ./dist/views",
"copy-timezones": "cp ./src/services/nnas/timezones.json ./dist/services/nnas/timezones.json",
"copy-static": "copyfiles -e \"src/**/*.ts\" -u 1 \"src/**/*\" dist",
"start": "node .",
"start:dev": "NODE_ENV=development node ."
"start:dev": "cross-env NODE_ENV=development node ."
},
"repository": {
"type": "git",
@@ -26,9 +24,9 @@
},
"homepage": "https://github.com/PretendoNetwork/account#readme",
"dependencies": {
"@aws-sdk/client-s3": "^3.657.0",
"@aws-sdk/client-ses": "^3.515.0",
"@pretendonetwork/grpc": "^1.0.5",
"aws-sdk": "^2.978.0",
"bcrypt": "^5.0.0",
"buffer-crc32": "^0.2.13",
"colors": "^1.4.0",
@@ -40,11 +38,11 @@
"email-validator": "^2.0.4",
"express": "^4.17.1",
"express-rate-limit": "^6.7.0",
"express-subdomain": "^1.0.5",
"fs-extra": "^8.1.0",
"got": "^11.8.2",
"hcaptcha": "^0.1.0",
"image-pixels": "^1.1.1",
"is-valid-hostname": "^1.0.2",
"joi": "^17.8.3",
"mii-js": "github:PretendoNetwork/mii-js",
"moment": "^2.29.4",
@@ -80,6 +78,8 @@
"@types/validator": "^13.7.14",
"@typescript-eslint/eslint-plugin": "^5.54.1",
"@typescript-eslint/parser": "^5.54.1",
"copyfiles": "^2.4.1",
"cross-env": "^7.0.3",
"eslint": "^8.35.0",
"ndarray": "^1.0.19",
"typescript": "^4.9.5"

View File

@@ -1,8 +1,9 @@
import fs from 'fs-extra';
import mongoose from 'mongoose';
import dotenv from 'dotenv';
import { LOG_INFO, LOG_WARN, LOG_ERROR } from '@/logger';
import { Config } from '@/types/common/config';
import isValidHostname from 'is-valid-hostname';
import { LOG_INFO, LOG_WARN, LOG_ERROR, formatHostnames } from '@/logger';
import { type Config, domainServices } from '@/types/common/config';
dotenv.config();
@@ -10,7 +11,8 @@ export const disabledFeatures = {
redis: false,
email: false,
captcha: false,
s3: false
s3: false,
datastore: false
};
const hexadecimalStringRegex = /^[0-9a-f]+$/i;
@@ -60,7 +62,7 @@ export const config: Config = {
secret: process.env.PN_ACT_CONFIG_HCAPTCHA_SECRET || ''
},
cdn: {
subdomain: process.env.PN_ACT_CONFIG_CDN_SUBDOMAIN || '',
subdomain: process.env.PN_ACT_CONFIG_CDN_SUBDOMAIN,
disk_path: process.env.PN_ACT_CONFIG_CDN_DISK_PATH || '',
base_url: process.env.PN_ACT_CONFIG_CDN_BASE_URL || ''
},
@@ -76,6 +78,16 @@ export const config: Config = {
server_environment: process.env.PN_ACT_CONFIG_SERVER_ENVIRONMENT || '',
datastore: {
signature_secret: process.env.PN_ACT_CONFIG_DATASTORE_SIGNATURE_SECRET || ''
},
domains: {
api: (process.env.PN_ACT_CONFIG_DOMAINS_API || 'api.pretendo.cc').split(','),
assets: (process.env.PN_ACT_CONFIG_DOMAINS_ASSETS || 'assets.pretendo.cc').split(','),
cbvc: (process.env.PN_ACT_CONFIG_DOMAINS_CBVC || 'cbvc.cdn.pretendo.cc').split(','),
conntest: (process.env.PN_ACT_CONFIG_DOMAINS_CONNTEST || 'conntest.pretendo.cc').split(','),
datastore: (process.env.PN_ACT_CONFIG_DOMAINS_DATASTORE || 'datastore.pretendo.cc').split(','),
cdn: (process.env.PN_ACT_CONFIG_DOMAINS_CDN || '').split(','),
nasc: (process.env.PN_ACT_CONFIG_DOMAINS_NASC || 'nasc.pretendo.cc').split(','),
nnas: (process.env.PN_ACT_CONFIG_DOMAINS_NNAS || 'c.account.pretendo.cc,account.pretendo.cc').split(','),
}
};
@@ -85,21 +97,50 @@ if (process.env.PN_ACT_CONFIG_STRIPE_SECRET_KEY) {
};
}
// * Add the old config option for backwards compatibility
if (config.cdn.subdomain) {
config.domains.cdn.push(config.cdn.subdomain);
}
let configValid = true;
LOG_INFO('Config loaded, checking integrity');
for (const service of domainServices) {
const validDomains: string[] = [];
const invalidDomains: string[] = [];
const uniqueDomains = [...new Set(config.domains[service])];
for (const domain of uniqueDomains) {
isValidHostname(domain) ? validDomains.push(domain) : invalidDomains.push(domain);
}
if (validDomains.length === 0) {
LOG_ERROR(`No valid domains found for ${service}. Set the PN_ACT_CONFIG_DOMAINS_${service.toUpperCase()} environment variable to a valid domain`);
configValid = false;
}
if (invalidDomains.length) {
LOG_WARN(`Invalid domain(s) skipped for ${service}: ${formatHostnames(invalidDomains)}`);
}
config.domains[service] = validDomains;
}
if (!config.http.port) {
LOG_ERROR('Failed to find HTTP port. Set the PN_ACT_CONFIG_HTTP_PORT environment variable');
process.exit(0);
configValid = false;
}
if (!config.mongoose.connection_string) {
LOG_ERROR('Failed to find MongoDB connection string. Set the PN_ACT_CONFIG_MONGO_CONNECTION_STRING environment variable');
process.exit(0);
configValid = false;
}
if (!config.cdn.base_url) {
LOG_ERROR('Failed to find asset CDN base URL. Set the PN_ACT_CONFIG_CDN_BASE_URL environment variable');
process.exit(0);
configValid = false;
}
if (!config.redis.client.url) {
@@ -130,7 +171,7 @@ if (!config.email.from) {
if (!disabledFeatures.email) {
if (!config.website_base) {
LOG_ERROR('Email sending is enabled and no website base was configured. Set the PN_ACT_CONFIG_WEBSITE_BASE environment variable');
process.exit(0);
configValid = false;
}
}
@@ -140,17 +181,17 @@ if (!config.hcaptcha.secret) {
}
if (!config.s3.endpoint) {
LOG_WARN('Failed to find s3 endpoint config. Disabling feature. To enable feature set the PN_ACT_CONFIG_S3_ENDPOINT environment variable');
LOG_WARN('Failed to find S3 endpoint config. Disabling feature. To enable feature set the PN_ACT_CONFIG_S3_ENDPOINT environment variable');
disabledFeatures.s3 = true;
}
if (!config.s3.key) {
LOG_WARN('Failed to find s3 access key config. Disabling feature. To enable feature set the PN_ACT_CONFIG_S3_ACCESS_KEY environment variable');
LOG_WARN('Failed to find S3 access key config. Disabling feature. To enable feature set the PN_ACT_CONFIG_S3_ACCESS_KEY environment variable');
disabledFeatures.s3 = true;
}
if (!config.s3.secret) {
LOG_WARN('Failed to find s3 secret key config. Disabling feature. To enable feature set the PN_ACT_CONFIG_S3_ACCESS_SECRET environment variable');
LOG_WARN('Failed to find S3 secret key config. Disabling feature. To enable feature set the PN_ACT_CONFIG_S3_ACCESS_SECRET environment variable');
disabledFeatures.s3 = true;
}
@@ -160,41 +201,41 @@ if (!config.server_environment) {
}
if (disabledFeatures.s3) {
if (!config.cdn.subdomain) {
LOG_ERROR('s3 file storage is disabled and no CDN subdomain was set. Set the PN_ACT_CONFIG_CDN_SUBDOMAIN environment variable');
process.exit(0);
if (config.domains.cdn.length === 0) {
LOG_ERROR('S3 file storage is disabled and no CDN subdomain was set. Set the PN_ACT_CONFIG_DOMAINS_CDN environment variable');
configValid = false;
}
if (!config.cdn.disk_path) {
LOG_ERROR('s3 file storage is disabled and no CDN disk path was set. Set the PN_ACT_CONFIG_CDN_DISK_PATH environment variable');
process.exit(0);
LOG_ERROR('S3 file storage is disabled and no CDN disk path was set. Set the PN_ACT_CONFIG_CDN_DISK_PATH environment variable');
configValid = false;
}
LOG_WARN(`s3 file storage disabled. Using disk-based file storage. Please ensure cdn.base_url config or PN_ACT_CONFIG_CDN_BASE env variable is set to point to this server with the subdomain being ${config.cdn.subdomain}`);
LOG_WARN(`S3 file storage disabled. Using disk-based file storage. Please ensure cdn.base_url config or PN_ACT_CONFIG_CDN_BASE env variable is set to point to this server with the subdomain being ${config.cdn.subdomain}`);
if (disabledFeatures.redis) {
LOG_WARN('Both s3 and Redis are disabled. Large CDN files will use the in-memory cache, which may result in high memory use. Please enable s3 if you\'re running a production server.');
LOG_WARN('Both S3 and Redis are disabled. Large CDN files will use the in-memory cache, which may result in high memory use. Please enable S3 if you\'re running a production server.');
}
}
if (!config.aes_key) {
LOG_ERROR('Token AES key is not set. Set the PN_ACT_CONFIG_AES_KEY environment variable to your AES-256-CBC key');
process.exit(0);
configValid = false;
}
if (!config.grpc.master_api_keys.account) {
LOG_ERROR('Master gRPC API key for the account service is not set. Set the PN_ACT_CONFIG_GRPC_MASTER_API_KEY_ACCOUNT environment variable');
process.exit(0);
configValid = false;
}
if (!config.grpc.master_api_keys.api) {
LOG_ERROR('Master gRPC API key for the api service is not set. Set the PN_ACT_CONFIG_GRPC_MASTER_API_KEY_API environment variable');
process.exit(0);
configValid = false;
}
if (!config.grpc.port) {
LOG_ERROR('Failed to find gRPC port. Set the PN_ACT_CONFIG_GRPC_PORT environment variable');
process.exit(0);
configValid = false;
}
if (!config.stripe?.secret_key) {
@@ -202,10 +243,17 @@ if (!config.stripe?.secret_key) {
}
if (!config.datastore.signature_secret) {
LOG_ERROR('Datastore signature secret key is not set. Set the PN_ACT_CONFIG_DATASTORE_SIGNATURE_SECRET environment variable');
process.exit(0);
LOG_WARN('Datastore signature secret key is not set. Disabling feature. To enable feature set the PN_ACT_CONFIG_DATASTORE_SIGNATURE_SECRET environment variable');
disabledFeatures.datastore = true;
} else {
if (config.datastore.signature_secret.length !== 32 || !hexadecimalStringRegex.test(config.datastore.signature_secret)) {
LOG_ERROR('Datastore signature secret key must be a 32-character hexadecimal string.');
configValid = false;
}
}
if (config.datastore.signature_secret.length !== 32 || !hexadecimalStringRegex.test(config.datastore.signature_secret)) {
LOG_ERROR('Datastore signature secret key must be a 32-character hexadecimal string.');
if (!configValid) {
LOG_ERROR('Config is invalid. Exiting');
process.exit(0);
}
}

View File

@@ -44,4 +44,8 @@ export function LOG_INFO(input: string): void {
streams.info.write(`${input}\n`);
console.log(`${input}`.cyan.bold);
}
export function formatHostnames(hostnames: string[]): string {
return hostnames.map(d => `'${d}'`).join(', ');
}

View File

@@ -0,0 +1,14 @@
import express from 'express';
export function restrictHostnames<TFn extends express.Router>(
allowedHostnames: string[],
fn: TFn
): (request: express.Request, response: express.Response, next: () => void) => void | TFn {
return (request: express.Request, response: express.Response, next: () => void) => {
if (!allowedHostnames.includes(request.hostname)) {
return fn(request, response, next);
}
return next();
};
}

View File

@@ -26,7 +26,7 @@ import api from '@/services/api';
import localcdn from '@/services/local-cdn';
import assets from '@/services/assets';
import { config } from '@/config-manager';
import { config, disabledFeatures } from '@/config-manager';
const app = express();
@@ -48,11 +48,14 @@ app.use(conntest);
app.use(cbvc);
app.use(nnas);
app.use(nasc);
app.use(datastore);
app.use(api);
app.use(localcdn);
app.use(assets);
if (!disabledFeatures.datastore) {
app.use(datastore);
}
// * 404 handler
LOG_INFO('Creating 404 status handler');
app.use((request: express.Request, response: express.Response): void => {

View File

@@ -1,10 +1,11 @@
import express from 'express';
import subdomain from 'express-subdomain';
import cors from 'cors';
import APIMiddleware from '@/middleware/api';
import { LOG_INFO } from '@/logger';
import { formatHostnames, LOG_INFO } from '@/logger';
import { V1 } from '@/services/api/routes';
import { config } from '@/config-manager';
import { restrictHostnames } from '@/middleware/host-limit';
// * Router to handle the subdomain restriction
const api = express.Router();
@@ -28,8 +29,8 @@ api.use('/v1/user', V1.USER);
// * Main router for endpoints
const router = express.Router();
// * Create subdomains
LOG_INFO('[USER API] Creating \'api\' subdomain');
router.use(subdomain('api', api));
// * Create domains
LOG_INFO(`[USER API] Registering api router with domains: ${formatHostnames(config.domains.api)}`);
router.use(restrictHostnames(config.domains.api, api));
export default router;

View File

@@ -2,8 +2,9 @@
import path from 'node:path';
import express from 'express';
import subdomain from 'express-subdomain';
import { LOG_INFO } from '@/logger';
import { LOG_INFO, formatHostnames } from '@/logger';
import { config } from '@/config-manager';
import { restrictHostnames } from '@/middleware/host-limit';
// * Router to handle the subdomain restriction
const assets = express.Router();
@@ -15,8 +16,8 @@ assets.use(express.static(path.join(__dirname, '../../assets')));
// * Main router for endpoints
const router = express.Router();
// * Create subdomains
LOG_INFO('[conntest] Creating \'assets\' subdomain');
router.use(subdomain('assets', assets));
// * Create domains
LOG_INFO(`[assets] Creating assets router with domains: ${formatHostnames(config.domains.assets)}`);
router.use(restrictHostnames(config.domains.assets, assets));
export default router;

View File

@@ -1,14 +1,15 @@
// * handles CBVC (CTR Browser Version Check?) endpoints
import express from 'express';
import subdomain from 'express-subdomain';
import { LOG_INFO } from '@/logger';
import { LOG_INFO, formatHostnames } from '@/logger';
import { config } from '@/config-manager';
import { restrictHostnames } from '@/middleware/host-limit';
// * Router to handle the subdomain restriction
const cbvc = express.Router();
// * Setup route
LOG_INFO('[cbvc] Applying imported routes');
LOG_INFO('[CBVC] Applying imported routes');
cbvc.get('/:consoleType/:unknown/:region', (request: express.Request, response: express.Response): void => {
response.set('Content-Type', 'text/plain');
@@ -24,8 +25,8 @@ cbvc.get('/:consoleType/:unknown/:region', (request: express.Request, response:
// * Main router for endpoints
const router = express.Router();
// * Create subdomains
LOG_INFO('[cbvc] Creating \'cbvc\' subdomain');
router.use(subdomain('cbvc.cdn', cbvc));
// * Create domains
LOG_INFO(`[CBVC] Creating cbvc router with domains: ${formatHostnames(config.domains.cbvc)}`);
router.use(restrictHostnames(config.domains.cbvc, cbvc));
export default router;

View File

@@ -1,8 +1,9 @@
// * handles conntest endpoints
import express from 'express';
import subdomain from 'express-subdomain';
import { LOG_INFO } from '@/logger';
import { LOG_INFO, formatHostnames } from '@/logger';
import { restrictHostnames } from '@/middleware/host-limit';
import { config } from '@/config-manager';
// * Router to handle the subdomain restriction
const conntest = express.Router();
@@ -29,8 +30,8 @@ This is test.html page
// * Main router for endpoints
const router = express.Router();
// * Create subdomains
LOG_INFO('[conntest] Creating \'conntest\' subdomain');
router.use(subdomain('conntest', conntest));
// * Create domains
LOG_INFO(`[conntest] Creating conntest router with domains: ${formatHostnames(config.domains.conntest)}`);
router.use(restrictHostnames(config.domains.conntest, conntest));
export default router;

View File

@@ -1,8 +1,9 @@
import express from 'express';
import subdomain from 'express-subdomain';
import { LOG_INFO } from '@/logger';
import { LOG_INFO, formatHostnames } from '@/logger';
import upload from '@/services/datastore/routes/upload';
import { restrictHostnames } from '@/middleware/host-limit';
import { config } from '@/config-manager';
// * Router to handle the subdomain
const datastore = express.Router();
@@ -14,8 +15,8 @@ datastore.use(upload);
// * Main router for endpoints
const router = express.Router();
// * Create subdomains
LOG_INFO('[DATASTORE] Creating \'datastore\' subdomain');
router.use(subdomain('datastore', datastore));
// * Create domains
LOG_INFO(`[DATASTORE] Creating datastore router with domains: ${formatHostnames(config.domains.datastore)}`);
router.use(restrictHostnames(config.domains.datastore, datastore));
export default router;

View File

@@ -1,9 +1,9 @@
import express from 'express';
import subdomain from 'express-subdomain';
import { config, disabledFeatures } from '@/config-manager';
import { LOG_INFO } from '@/logger';
import { LOG_INFO, formatHostnames } from '@/logger';
import get from '@/services/local-cdn/routes/get';
import { restrictHostnames } from '@/middleware/host-limit';
const router = express.Router();
@@ -17,9 +17,9 @@ if (disabledFeatures.s3) {
LOG_INFO('[LOCAL-CDN] Applying imported routes');
localcdn.use(get);
// * Create subdomains
LOG_INFO(`[LOCAL-CDN] Creating '${config.cdn.subdomain}' subdomain`);
router.use(subdomain(config.cdn.subdomain, localcdn));
// * Create domains
LOG_INFO(`[LOCAL-CDN] Creating cdn router with domains: ${formatHostnames(config.domains.cdn)}`);
router.use(restrictHostnames(config.domains.cdn, localcdn));
} else {
LOG_INFO('[LOCAL-CDN] s3 enabled, skipping local CDN');
}

View File

@@ -1,11 +1,12 @@
// * handles NASC endpoints
import express from 'express';
import subdomain from 'express-subdomain';
import NASCMiddleware from '@/middleware/nasc';
import { LOG_INFO } from '@/logger';
import { LOG_INFO, formatHostnames } from '@/logger';
import ac from '@/services/nasc/routes/ac';
import { restrictHostnames } from '@/middleware/host-limit';
import { config } from '@/config-manager';
// * Router to handle the subdomain restriction
const nasc = express.Router();
@@ -20,8 +21,8 @@ nasc.use('/ac', ac);
// * Main router for endpoints
const router = express.Router();
// * Create subdomains
LOG_INFO('[NASC] Creating \'nasc\' subdomain');
router.use(subdomain('nasc', nasc));
// * Create domains
LOG_INFO(`[NASC] Creating nasc router with domains: ${formatHostnames(config.domains.nasc)}`);
router.use(restrictHostnames(config.domains.nasc, nasc));
export default router;

View File

@@ -2,11 +2,10 @@
import path from 'node:path';
import express from 'express';
import subdomain from 'express-subdomain';
import clientHeaderCheck from '@/middleware/client-header';
import cemuMiddleware from '@/middleware/cemu';
import pnidMiddleware from '@/middleware/pnid';
import { LOG_INFO } from '@/logger';
import { LOG_INFO, formatHostnames } from '@/logger';
import admin from '@/services/nnas/routes/admin';
import content from '@/services/nnas/routes/content';
@@ -17,6 +16,8 @@ import people from '@/services/nnas/routes/people';
import provider from '@/services/nnas/routes/provider';
import support from '@/services/nnas/routes/support';
import settings from '@/services/nnas/routes/account-settings';
import { config } from '@/config-manager';
import { restrictHostnames } from '@/middleware/host-limit';
// * Router to handle the subdomain restriction
const nnas = express.Router();
@@ -61,11 +62,8 @@ nnas.use('/v1/api/support', support);
// * Main router for endpoints
const router = express.Router();
// * Create subdomains
LOG_INFO('[NNAS] Creating \'account\' subdomain');
router.use(subdomain('account', nnas));
LOG_INFO('[NNAS] Creating \'c.account\' subdomain');
router.use(subdomain('c.account', nnas));
// * Create domains
LOG_INFO(`[NNAS] Creating nnas router with domains: ${formatHostnames(config.domains.nnas)}`);
router.use(restrictHostnames(config.domains.nnas, nnas));
export default router;

View File

@@ -1,5 +1,8 @@
import mongoose from 'mongoose';
export const domainServices = ['api', 'assets', 'cbvc', 'conntest', 'datastore', 'nasc', 'nnas', 'cdn'] as const;
export type DomainService = typeof domainServices[number];
export interface Config {
http: {
port: number;
@@ -30,7 +33,10 @@ export interface Config {
secret: string;
};
cdn: {
subdomain: string;
/**
* @deprecated Use `domains.cdn` instead
*/
subdomain?: string;
disk_path: string;
base_url: string;
};
@@ -50,4 +56,5 @@ export interface Config {
datastore: {
signature_secret: string;
};
domains: Record<DomainService, string[]>;
}

View File

@@ -1,16 +0,0 @@
// * Credit to https://github.com/bmullan91/express-subdomain/pull/61 for the types!
declare module 'express-subdomain'{
import type { Request, Response, Router } from 'express';
/**
* @description The subdomain function.
* @param subdomain The subdomain to listen on.
* @param fn The listener function, takes a response and request.
* @returns A function call to the value passed as FN, or void (the next function).
*/
export default function subdomain(
subdomain: string,
fn: Router | ((req: Request, res: Response) => void | any)
): (req: Request, res: Response, next: () => void) => void | typeof fn;
}

View File

@@ -1,7 +1,7 @@
import crypto from 'node:crypto';
import path from 'node:path';
import { IncomingHttpHeaders } from 'node:http';
import aws from 'aws-sdk';
import { ObjectCannedACL, S3 } from '@aws-sdk/client-s3';
import fs from 'fs-extra';
import express from 'express';
import mongoose from 'mongoose';
@@ -15,13 +15,16 @@ import { Token } from '@/types/common/token';
import { IPNID, IPNIDMethods } from '@/types/mongoose/pnid';
import { SafeQs } from '@/types/common/safe-qs';
let s3: aws.S3;
let s3: S3;
if (!disabledFeatures.s3) {
s3 = new aws.S3({
endpoint: new aws.Endpoint(config.s3.endpoint),
accessKeyId: config.s3.key,
secretAccessKey: config.s3.secret
s3 = new S3({
endpoint: config.s3.endpoint,
credentials: {
accessKeyId: config.s3.key,
secretAccessKey: config.s3.secret,
},
});
}
@@ -150,7 +153,7 @@ export function fullUrl(request: express.Request): string {
return `${protocol}://${host}${opath}`;
}
export async function uploadCDNAsset(bucket: string, key: string, data: Buffer, acl: string): Promise<void> {
export async function uploadCDNAsset(bucket: string, key: string, data: Buffer, acl: ObjectCannedACL): Promise<void> {
if (disabledFeatures.s3) {
await writeLocalCDNFile(key, data);
} else {
@@ -159,7 +162,7 @@ export async function uploadCDNAsset(bucket: string, key: string, data: Buffer,
Key: key,
Bucket: bucket,
ACL: acl
}).promise();
});
}
}