Merge pull request #97 from MatthewL246/docker-updates

This commit is contained in:
Jonathan Barrow
2024-07-01 16:21:35 -04:00
committed by GitHub
14 changed files with 127 additions and 413 deletions

View File

@@ -1,6 +1,5 @@
.git
config.json
logs
certs
cdn
node_modules
.git
.env
node_modules
dist
logs

47
.github/workflows/docker.yml vendored Normal file
View File

@@ -0,0 +1,47 @@
name: Build and Publish Docker Image
on:
push:
pull_request:
workflow_dispatch:
jobs:
build-publish:
env:
SHOULD_PUSH_IMAGE: ${{ (github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev')) || github.event_name == 'workflow_dispatch' }}
runs-on: ubuntu-latest
steps:
- name: Set up QEMU for Docker
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log into the Docker container registry
if: ${{ env.SHOULD_PUSH_IMAGE == 'true' }}
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ github.repository }}
tags: |
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
type=raw,value=edge,enable=${{ github.ref == 'refs/heads/dev' }}
type=sha
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v6
with:
platforms: linux/amd64,linux/arm64
push: ${{ env.SHOULD_PUSH_IMAGE }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max

View File

@@ -1,16 +1,47 @@
FROM node:18-alpine
RUN apk add --no-cache python3 make gcc g++
WORKDIR /app
COPY "docker/entrypoint.sh" ./
COPY package*.json ./
RUN npm install bcrypt && npm rebuild bcrypt --build-from-source
RUN npm install
COPY . ./
VOLUME [ "/app/config.json", "/app/certs" ]
CMD ["sh", "entrypoint.sh"]
# syntax=docker/dockerfile:1
ARG app_dir="/home/node/app"
# * Base Node.js image
FROM node:20-alpine AS base
ARG app_dir
WORKDIR ${app_dir}
# * Installing production dependencies
FROM base AS dependencies
RUN --mount=type=bind,source=package.json,target=package.json \
--mount=type=bind,source=package-lock.json,target=package-lock.json \
--mount=type=cache,target=/root/.npm \
npm ci --omit=dev
# * Installing development dependencies and building the application
FROM base AS build
RUN --mount=type=bind,source=package.json,target=package.json \
--mount=type=bind,source=package-lock.json,target=package-lock.json \
--mount=type=cache,target=/root/.npm \
npm ci
COPY . .
RUN npm run build
# * Running the final application
FROM base AS final
ARG app_dir
RUN mkdir -p ${app_dir}/logs && chown node:node ${app_dir}/logs
ENV NODE_ENV=production
USER node
COPY package.json .
COPY --from=dependencies ${app_dir}/node_modules ${app_dir}/node_modules
COPY --from=build ${app_dir}/dist ${app_dir}/dist
CMD ["node", "."]

View File

@@ -75,6 +75,7 @@ Configurations are loaded through environment variables. `.env` files are suppor
| `PN_ACT_CONFIG_CDN_BASE_URL` | URL for serving CDN contents (usually the same as s3 endpoint) | No |
| `PN_ACT_CONFIG_WEBSITE_BASE` | Website URL | Yes |
| `PN_ACT_CONFIG_AES_KEY` | AES-256 key used for encrypting tokens | No |
| `PN_ACT_CONFIG_DATASTORE_SIGNATURE_SECRET` | HMAC secret key (16 bytes in hex format) used to sign uploaded DataStore files | No |
| `PN_ACT_CONFIG_GRPC_MASTER_API_KEY_ACCOUNT` | Master API key to interact with the account gRPC service | No |
| `PN_ACT_CONFIG_GRPC_MASTER_API_KEY_API` | Master API key to interact with the API gRPC service | No |
| `PN_ACT_CONFIG_GRPC_PORT` | gRPC server port | No |

View File

@@ -1,79 +0,0 @@
const prompt = require('prompt');
const crypto = require('crypto');
const database = require('./src/database');
const { PNID } = require('./src/models/pnid');
prompt.message = '';
const properties = [
'username',
'email',
{
name: 'password',
hidden: true
}
];
prompt.get(properties, (error, { username, email, password }) => {
const date = new Date().toISOString();
// Sample Mii data
const miiData = 'AwAAQOlVognnx0GC2X0LLQOzuI0n2QAAAUBiAGUAbABsAGEAAABFAAAAAAAAAEBAEgCBAQRoQxggNEYUgRIXaA0AACkDUkhQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP6G';
const document = {
pid: 1,
creation_date: date.split('.')[0],
updated: date,
username: username,
password: password,
birthdate: '1990-01-01',
gender: 'M',
country: 'US',
language: 'en',
email: {
address: email,
primary: true,
parent: true,
reachable: true,
validated: true,
id: crypto.randomBytes(4).readUInt32LE()
},
region: 0x310B0000,
timezone: {
name: 'America/New_York',
offset: -14400
},
mii: {
name: 'bella',
primary: true,
data: miiData,
id: crypto.randomBytes(4).readUInt32LE(),
hash: crypto.randomBytes(7).toString('hex'),
image_url: '', // Deprecated, will be removed
image_id: crypto.randomBytes(4).readUInt32LE()
},
flags: {
active: true,
marketing: false,
off_device: true
},
validation: {
// These values are temp and will be overwritten before the document saves
// These values are only being defined to get around the `E11000 duplicate key error collection` error
email_code: 1,
email_token: ''
}
};
const newUser = new PNID(document);
newUser.save(async (error, newUser) => {
if (error) {
throw error;
}
console.log(newUser);
console.log('New user created');
});
});
database.connect().then(prompt.start);

View File

@@ -1,28 +0,0 @@
#!/bin/sh
# this doesnt check game server specific certs, only static file paths
files='config.json'
for file in $files; do
if [ ! -f $file ]; then
echo "$PWD/$file file does not exist. Please mount and try again."
exit 1
fi
done
# check for keys
keys='certs/nex/datastore/secret.key certs/service/account/secret.key certs/service/account/aes.key certs/service/account/private.pem certs/service/account/public.pem'
for file in $keys; do
if [ ! -f "$file" ]; then
if [ x"${GENERATE_NEW_KEYS}" = "x" ]; then
echo "$PWD/$file file does not exist. Please mount and try again."
exit 1
else
echo "$PWD/$file file does not exist. Generating a temporary one"
node generate-keys.js nex datastore
node generate-keys.js account
fi
fi
done
exec node src/server.js

View File

@@ -1,145 +0,0 @@
const NodeRSA = require('node-rsa');
const crypto = require('crypto');
const fs = require('fs-extra');
const yesno = require('yesno');
const logger = require('./logger');
require('colors');
const ALLOWED_CHARS_REGEX = /[^a-zA-Z0-9_-]/g;
async function main() {
const args = process.argv.slice(2);
if (args.length < 1) {
logger.error('Must pass in type and optional name');
usage();
return;
}
let [type, name] = args;
type = type.toLowerCase().trim();
if (name) {
name = name.toLowerCase().trim();
if (ALLOWED_CHARS_REGEX.test(name)) {
logger.error(`Invalid name. Names must only contain [^a-zA-Z0-9_-]. Got ${name}`);
return;
}
}
if (!['nex', 'service', 'account'].includes(type)) {
logger.error(`Invalid type. Expected nex, service, or account. Got ${type}`);
usage();
return;
}
if (type !== 'account' && (!name || name === '')) {
logger.error('If type is not account, a name MUST be passed');
usage();
return;
}
if (type === 'service' && name === 'account') {
logger.error('Cannot use service name \'account\'. Reserved');
usage();
return;
}
let path;
if (type === 'account') {
path = `${__dirname}/certs/service/account`;
} else {
path = `${__dirname}/certs/${type}/${name}`;
}
if (fs.pathExistsSync(path)) {
const overwrite = await yesno({
question: 'Keys found for type name, overwrite existing keys?'
});
if (!overwrite) {
logger.info('Not overwriting existing keys. Exiting program');
return;
}
}
const publicKeyPath = `${path}/public.pem`;
const privateKeyPath = `${path}/private.pem`;
const aesKeyPath = `${path}/aes.key`;
const secretKeyPath = `${path}/secret.key`;
// Ensure the output directories exist
logger.info('Creating output directories...');
fs.ensureDirSync(path);
logger.success('Created output directories!');
const key = new NodeRSA({ b: 1024 }, null, {
environment: 'browser',
encryptionScheme: {
'hash': 'sha256',
}
});
// Generate new key pair
logger.info('Generating RSA key pair...');
logger.warn('(this may take a while)')
key.generateKeyPair(1024);
logger.success('Generated RSA key pair!');
// Export the keys
logger.info('Exporting public key...');
const publicKey = key.exportKey('public');
logger.success('Exported public key!');
// Saving public key
logger.info('Saving public key to disk...');
fs.writeFileSync(publicKeyPath, publicKey);
logger.success(`Saved public key to ${publicKeyPath}!`);
logger.info('Exporting private key...');
const privateKey = key.exportKey('private');
logger.success('Exported private key!');
// Saving private key
logger.info('Saving private key to disk...');
fs.writeFileSync(privateKeyPath, privateKey);
logger.success(`Saved private key to ${privateKeyPath}!`);
// Generate new AES key
logger.info('Generating AES key...');
const aesKey = crypto.randomBytes(16);
logger.success('Generated AES key!');
// Saving AES key
logger.info('Saving AES key to disk...');
fs.writeFileSync(aesKeyPath, aesKey.toString('hex'));
logger.success(`Saved AES key to ${aesKeyPath}!`);
// Create HMAC secret key
logger.info('Generating HMAC secret...');
const secret = crypto.randomBytes(16);
logger.success('Generated RSA key pair!');
logger.info('Saving HMAC secret to disk...');
fs.writeFileSync(secretKeyPath, secret.toString('hex'));
logger.success(`Saved HMAC secret to ${secretKeyPath}!`);
logger.success('Keys generated successfully');
}
// Display usage information
function usage() {
console.log('Usage: node generate-keys.js type [name]');
console.log('Types:');
console.log(' - nex');
console.log(' - service');
console.log(' - account');
console.log('Name: Service or NEX server name. Not used in account type');
}
main().catch(logger.error);

130
package-lock.json generated
View File

@@ -65,9 +65,7 @@
"@typescript-eslint/parser": "^5.54.1",
"eslint": "^8.35.0",
"ndarray": "^1.0.19",
"prompt": "^1.0.0",
"typescript": "^4.9.5",
"yesno": "^0.4.0"
"typescript": "^4.9.5"
}
},
"node_modules/@aashutoshrathi/word-wrap": {
@@ -769,15 +767,6 @@
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
},
"node_modules/@colors/colors": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz",
"integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==",
"dev": true,
"engines": {
"node": ">=0.1.90"
}
},
"node_modules/@eslint-community/eslint-utils": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
@@ -3257,15 +3246,6 @@
"node": ">= 8"
}
},
"node_modules/cycle": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz",
"integrity": "sha512-TVF6svNzeQCOpjCqsy0/CSy8VgObG3wXusJ73xW2GbG5rGx7lC8zxDSURicsXI2UsGdi2L0QNRCi745/wUDvsA==",
"dev": true,
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/d": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz",
@@ -4023,15 +4003,6 @@
"node >=0.6.0"
]
},
"node_modules/eyes": {
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz",
"integrity": "sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ==",
"dev": true,
"engines": {
"node": "> 0.1.90"
}
},
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -5136,12 +5107,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/lodash": {
"version": "4.17.21",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
"dev": true
},
"node_modules/lodash.camelcase": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
@@ -5547,12 +5512,6 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/mute-stream": {
"version": "0.0.8",
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
"integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==",
"dev": true
},
"node_modules/natural-compare": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@@ -5945,22 +5904,6 @@
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/prompt": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/prompt/-/prompt-1.3.0.tgz",
"integrity": "sha512-ZkaRWtaLBZl7KKAKndKYUL8WqNT+cQHKRZnT4RYYms48jQkFw3rrBL+/N5K/KtdEveHkxs982MX2BkDKub2ZMg==",
"dev": true,
"dependencies": {
"@colors/colors": "1.5.0",
"async": "3.2.3",
"read": "1.0.x",
"revalidator": "0.1.x",
"winston": "2.x"
},
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/protobufjs": {
"version": "7.2.4",
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.4.tgz",
@@ -6107,18 +6050,6 @@
"node": ">= 0.8"
}
},
"node_modules/read": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/read/-/read-1.0.7.tgz",
"integrity": "sha512-rSOKNYUmaxy0om1BNjMN4ezNT6VKK+2xF4GBhc81mkH7L60i6dp8qPYrkndNLT3QPphoII3maL9PVC9XmhHwVQ==",
"dev": true,
"dependencies": {
"mute-stream": "~0.0.4"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
@@ -6261,15 +6192,6 @@
"node": ">=0.10.0"
}
},
"node_modules/revalidator": {
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/revalidator/-/revalidator-0.1.8.tgz",
"integrity": "sha512-xcBILK2pA9oh4SiinPEZfhP8HfrB/ha+a2fTMyl7Om2WjlDVrOQy99N2MXXlUHqGJz4qEu2duXxHJjDWuK/0xg==",
"dev": true,
"engines": {
"node": ">= 0.4.0"
}
},
"node_modules/rimraf": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
@@ -6557,15 +6479,6 @@
"node": ">=0.10.0"
}
},
"node_modules/stack-trace": {
"version": "0.0.10",
"resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
"integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==",
"dev": true,
"engines": {
"node": "*"
}
},
"node_modules/static-eval": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/static-eval/-/static-eval-2.1.0.tgz",
@@ -7160,41 +7073,6 @@
"string-width": "^1.0.2 || 2 || 3 || 4"
}
},
"node_modules/winston": {
"version": "2.4.7",
"resolved": "https://registry.npmjs.org/winston/-/winston-2.4.7.tgz",
"integrity": "sha512-vLB4BqzCKDnnZH9PHGoS2ycawueX4HLqENXQitvFHczhgW2vFpSOn31LZtVr1KU8YTw7DS4tM+cqyovxo8taVg==",
"dev": true,
"dependencies": {
"async": "^2.6.4",
"colors": "1.0.x",
"cycle": "1.0.x",
"eyes": "0.1.x",
"isstream": "0.1.x",
"stack-trace": "0.0.x"
},
"engines": {
"node": ">= 0.10.0"
}
},
"node_modules/winston/node_modules/async": {
"version": "2.6.4",
"resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz",
"integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==",
"dev": true,
"dependencies": {
"lodash": "^4.17.14"
}
},
"node_modules/winston/node_modules/colors": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
"integrity": "sha512-pFGrxThWcWQ2MsAz6RtgeWe4NK2kUE1WfsrvvlctdII745EW9I0yflqhe7++M5LEc7bV2c/9/5zc8sFcpL0Drw==",
"dev": true,
"engines": {
"node": ">=0.1.90"
}
},
"node_modules/word-wrap": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
@@ -7311,12 +7189,6 @@
"node": ">=12"
}
},
"node_modules/yesno": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/yesno/-/yesno-0.4.0.tgz",
"integrity": "sha512-tdBxmHvbXPBKYIg81bMCB7bVeDmHkRzk5rVJyYYXurwKkHq/MCd8rz4HSJUP7hW0H2NlXiq8IFiWvYKEHhlotA==",
"dev": true
},
"node_modules/yocto-queue": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",

View File

@@ -82,8 +82,6 @@
"@typescript-eslint/parser": "^5.54.1",
"eslint": "^8.35.0",
"ndarray": "^1.0.19",
"prompt": "^1.0.0",
"typescript": "^4.9.5",
"yesno": "^0.4.0"
"typescript": "^4.9.5"
}
}

View File

@@ -1,5 +1,5 @@
import fs from 'fs-extra';
import redis from 'redis';
import * as redis from 'redis';
import { config, disabledFeatures } from '@/config-manager';
let client: redis.RedisClientType;

View File

@@ -13,6 +13,8 @@ export const disabledFeatures = {
s3: false
};
const hexadecimalStringRegex = /^[0-9a-f]+$/i;
LOG_INFO('Loading config');
let mongooseConnectOptions: mongoose.ConnectOptions = {};
@@ -71,7 +73,10 @@ export const config: Config = {
},
port: Number(process.env.PN_ACT_CONFIG_GRPC_PORT || ''),
},
server_environment: process.env.PN_ACT_CONFIG_SERVER_ENVIRONMENT || ''
server_environment: process.env.PN_ACT_CONFIG_SERVER_ENVIRONMENT || '',
datastore: {
signature_secret: process.env.PN_ACT_CONFIG_DATASTORE_SIGNATURE_SECRET || ''
}
};
if (process.env.PN_ACT_CONFIG_STRIPE_SECRET_KEY) {
@@ -194,4 +199,13 @@ if (!config.grpc.port) {
if (!config.stripe?.secret_key) {
LOG_WARN('Failed to find Stripe api key! If a PNID is deleted with an active subscription, the subscription will *NOT* be canceled! Set the PN_ACT_CONFIG_STRIPE_SECRET_KEY environment variable to enable');
}
}
if (!config.datastore.signature_secret) {
LOG_ERROR('Datastore signature secret key is not set. Set the PN_ACT_CONFIG_DATASTORE_SIGNATURE_SECRET environment variable');
process.exit(0);
}
if (config.datastore.signature_secret.length !== 32 || !hexadecimalStringRegex.test(config.datastore.signature_secret)) {
LOG_ERROR('Datastore signature secret key must be a 32-character hexadecimal string.');
process.exit(0);
}

View File

@@ -3,6 +3,9 @@ process.on('uncaughtException', (err, origin) => {
console.log(err);
console.log(origin);
});
process.on('SIGTERM', () => {
process.exit(0);
});
import express from 'express';
import morgan from 'morgan';

View File

@@ -1,13 +1,11 @@
import fs from 'node:fs';
import crypto from 'node:crypto';
import express from 'express';
import Dicer from 'dicer';
import { uploadCDNAsset } from '@/util';
import { config } from '@/config-manager';
const router = express.Router();
const signatureSecret = fs.readFileSync(`${__dirname}/../../../../certs/nex/datastore/secret.key`);
function multipartParser(request: express.Request, response: express.Response, next: express.NextFunction): void {
const RE_BOUNDARY = /^multipart\/.+?(?:; boundary=(?:(?:"(.+)")|(?:([^\s]+))))$/i;
const RE_FILE_NAME = /name="(.*)"/;
@@ -86,7 +84,7 @@ router.post('/upload', multipartParser, async (request: express.Request, respons
const data = `${pid}${bucket}${key}${date}`;
const hmac = crypto.createHmac('sha256', signatureSecret).update(data).digest('hex');
const hmac = crypto.createHmac('sha256', config.datastore.signature_secret).update(data).digest('hex');
console.log(hmac, signature);

View File

@@ -47,4 +47,7 @@ export interface Config {
secret_key: string;
};
server_environment: string;
datastore: {
signature_secret: string;
};
}