Merge pull request #30 from PretendoNetwork/dev

Release
This commit is contained in:
mrjvs 2025-09-16 21:13:07 +02:00 committed by GitHub
commit ea786616eb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
335 changed files with 18213 additions and 10270 deletions

11
.editorconfig Normal file
View File

@ -0,0 +1,11 @@
[*]
indent_style = tab
end_of_line = lf
insert_final_newline = true
[*.json]
indent_size = 2
[*.yml]
indent_style = space
indent_size = 2

View File

@ -1,2 +0,0 @@
dist
*.js

View File

@ -1,60 +0,0 @@
{
"env": {
"node": true,
"commonjs": true,
"es6": true
},
"parser": "@typescript-eslint/parser",
"globals": {
"BigInt": true
},
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/recommended"
],
"plugins": [
"@typescript-eslint"
],
"rules": {
"require-atomic-updates": "warn",
"no-case-declarations": "off",
"no-empty": "off",
"no-console": "off",
"linebreak-style": "off",
"no-global-assign": "off",
"prefer-const": "error",
"no-var": "error",
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }],
"no-extra-semi": "off",
"@typescript-eslint/no-extra-semi": "error",
"@typescript-eslint/no-empty-interface": "warn",
"@typescript-eslint/no-inferrable-types": "off",
"@typescript-eslint/typedef": "error",
"@typescript-eslint/explicit-function-return-type": "error",
"@typescript-eslint/no-explicit-any": "warn",
"keyword-spacing": "off",
"@typescript-eslint/keyword-spacing": "error",
"curly": "error",
"brace-style": "error",
"one-var": [
"error",
"never"
],
"indent": [
"error",
"tab",
{
"SwitchCase": 1
}
],
"quotes": [
"error",
"single"
],
"semi": [
"error",
"always"
]
}
}

View File

@ -8,11 +8,12 @@ on:
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
DEFAULT_BRANCH: ${{ format('refs/heads/{0}', github.event.repository.default_branch) }}
SHOULD_PUSH_IMAGE: ${{ (github.event_name == 'push' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.ref == 'refs/heads/dev')) || github.event_name == 'workflow_dispatch' }}
jobs:
build-publish:
env:
SHOULD_PUSH_IMAGE: ${{ (github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev')) || github.event_name == 'workflow_dispatch' }}
build-publish-amd64:
name: Build and Publish Docker Image (amd64)
runs-on: ubuntu-latest
permissions:
@ -40,7 +41,7 @@ jobs:
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
type=raw,value=latest,enable=${{ github.ref == env.DEFAULT_BRANCH }}
type=raw,value=edge,enable=${{ github.ref == 'refs/heads/dev' }}
type=sha
@ -48,7 +49,51 @@ jobs:
id: build-and-push
uses: docker/build-push-action@v6
with:
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
push: ${{ env.SHOULD_PUSH_IMAGE }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
build-publish-arm64:
name: Build and Publish Docker Image (arm64)
runs-on: ubuntu-24.04-arm
permissions:
contents: read
packages: write
steps:
- name: Set up QEMU for Docker
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log into the container registry
if: ${{ env.SHOULD_PUSH_IMAGE == 'true' }}
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=raw,value=latest-arm,enable=${{ github.ref == env.DEFAULT_BRANCH }}
type=raw,value=edge-arm,enable=${{ github.ref == 'refs/heads/dev' }}
type=sha,suffix=-arm
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v6
with:
platforms: linux/arm64
push: ${{ env.SHOULD_PUSH_IMAGE }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

23
.github/workflows/lint.yml vendored Normal file
View File

@ -0,0 +1,23 @@
name: Lint
on:
pull_request: {}
jobs:
lint:
name: Lint
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
- name: Install dependencies
run: npm ci
- name: Lint
run: npm run lint

130
.gitignore vendored
View File

@ -1,63 +1,67 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
# nyc test coverage
.nyc_output
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (http://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Typescript v1 declaration files
typings/
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
# custom
.vscode
dist
cdn/storage
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
# nyc test coverage
.nyc_output
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (http://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Typescript v1 declaration files
typings/
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
# custom
.vscode
dist
cdn/storage
*.byaml
*.byaml.checksum
*.bak
!/seeding/**/*

View File

@ -34,14 +34,16 @@ RUN npm run build
FROM base AS final
ARG app_dir
RUN mkdir -p ${app_dir}/logs && chown node:node ${app_dir}/logs
RUN chown node:node ${app_dir}
ENV NODE_ENV=production
USER node
COPY package.json .
COPY --chown=node:node update-rotation.mjs ${app_dir}
COPY --chown=node:node ./boss ${app_dir}
COPY --chown=node:node package.json .
COPY --from=dependencies ${app_dir}/node_modules ${app_dir}/node_modules
COPY --from=build ${app_dir}/dist ${app_dir}/dist
COPY --from=dependencies --chown=node:node ${app_dir}/node_modules ${app_dir}/node_modules
COPY --from=build --chown=node:node ${app_dir}/dist ${app_dir}/dist
CMD ["node", "."]

View File

@ -1,5 +1,70 @@
# BOSS
### Pretendo BOSS server implementation
## About
Handles all BOSS (SpotPass on 3DS) related tasks for the WiiU and 3DS
Handles all BOSS (Background Online Storage Service) related tasks for the Pretendo network.
## What does BOSS handle?
- SpotPass on 3DS
- Tasksheets and policy files for both WiiU and 3DS
- Streetpass relay
## Configuration
Configurations are loaded through environment variables. `.env` files are supported.
| Environment variable | Description | Default |
| -------------------------------------------- | ----------------------------------------------------------------- | --------------------------------------------- |
| `PN_BOSS_CONFIG_HTTP_PORT` | The HTTP port the server listens on | None |
| `PN_BOSS_CONFIG_LOG_FORMAT` | What logging format to use, possible options: `pretty` or `json` | `pretty` |
| `PN_BOSS_CONFIG_LOG_LEVEL` | What log level to use | `info` |
| `PN_BOSS_CONFIG_BOSS_WIIU_AES_KEY` | The BOSS WiiU AES key, needs to be dumped from a console | None |
| `PN_BOSS_CONFIG_BOSS_WIIU_HMAC_KEY` | The BOSS WiiU HMAC key, needs to be dumped from a console | None |
| `PN_BOSS_CONFIG_BOSS_3DS_AES_KEY` | The BOSS 3DS AES key, needs to be dumped from a console | None |
| `PN_BOSS_CONFIG_MONGO_CONNECTION_STRING` | MongoDB connection string | None |
| `PN_BOSS_CONFIG_GRPC_BOSS_SERVER_ADDRESS` | Address for the GRPC server to listen on | None |
| `PN_BOSS_CONFIG_GRPC_BOSS_SERVER_PORT` | Port for the GRPC server to listen on | None |
| `PN_BOSS_CONFIG_GRPC_BOSS_SERVER_API_KEY` | API key that services will use to connect to the BOSS GRPC server | None |
| `PN_BOSS_CONFIG_GRPC_ACCOUNT_SERVER_ADDRESS` | Address of the account GRPC server | None |
| `PN_BOSS_CONFIG_GRPC_ACCOUNT_SERVER_PORT` | Port of the account GRPC server | None |
| `PN_BOSS_CONFIG_GRPC_ACCOUNT_SERVER_API_KEY` | API key of the account GRPC server | None |
| `PN_BOSS_CONFIG_GRPC_FRIENDS_SERVER_ADDRESS` | Address of the friends GRPC server | None |
| `PN_BOSS_CONFIG_GRPC_FRIENDS_SERVER_PORT` | Port of the friends GRPC server | None |
| `PN_BOSS_CONFIG_GRPC_FRIENDS_SERVER_API_KEY` | API key of the friends GRPC server | None |
| `PN_BOSS_CONFIG_S3_ENDPOINT` | S3 server endpoint | None |
| `PN_BOSS_CONFIG_S3_REGION` | S3 server region | None |
| `PN_BOSS_CONFIG_S3_BUCKET` | S3 server bucket | None |
| `PN_BOSS_CONFIG_S3_ACCESS_KEY` | S3 access key | None |
| `PN_BOSS_CONFIG_S3_ACCESS_SECRET` | S3 access key secret | None |
| `PN_BOSS_CONFIG_CDN_DISK_PATH` | Storage path for the CDN, use as alternative for S3 | None |
| `PN_BOSS_CONFIG_STREETPASS_RELAY_ENABLED` | Should Streetpass Relay be enabled? | `false` |
| `PN_BOSS_CONFIG_DOMAINS_NPDI` | What domain should the NPDI component use? | `npdi.cdn.pretendo.cc` |
| `PN_BOSS_CONFIG_DOMAINS_NPDL` | What domain should the NPDL component use? | `npdl.cdn.pretendo.cc` |
| `PN_BOSS_CONFIG_DOMAINS_NPFL` | What domain should the NPFL component use? | `npfl.c.app.pretendo.cc` |
| `PN_BOSS_CONFIG_DOMAINS_NPPL` | What domain should the NPPL component use? | `nppl.app.pretendo.cc,nppl.c.app.pretendo.cc` |
| `PN_BOSS_CONFIG_DOMAINS_NPTS` | What domain should the NPTS component use? | `npts.app.pretendo.cc` |
| `PN_BOSS_CONFIG_DOMAINS_SPR` | What domain should the SPR component use? | `service.spr.app.pretendo.cc` |
## S3 server
The S3 server is optional, you can set `PN_BOSS_CONFIG_CDN_DISK_PATH` if you want to use a local folder as CDN source instead.
# BOSS CLI
The CLI is a helper to interact with the content of the BOSS server.
```sh
npm run build
./boss --help
```
## CLI configuration
Configurations are loaded through environment variables. `.env` files are supported.
| Environment variable | Description | |
| --------------------------- | ------------------------------------------------------------------------------------------- | -------- |
| `PN_BOSS_CLI_GRPC_HOST` | The Host that the BOSS GRPC server is on. Example: `localhost:5678` | Required |
| `PN_BOSS_CLI_GRPC_APIKEY` | Master API key of the BOSS GRPC server. | Required |
| `PN_BOSS_CLI_WIIU_AES_KEY` | The BOSS WiiU AES key, needs to be dumped from a console | Optional |
| `PN_BOSS_CLI_WIIU_HMAC_KEY` | The BOSS WiiU HMAC key, needs to be dumped from a console | Optional |
| `PN_BOSS_CLI_NPDI_URL` | The URL of the NPDI part the BOSS HTTP server, only needed when downloading | Optional |
| `PN_BOSS_CLI_NPDI_HOST` | The Host header for the NPDI requests. Use when you don't have NPDI exposed to the internet | Optional |

2
boss Executable file
View File

@ -0,0 +1,2 @@
#!/bin/bash
exec node --enable-source-maps dist/cli/cli.js $*

View File

@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<TaskSheet>
<TitleId>000500001018dc00</TitleId>
<TaskId>CHARA</TaskId>
<ServiceStatus>open</ServiceStatus>
<Files>
<File>
<Filename>Boss002.pack</Filename>
<DataId>477</DataId>
<Type>AppData</Type>
<Url>https://npdi.cdn.pretendo.cc/p01/data/1/vGwChBW1ExOoHDsm/477/7255c7d67475057abd957577e308599d</Url>
<Size>11837</Size>
<Notify>
<New>app</New>
<LED>false</LED>
</Notify>
</File>
</Files>
</TaskSheet>

View File

@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<TaskSheet>
<TitleId>000500001018dc00</TitleId>
<TaskId>CHARA</TaskId>
<ServiceStatus>open</ServiceStatus>
<Files>
<File>
<Filename>BossStatic002.pack</Filename>
<DataId>480</DataId>
<Type>AppData</Type>
<Url>https://npdi.cdn.pretendo.cc/p01/data/1/vGwChBW1ExOoHDsm/480/2b41543799f046ccdcf54c02cf2608cb</Url>
<Size>4669</Size>
<Notify>
<New>app</New>
<LED>false</LED>
</Notify>
</File>
</Files>
</TaskSheet>

View File

@ -1,22 +0,0 @@
const BOSS = require('boss-js');
const path = require('path');
const fs = require('fs-extra');
const crypto = require('crypto');
require('dotenv').config();
const { PN_BOSS_CONFIG_BOSS_WIIU_AES_KEY, PN_BOSS_CONFIG_BOSS_WIIU_HMAC_KEY } = process.env;
const decryptedFilePath = path.normalize(path.resolve(__dirname, process.argv[2]));
const encryptedFolderName = path.basename(path.dirname(decryptedFilePath));
const encryptedFolderPath = path.normalize(path.resolve(decryptedFilePath, '../../../encrypted', encryptedFolderName));
fs.ensureDirSync(encryptedFolderPath);
const encryptedContents = BOSS.encryptWiiU(decryptedFilePath, PN_BOSS_CONFIG_BOSS_WIIU_AES_KEY, PN_BOSS_CONFIG_BOSS_WIIU_HMAC_KEY);
const hash = crypto.createHash('md5').update(encryptedContents).digest('hex');
const encryptedFilePath = path.normalize(path.resolve(encryptedFolderPath, hash));
fs.writeFileSync(encryptedFilePath, encryptedContents);
console.log(`Encrypted ${process.argv[2].split('/').pop()} to ${encryptedFilePath}, (Length ${encryptedContents.length})`);

10
eslint.config.mjs Normal file
View File

@ -0,0 +1,10 @@
import eslintConfig from '@pretendonetwork/eslint-config';
export default [
...eslintConfig,
{
ignores: [
'scripts/*'
]
}
];

View File

@ -1,52 +0,0 @@
const fs = require('fs-extra');
require('colors');
const root = __dirname;
fs.ensureDirSync(`${root}/logs`);
const streams = {
latest: fs.createWriteStream(`${root}/logs/latest.log`),
success: fs.createWriteStream(`${root}/logs/success.log`),
error: fs.createWriteStream(`${root}/logs/error.log`),
warn: fs.createWriteStream(`${root}/logs/warn.log`),
info: fs.createWriteStream(`${root}/logs/info.log`)
};
function success(input) {
const time = new Date();
input = `[${time.getHours()}:${time.getMinutes()}:${time.getSeconds()}] [SUCCESS]: ${input}`;
streams.success.write(`${input}\n`);
console.log(`${input}`.green.bold);
}
function error(input) {
const time = new Date();
input = `[${time.getHours()}:${time.getMinutes()}:${time.getSeconds()}] [ERROR]: ${input}`;
streams.error.write(`${input}\n`);
console.log(`${input}`.red.bold);
}
function warn(input) {
const time = new Date();
input = `[${time.getHours()}:${time.getMinutes()}:${time.getSeconds()}] [WARN]: ${input}`;
streams.warn.write(`${input}\n`);
console.log(`${input}`.yellow.bold);
}
function info(input) {
const time = new Date();
input = `[${time.getHours()}:${time.getMinutes()}:${time.getSeconds()}] [INFO]: ${input}`;
streams.info.write(`${input}\n`);
console.log(`${input}`.cyan.bold);
}
module.exports = {
success,
error,
warn,
info
};

25985
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,48 +1,48 @@
{
"name": "boss",
"version": "2.0.0",
"description": "",
"main": "dist/server.js",
"scripts": {
"lint": "npx eslint .",
"build": "npm run lint && npm run clean && npx tsc && npx tsc-alias",
"clean": "rimraf ./dist",
"start": "node --enable-source-maps ."
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@aws-sdk/client-s3": "^3.395.0",
"@pretendonetwork/boss-crypto": "^1.0.0",
"@pretendonetwork/grpc": "^1.0.6",
"@typegoose/auto-increment": "^3.6.1",
"boss-js": "github:PretendoNetwork/boss-js",
"cacache": "^18.0.0",
"colors": "^1.4.0",
"dicer": "^0.3.1",
"dotenv": "^10.0.0",
"express": "^4.17.1",
"express-subdomain": "^1.0.5",
"fs-extra": "^11.2.0",
"moment": "^2.29.4",
"mongoose": "^7.4.3",
"morgan": "^1.10.0",
"nice-grpc": "^2.1.5",
"xmlbuilder": "^15.1.1"
},
"devDependencies": {
"@types/dicer": "^0.2.4",
"@types/express": "^4.17.17",
"@types/fs-extra": "^11.0.1",
"@types/morgan": "^1.9.4",
"@types/node": "^20.5.0",
"@typescript-eslint/eslint-plugin": "^6.4.0",
"@typescript-eslint/parser": "^6.4.0",
"axios": "^1.6.2",
"eslint": "^8.47.0",
"tsc-alias": "^1.8.7",
"typescript": "^5.1.6",
"xmlbuilder2": "^3.0.2"
}
"name": "boss",
"description": "Pretendo BOSS server implementation",
"version": "2.0.0",
"license": "AGPL-3.0-only",
"main": "dist/server.js",
"scripts": {
"dev": "tsup --watch --onSuccess \"node dist/server.js\"",
"lint": "eslint .",
"lint:fix": "eslint . --fix",
"build": "tsup && tsc --noEmit",
"start": "node --enable-source-maps dist/server.js",
"cli": "node --enable-source-maps dist/cli/cli.js"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.723.0",
"@pretendonetwork/boss-crypto": "^1.0.0",
"@pretendonetwork/grpc": "^1.0.6",
"@typegoose/auto-increment": "^3.6.1",
"commander": "^14.0.0",
"dicer": "^0.3.1",
"dotenv": "^16.4.7",
"express": "^5.1.0",
"fs-extra": "^11.2.0",
"moment": "^2.30.1",
"mongoose": "~7.6.1",
"nice-grpc": "^2.1.10",
"pino": "^9.9.1",
"pino-http": "^10.5.0",
"pino-pretty": "^13.1.1",
"undici": "^7.16.0",
"xml-js": "^1.6.11",
"xmlbuilder": "^15.1.1"
},
"devDependencies": {
"@pretendonetwork/eslint-config": "^0.1.1",
"@smithy/types": "^4.0.0",
"@types/dicer": "^0.2.4",
"@types/express": "^4.17.21",
"@types/fs-extra": "^11.0.4",
"@types/node": "^22.10.5",
"axios": "^1.7.9",
"eslint": "^9.17.0",
"tsup": "^8.3.5",
"typescript": "^5.7.2",
"xmlbuilder2": "^3.1.1"
}
}

16
seeding/README.md Normal file
View File

@ -0,0 +1,16 @@
# Seeding
The CLI of the BOSS server can seed data from this folder. This folder contains some default tasksheets that are expected from the system.
```sh
npm run cli -- import seed
```
## Notes about seeding
1. Only files referenced by tasksheets are processed
2. You can safely run seeding as many times as seeded, it ignores unchanged data.
3. Unencrypted task files must follow this syntax: `<DATA_ID>.<FILENAME>` - For example: `39015.Festival.byaml` (The name unused, but good practice to set it appropiately)
4. Encrypted task files must follow this syntax: `<DATA_ID>.enc.<FILENAME>` - For example: `39015.enc.Festival.byaml` (The name unused, but good practice to set it appropiately)
5. Tasksheets must follow this syntax: `1.<BOSS_APP_ID>.<TASKNAME>.taskheet.xml`
6. Seeding only adds and updates data. Tasksheets or files that are removed are not deleted.

Some files were not shown because too many files have changed in this diff Show More