From 4c07a01ed77d28b51e759da2f224b78b7ab54066 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Mon, 20 Oct 2025 14:13:56 -0400 Subject: [PATCH] feat: full upgrade to boss grpc v2 --- src/database.ts | 111 ++++++++++-- src/models/file-ctr.ts | 48 +++++ src/models/{file.ts => file-wup.ts} | 14 +- src/services/grpc/boss/v1/delete-file.ts | 4 +- src/services/grpc/boss/v1/list-files.ts | 4 +- .../grpc/boss/v1/update-file-metadata.ts | 4 +- src/services/grpc/boss/v1/upload-file.ts | 8 +- src/services/grpc/boss/v2/delete-file.ts | 15 +- src/services/grpc/boss/v2/implementation.ts | 8 +- src/services/grpc/boss/v2/list-files-ctr.ts | 28 +-- src/services/grpc/boss/v2/list-files-wup.ts | 8 +- .../grpc/boss/v2/update-file-metadata-ctr.ts | 55 ++++++ .../grpc/boss/v2/update-file-metadata-wup.ts | 6 +- src/services/grpc/boss/v2/upload-file-ctr.ts | 168 ++++++++++++++++++ src/services/grpc/boss/v2/upload-file-wup.ts | 15 +- src/services/npdi.ts | 4 +- src/services/npdl.ts | 4 +- src/services/npfl.ts | 4 +- src/services/npts.ts | 10 +- src/types/mongoose/file-ctr.ts | 41 +++++ src/types/mongoose/{file.ts => file-wup.ts} | 12 +- 21 files changed, 496 insertions(+), 75 deletions(-) create mode 100644 src/models/file-ctr.ts rename src/models/{file.ts => file-wup.ts} (58%) create mode 100644 src/services/grpc/boss/v2/update-file-metadata-ctr.ts create mode 100644 src/services/grpc/boss/v2/upload-file-ctr.ts create mode 100644 src/types/mongoose/file-ctr.ts rename src/types/mongoose/{file.ts => file-wup.ts} (60%) diff --git a/src/database.ts b/src/database.ts index 0c41186..5161e58 100644 --- a/src/database.ts +++ b/src/database.ts @@ -2,12 +2,14 @@ import mongoose from 'mongoose'; import { CECData } from '@/models/cec-data'; import { CECSlot } from '@/models/cec-slot'; import { Task } from '@/models/task'; -import { File } from '@/models/file'; +import { FileCTR } from '@/models/file-ctr'; +import { FileWUP } from '@/models/file-wup'; import { config } from '@/config-manager'; import type { HydratedCECDataDocument } from '@/types/mongoose/cec-data'; import type { HydratedCECSlotDocument, ICECSlot } from '@/types/mongoose/cec-slot'; import type { HydratedTaskDocument, ITask } from '@/types/mongoose/task'; -import type { HydratedFileDocument, IFile } from '@/types/mongoose/file'; +import type { HydratedFileCTRDocument, IFileCTR } from '@/types/mongoose/file-ctr'; +import type { HydratedFileWUPDocument, IFileWUP } from '@/types/mongoose/file-wup'; const connection_string: string = config.mongoose.connection_string; @@ -52,10 +54,10 @@ export function getTask(bossAppID: string, taskID: string): Promise { +export function getCTRTaskFiles(allowDeleted: boolean, bossAppID: string, taskID: string, country?: string, language?: string): Promise { verifyConnected(); - const filter: mongoose.FilterQuery = { + const filter: mongoose.FilterQuery = { task_id: taskID.slice(0, 7), boss_app_id: bossAppID, $and: [] @@ -87,13 +89,51 @@ export function getTaskFiles(allowDeleted: boolean, bossAppID: string, taskID: s delete filter.$and; } - return File.find(filter); + return FileCTR.find(filter); } -export function getTaskFilesWithAttributes(allowDeleted: boolean, bossAppID: string, taskID: string, country?: string, language?: string, attribute1?: string, attribute2?: string, attribute3?: string): Promise { +export function getWUPTaskFiles(allowDeleted: boolean, bossAppID: string, taskID: string, country?: string, language?: string): Promise { verifyConnected(); - const filter: mongoose.FilterQuery = { + const filter: mongoose.FilterQuery = { + task_id: taskID.slice(0, 7), + boss_app_id: bossAppID, + $and: [] + }; + + if (!allowDeleted) { + filter.deleted = false; + } + + if (country) { + filter.$and?.push({ + $or: [ + { supported_countries: { $eq: [] } }, + { supported_countries: country } + ] + }); + } + + if (language) { + filter.$and?.push({ + $or: [ + { supported_languages: { $eq: [] } }, + { supported_languages: language } + ] + }); + } + + if (filter.$and?.length === 0) { + delete filter.$and; + } + + return FileWUP.find(filter); +} + +export function getCTRTaskFilesWithAttributes(allowDeleted: boolean, bossAppID: string, taskID: string, country?: string, language?: string, attribute1?: string, attribute2?: string, attribute3?: string): Promise { + verifyConnected(); + + const filter: mongoose.FilterQuery = { task_id: taskID.slice(0, 7), boss_app_id: bossAppID, $and: [] @@ -137,13 +177,13 @@ export function getTaskFilesWithAttributes(allowDeleted: boolean, bossAppID: str delete filter.$and; } - return File.find(filter); + return FileWUP.find(filter); } -export function getTaskFile(bossAppID: string, taskID: string, name: string, country?: string, language?: string): Promise { +export function getCTRTaskFile(bossAppID: string, taskID: string, name: string, country?: string, language?: string): Promise { verifyConnected(); - const filter: mongoose.FilterQuery = { + const filter: mongoose.FilterQuery = { deleted: false, boss_app_id: bossAppID, task_id: taskID.slice(0, 7), @@ -173,13 +213,58 @@ export function getTaskFile(bossAppID: string, taskID: string, name: string, cou delete filter.$and; } - return File.findOne(filter); + return FileCTR.findOne(filter); } -export function getTaskFileByDataID(dataID: bigint): Promise { +export function getWUPTaskFile(bossAppID: string, taskID: string, name: string, country?: string, language?: string): Promise { verifyConnected(); - return File.findOne({ + const filter: mongoose.FilterQuery = { + deleted: false, + boss_app_id: bossAppID, + task_id: taskID.slice(0, 7), + name: name, + $and: [] + }; + + if (country) { + filter.$and?.push({ + $or: [ + { supported_countries: { $eq: [] } }, + { supported_countries: country } + ] + }); + } + + if (language) { + filter.$and?.push({ + $or: [ + { supported_languages: { $eq: [] } }, + { supported_languages: language } + ] + }); + } + + if (filter.$and?.length === 0) { + delete filter.$and; + } + + return FileWUP.findOne(filter); +} + +export function getCTRTaskFileBySerialNumber(serialNumber: number): Promise { + verifyConnected(); + + return FileCTR.findOne({ + deleted: false, + serial_number: serialNumber + }); +} + +export function getWUPTaskFileByDataID(dataID: bigint): Promise { + verifyConnected(); + + return FileWUP.findOne({ deleted: false, data_id: Number(dataID) }); diff --git a/src/models/file-ctr.ts b/src/models/file-ctr.ts new file mode 100644 index 0000000..dfcbdd7 --- /dev/null +++ b/src/models/file-ctr.ts @@ -0,0 +1,48 @@ +import mongoose from 'mongoose'; +import { AutoIncrementID } from '@typegoose/auto-increment'; +import type { IFileCTR, IFileCTRMethods, FileCTRModel } from '@/types/mongoose/file-ctr'; + +const FileCTRSchema = new mongoose.Schema({ + deleted: { + type: Boolean, + default: false + }, + creator_pid: Number, + hash: String, + file_key: String, + size: BigInt, + task_id: String, + boss_app_id: String, + supported_countries: [String], + supported_languages: [String], + attributes: { + attribute1: String, + attribute2: String, + attribute3: String, + description: String + }, + name: String, + serial_number: Number, // * This is effectively the predecessor of the Wii U DataID. TODO - 3DBrew says this is a uint64? + payload_contents: [{ + title_id: BigInt, + content_datatype: Number, + ns_data_id: Number, // * Should payload contents be put in their own collection with their own autoincrementing IDs? + version: Number, + size: Number + }], + flags: { + mark_arrived_privileged: Boolean + }, + created: BigInt, + updated: BigInt +}, { id: false }); + +FileCTRSchema.plugin(AutoIncrementID, { + startAt: 50000, // * Start very high to avoid conflicts with Nintendo Data IDs + field: 'serial_number' +}); + +FileCTRSchema.index({ task_id: 1, boss_app_id: 1 }); +FileCTRSchema.index({ task_id: 1, boss_app_id: 1, name: 1 }); + +export const FileCTR = mongoose.model('FileCTR', FileCTRSchema, 'files-ctr'); diff --git a/src/models/file.ts b/src/models/file-wup.ts similarity index 58% rename from src/models/file.ts rename to src/models/file-wup.ts index d918cb8..b329a18 100644 --- a/src/models/file.ts +++ b/src/models/file-wup.ts @@ -1,8 +1,8 @@ import mongoose from 'mongoose'; import { AutoIncrementID } from '@typegoose/auto-increment'; -import type { IFile, IFileMethods, FileModel } from '@/types/mongoose/file'; +import type { IFileWUP, IFileWUPMethods, FileWUPModel } from '@/types/mongoose/file-wup'; -const FileSchema = new mongoose.Schema({ +const FileWUPSchema = new mongoose.Schema({ deleted: { type: Boolean, default: false @@ -24,16 +24,18 @@ const FileSchema = new mongoose.Schema({ size: BigInt, notify_on_new: [String], notify_led: Boolean, + condition_played: BigInt, + auto_delete: Boolean, // * We don't know what this does, but it exists on WUP tasks. So track it created: BigInt, updated: BigInt }, { id: false }); -FileSchema.plugin(AutoIncrementID, { +FileWUPSchema.plugin(AutoIncrementID, { startAt: 50000, // * Start very high to avoid conflicts with Nintendo Data IDs field: 'data_id' }); -FileSchema.index({ task_id: 1, boss_app_id: 1 }); -FileSchema.index({ task_id: 1, boss_app_id: 1, name: 1 }); +FileWUPSchema.index({ task_id: 1, boss_app_id: 1 }); +FileWUPSchema.index({ task_id: 1, boss_app_id: 1, name: 1 }); -export const File = mongoose.model('File', FileSchema); +export const FileWUP = mongoose.model('FileWUP', FileWUPSchema, 'files-wup'); diff --git a/src/services/grpc/boss/v1/delete-file.ts b/src/services/grpc/boss/v1/delete-file.ts index e1da284..0947aad 100644 --- a/src/services/grpc/boss/v1/delete-file.ts +++ b/src/services/grpc/boss/v1/delete-file.ts @@ -1,5 +1,5 @@ import { Status, ServerError } from 'nice-grpc'; -import { getTaskFileByDataID } from '@/database'; +import { getWUPTaskFileByDataID } from '@/database'; import { hasPermission } from '@/services/grpc/boss/v1/middleware/authentication-middleware'; import type { AuthenticationCallContextExt } from '@/services/grpc/boss/v1/middleware/authentication-middleware'; import type { CallContext } from 'nice-grpc'; @@ -18,7 +18,7 @@ export async function deleteFile(request: DeleteFileRequest, context: CallContex throw new ServerError(Status.INVALID_ARGUMENT, 'Missing file data ID'); } - const file = await getTaskFileByDataID(dataID); + const file = await getWUPTaskFileByDataID(dataID); if (!file || file.boss_app_id !== bossAppID) { throw new ServerError(Status.INVALID_ARGUMENT, `File ${dataID} not found for BOSS app ${bossAppID}`); diff --git a/src/services/grpc/boss/v1/list-files.ts b/src/services/grpc/boss/v1/list-files.ts index 112caa9..d71f819 100644 --- a/src/services/grpc/boss/v1/list-files.ts +++ b/src/services/grpc/boss/v1/list-files.ts @@ -1,6 +1,6 @@ import { Status, ServerError } from 'nice-grpc'; import { isValidCountryCode, isValidLanguage } from '@/util'; -import { getTaskFiles } from '@/database'; +import { getWUPTaskFiles } from '@/database'; import type { ListFilesRequest, ListFilesResponse } from '@pretendonetwork/grpc/boss/list_files'; const BOSS_APP_ID_FILTER_REGEX = /^[A-Za-z0-9]*$/; @@ -35,7 +35,7 @@ export async function listFiles(request: ListFilesRequest): Promise ({ diff --git a/src/services/grpc/boss/v1/update-file-metadata.ts b/src/services/grpc/boss/v1/update-file-metadata.ts index 0a23437..406fe38 100644 --- a/src/services/grpc/boss/v1/update-file-metadata.ts +++ b/src/services/grpc/boss/v1/update-file-metadata.ts @@ -1,5 +1,5 @@ import { Status, ServerError } from 'nice-grpc'; -import { getTaskFileByDataID } from '@/database'; +import { getWUPTaskFileByDataID } from '@/database'; import { isValidFileNotifyCondition, isValidFileType } from '@/util'; import { hasPermission } from '@/services/grpc/boss/v1/middleware/authentication-middleware'; import type { AuthenticationCallContextExt } from '@/services/grpc/boss/v1/middleware/authentication-middleware'; @@ -23,7 +23,7 @@ export async function updateFileMetadata(request: UpdateFileMetadataRequest, con throw new ServerError(Status.INVALID_ARGUMENT, 'Missing file update data'); } - const file = await getTaskFileByDataID(dataID); + const file = await getWUPTaskFileByDataID(dataID); if (!file || file.deleted) { throw new ServerError(Status.INVALID_ARGUMENT, `File ${dataID} not found`); diff --git a/src/services/grpc/boss/v1/upload-file.ts b/src/services/grpc/boss/v1/upload-file.ts index be7d873..6fc63f6 100644 --- a/src/services/grpc/boss/v1/upload-file.ts +++ b/src/services/grpc/boss/v1/upload-file.ts @@ -1,8 +1,8 @@ import { Status, ServerError } from 'nice-grpc'; import { encryptWiiU } from '@pretendonetwork/boss-crypto'; import { isValidCountryCode, isValidFileNotifyCondition, isValidFileType, isValidLanguage, md5 } from '@/util'; -import { getTask, getTaskFile } from '@/database'; -import { File } from '@/models/file'; +import { getTask, getWUPTaskFile } from '@/database'; +import { FileWUP } from '@/models/file-wup'; import { config } from '@/config-manager'; import { uploadCDNFile } from '@/cdn'; import { hasPermission } from '@/services/grpc/boss/v1/middleware/authentication-middleware'; @@ -113,7 +113,7 @@ export async function uploadFile(request: UploadFileRequest, context: CallContex throw new ServerError(Status.ABORTED, message); } - let file = await getTaskFile(bossAppID, taskID, name); + let file = await getWUPTaskFile(bossAppID, taskID, name); if (file) { file.deleted = true; @@ -122,7 +122,7 @@ export async function uploadFile(request: UploadFileRequest, context: CallContex await file.save(); } - file = await File.create({ + file = await FileWUP.create({ task_id: taskID.slice(0, 7), boss_app_id: bossAppID, file_key: key, diff --git a/src/services/grpc/boss/v2/delete-file.ts b/src/services/grpc/boss/v2/delete-file.ts index 462194e..740976c 100644 --- a/src/services/grpc/boss/v2/delete-file.ts +++ b/src/services/grpc/boss/v2/delete-file.ts @@ -1,10 +1,13 @@ import { Status, ServerError } from 'nice-grpc'; -import { getTaskFileByDataID } from '@/database'; +import { getCTRTaskFileBySerialNumber, getWUPTaskFileByDataID } from '@/database'; import { hasPermission } from '@/services/grpc/boss/v2/middleware/authentication-middleware'; +import { PlatformType } from '@pretendonetwork/grpc/boss/v2/platform_type'; import type { AuthenticationCallContextExt } from '@/services/grpc/boss/v2/middleware/authentication-middleware'; import type { CallContext } from 'nice-grpc'; import type { DeleteFileRequest } from '@pretendonetwork/grpc/boss/v2/delete_file'; import type { Empty } from '@pretendonetwork/grpc/google/protobuf/empty'; +import type { HydratedFileCTRDocument } from '@/types/mongoose/file-ctr'; +import type { HydratedFileWUPDocument } from '@/types/mongoose/file-wup'; export async function deleteFile(request: DeleteFileRequest, context: CallContext & AuthenticationCallContextExt): Promise { if (!hasPermission(context, 'deleteBossFiles')) { @@ -18,7 +21,15 @@ export async function deleteFile(request: DeleteFileRequest, context: CallContex throw new ServerError(Status.INVALID_ARGUMENT, 'Missing file data ID'); } - const file = await getTaskFileByDataID(dataID); + let file: HydratedFileCTRDocument | HydratedFileWUPDocument | null; + + if (request.platformType === PlatformType.PLATFORM_TYPE_CTR) { + file = await getCTRTaskFileBySerialNumber(Number(dataID)); + } else if (request.platformType === PlatformType.PLATFORM_TYPE_WUP) { + file = await getWUPTaskFileByDataID(dataID); + } else { + throw new ServerError(Status.INVALID_ARGUMENT, 'Invalid platform type'); + } if (!file || file.boss_app_id !== bossAppID) { throw new ServerError(Status.INVALID_ARGUMENT, `File ${dataID} not found for BOSS app ${bossAppID}`); diff --git a/src/services/grpc/boss/v2/implementation.ts b/src/services/grpc/boss/v2/implementation.ts index 86d45da..11376a7 100644 --- a/src/services/grpc/boss/v2/implementation.ts +++ b/src/services/grpc/boss/v2/implementation.ts @@ -7,8 +7,8 @@ import { deleteFile } from '@/services/grpc/boss/v2/delete-file'; import { listFilesWUP } from '@/services/grpc/boss/v2/list-files-wup'; import { uploadFileWUP } from '@/services/grpc/boss/v2/upload-file-wup'; import { listFilesCTR } from '@/services/grpc/boss/v2/list-files-ctr'; -// import { uploadFileCTR } from '@/services/grpc/boss/v2/upload-file-ctr'; -// import { updateFileMetadataCTR } from '@/services/grpc/boss/v2/update-file-metadata-ctr'; +import { uploadFileCTR } from '@/services/grpc/boss/v2/upload-file-ctr'; +import { updateFileMetadataCTR } from '@/services/grpc/boss/v2/update-file-metadata-ctr'; import { updateFileMetadataWUP } from '@/services/grpc/boss/v2/update-file-metadata-wup'; import type { BossServiceImplementation } from '@pretendonetwork/grpc/boss/v2/boss_service'; @@ -22,7 +22,7 @@ export const bossServiceImplementationV2: BossServiceImplementation = { listFilesWUP, uploadFileWUP, listFilesCTR, - // uploadFileCTR, - // updateFileMetadataCTR, + uploadFileCTR, + updateFileMetadataCTR, updateFileMetadataWUP }; diff --git a/src/services/grpc/boss/v2/list-files-ctr.ts b/src/services/grpc/boss/v2/list-files-ctr.ts index 1b7bd9c..c2a79e0 100644 --- a/src/services/grpc/boss/v2/list-files-ctr.ts +++ b/src/services/grpc/boss/v2/list-files-ctr.ts @@ -1,6 +1,6 @@ import { Status, ServerError } from 'nice-grpc'; import { isValidCountryCode, isValidLanguage } from '@/util'; -import { getTaskFiles } from '@/database'; +import { getCTRTaskFiles } from '@/database'; import type { ListFilesCTRRequest, ListFilesCTRResponse } from '@pretendonetwork/grpc/boss/v2/list_files_ctr'; const BOSS_APP_ID_FILTER_REGEX = /^[A-Za-z0-9]*$/; @@ -35,30 +35,34 @@ export async function listFilesCTR(request: ListFilesCTRRequest): Promise ({ deleted: file.deleted, - dataId: file.data_id, + dataId: BigInt(file.serial_number), // TODO - Is this okay? taskId: file.task_id, bossAppId: file.boss_app_id, supportedCountries: file.supported_countries, supportedLanguages: file.supported_languages, - attributes: { - attribute1: file.attribute1, - attribute2: file.attribute2, - attribute3: file.attribute3, - description: file.password - }, + attributes: file.attributes, creatorPid: file.creator_pid, name: file.name, hash: file.hash, - serialNumber: 0, // TODO - Don't stub this - payloadContents: [], // TODO - Don't stub this + serialNumber: file.serial_number, + payloadContents: file.payload_contents.map(payloadContentInfo => ({ + titleId: payloadContentInfo.title_id, + contentDatatype: payloadContentInfo.content_datatype, + nsDataId: payloadContentInfo.ns_data_id, + version: payloadContentInfo.version, + size: payloadContentInfo.size + })), size: file.size, createdTimestamp: file.created, - updatedTimestamp: file.updated + updatedTimestamp: file.updated, + flags: { + markArrivedPrivileged: file.flags.mark_arrived_privileged + } })) }; } diff --git a/src/services/grpc/boss/v2/list-files-wup.ts b/src/services/grpc/boss/v2/list-files-wup.ts index 80f429c..f2ec63a 100644 --- a/src/services/grpc/boss/v2/list-files-wup.ts +++ b/src/services/grpc/boss/v2/list-files-wup.ts @@ -1,6 +1,6 @@ import { Status, ServerError } from 'nice-grpc'; import { isValidCountryCode, isValidLanguage } from '@/util'; -import { getTaskFiles } from '@/database'; +import { getWUPTaskFiles } from '@/database'; import type { ListFilesWUPRequest, ListFilesWUPResponse } from '@pretendonetwork/grpc/boss/v2/list_files_wup'; const BOSS_APP_ID_FILTER_REGEX = /^[A-Za-z0-9]*$/; @@ -35,7 +35,7 @@ export async function listFilesWUP(request: ListFilesWUPRequest): Promise ({ @@ -58,8 +58,8 @@ export async function listFilesWUP(request: ListFilesWUPRequest): Promise { + if (!hasPermission(context, 'updateBossFiles')) { + throw new ServerError(Status.PERMISSION_DENIED, 'PNID not authorized to update file metadata'); + } + + const serialNumber = request.dataId; + const updateData = request.updateData; + + if (!serialNumber) { + throw new ServerError(Status.INVALID_ARGUMENT, 'Missing file serial number'); + } + + if (!updateData) { + throw new ServerError(Status.INVALID_ARGUMENT, 'Missing file update data'); + } + + const file = await getCTRTaskFileBySerialNumber(Number(serialNumber)); + + if (!file || file.deleted) { + throw new ServerError(Status.INVALID_ARGUMENT, `File ${serialNumber} not found`); + } + + file.task_id = updateData.taskId.slice(0, 7); + file.boss_app_id = updateData.bossAppId; + file.supported_countries = updateData.supportedCountries; + file.supported_languages = updateData.supportedLanguages; + file.attributes.attribute1 = updateData.attributes ? updateData.attributes.attribute1 : file.attributes.attribute1; + file.attributes.attribute2 = updateData.attributes ? updateData.attributes.attribute2 : file.attributes.attribute2; + file.attributes.attribute3 = updateData.attributes ? updateData.attributes.attribute3 : file.attributes.attribute3; + file.attributes.description = updateData.attributes ? updateData.attributes.description : file.attributes.description; + file.name = updateData.name; + file.updated = BigInt(Date.now()); + + if (updateData.payloadContents.length !== 0) { + file.payload_contents = updateData.payloadContents.map(payload => ({ + title_id: payload.titleId, + content_datatype: payload.contentDatatype, + ns_data_id: payload.nsDataId, + version: payload.version, + size: payload.size + })); + } + + await file.save(); + + return {}; +} diff --git a/src/services/grpc/boss/v2/update-file-metadata-wup.ts b/src/services/grpc/boss/v2/update-file-metadata-wup.ts index 4fd7e72..2a4e39a 100644 --- a/src/services/grpc/boss/v2/update-file-metadata-wup.ts +++ b/src/services/grpc/boss/v2/update-file-metadata-wup.ts @@ -1,5 +1,5 @@ import { Status, ServerError } from 'nice-grpc'; -import { getTaskFileByDataID } from '@/database'; +import { getWUPTaskFileByDataID } from '@/database'; import { isValidFileNotifyCondition, isValidFileType } from '@/util'; import { hasPermission } from '@/services/grpc/boss/v2/middleware/authentication-middleware'; import type { AuthenticationCallContextExt } from '@/services/grpc/boss/v2/middleware/authentication-middleware'; @@ -23,7 +23,7 @@ export async function updateFileMetadataWUP(request: UpdateFileMetadataWUPReques throw new ServerError(Status.INVALID_ARGUMENT, 'Missing file update data'); } - const file = await getTaskFileByDataID(dataID); + const file = await getWUPTaskFileByDataID(dataID); if (!file || file.deleted) { throw new ServerError(Status.INVALID_ARGUMENT, `File ${dataID} not found`); @@ -51,6 +51,8 @@ export async function updateFileMetadataWUP(request: UpdateFileMetadataWUPReques file.type = updateData.type; file.notify_on_new = updateData.notifyOnNew; file.notify_led = updateData.notifyLed; + file.condition_played = updateData.conditionPlayed; + file.auto_delete = updateData.autoDelete; file.updated = BigInt(Date.now()); await file.save(); diff --git a/src/services/grpc/boss/v2/upload-file-ctr.ts b/src/services/grpc/boss/v2/upload-file-ctr.ts new file mode 100644 index 0000000..31c6123 --- /dev/null +++ b/src/services/grpc/boss/v2/upload-file-ctr.ts @@ -0,0 +1,168 @@ +import { Status, ServerError } from 'nice-grpc'; +import { encrypt3DS } from '@pretendonetwork/boss-crypto'; +import { isValidCountryCode, isValidLanguage, md5 } from '@/util'; +import { connection as databaseConnection, getTask, getCTRTaskFile } from '@/database'; +import { FileCTR } from '@/models/file-ctr'; +import { config } from '@/config-manager'; +import { uploadCDNFile } from '@/cdn'; +import { hasPermission } from '@/services/grpc/boss/v2/middleware/authentication-middleware'; +import type { AuthenticationCallContextExt } from '@/services/grpc/boss/v2/middleware/authentication-middleware'; +import type { CallContext } from 'nice-grpc'; +import type { UploadFileCTRRequest, UploadFileCTRResponse } from '@pretendonetwork/grpc/boss/v2/upload_file_ctr'; +import type { HydratedFileCTRDocument } from '@/types/mongoose/file-ctr'; + +const BOSS_APP_ID_FILTER_REGEX = /^[A-Za-z0-9]*$/; + +export async function uploadFileCTR(request: UploadFileCTRRequest, context: CallContext & AuthenticationCallContextExt): Promise { + if (!hasPermission(context, 'uploadBossFiles')) { + throw new ServerError(Status.PERMISSION_DENIED, 'PNID not authorized to upload new files'); + } + + const taskID = request.taskId.trim(); + const bossAppID = request.bossAppId.trim(); + const supportedCountries = request.supportedCountries; + const supportedLanguages = request.supportedLanguages; + const name = request.name.trim(); + const payloads = request.payloadContents; + + if (!taskID) { + throw new ServerError(Status.INVALID_ARGUMENT, 'Missing task ID'); + } + + if (!bossAppID) { + throw new ServerError(Status.INVALID_ARGUMENT, 'Missing BOSS app ID'); + } + + if (bossAppID.length !== 16) { + throw new ServerError(Status.INVALID_ARGUMENT, 'BOSS app ID must be 16 characters'); + } + + if (!BOSS_APP_ID_FILTER_REGEX.test(bossAppID)) { + throw new ServerError(Status.INVALID_ARGUMENT, 'BOSS app ID must only contain letters and numbers'); + } + + if (!(await getTask(bossAppID, taskID))) { + throw new ServerError(Status.NOT_FOUND, `Task ${taskID} does not exist for BOSS app ${bossAppID}`); + } + + for (const country of supportedCountries) { + if (!isValidCountryCode(country)) { + throw new ServerError(Status.INVALID_ARGUMENT, `${country} is not a valid country`); + } + } + + for (const language of supportedLanguages) { + if (!isValidLanguage(language)) { + throw new ServerError(Status.INVALID_ARGUMENT, `${language} is not a valid language`); + } + } + + if (payloads.length === 0) { + throw new ServerError(Status.INVALID_ARGUMENT, 'Cannot upload empty file'); + } + + const session = await databaseConnection().startSession(); + await session.startTransaction(); + + let file: HydratedFileCTRDocument | null; + + try { + // * Create the FileCTR first since encrypt3DS relies on the serial number + file = await getCTRTaskFile(bossAppID, taskID, name); + + if (file) { + file.deleted = true; + file.updated = BigInt(Date.now()); + + await file.save({ session }); + } + + file = await FileCTR.create({ + creator_pid: context.user?.pid, + // * hash: String, + // * file_key: String, + // * size: BigInt, + task_id: taskID, + boss_app_id: bossAppID, + supported_countries: supportedCountries, + supported_languages: supportedLanguages, + attributes: request.attributes, + name: name, + payload_contents: payloads.map(payload => ({ + title_id: payload.titleId, + content_datatype: payload.contentDatatype, + ns_data_id: payload.nsDataId, + version: payload.version, + size: payload.content.length + })), + flags: { + mark_arrived_privileged: request.flags?.markArrivedPrivileged || false + }, + created: Date.now(), + updated: Date.now() + }); + + const cryptoOptions = payloads.map(payload => ({ + program_id: payload.titleId, + content_datatype: payload.contentDatatype, + ns_data_id: payload.nsDataId, + version: payload.version, + content: payload.content + })); + + // TODO - Add flags here. @pretendonetwork/boss-crypto does not export CTR_BOSS_FLAGS at the moment + + // TODO - Somehow support pre-encrypted content? + const encryptedData = encrypt3DS(config.crypto.ctr.aes_key, BigInt(file.serial_number), cryptoOptions); + const contentHash = md5(encryptedData); + const key = `${bossAppID}/${taskID}/${contentHash}`; + + await uploadCDNFile('taskFile', key, encryptedData); + + file.hash = contentHash; + file.file_key = key; + file.size = BigInt(encryptedData.length); + + await file.save({ session }); + await session.commitTransaction(); + } catch (error: unknown) { + let message = 'Unknown file upload error'; + + if (error instanceof Error) { + message = error.message; + } + + throw new ServerError(Status.ABORTED, message); + } finally { + await session.endSession(); + } + + return { + file: { + deleted: file.deleted, + dataId: BigInt(file.serial_number), // TODO - Is this okay? + taskId: file.task_id, + bossAppId: file.boss_app_id, + supportedCountries: file.supported_countries, + supportedLanguages: file.supported_languages, + attributes: file.attributes, + creatorPid: file.creator_pid, + name: file.name, + hash: file.hash, + serialNumber: file.serial_number, + payloadContents: file.payload_contents.map(payloadContentInfo => ({ + titleId: payloadContentInfo.title_id, + contentDatatype: payloadContentInfo.content_datatype, + nsDataId: payloadContentInfo.ns_data_id, + version: payloadContentInfo.version, + size: payloadContentInfo.size + })), + size: file.size, + createdTimestamp: file.created, + updatedTimestamp: file.updated, + flags: { + markArrivedPrivileged: file.flags.mark_arrived_privileged + } + } + }; +} diff --git a/src/services/grpc/boss/v2/upload-file-wup.ts b/src/services/grpc/boss/v2/upload-file-wup.ts index 26959b0..58da35d 100644 --- a/src/services/grpc/boss/v2/upload-file-wup.ts +++ b/src/services/grpc/boss/v2/upload-file-wup.ts @@ -1,8 +1,8 @@ import { Status, ServerError } from 'nice-grpc'; import { encryptWiiU } from '@pretendonetwork/boss-crypto'; import { isValidCountryCode, isValidFileNotifyCondition, isValidFileType, isValidLanguage, md5 } from '@/util'; -import { getTask, getTaskFile } from '@/database'; -import { File } from '@/models/file'; +import { getTask, getWUPTaskFile } from '@/database'; +import { FileWUP } from '@/models/file-wup'; import { config } from '@/config-manager'; import { uploadCDNFile } from '@/cdn'; import { hasPermission } from '@/services/grpc/boss/v2/middleware/authentication-middleware'; @@ -81,6 +81,7 @@ export async function uploadFileWUP(request: UploadFileWUPRequest, context: Call let encryptedData: Buffer; try { + // TODO - Check the first few bytes of the uploaded content to see if it's encrypted already, to support pre-encrypted content? encryptedData = encryptWiiU(data, config.crypto.wup.aes_key, config.crypto.wup.hmac_key); } catch (error: unknown) { let message = 'Unknown file encryption error'; @@ -113,7 +114,7 @@ export async function uploadFileWUP(request: UploadFileWUPRequest, context: Call throw new ServerError(Status.ABORTED, message); } - let file = await getTaskFile(bossAppID, taskID, name); + let file = await getWUPTaskFile(bossAppID, taskID, name); if (file) { file.deleted = true; @@ -122,7 +123,7 @@ export async function uploadFileWUP(request: UploadFileWUPRequest, context: Call await file.save(); } - file = await File.create({ + file = await FileWUP.create({ task_id: taskID.slice(0, 7), boss_app_id: bossAppID, file_key: key, @@ -135,6 +136,8 @@ export async function uploadFileWUP(request: UploadFileWUPRequest, context: Call size: BigInt(encryptedData.length), notify_on_new: notifyOnNew, notify_led: notifyLed, + condition_played: request.conditionPlayed, + auto_delete: request.autoDelete, created: Date.now(), updated: Date.now() }); @@ -165,8 +168,8 @@ export async function uploadFileWUP(request: UploadFileWUPRequest, context: Call size: file.size, notifyOnNew: file.notify_on_new, notifyLed: file.notify_led, - conditionPlayed: 0n, // TODO - Don't stub this - autoDelete: false, // TODO - Don't stub this + conditionPlayed: request.conditionPlayed, + autoDelete: request.autoDelete, createdTimestamp: file.created, updatedTimestamp: file.updated } diff --git a/src/services/npdi.ts b/src/services/npdi.ts index 5af9c89..df3a9b2 100644 --- a/src/services/npdi.ts +++ b/src/services/npdi.ts @@ -2,7 +2,7 @@ import express from 'express'; import { restrictHostnames } from '@/middleware/host-limit'; import { config } from '@/config-manager'; import { getCDNFileAsStream, streamFileToResponse } from '@/cdn'; -import { getTaskFileByDataID } from '@/database'; +import { getWUPTaskFileByDataID } from '@/database'; import { handleEtag, sendEtagCacheResponse } from '@/util'; const npdi = express.Router(); @@ -10,7 +10,7 @@ const npdi = express.Router(); npdi.get('/p01/data/1/:bossAppId/:dataId/:fileHash', async (request, response) => { const { dataId, fileHash, bossAppId } = request.params; - const file = await getTaskFileByDataID(BigInt(dataId)); + const file = await getWUPTaskFileByDataID(BigInt(dataId)); if (!file) { return response.sendStatus(404); } diff --git a/src/services/npdl.ts b/src/services/npdl.ts index d41a998..c9a5d5d 100644 --- a/src/services/npdl.ts +++ b/src/services/npdl.ts @@ -1,5 +1,5 @@ import express from 'express'; -import { getTaskFile } from '@/database'; +import { getCTRTaskFile } from '@/database'; import { config } from '@/config-manager'; import { restrictHostnames } from '@/middleware/host-limit'; import { getCDNFileAsStream, streamFileToResponse } from '@/cdn'; @@ -23,7 +23,7 @@ npdl.get([ }>, response) => { const { appID, taskID, countryCode, languageCode, fileName } = request.params; - const file = await getTaskFile(appID, taskID, fileName, countryCode, languageCode); + const file = await getCTRTaskFile(appID, taskID, fileName, countryCode, languageCode); if (!file) { response.sendStatus(404); diff --git a/src/services/npfl.ts b/src/services/npfl.ts index c4329b1..ece6132 100644 --- a/src/services/npfl.ts +++ b/src/services/npfl.ts @@ -1,6 +1,6 @@ import crypto from 'node:crypto'; import express from 'express'; -import { getTaskFilesWithAttributes } from '@/database'; +import { getCTRTaskFilesWithAttributes } from '@/database'; import { restrictHostnames } from '@/middleware/host-limit'; import { config } from '@/config-manager'; @@ -40,7 +40,7 @@ npfl.get('/p01/filelist/:appID/:taskID', async (request: express.Request<{ const attribute2 = request.query.a2; const attribute3 = request.query.a3; - const files = await getTaskFilesWithAttributes(false, appID, taskID, country, language, attribute1, attribute2, attribute3); + const files = await getCTRTaskFilesWithAttributes(false, appID, taskID, country, language, attribute1, attribute2, attribute3); // * https://gist.github.com/DaniElectra/ada7ecc930a84432f2045f6fcabac84f#nintendo-boss-file-list-server-npfl // * diff --git a/src/services/npts.ts b/src/services/npts.ts index 93c3753..83f5e9e 100644 --- a/src/services/npts.ts +++ b/src/services/npts.ts @@ -2,15 +2,15 @@ import express from 'express'; import xmlbuilder from 'xmlbuilder'; import { config } from '@/config-manager'; import { restrictHostnames } from '@/middleware/host-limit'; -import { getTask, getTaskFile, getTaskFiles } from '@/database'; -import type { HydratedFileDocument } from '@/types/mongoose/file'; +import { getTask, getWUPTaskFile, getWUPTaskFiles } from '@/database'; +import type { HydratedFileWUPDocument } from '@/types/mongoose/file-wup'; import type { HydratedTaskDocument } from '@/types/mongoose/task'; const npts = express.Router(); const xmlHeadSettings = { encoding: 'UTF-8', version: '1.0' }; -function buildFile(task: HydratedTaskDocument, file: HydratedFileDocument): any { +function buildFile(task: HydratedTaskDocument, file: HydratedFileWUPDocument): any { return { Filename: file.name, DataId: file.data_id, @@ -32,7 +32,7 @@ npts.get('/p01/tasksheet/:id/:bossAppId/:taskId', async (request, response) => { return response.sendStatus(404); } - const files = await getTaskFiles(false, bossAppId, taskId); + const files = await getWUPTaskFiles(false, bossAppId, taskId); const xmlContent = { TaskSheet: { @@ -57,7 +57,7 @@ npts.get('/p01/tasksheet/:id/:bossAppId/:taskId/:fileName', async (request, resp return response.sendStatus(404); } - const file = await getTaskFile(bossAppId, taskId, fileName); + const file = await getWUPTaskFile(bossAppId, taskId, fileName); if (!file) { return response.sendStatus(404); } diff --git a/src/types/mongoose/file-ctr.ts b/src/types/mongoose/file-ctr.ts new file mode 100644 index 0000000..0e9c796 --- /dev/null +++ b/src/types/mongoose/file-ctr.ts @@ -0,0 +1,41 @@ +import type { Model, HydratedDocument } from 'mongoose'; + +export interface IFileCTR { + deleted: boolean; + creator_pid: number; + hash: string; + file_key: string; + size: bigint; + task_id: string; + boss_app_id: string; + supported_countries: string[]; + supported_languages: string[]; + attributes: { + attribute1: string; + attribute2: string; + attribute3: string; + description: string; + }; + name: string; + serial_number: number; // * This is effectively the predecessor of the Wii U DataID + payload_contents: { + title_id: bigint; + content_datatype: number; + ns_data_id: number; + version: number; + size: number; + }[]; + flags: { + mark_arrived_privileged: boolean; + }; + created: bigint; + updated: bigint; +} + +export interface IFileCTRMethods {} + +interface IFileCTRQueryHelpers {} + +export type FileCTRModel = Model; + +export type HydratedFileCTRDocument = HydratedDocument; diff --git a/src/types/mongoose/file.ts b/src/types/mongoose/file-wup.ts similarity index 60% rename from src/types/mongoose/file.ts rename to src/types/mongoose/file-wup.ts index 845f3c8..68b116e 100644 --- a/src/types/mongoose/file.ts +++ b/src/types/mongoose/file-wup.ts @@ -1,6 +1,6 @@ import type { Model, HydratedDocument } from 'mongoose'; -export interface IFile { +export interface IFileWUP { deleted: boolean; file_key: string; data_id: bigint; @@ -19,14 +19,16 @@ export interface IFile { size: bigint; notify_on_new: string[]; notify_led: boolean; + condition_played: bigint; + auto_delete: boolean; created: bigint; updated: bigint; } -export interface IFileMethods {} +export interface IFileWUPMethods {} -interface IFileQueryHelpers {} +interface IFileWUPQueryHelpers {} -export type FileModel = Model; +export type FileWUPModel = Model; -export type HydratedFileDocument = HydratedDocument; +export type HydratedFileWUPDocument = HydratedDocument;