diff --git a/.gitignore b/.gitignore index 7e3dce6..f515632 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,4 @@ cdn/storage *.byaml *.byaml.checksum *.bak +!/seeding/**/* diff --git a/seeding/README.md b/seeding/README.md new file mode 100644 index 0000000..5d47b77 --- /dev/null +++ b/seeding/README.md @@ -0,0 +1,15 @@ +# Seeding + +The CLI of the BOSS server can seed data from this folder. This folder contains some default tasksheets that are expected from the system. + +```sh +npm run cli -- import seed +``` + +## Notes about seeding + +1. Only files referenced by tasksheets are processed +2. You can safely run seeding as many times as seeded, it ignores unchanged data. +3. Files must be prefixed with their Data ID - followed by a `.`. For example: `39015.Festival.byaml` (Everything after the first dot is not enforced, name it appropiately) +4. Tasksheets must follow this syntax: `1...taskheet.xml` +5. Seeding only adds and updates data. Tasksheets or files that are removed are not deleted. diff --git a/src/cli/cli.ts b/src/cli/cli.ts index ead2439..03c0e11 100644 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -2,12 +2,14 @@ import { program as baseProgram } from 'commander'; import { taskCmd } from './tasks.cmd'; import { fileCmd } from './files.cmd'; import { appCmd } from './apps.cmd'; +import { importCmd } from './import.cmd'; const program = baseProgram .name('BOSS') .description('CLI to manage and view BOSS data') .addCommand(appCmd) .addCommand(taskCmd) + .addCommand(importCmd) .addCommand(fileCmd); program.parseAsync(process.argv) diff --git a/src/cli/import.cmd.ts b/src/cli/import.cmd.ts new file mode 100644 index 0000000..3dbf620 --- /dev/null +++ b/src/cli/import.cmd.ts @@ -0,0 +1,14 @@ +import { Command } from 'commander'; +import { seedCmd } from './seed.cmd'; + +const oldCdnCmd = new Command('old-cdn') + .description('Copy old CDN format data into new database') + .argument('', 'CDN folder to import') + .action(async (_folder: string) => { + // const ctx = getCliContext(); + }); + +export const importCmd = new Command('import') + .description('Import existing files into BOSS') + .addCommand(oldCdnCmd) + .addCommand(seedCmd); diff --git a/src/cli/root.ts b/src/cli/root.ts new file mode 100644 index 0000000..68b405a --- /dev/null +++ b/src/cli/root.ts @@ -0,0 +1,3 @@ +import path from 'path'; + +export const seedFolder = path.join(__dirname, '../../seeding'); diff --git a/src/cli/seed.cmd.ts b/src/cli/seed.cmd.ts new file mode 100644 index 0000000..e2e30cf --- /dev/null +++ b/src/cli/seed.cmd.ts @@ -0,0 +1,120 @@ +import path from 'path'; +import fs from 'fs/promises'; +import { Command } from 'commander'; +import { xml2js } from 'xml-js'; +import { getCliContext } from './utils'; +import { seedFolder } from './root'; +import type { CliContext } from './utils'; + +type UploadFileOptions = { + ctx: CliContext; + taskFiles: string[]; + fileXml: Record; + bossAppId: string; + taskId: string; + dataId: string; +}; + +export async function uploadFileIfChanged(ops: UploadFileOptions): Promise { + const newTaskFileName = ops.taskFiles.find(v => v.startsWith(`${ops.dataId}.`)); + if (!newTaskFileName) { + console.warn(`${ops.dataId}: Could not find file on disk the specified data ID - skipping`); + return; + } + const fileContents = await fs.readFile(path.join(seedFolder, 'files', newTaskFileName)); + + const allExistingTaskFiles = await ops.ctx.grpc.listFiles({ + bossAppId: ops.bossAppId, + taskId: ops.taskId + }); + const existingTaskFile = allExistingTaskFiles.files.find(v => v.dataId === BigInt(ops.dataId)); + if (existingTaskFile) { + console.warn(`${ops.dataId}: File already uploaded, reuploading`); + await ops.ctx.grpc.deleteFile({ + bossAppId: ops.bossAppId, + dataId: BigInt(ops.dataId) + }); + } + + await ops.ctx.grpc.uploadFile({ + bossAppId: ops.bossAppId, + taskId: ops.taskId, + name: ops.fileXml.Filename._text, + type: ops.fileXml.Type._text, + notifyLed: ops.fileXml.Notify.LED._text === 'true', + notifyOnNew: ops.fileXml.Notify.New._text.split(','), + data: fileContents, + supportedLanguages: [], + supportedCountries: [] + }); + console.log(`${ops.dataId}: Uploaded file!`); +} + +export async function processTasksheet(ctx: CliContext, taskFiles: string[], filename: string, contents: string): Promise { + console.log(`${filename}: Processing tasksheet`); + const [unknownNumber, bossAppId, taskName] = filename.split('.'); + if (!unknownNumber || !bossAppId || !taskName) { + console.warn(`${filename}: Invalid syntax tasksheet - skipping`); + return; + } + + const xmlContents: any = xml2js(contents, { compact: true }); + if (xmlContents.TaskSheet.TaskId._text !== taskName) { + console.warn(`${filename}: Taskname in tasksheet doesn't match filename - skipping`); + return; + } + + const res = await ctx.grpc.listTasks({}); + const existingTask = res.tasks.find(v => v.bossAppId === bossAppId && v.id === taskName); + if (!existingTask) { + await ctx.grpc.registerTask({ + bossAppId: bossAppId, + id: taskName, + titleId: xmlContents.TaskSheet.TitleId._text, + country: 'This value isnt used' + }); + console.log(`${filename}: Created task`); + } + await ctx.grpc.updateTask({ + bossAppId: bossAppId, + id: taskName, + updateData: { + titleId: xmlContents.TaskSheet.TitleId._text, + status: xmlContents.TaskSheet.ServiceStatus._text + } + }); + console.log(`${filename}: Updated title ID and status`); + + for (const file of xmlContents.TaskSheet.Files?.File ?? []) { + await uploadFileIfChanged({ + bossAppId: bossAppId, + ctx: ctx, + dataId: file.DataId._text, + fileXml: file, + taskFiles: taskFiles, + taskId: taskName + }); + } + + console.log(`${filename}: Successfully processed tasksheet`); +} + +export const seedCmd = new Command('seed') + .description('Seed the BOSS database with initial data') + .action(async () => { + const ctx = getCliContext(); + + const taskFilesDirEntries = await fs.readdir(path.join(seedFolder, 'files'), { withFileTypes: true }); + const taskFiles = taskFilesDirEntries.filter(v => v.isFile()).map(v => v.name); + + const tasksheetDirEntries = await fs.readdir(path.join(seedFolder, 'tasksheets'), { withFileTypes: true }); + const tasksheets = tasksheetDirEntries.filter(v => v.isFile() && v.name.endsWith('.tasksheet.xml')).map(v => v.name); + console.log(`Found ${tasksheets.length} tasksheets - starting seeding`); + + for (const tasksheetFilename of tasksheets) { + const contents = await fs.readFile(path.join(seedFolder, 'tasksheets', tasksheetFilename), { encoding: 'utf-8' }); + await processTasksheet(ctx, taskFiles, tasksheetFilename, contents); + } + + console.log(`Completed seeding!`); + }); diff --git a/src/cli/tasks.cmd.ts b/src/cli/tasks.cmd.ts index 07ec73e..bc70229 100644 --- a/src/cli/tasks.cmd.ts +++ b/src/cli/tasks.cmd.ts @@ -45,16 +45,15 @@ const createCmd = new Command('create') .argument('', 'BOSS app to store the task in') .requiredOption('--id ', 'Id of the task') .requiredOption('--title-id ', 'Title ID for the task') - .requiredOption('--country ', 'Description of the task') .option('--desc [desc]', 'Description of the task') - .action(async (appId: string, opts: { id: string; titleId: string; desc?: string; country: string }) => { + .action(async (appId: string, opts: { id: string; titleId: string; desc?: string }) => { const ctx = getCliContext(); const { task } = await ctx.grpc.registerTask({ bossAppId: appId, id: opts.id, titleId: opts.titleId, description: opts.desc ?? '', - country: opts.country + country: 'This value isnt used' }); if (!task) { console.log(`Failed to create task!`);