feat: add fully featured seeding command

.gitignore

@@ -64,3 +64,4 @@ cdn/storage
 *.byaml
 *.byaml.checksum
 *.bak
+!/seeding/**/*

seeding/README.md

@@ -0,0 +1,15 @@
+# Seeding
+
+The CLI of the BOSS server can seed data from this folder. This folder contains some default tasksheets that are expected from the system.
+
+```sh
+npm run cli -- import seed
+```
+
+## Notes about seeding
+
+1. Only files referenced by tasksheets are processed
+2. You can safely run seeding as many times as seeded, it ignores unchanged data.
+3. Files must be prefixed with their Data ID - followed by a `.`. For example: `39015.Festival.byaml` (Everything after the first dot is not enforced, name it appropiately)
+4. Tasksheets must follow this syntax: `1.<BOSS_APP_ID>.<TASKNAME>.taskheet.xml`
+5. Seeding only adds and updates data. Tasksheets or files that are removed are not deleted.

src/cli/cli.ts

@@ -2,12 +2,14 @@ import { program as baseProgram } from 'commander';
 import { taskCmd } from './tasks.cmd';
 import { fileCmd } from './files.cmd';
 import { appCmd } from './apps.cmd';
+import { importCmd } from './import.cmd';
 
 const program = baseProgram
 	.name('BOSS')
 	.description('CLI to manage and view BOSS data')
 	.addCommand(appCmd)
 	.addCommand(taskCmd)
+	.addCommand(importCmd)
 	.addCommand(fileCmd);
 
 program.parseAsync(process.argv)

src/cli/import.cmd.ts

@@ -0,0 +1,14 @@
+import { Command } from 'commander';
+import { seedCmd } from './seed.cmd';
+
+const oldCdnCmd = new Command('old-cdn')
+	.description('Copy old CDN format data into new database')
+	.argument('<folder>', 'CDN folder to import')
+	.action(async (_folder: string) => {
+		// const ctx = getCliContext();
+	});
+
+export const importCmd = new Command('import')
+	.description('Import existing files into BOSS')
+	.addCommand(oldCdnCmd)
+	.addCommand(seedCmd);

src/cli/root.ts

@@ -0,0 +1,3 @@
+import path from 'path';
+
+export const seedFolder = path.join(__dirname, '../../seeding');

src/cli/seed.cmd.ts

@@ -0,0 +1,120 @@
+import path from 'path';
+import fs from 'fs/promises';
+import { Command } from 'commander';
+import { xml2js } from 'xml-js';
+import { getCliContext } from './utils';
+import { seedFolder } from './root';
+import type { CliContext } from './utils';
+
+type UploadFileOptions = {
+	ctx: CliContext;
+	taskFiles: string[];
+	fileXml: Record<string, any>;
+	bossAppId: string;
+	taskId: string;
+	dataId: string;
+};
+
+export async function uploadFileIfChanged(ops: UploadFileOptions): Promise<void> {
+	const newTaskFileName = ops.taskFiles.find(v => v.startsWith(`${ops.dataId}.`));
+	if (!newTaskFileName) {
+		console.warn(`${ops.dataId}: Could not find file on disk the specified data ID - skipping`);
+		return;
+	}
+	const fileContents = await fs.readFile(path.join(seedFolder, 'files', newTaskFileName));
+
+	const allExistingTaskFiles = await ops.ctx.grpc.listFiles({
+		bossAppId: ops.bossAppId,
+		taskId: ops.taskId
+	});
+	const existingTaskFile = allExistingTaskFiles.files.find(v => v.dataId === BigInt(ops.dataId));
+	if (existingTaskFile) {
+		console.warn(`${ops.dataId}: File already uploaded, reuploading`);
+		await ops.ctx.grpc.deleteFile({
+			bossAppId: ops.bossAppId,
+			dataId: BigInt(ops.dataId)
+		});
+	}
+
+	await ops.ctx.grpc.uploadFile({
+		bossAppId: ops.bossAppId,
+		taskId: ops.taskId,
+		name: ops.fileXml.Filename._text,
+		type: ops.fileXml.Type._text,
+		notifyLed: ops.fileXml.Notify.LED._text === 'true',
+		notifyOnNew: ops.fileXml.Notify.New._text.split(','),
+		data: fileContents,
+		supportedLanguages: [],
+		supportedCountries: []
+	});
+	console.log(`${ops.dataId}: Uploaded file!`);
+}
+
+export async function processTasksheet(ctx: CliContext, taskFiles: string[], filename: string, contents: string): Promise<void> {
+	console.log(`${filename}: Processing tasksheet`);
+	const [unknownNumber, bossAppId, taskName] = filename.split('.');
+	if (!unknownNumber || !bossAppId || !taskName) {
+		console.warn(`${filename}: Invalid syntax tasksheet - skipping`);
+		return;
+	}
+
+	const xmlContents: any = xml2js(contents, { compact: true });
+	if (xmlContents.TaskSheet.TaskId._text !== taskName) {
+		console.warn(`${filename}: Taskname in tasksheet doesn't match filename - skipping`);
+		return;
+	}
+
+	const res = await ctx.grpc.listTasks({});
+	const existingTask = res.tasks.find(v => v.bossAppId === bossAppId && v.id === taskName);
+	if (!existingTask) {
+		await ctx.grpc.registerTask({
+			bossAppId: bossAppId,
+			id: taskName,
+			titleId: xmlContents.TaskSheet.TitleId._text,
+			country: 'This value isnt used'
+		});
+		console.log(`${filename}: Created task`);
+	}
+	await ctx.grpc.updateTask({
+		bossAppId: bossAppId,
+		id: taskName,
+		updateData: {
+			titleId: xmlContents.TaskSheet.TitleId._text,
+			status: xmlContents.TaskSheet.ServiceStatus._text
+		}
+	});
+	console.log(`${filename}: Updated title ID and status`);
+
+	for (const file of xmlContents.TaskSheet.Files?.File ?? []) {
+		await uploadFileIfChanged({
+			bossAppId: bossAppId,
+			ctx: ctx,
+			dataId: file.DataId._text,
+			fileXml: file,
+			taskFiles: taskFiles,
+			taskId: taskName
+		});
+	}
+
+	console.log(`${filename}: Successfully processed tasksheet`);
+}
+
+export const seedCmd = new Command('seed')
+	.description('Seed the BOSS database with initial data')
+	.action(async () => {
+		const ctx = getCliContext();
+
+		const taskFilesDirEntries = await fs.readdir(path.join(seedFolder, 'files'), { withFileTypes: true });
+		const taskFiles = taskFilesDirEntries.filter(v => v.isFile()).map(v => v.name);
+
+		const tasksheetDirEntries = await fs.readdir(path.join(seedFolder, 'tasksheets'), { withFileTypes: true });
+		const tasksheets = tasksheetDirEntries.filter(v => v.isFile() && v.name.endsWith('.tasksheet.xml')).map(v => v.name);
+		console.log(`Found ${tasksheets.length} tasksheets - starting seeding`);
+
+		for (const tasksheetFilename of tasksheets) {
+			const contents = await fs.readFile(path.join(seedFolder, 'tasksheets', tasksheetFilename), { encoding: 'utf-8' });
+			await processTasksheet(ctx, taskFiles, tasksheetFilename, contents);
+		}
+
+		console.log(`Completed seeding!`);
+	});

src/cli/tasks.cmd.ts

@@ -45,16 +45,15 @@ const createCmd = new Command('create')
 	.argument('<app_id>', 'BOSS app to store the task in')
 	.requiredOption('--id <id>', 'Id of the task')
 	.requiredOption('--title-id <titleId>', 'Title ID for the task')
-	.requiredOption('--country <country>', 'Description of the task')
 	.option('--desc [desc]', 'Description of the task')
-	.action(async (appId: string, opts: { id: string; titleId: string; desc?: string; country: string }) => {
+	.action(async (appId: string, opts: { id: string; titleId: string; desc?: string }) => {
 		const ctx = getCliContext();
 		const { task } = await ctx.grpc.registerTask({
 			bossAppId: appId,
 			id: opts.id,
 			titleId: opts.titleId,
 			description: opts.desc ?? '',
-			country: opts.country
+			country: 'This value isnt used'
 		});
 		if (!task) {
 			console.log(`Failed to create task!`);