From 0bc74044c66ca90b0e7f93f6d1facbba66dad25d Mon Sep 17 00:00:00 2001 From: Christopher Monsanto Date: Wed, 29 Mar 2023 02:58:40 -0400 Subject: [PATCH] deploy: add support for writing tar archives --- pnpm-lock.yaml | 88 +++++++++++++++++++++++++++++++++++++++ tools/deploy/index.ts | 14 ++++--- tools/deploy/package.json | 4 +- tools/deploy/script.ts | 36 ++++++++++++---- 4 files changed, 127 insertions(+), 15 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eca09100..2dfedf78 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,13 +29,17 @@ importers: tools/deploy: specifiers: + '@types/tar-stream': ^2.2.2 commander: ^5.1.0 debug: ^4.1.1 expect: ^26.4.1 mocha: ^8.1.1 + tar-stream: ^3.0.0 dependencies: + '@types/tar-stream': 2.2.2 commander: 5.1.0 debug: 4.1.1 + tar-stream: 3.0.0 devDependencies: expect: 26.4.1 mocha: 8.1.1 @@ -169,6 +173,12 @@ packages: resolution: {integrity: sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw==} dev: true + /@types/tar-stream/2.2.2: + resolution: {integrity: sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==} + dependencies: + '@types/node': 18.15.8 + dev: false + /@types/vinyl/2.0.7: resolution: {integrity: sha512-4UqPv+2567NhMQuMLdKAyK4yzrfCqwaTt6bLhHEs8PFcxbHILsrxaY63n4wgE/BRLDWDQeI+WcTmkXKExh9hQg==} dependencies: @@ -186,6 +196,13 @@ packages: '@types/yargs-parser': 15.0.0 dev: true + /abort-controller/3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + dependencies: + event-target-shim: 5.0.1 + dev: false + /ajv/6.12.3: resolution: {integrity: sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==} dependencies: @@ -296,6 +313,10 @@ packages: resolution: {integrity: sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==} dev: false + /b4a/1.6.3: + resolution: {integrity: sha512-aX6/FqpWQve8VN9kyTExy7GlmwNShvxcCWWD5QVR3ZbRlyBGtCrG5Autu95xxSPH4CRs+5PSV4d7PRnWpmqFlA==} + dev: false + /balanced-match/1.0.0: resolution: {integrity: sha1-ibTRmasr7kneFk6gK4nORi1xt2c=} dev: true @@ -342,6 +363,14 @@ packages: readable-stream: 3.6.0 dev: false + /bl/6.0.1: + resolution: {integrity: sha512-zk1P1eAEBHhhB+4NfGxqmuV6NgwECnIoRgsOq2ObdEsmoFVIYzJ/Jjcgaj7JOY/8ekH27bIHSV4Si2T+evqu+Q==} + dependencies: + buffer: 6.0.3 + inherits: 2.0.4 + readable-stream: 4.3.0 + dev: false + /brace-expansion/1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -367,6 +396,13 @@ packages: ieee754: 1.1.13 dev: false + /buffer/6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + dependencies: + base64-js: 1.3.1 + ieee754: 1.2.1 + dev: false + /camelcase/5.3.1: resolution: {integrity: sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==} engines: {node: '>=6'} @@ -659,6 +695,16 @@ packages: hasBin: true dev: true + /event-target-shim/5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + dev: false + + /events/3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + dev: false + /expand-template/2.0.3: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} @@ -689,6 +735,10 @@ packages: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: false + /fast-fifo/1.1.0: + resolution: {integrity: sha512-Kl29QoNbNvn4nhDsLYjyIAaIqaJB6rBx5p3sL9VjaefJ+eMFBWVZiaoguaoZfzEKr5RhAti0UgM8703akGPJ6g==} + dev: false + /fast-json-stable-stringify/2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} dev: false @@ -907,6 +957,10 @@ packages: resolution: {integrity: sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==} dev: false + /ieee754/1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + dev: false + /inflight/1.0.6: resolution: {integrity: sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=} dependencies: @@ -1491,6 +1545,11 @@ packages: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} dev: false + /process/0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + dev: false + /promise.allsettled/1.0.2: resolution: {integrity: sha512-UpcYW5S1RaNKT6pd+s9jp9K9rlQge1UXKskec0j6Mmuq7UJCvlS2J2/s/yuPN8ehftf9HXMxWlKiPbGGUzpoRg==} engines: {node: '>= 0.4'} @@ -1523,6 +1582,10 @@ packages: engines: {node: '>=0.6'} dev: false + /queue-tick/1.0.1: + resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} + dev: false + /randombytes/2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} dependencies: @@ -1593,6 +1656,16 @@ packages: util-deprecate: 1.0.2 dev: false + /readable-stream/4.3.0: + resolution: {integrity: sha512-MuEnA0lbSi7JS8XM+WNJlWZkHAAdm7gETHdFK//Q/mChGyj2akEFtdLZh32jSdkWGbRwCW9pn6g3LWDdDeZnBQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + dev: false + /readdirp/3.3.0: resolution: {integrity: sha512-zz0pAkSPOXXm1viEwygWIPSPkcBYjW1xU5j/JBh5t9bGCJwa6f9+BJa6VaB2g+b55yVrmXzqkyLf4xaWYM0IkQ==} engines: {node: '>=8.10.0'} @@ -1746,6 +1819,13 @@ packages: escape-string-regexp: 2.0.0 dev: true + /streamx/2.13.2: + resolution: {integrity: sha512-+TWqixPhGDXEG9L/XczSbhfkmwAtGs3BJX5QNU6cvno+pOLKeszByWcnaTu6dg8efsTYqR8ZZuXWHhZfgrxMvA==} + dependencies: + fast-fifo: 1.1.0 + queue-tick: 1.0.1 + dev: false + /string-width/1.0.2: resolution: {integrity: sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=} engines: {node: '>=0.10.0'} @@ -1879,6 +1959,14 @@ packages: readable-stream: 3.6.0 dev: false + /tar-stream/3.0.0: + resolution: {integrity: sha512-O6OfUKBbQOqAhh6owTWmA730J/yZCYcpmZ1DBj2YX51ZQrt7d7NgzrR+CnO9wP6nt/viWZW2XeXLavX3/ZEbEg==} + dependencies: + b4a: 1.6.3 + bl: 6.0.1 + streamx: 2.13.2 + dev: false + /tar/4.4.10: resolution: {integrity: sha512-g2SVs5QIxvo6OLp0GudTqEf05maawKUxXru104iaayWA09551tFCTI8f1Asb4lPfkBr91k07iL4c11XO3/b0tA==} engines: {node: '>=4.5'} diff --git a/tools/deploy/index.ts b/tools/deploy/index.ts index 84704b69..5e2a1a4a 100644 --- a/tools/deploy/index.ts +++ b/tools/deploy/index.ts @@ -7,14 +7,14 @@ function collect(value : string, previous : string[]) { return previous.concat([value]); } -function runAq(aq : script.ActionQueue, mode: 'copy' | 'link', outputDir : undefined | string, verbose : undefined | true) { +async function runAq(aq : script.ActionQueue, mode: 'copy' | 'link' | 'tar', outputDir : undefined | string, verbose : undefined | true) { const level = verbose ? 'all' : 'errors'; if (!aq.valid) { aq.print(level); process.exit(1); } if (outputDir !== undefined) { - aq.run(outputDir, mode); + await aq.run(outputDir, mode); } else { if (level === 'errors') { console.log(`Success, but nothing to do. Please rerun with -v or -o`); @@ -32,10 +32,11 @@ program .option('-m, --module ', 'Module') .option('-v, --verbose', 'Verbose') .option('--link', 'Link') + .option('--tar', 'Tar') // TODO // .option('-t, --tag ', 'Tag', collect, []) // from rename(1) - .action(async (files : string[], {eval: expr, module: mod, output: outputDir, verbose, link}) => { + .action(async (files : string[], {eval: expr, module: mod, output: outputDir, verbose, link, tar}) => { let scr; if (expr !== undefined) { scr = new script.Script(expr, 'expr'); @@ -51,7 +52,7 @@ program script.runOnFile(scr, src, aq); } - runAq(aq, link ? 'link' : 'copy', outputDir, verbose); + await runAq(aq, tar ? 'tar' : link ? 'link' : 'copy', outputDir, verbose); }); program @@ -59,7 +60,8 @@ program .option('-o, --output ', 'Output directory') .option('-v, --verbose', 'Verbose') .option('--link', 'Link') - .action((scripts : string[], {output: outputDir, verbose, link}) => { + .option('--tar', 'Tar') + .action(async (scripts : string[], {output: outputDir, verbose, link, tar}) => { const aq = new script.ActionQueue; for (const file of scripts) { @@ -67,7 +69,7 @@ program script.run(scr, nodePath.dirname(file), aq); } - runAq(aq, link ? 'link' : 'copy', outputDir, verbose); + await runAq(aq, tar ? 'tar' : link ? 'link' : 'copy', outputDir, verbose); }); program.parse(process.argv); diff --git a/tools/deploy/package.json b/tools/deploy/package.json index 79fe0875..1a2c09ed 100644 --- a/tools/deploy/package.json +++ b/tools/deploy/package.json @@ -1,7 +1,9 @@ { "dependencies": { + "@types/tar-stream": "^2.2.2", "commander": "^5.1.0", - "debug": "^4.1.1" + "debug": "^4.1.1", + "tar-stream": "^3.0.0" }, "devDependencies": { "expect": "^26.4.1", diff --git a/tools/deploy/script.ts b/tools/deploy/script.ts index 12a27d13..a7beee19 100644 --- a/tools/deploy/script.ts +++ b/tools/deploy/script.ts @@ -4,6 +4,7 @@ import nodePath from 'path'; import vm from 'vm'; import * as pathlib from './path.js'; import * as spritedata from '@smogon/sprite-data'; +import tar from 'tar-stream'; type Op = { @@ -36,7 +37,7 @@ export class ActionQueue { public log : LogEntry[]; public valid : boolean; private debugBuffer : unknown[]; - + constructor() { this.seen = new Map; this.log = []; @@ -99,7 +100,7 @@ export class ActionQueue { } this.debugBuffer = []; } - + print(level : 'errors' | 'all') { for (const entry of this.log) { if (entry.type === 'Op') { @@ -128,11 +129,13 @@ export class ActionQueue { } } - run(dir : string, mode : 'link' | 'copy') { + async run(dir : string, mode : 'link' | 'copy' | 'tar') { if (!this.valid) throw new Error(`Invalid ActionQueue`); - for (const entry of this.log) { - if (entry.type === 'Op') { + if (mode !== 'tar') { + for (const entry of this.log) { + if (entry.type !== 'Op') + continue; const op = entry.op; const dst = nodePath.join(dir, entry.dst); fs.mkdirSync(nodePath.dirname(dst), {recursive: true}); @@ -146,6 +149,23 @@ export class ActionQueue { fs.writeFileSync(dst, op.data); } } + } else { + let t = tar.pack(); + for (const entry of this.log) { + if (entry.type !== 'Op') + continue; + const op = entry.op; + if (op.type === 'Copy'){ + t.entry({name: entry.dst}, fs.readFileSync(op.src)); + } else if (op.type === 'Write') { + t.entry({name: entry.dst}, op.data); + } + } + // In this case, I guess its a file rather than a dir. + t.pipe(fs.createWriteStream(dir)) + return new Promise(resolve => { + t.on('close', () => resolve()) + }) } } } @@ -185,7 +205,7 @@ function makeEnv1(queue: ActionQueue) { debug(obj : unknown) { queue.debug(obj); }, - + gdebug(obj : unknown) { queue.gdebug(obj, false); } @@ -195,7 +215,7 @@ function makeEnv1(queue: ActionQueue) { function makeEnv2(srcDir : string, queue: ActionQueue) { return { __proto__: makeEnv1(queue), - + list(dir : string) : pathlib.Path[] { const result = []; for (const filename of fs.readdirSync(nodePath.join(srcDir, dir))) { @@ -203,7 +223,7 @@ function makeEnv2(srcDir : string, queue: ActionQueue) { } return result; }, - + copy(srcp : pathlib.PathLike, dstp : string | pathlib.Delta /* todo deltalike */) { const src = pathlib.format(pathlib.path(srcp)); let dst : string;