Add support for DeflOpt

- fuse and wine do not get along.

- This is the absolute final release of DeflOpt, and it is only
accessible in binary form via archive.org, so we vendored it.

README.md

@@ -12,6 +12,7 @@ This project depends on
 - [OptiPNG](http://optipng.sourceforge.net/) (optional)
 - [pnpm](https://pnpm.js.org)
 - [node.js](https://nodejs.org) >= 13
+- [wine](https://www.winehq.org/) (optional)
 
 ### Windows
 
@@ -20,7 +21,7 @@ Windows binaries of these dependencies can be found on the download pages of the
 ### Linux
 
 ```
-$ sudo apt install nodejs imagemagick advancecomp
+$ sudo apt install nodejs imagemagick advancecomp wine
 $ sudo npm install -g pnpm
 ```
 
@@ -52,7 +53,7 @@ $ sudo ldconfig /usr/local/lib
 Using [`brew`](https://brew.sh/) on  a macOS:
 
 ```
-$ brew cask install osxfuse
+$ brew cask install osxfuse wine-stable
 $ brew install tup imagemagick advancecomp
 ```
 
@@ -70,14 +71,19 @@ Build settings are configurable in `tup.config`.
 
 - `CONFIG_DEFAULT_OPTIPNG`: Command line to pass to `optipng`.
 - `CONFIG_DEFAULT_ADVPNG`: Command line to pass to `advpng`.
-- `CONFIG_TRAINERS_{OPTIPNG,ADVPNG}`: Compression options for `trainers/` only.
-- `CONFIG_DEX_{OPTIPNG,ADVPNG}`: Compression options for `dex/` only.
-- `CONFIG_MODELS_{OPTIPNG,ADVPNG}`: Compression options for `models/` only.
+- `CONFIG_USE_DEFLOPT`: Path to `DeflOpt.exe`. If using Wine, you have to copy it somewhere outside the Tup tree.
+
+There are src-specific versions of these settings:
+
+- `CONFIG_TRAINERS_<PROGRAM>`: Compression options for `trainers/` only.
+- `CONFIG_DEX_<PROGRAM>`: Compression options for `dex/` only.
+- `CONFIG_MODELS_<PROGRAM>`: Compression options for `models/` only.
 
 For example, these settings reflect the compression settings for the files chaos uploaded in `src/`:
 ```
 CONFIG_DEFAULT_OPTIPNG=-o7
 CONFIG_DEFAULT_ADVPNG=-z4 -i5000
+CONFIG_USE_DEFLOPT=/home/monsanto/.local/bin/DeflOpt.exe
 ```
 
 ## Filename Scheme

Tuprules.lua

@@ -3,6 +3,8 @@ tup.include("util/strict.lua")
 tup.include("util/lua-ext.lua")
 tup.include("util/tup-ext.lua")
 
+ROOTDIR = tup.getcwd()
+
 function pad(w, h, input, output)
     return rep{
         "convert {input} -background transparent -gravity center -extent {w}x{h} {output}",
@@ -15,23 +17,32 @@ end
 
 local DEFAULT_OPTIPNG = getconfig("DEFAULT_OPTIPNG")
 local DEFAULT_ADVPNG = getconfig("DEFAULT_ADVPNG")
+local USE_DEFLOPT = getconfig("USE_DEFLOPT")
 
 function compresspng(opts)
     local cmds = {}
     local output = opts.output or "%o"
     local optipng = DEFAULT_OPTIPNG
+    local advpng = DEFAULT_ADVPNG
+    local deflopt = USE_DEFLOPT
     if opts.config then
         optipng = getconfig(opts.config .. "_OPTIPNG") or optipng;
-    end
-    local advpng = DEFAULT_ADVPNG
-    if opts.config then
         advpng = getconfig(opts.config .. "_ADVPNG") or advpng;
+        deflopt = getconfig(opts.config .. "_DEFLOPT") or deflopt;
     end
+
     if optipng then
         cmds += rep{"optipng -q {opts} {output}", opts=optipng, output=output}
     end
     if advpng then
         cmds += rep{"advpng -q {opts} {output}", opts=advpng, output=output}
     end
+    if deflopt then
+        cmds += rep{"node {root}/tools/deflopt {deflopt} {output}",
+                    root=ROOTDIR,
+                    deflopt=deflopt,
+                    output=output}
+    end
+    
     return cmds
 end

pnpm-lock.yaml

@@ -1,4 +1,9 @@
 importers:
+  tools/deflopt:
+    dependencies:
+      debug: 4.1.1
+    specifiers:
+      debug: ^4.1.1
   tools/deploy:
     dependencies:
       commander: 5.1.0

pnpm-workspace.yaml

@@ -2,3 +2,4 @@ packages:
     - 'tools/sprites'
     - 'tools/tupctime'
     - 'tools/deploy'
+    - 'tools/deflopt'

tools/deflopt/debug.js

@@ -0,0 +1,4 @@
+
+import debugfn from 'debug';
+
+export default debugfn('deflopt');

tools/deflopt/deflopt.js

@@ -0,0 +1,48 @@
+
+import cp from 'child_process';
+import debug from './debug.js';
+
+/*
+  Output looks something like:
+
+***                 DeflOpt V2.07                 ***
+***       Built on Wed Sep  5 18:56:30 2007       ***
+***  Copyright (C) 2003-2007 by Ben Jos Walbeehm  ***
+
+
+
+
+Number of files processed  :        0
+Number of files rewritten  :        0
+Total number of bytes saved:        0
+2,097,357 cycles.
+ */
+function parse(statusLine, output) {
+    const re = new RegExp(statusLine + " *: *([0-9]+)");
+    const m = output.match(re);
+    if (!m) {
+        throw new Error(`Can't match output: ` + output);
+    }
+    const num = parseInt(m[1], 10);
+    return num;
+}
+
+export function deflopt(exe, file) {
+    // DeflOpt doesn't like absolute paths, it thinks they are Windows-style
+    // command line switches.
+    file = file.replace(/\//g, "\\");
+    
+    debug("File at: " + file);
+    const output = cp.execFileSync(
+        "wine",
+        [exe, file],
+        {encoding: 'utf8',
+         // Wine doesn't even like it when the current directory is on fuse!
+         // What a persnickety program!
+         cwd: '/'});
+    debug("Output: " + JSON.stringify(output));
+    const processed = parse("Number of files processed", output);
+    const rewritten = parse("Number of files rewritten", output);
+    debug(`Processed ${processed}, rewritten ${rewritten}`);
+    return {processed, rewritten};
+}

tools/deflopt/index.js

@@ -0,0 +1,20 @@
+// Run DeflOpt.exe.
+// Wine and Tup's fuse filesystem don't play nicely together.
+// Copy inputs to /tmp first.
+
+import {asTmp} from './tmp.js';
+import {deflopt} from './deflopt.js';
+
+const exe = process.argv[2];
+const src = process.argv[3];
+if (src === undefined || exe === undefined) {
+    throw new Error('tools/deflopt <path to DeflOpt.exe> <file>');
+}
+
+asTmp("deflopt", src, dst => {
+    const {processed, rewritten} = deflopt(exe, dst);
+    if (processed !== 1) {
+        throw new Error(`Didn't process any files.`);
+    }
+    return {changed: rewritten > 0};
+});

tools/deflopt/package.json

@@ -0,0 +1 @@
+{"type":"module","dependencies":{"debug":"^4.1.1"}}

tools/deflopt/tmp.js

@@ -0,0 +1,31 @@
+
+import pathlib from 'path';
+import fs from 'fs';
+import crypto from 'crypto';
+import os from 'os';
+
+function join(...paths) {
+    return pathlib.join(os.tmpdir(), ...paths);
+}
+
+// Adapted from https://stackoverflow.com/a/61312694
+function mktemp(prefix) {
+    return join(`${prefix}.${crypto.randomBytes(6).readUIntLE(0,6).toString(36)}`);
+}
+
+export function asTmp(prefix, src, f) {
+    const parsed = pathlib.parse(src);
+    const dst = mktemp(prefix + "." + parsed.name) + parsed.ext;
+    // This has an astronomically low chance of throwing, but if it does, you
+    // can just restart the build.
+    fs.copyFileSync(src, dst, fs.constants.COPYFILE_EXCL);
+    try {
+        const {changed, ret} = f(dst);
+        if (changed) {
+            fs.copyFileSync(dst, src);
+        }
+        return ret;
+    } finally {
+        fs.unlinkSync(dst);
+    }
+}