Build refactor

There's still more implementation cleanup to do, but I at least wanted
to publish the new build DSL, which is stable-ish at this point

Tupfile.lua

@@ -1,28 +1,26 @@
 
 -- Generate uniform size minisprites
 
-for canon in iter{"canonical", "cap"} do 
-    for dir in iter{"asymmetrical", "custom", "misc", "pokemon"} do
-        tup.foreach_rule(
-            rep{"src/{canon}/minisprites/gen6/{dir}/*.png", canon=canon, dir=dir},
-            makecmd{
-                display="pad g6 minisprite %f",
-                pad{w=40, h=30}
-            },
-            rep{"build/gen6-minisprites-padded/{canon}/{dir}/%b",canon=canon,dir=dir}
-        )
-        
-        ::continue::
-    end
-end
+foreach_rule{
+    display="pad g6 minisprite %f",
+    input="src/%{canon}/minisprites/gen6/%{dir}/*.png",
+    command=pad{w=40, h=30},
+    output="build/gen6-minisprites-padded/%{canon}/%{dir}/%b",
+    dimensions={
+        canon={"canonical", "cap"},
+        dir={"asymmetrical", "custom", "misc", "pokemon"}
+    }
+}
 
 -- PS spritesheet
 
-tup.rule(
-    {"build/gen6-minisprites-padded/canonical/pokemon/*", "build/gen6-minisprites-padded/cap/pokemon/*"},
-    "node tools/sprites/ps.js build/gen6-minisprites-padded/canonical/pokemon/ build/gen6-minisprites-padded/cap/pokemon/ --output-image build/ps/pokemonicons-sheet.png --output-metadata build/ps/pokemonicons.json",
-    {"build/ps/pokemonicons-sheet.png", "build/ps/pokemonicons.json"}
-)
+rule{
+    input={"build/gen6-minisprites-padded/canonical/pokemon/*",
+           "build/gen6-minisprites-padded/cap/pokemon/*"},
+    command="node tools/sprites/ps.js build/gen6-minisprites-padded/canonical/pokemon/ build/gen6-minisprites-padded/cap/pokemon/ --output-image build/ps/pokemonicons-sheet.png --output-metadata build/ps/pokemonicons.json",
+    output={"build/ps/pokemonicons-sheet.png",
+            "build/ps/pokemonicons.json"}
+}
 
 -- PS pokeball icons
 
@@ -32,99 +30,86 @@ local balls = {
     "src/noncanonical/ui/battle/Ball-Null.png",
 }
 
-tup.rule(
-    balls,
-    makecmd{
-        display="pokemonicons-pokeball-sheet",
-        [[
-convert -background transparent -gravity center -extent 40x30 %f +append %o
-]],
+rule{
+    display="pokemonicons-pokeball-sheet",
+    input=balls,
+    command={
+        "convert -background transparent -gravity center -extent 40x30 %f +append %o",
         compresspng{config="SPRITESHEET"}
     },
-    {"build/ps/pokemonicons-pokeball-sheet.png"}
-)
+    output={"build/ps/pokemonicons-pokeball-sheet.png"}
+}
 
 -- Smogdex social images
 
-function fbsprite(input, output)
-    tup.foreach_rule(
-        input,
-        makecmd{
-            display="fbsprite %f",
-            "tools/fbsprite.sh %f %o",
-            compresspng{config="MODELS"}
-        },
-        output
-    )
-end
+foreach_rule{
+    display="fbsprite %f",
+    input={"src/cap/models/front/*", "src/cap/sprites/gen5/front/*", "src/canonical/models/front/*"},
+    key="%B",
+    command={
+        "tools/fbsprite.sh %f %o",
+        compresspng{config="MODELS"}
+    },
+    output="build/smogon/fbsprites/xy/%B.png"
+}
 
-function twittersprite(input, output)
-    tup.foreach_rule(
-        input,
-        makecmd{
-            display="twittersprite %f",
-            "tools/twittersprite.sh %f %o",
-            compresspng{config="MODELS"}
-        },
-        output
-    )
-end
+foreach_rule{
+    display="twittersprite %f",
+    input={"src/cap/models/front/*", "src/cap/sprites/gen5/front/*", "src/canonical/models/front/*"},
+    key="%B",
+    command={
+        "tools/twittersprite.sh %f %o",
+        compresspng{config="MODELS"}
+    },
+    output="build/smogon/twittersprites/xy/%B.png"
+}
 
--- TODO: nicer API
-local files = mergededup(
-    glob("src/cap/sprites/gen5/front/*"),
-    glob{"src/canonical/models/front/*", "src/cap/models/front/*"},
-    tup.base
-)
-
-fbsprite(files, "build/smogon/fbsprites/xy/%B.png")
-twittersprite(files, "build/smogon/twittersprites/xy/%B.png")
 
 -- Trainers
 
-tup.foreach_rule(
-    {"src/canonical/trainers/*"},
-    makecmd{
-        display="pad trainer %f",
+foreach_rule{
+    display="pad trainer %f",
+    input={"src/canonical/trainers/*"},
+    command={
         pad{w=80, h=80},
         compresspng{config="TRAINERS"}
     },
-    {"build/padded-trainers/canonical/%b"}
-)
+    output={"build/padded-trainers/canonical/%b"}
+}
 
 -- Padded Dex
 
-for canon in iter{"canonical", "cap"} do
-    for dir in iter{"front", "front-cosmetic", "front-shiny", "front-shiny-cosmetic"} do
-        tup.foreach_rule(
-            rep{"src/{canon}/dex/{dir}/*", canon=canon, dir=dir},
-            makecmd{
-                display="pad dex %f",
-                pad{w=120, h=120},
-                compresspng{config="DEX"}
-            },
-            rep{"build/padded-dex/{canon}/{dir}/%b", canon=canon, dir=dir}
-        )
-    end
-end
+foreach_rule{
+    display="pad dex %f",
+    input="src/%{canon}/dex/%{dir}/*",
+    command={
+        pad{w=120, h=120},
+        compresspng{config="DEX"}
+    },
+    output="build/padded-dex/%{canon}/%{dir}/%b",
+    dimensions={
+        canon={"canonical", "cap"},
+        dir={"front", "front-cosmetic", "front-shiny", "front-shiny-cosmetic"}
+    }
+}
 
 -- Build missing CAP dex
 
-for folder in iter({"front", "front-shiny", "front-cosmetic", "front-shiny-cosmetic"}) do
-    for file in iter(mergededup(glob(rep{"src/cap/models/{folder}/*.gif", folder=folder}),
-                                glob(rep{"src/cap/sprites/gen5/{folder}/*.gif", folder=folder}),
-                                tup.base)) do
-        if not fileexists(rep{"src/cap/dex/{folder}/{base}.png", folder=folder, base=tup.base(file)}) then
-            tup.rule(
-                file,
-                makecmd{
-                    display=rep{"missing dex cap {base} -> {folder}/{file}", folder=folder, base=tup.base(file), file=file},
-                    "convert %f'[0]' -trim %o",
-                    "mogrify -background transparent -gravity center -extent 120x120 %o",
-                    compresspng{config="DEX"}
-                },
-                rep{"build/padded-dex/cap/{folder}/{base}.png", folder=folder,base=tup.base(file)}
-            )
-        end
-    end
-end
+foreach_rule{
+    input={"src/cap/sprites/gen5/%{folder}/*.gif", "src/cap/models/%{folder}/*.gif"},
+    display="missing dex cap %B -> %{folder}/%f",
+    command={
+        "convert %f'[0]' -trim %o",
+        "mogrify -background transparent -gravity center -extent 120x120 %o",
+        compresspng{config="DEX"}
+    },
+    key="%B",
+    filter=function()
+        return not glob_matches("src/cap/dex/%{folder}/%B.png")
+    end,
+    output="build/padded-dex/cap/%{folder}/%B.png",
+    dimensions={
+        folder={"front", "front-shiny", "front-cosmetic", "front-shiny-cosmetic"}
+    }
+}
+

Tuprules.lua

@@ -1,13 +1,14 @@
 
 tup.include("util/strict.lua")
 tup.include("util/lua-ext.lua")
+tup.include("util/interpolation.lua")
 tup.include("util/tup-ext.lua")
 
 ROOTDIR = tup.getcwd()
 
 function pad(opts)
     return rep{
-        "convert {input} -background transparent -gravity center -extent {w}x{h} {output}",
+        "convert ${input} -background transparent -gravity center -extent ${w}x${h} ${output}",
         input = opts.input or "%f",
         output = opts.output or "%o",
         w = opts.w,
@@ -43,16 +44,16 @@ function compresspng(opts)
     end
 
     if optipng then
-        cmds += rep{"optipng -q {opts} {output}", opts=optipng, output=output}
+        cmds += rep{"optipng -q ${opts} ${output}", opts=optipng, output=output}
     end
     if advpng then
-        cmds += rep{"advpng -q {opts} {output}", opts=advpng, output=output}
+        cmds += rep{"advpng -q ${opts} ${output}", opts=advpng, output=output}
     end
     if deflopt then
         if DEFLOPT_PATH == nil then
             error("Need to set CONFIG_DEFLOPT_PATH")
         end
-        cmds += rep{"node {root}/tools/deflopt {deflopt} {output}",
+        cmds += rep{"node ${root}/tools/deflopt ${deflopt} ${output}",
                     root=ROOTDIR,
                     deflopt=DEFLOPT_PATH,
                     output=output}

util/interpolation.lua

@@ -0,0 +1,134 @@
+--
+
+local function rep_fn(f, var, modifier)
+    local v = f(var)
+    if v == nil then
+        error("unknown substitution: " .. var)
+    end
+    if modifier ~= nil then
+        if modifier == "b" then
+            return tup.file(v)
+        elseif modifier == "B" then
+            return tup.base(v)
+        elseif modifier == "e" then
+            return tup.ext(v)
+        else
+            error("unknown modifier: " .. modifier)
+        end
+    else
+        return v
+    end
+end
+
+-- Lua doesn't have alternation...
+local function do_rep(prefix, str, fn)
+    str = str:gsub(prefix .. "{(%a+)}",
+                   function(var)
+                       return rep_fn(fn, var, nil)
+                   end
+    )
+    str = str:gsub(prefix .. "{(%a+):(%a)}",
+                   function(var, modifier)
+                       return rep_fn(fn, var, modifier)
+                   end
+    )
+    str = str:gsub(prefix .. "(%a)",
+                   function(var)
+                       if var == "f" then
+                           return rep_fn(fn, "input", nil)
+                       elseif var == "b" then
+                           return rep_fn(fn, "input", "b")
+                       elseif var == "B" then
+                           return rep_fn(fn, "input", "B")
+                       elseif var == "o" then
+                           return rep_fn(fn, "output", nil)
+                       end
+                   end
+    )
+    return str
+end
+
+function rep(args)
+    local str = args[1]
+    local vars = args
+    return do_rep(
+        "$",
+        str,
+        function(var)
+            return vars[var]
+        end
+    )
+end
+
+-- Variable interpolation with dynamic scoping
+
+local FRAMES = {}
+
+function push_frame()
+    local frame = {}
+    table.insert(FRAMES, frame)
+    return frame
+end
+
+function pop_frame()
+    table.remove(FRAMES)
+end
+
+function adjust_frame(t)
+    local last_frame = FRAMES[#FRAMES]
+    for k, v in pairs(t) do
+        last_frame[k] = v
+    end
+end
+
+function print_frame()
+    local last_frame = FRAMES[#FRAMES]
+    for k, v in pairs(last_frame) do
+        print("frame", k, v)
+    end
+end
+
+function dim(var)
+    for i = #FRAMES, 1, -1 do
+        if FRAMES[i][var] ~= nil then
+            return FRAMES[i][var]
+        end
+    end
+end
+
+function expand(str)
+    return do_rep(
+        "%%",
+        str,
+        function(var)
+            return dim(var)
+        end
+    )
+end
+
+function with_rep(vars, f)
+    push_frame()
+    adjust_frame(vars)
+    local ret = f()
+    pop_frame()
+    return ret
+end
+
+function iter_rep(varspec, f)
+    local frame = push_frame()
+    local keys = table_keys(varspec)
+    local function loop(i)
+        if #keys + 1 == i then
+            f()
+        else
+            local k = keys[i]
+            for v in iter(varspec[k]) do
+                frame[k] = v
+                loop(i+1)
+            end
+        end
+    end
+    loop(1)
+    pop_frame()
+end
+

util/lua-ext.lua

@@ -22,7 +22,7 @@ end
 
 -- Adapted from premake
 function flatten(arr)
-    local result = { }
+    local result = {}
     
     local function flatten(arr)
         for v in iter(arr) do
@@ -38,36 +38,14 @@ function flatten(arr)
     return result
 end
 
--- Adapted from lua wiki, originally called replace_vars
-function rep(str, vars)
-  -- Allow replace_vars{str, vars} syntax as well as replace_vars(str, {vars})
-  if not vars then
-    vars = str
-    str = vars[1]
-  end
-  return (str:gsub("({([^}]+)})",
-    function(whole,i)
-      return vars[i] or whole
-    end))
-end
-
--- Merge, preferring elem2 when f(elem1) == f(elem2) 
-function mergededup(table1, table2, f)
-    local seen = {}
-    for v in iter(table1) do
-        seen[f(v)] = v
-    end
-    for v in iter(table2) do
-        seen[f(v)] = v
-    end
-    local result = {}
-    -- Can't be iter, because non-numeric keys
-    for _, v in pairs(seen) do
-        table.insert(result, v)
-    end
-    return result
-end
-
 function trim(s)
     return s:gsub("^%s*(.-)%s*$", "%1")
 end
+
+function table_keys(t)
+    local result = {}
+    for k, v in pairs(t) do
+        table.insert(result, k)
+    end
+    return result
+end

util/tup-ext.lua

@@ -1,25 +1,7 @@
 
--- Some Tup convenience functions extracted from Smogon's old build system
-
--- TODO: extract treerule() for naming intermediates?
-
-function glob(pats)
-    local results = {}
-    for pat in iter(astable(pats)) do
-        for file in iter(tup.glob(pat)) do
-            -- Workaround a weird issue pre reported
-            file = file:gsub("//", "/")
-            table.insert(results, file)
-        end
-    end
-    return results
-end
-
--- Convenience iterator so you can do
--- `for path in iglob{...}` instead of `for path in iter(glob{...})`
-function iglob(pats)
-    return iter(glob(pats))
-end
+--
+-- Configuration
+--
 
 -- getconfig returning an empty string on absence is inconvenient
 function getconfig(str)
@@ -31,35 +13,147 @@ function getconfig(str)
     end
 end
 
-function symlink(input, output)
-    -- The path must be relative from output, walk back to the root
-    local prefix = ""
-    for _ in output:gmatch("/") do
-        prefix = "../" .. prefix
+--
+-- Globs
+--
+
+-- A glob pattern is either an interpolated string, or a table of glob patterns
+--
+-- A normalized glob pattern is a table of strings, with interpolation resolved.
+-- globpat_normalize("foo/*") --> {"foo/*"}
+-- globpat_normalize({"foo/*"}) --> {"foo/*"}
+-- globpat_normalize({"foo/*", {"{bar}/*"}}) --> {"foo/*", "baz_value/*"}
+local function globpat_normalize(pat)
+    if type(pat) == "string" then
+        return {expand(pat)}
+    elseif type(pat) == 'table' then
+        local result = {}
+        for x in iter(pat) do
+            result += globpat_normalize(x)
+        end
+        return result
+    else
+        error("bad globpat")
     end
-    tup.foreach_rule(
-        input,
-        "^ symlink %f -> %o^ ln -s " .. prefix .. "%f %o",
-        output
-    )
 end
 
-function makecmd(cmds)
+function glob(pat, opts)
+    local results = {}
+    local filter = opts and opts.filter
+    local key = opts and opts.key
+    local seen = {}
+    for pat in iter(globpat_normalize(pat)) do
+        local frame = push_frame()
+        for file in iter(tup.glob(pat)) do
+            -- Workaround a weird issue pre reported
+            file = file:gsub("//", "/")
+            frame.input = file
+            
+            if key then
+                local k = expand(key)
+                if seen[k] then
+                    goto continue
+                end
+                seen[k] = true
+            end
+
+            if filter then
+                if not filter() then
+                    goto continue
+                end
+            end
+
+            table.insert(results, file)
+            ::continue::
+        end
+        pop_frame()
+    end
+    return results
+end
+
+-- Convenience iterator so you can do
+-- `for path in iglob{...}` instead of `for path in iter(glob{...})`
+function iglob(pat, opts)
+    return iter(glob(pat, opts))
+end
+
+
+function glob_matches(pat, opts)
+    local matched = glob(pat, opts)
+    return #matched > 0
+end
+
+-- Commands
+
+local function cspec2cmd(cmdSpec)
     local cmd = ""
-    for newcmd in iter(flatten(cmds)) do
+    for newcmd in iter(flatten(astable(cmdSpec))) do
         newcmd = trim(newcmd)
         if cmd ~= "" then
             cmd = cmd .. " &&\n"
         end
         cmd = cmd .. newcmd
     end
-    if cmds.display then
-        cmd = rep{"^ {display}^ {cmd}", display=cmds.display, cmd=cmd}
-    end
     return cmd
 end
 
-function fileexists(file)
-    local matched = glob(file)
-    return #matched > 0
+-- Abstract over tup's arcane display override
+local function display_cmd(display, cmd)
+    if display == nil then
+        return cmd
+    else
+        return rep{"^ ${display}^ ${cmd}", display=display, cmd=cmd}
+    end
 end
+
+local function do_rule_frame(opts, foreach)
+    local cspec = opts.command
+    local cmd = display_cmd(opts.display, cspec2cmd(cspec))
+    if foreach then
+        local input = glob(opts.input, opts)
+        for i in iter(input) do
+            local frame = push_frame()
+            frame.input = i
+            local output = {}
+            for v in iter(astable(opts.output)) do
+                table.insert(output, expand(v))
+            end
+            frame.output = tostring(output)
+            tup.rule(i, expand(cmd), output)
+            pop_frame()
+        end
+    else
+        local frame = push_frame()
+        local input = glob(opts.input)
+        frame.input = tostring(input)
+        local output = {}
+        for v in iter(astable(opts.output)) do
+            table.insert(output, expand(v))
+        end
+        frame.output = tostring(output)
+        tup.rule(input, expand(cmd), output)
+        pop_frame()
+    end
+end
+
+local function do_rule(opts, foreach)
+    local dimensions = opts.dimensions or {}
+    local ret = {}
+    iter_rep(dimensions,
+             function()
+                 for v in iter{do_rule_frame(opts, foreach, ret)} do
+                     table.insert(ret, v)
+                 end
+             end
+    )
+    return ret
+end
+
+function rule(opts)
+    return do_rule(opts, false)
+end
+
+function foreach_rule(opts)
+    return do_rule(opts, true)
+end
+