pokeyellow/dos_port/tools/work_queue

#!/usr/bin/env python3
"""
work_queue — Agent skill for querying and updating the translation work queue.

All output is JSON so agents can parse it without screen-scraping.

** NEVER access translation.db with raw SQL. Use this tool exclusively. **
Direct sqlite3 calls bypass the audit log and will use the wrong table name.

Status pipeline
---------------
  needs_translation → in_progress → translated → [wired → verified]
                                         ↑
                                   (swarm terminal)
  wired and verified are Claude Code session transitions only.

Commands
--------
  status                              Queue summary counts by category × status.
  list   [--category C] [--status S] [--limit N]
                                      List matching functions.
  claim  --agent ID [--count N] [--category C]
                                      Atomically claim N jobs (default 1).
  manifest --id ID                    Print the manifest header a worker must write.
  complete --id ID --scratch PATH [--agent ID]
                                      Worker marks job translated. Scratch file must
                                      exist and be different from the output path.
  place    --id ID --output PATH [--agent ID]
                                      Integration Agent records final dos_port/src/
                                      placement and deletes the scratch file.
  reset    --id ID                    Return any function to needs_translation
                                      (preserves category). Maintenance command.
  wire     --id ID [--agent ID]       Claude Code: translated → wired.
  verify   --id ID [--agent ID]       Claude Code: wired → verified.
  recategorize --id ID --category C [--agent ID] [--notes MSG]
                                      Move a job to a different category.
                                      Resets status to needs_translation. Logged.
  unclaim  --id ID                    Return a claimed job to needs_translation.
  fail     --id ID [--notes MSG]      Return a job with a failure note.
  log      --id ID                    Full audit log for one function.
  pending-placement                   List translated jobs waiting for Integration Agent.
  translation-log-entry --id ID       Emit a formatted docs/translation_log.md entry.

Exit codes: 0 = success, 1 = error (details in JSON 'error' key).
"""

import argparse
import json
import os
import re
import sqlite3
import sys
from datetime import datetime, timezone

SCRIPT_DIR  = os.path.dirname(os.path.abspath(__file__))
REPO_ROOT   = os.path.normpath(os.path.join(SCRIPT_DIR, '..', '..'))
DB_PATH     = os.path.join(SCRIPT_DIR, 'translation.db')
SCRATCH_DIR = os.path.join(REPO_ROOT, 'dos_port', 'scratch')

ALL_STATUSES = [
    'needs_translation', 'in_progress', 'translated', 'wired', 'verified', 'skip',
    # legacy values kept so old rows survive migration
    'complete', 'unverified',
]

# ── DB helpers ───────────────────────────────────────────────────────────────────

def db():
    if not os.path.exists(DB_PATH):
        err({'error': f'Database not found at {DB_PATH}. Run build_index first.'})
    con = sqlite3.connect(DB_PATH)
    con.row_factory = sqlite3.Row
    con.execute("PRAGMA journal_mode=WAL")
    _migrate(con)
    return con

def _migrate(con):
    cols = {r[1] for r in con.execute("PRAGMA table_info(functions)")}
    if 'scratch_file' not in cols:
        con.execute("ALTER TABLE functions ADD COLUMN scratch_file TEXT")
    if 'translation_notes' not in cols:
        con.execute("ALTER TABLE functions ADD COLUMN translation_notes TEXT")
    # Rename legacy statuses. Disable CHECK constraints temporarily because the
    # old schema only allows 'complete'/'unverified'; new names are 'translated'
    # and 'needs_translation'.
    con.execute("PRAGMA ignore_check_constraints = ON")
    con.execute("UPDATE functions SET status='translated'        WHERE status='complete'")
    con.execute("UPDATE functions SET status='needs_translation' WHERE status='unverified'")
    con.execute("PRAGMA ignore_check_constraints = OFF")
    con.commit()

def scratch_path(fid, label):
    safe = re.sub(r'[^A-Za-z0-9_]', '_', label)
    return os.path.join(SCRATCH_DIR, f'{fid}__{safe}.asm')

def now_utc():
    return datetime.now(timezone.utc).isoformat()

def today():
    return datetime.now(timezone.utc).strftime('%Y-%m-%d')

def out(obj):
    print(json.dumps(obj, indent=2))

def err(obj):
    print(json.dumps(obj, indent=2), file=sys.stderr)
    sys.exit(1)

def log_transition(cur, function_id, old_status, new_status, agent=None, notes=None):
    cur.execute("""
        INSERT INTO translation_log
            (function_id, old_status, new_status, agent, notes, timestamp)
        VALUES (?, ?, ?, ?, ?, ?)
    """, (function_id, old_status, new_status, agent, notes, now_utc()))

# ── Manifest / notes parsing ─────────────────────────────────────────────────────

def make_manifest(row):
    sp = scratch_path(row['id'], row['name'])
    rel_scratch = os.path.relpath(sp, REPO_ROOT)
    lines = [
        '; ╔══════════════════════════════════════════════════════════╗',
        '; ║              PKMNDOS TRANSLATION MANIFEST               ║',
        '; ╚══════════════════════════════════════════════════════════╝',
        f'; queue_id   : {row["id"]}',
        f'; label      : {row["name"]}',
        f'; source     : {row["source_file"]}',
        f'; category   : {row["category"]}',
        f'; scratch    : {rel_scratch}',
        '; -----------------------------------------------------------',
        '; target      : (Integration Agent fills this in)',
        '; aggregator  : (Integration Agent fills this in)',
        '; -----------------------------------------------------------',
        '; WORKER NOTES — fill in before calling work_queue complete',
        '; registers   : (e.g. HL→ESI for table ptr, A→AL, B→BH, C→BL)',
        '; hflag       : (not involved / lazy / computed)',
        '; bug_tags    : (none / BUG(critical) / BUG(cosmetic) / GLITCH:<name>)',
        '; notes       : (key decisions, edge cases, deviations — one line)',
        '; ╔══════════════════════════════════════════════════════════╗',
        '; ║  CODE BELOW — do not modify the header above            ║',
        '; ╚══════════════════════════════════════════════════════════╝',
        '',
    ]
    return '\n'.join(lines)

_WORKER_FIELDS = ('registers', 'hflag', 'bug_tags', 'notes')

def parse_notes_from_scratch(path):
    result = {f: '' for f in _WORKER_FIELDS}
    try:
        with open(path, encoding='utf-8', errors='replace') as fh:
            for line in fh:
                if 'CODE BELOW' in line:
                    break
                m = re.match(r';\s*(registers|hflag|bug_tags|notes)\s*:\s*(.*)', line)
                if m:
                    key, val = m.group(1), m.group(2).strip()
                    if not (val.startswith('(') and val.endswith(')')):
                        result[key] = val
    except OSError:
        pass
    return result

# ── sub-commands ─────────────────────────────────────────────────────────────────

def cmd_status(_args):
    con = db()
    rows = con.execute("""
        SELECT category, status, COUNT(*) AS n
        FROM functions GROUP BY category, status ORDER BY category, status
    """).fetchall()
    summary = {}
    for r in rows:
        summary.setdefault(r['category'], {})[r['status']] = r['n']
    totals = con.execute(
        "SELECT status, COUNT(*) AS n FROM functions GROUP BY status"
    ).fetchall()
    out({'summary': summary,
         'totals': {r['status']: r['n'] for r in totals},
         'db': DB_PATH})
    con.close()

def cmd_list(args):
    con = db()
    clauses, params = [], []
    if args.category:
        clauses.append("category = ?"); params.append(args.category)
    if args.status:
        clauses.append("status = ?");   params.append(args.status)
    where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
    limit = f"LIMIT {args.limit}" if args.limit else ""
    rows = con.execute(f"""
        SELECT id, name, source_file, category, status,
               scratch_file, output_file, assigned_agent, notes
        FROM functions {where} ORDER BY source_file, name {limit}
    """, params).fetchall()
    out({'count': len(rows), 'functions': [dict(r) for r in rows]})
    con.close()

def cmd_claim(args):
    category = args.category or 'simple'
    count    = args.count or 1
    con      = db()
    cur      = con.cursor()
    cur.execute("BEGIN EXCLUSIVE")
    rows = cur.execute("""
        SELECT id, name, source_file, category, status, notes
        FROM functions
        WHERE category = ? AND status = 'needs_translation'
        ORDER BY source_file, name
        LIMIT ?
    """, (category, count)).fetchall()

    if not rows:
        con.rollback(); con.close()
        out({'claimed': [], 'message': f'No jobs available in category={category}'})
        return

    ts = now_utc()
    claimed = []
    for r in rows:
        sp     = scratch_path(r['id'], r['name'])
        rel_sp = os.path.relpath(sp, REPO_ROOT)
        cur.execute("""
            UPDATE functions
            SET status='in_progress', assigned_agent=?, scratch_file=?, updated_at=?
            WHERE id=?
        """, (args.agent, rel_sp, ts, r['id']))
        log_transition(cur, r['id'], 'needs_translation', 'in_progress', args.agent)
        claimed.append({**dict(r), 'scratch_path': rel_sp,
                        'manifest_cmd': f'dos_port/tools/work_queue manifest --id {r["id"]}'})

    con.commit()
    out({'claimed': claimed, 'agent': args.agent, 'count': len(claimed),
         'scratch_dir': os.path.relpath(SCRATCH_DIR, REPO_ROOT)})
    con.close()

def cmd_manifest(args):
    con = db()
    row = con.execute(
        "SELECT id, name, source_file, category FROM functions WHERE id=?",
        (args.id,)
    ).fetchone()
    if not row:
        err({'error': f'No function with id={args.id}'})
    sp     = scratch_path(row['id'], row['name'])
    rel_sp = os.path.relpath(sp, REPO_ROOT)
    out({'id': row['id'], 'label': row['name'],
         'source': row['source_file'],
         'scratch_path': rel_sp,
         'header': make_manifest(row)})
    con.close()

def cmd_complete(args):
    """Worker marks job translated; scratch file must exist and differ from output."""
    con = db()
    cur = con.cursor()
    row = cur.execute("SELECT id, status, name FROM functions WHERE id=?",
                      (args.id,)).fetchone()
    if not row:
        err({'error': f'No function with id={args.id}'})
    full_scratch = os.path.join(REPO_ROOT, args.scratch)
    if not os.path.exists(full_scratch):
        err({'error': f'Scratch file not found: {args.scratch}',
             'hint': 'Worker must write the file before calling complete.'})
    worker_notes = parse_notes_from_scratch(full_scratch)
    notes_json   = json.dumps(worker_notes)
    old = row['status']
    cur.execute("""
        UPDATE functions
        SET status='translated', scratch_file=?, translation_notes=?,
            assigned_agent=?, updated_at=?
        WHERE id=?
    """, (args.scratch, notes_json, args.agent, now_utc(), args.id))
    log_transition(cur, args.id, old, 'translated', args.agent,
                   f'scratch: {args.scratch}')
    con.commit()
    out({'id': args.id, 'name': row['name'], 'old_status': old,
         'new_status': 'translated', 'scratch_file': args.scratch,
         'parsed_notes': worker_notes,
         'next_step': 'Integration Agent: run `place --id` after moving to dos_port/src/'})
    con.close()

def cmd_place(args):
    """Integration Agent records placement. Guards against scratch == output."""
    con = db()
    cur = con.cursor()
    row = cur.execute("SELECT id, status, name, scratch_file FROM functions WHERE id=?",
                      (args.id,)).fetchone()
    if not row:
        err({'error': f'No function with id={args.id}'})
    if row['status'] != 'translated':
        err({'error': f'Function {row["name"]} is not translated (status={row["status"]})',
             'hint': 'Only translated functions can be placed.'})

    full_output = os.path.join(REPO_ROOT, args.output)
    if row['scratch_file']:
        full_scratch = os.path.join(REPO_ROOT, row['scratch_file'])
        try:
            if os.path.realpath(full_scratch) == os.path.realpath(full_output):
                err({'error': 'scratch and output resolve to the same file — '
                              'create a separate scratch file in dos_port/scratch/',
                     'scratch': row['scratch_file'],
                     'output': args.output})
        except OSError:
            pass

    cur.execute("""
        UPDATE functions
        SET output_file=?, scratch_file=NULL, assigned_agent=?, updated_at=?
        WHERE id=?
    """, (args.output, args.agent, now_utc(), args.id))
    log_transition(cur, args.id, 'translated', 'translated', args.agent,
                   f'placed → {args.output}')
    con.commit()

    deleted_scratch = None
    if row['scratch_file']:
        full_scratch = os.path.join(REPO_ROOT, row['scratch_file'])
        try:
            os.remove(full_scratch)
            deleted_scratch = row['scratch_file']
        except OSError:
            pass

    out({'id': args.id, 'name': row['name'],
         'scratch_deleted': deleted_scratch,
         'output_file': args.output})
    con.close()

def cmd_reset(args):
    """Return any function to needs_translation, preserving category."""
    con = db()
    cur = con.cursor()
    row = cur.execute("SELECT id, status, name, assigned_agent FROM functions WHERE id=?",
                      (args.id,)).fetchone()
    if not row:
        err({'error': f'No function with id={args.id}'})
    old = row['status']
    cur.execute("""
        UPDATE functions
        SET status='needs_translation', assigned_agent=NULL,
            scratch_file=NULL, updated_at=?
        WHERE id=?
    """, (now_utc(), args.id))
    log_transition(cur, args.id, old, 'needs_translation',
                   row['assigned_agent'], 'reset by maintenance')
    con.commit()
    out({'id': args.id, 'name': row['name'],
         'old_status': old, 'new_status': 'needs_translation'})
    con.close()

def cmd_wire(args):
    """Claude Code session: translated → wired."""
    con = db()
    cur = con.cursor()
    row = cur.execute("SELECT id, status, name FROM functions WHERE id=?",
                      (args.id,)).fetchone()
    if not row:
        err({'error': f'No function with id={args.id}'})
    if row['status'] != 'translated':
        err({'error': f'{row["name"]} is not translated (status={row["status"]})',
             'hint': 'Only translated functions can be wired.'})
    cur.execute("""
        UPDATE functions SET status='wired', assigned_agent=?, updated_at=?
        WHERE id=?
    """, (args.agent, now_utc(), args.id))
    log_transition(cur, args.id, 'translated', 'wired', args.agent)
    con.commit()
    out({'id': args.id, 'name': row['name'],
         'old_status': 'translated', 'new_status': 'wired'})
    con.close()

def cmd_verify(args):
    """Claude Code session: wired → verified (confirmed working in DOSBox-X)."""
    con = db()
    cur = con.cursor()
    row = cur.execute("SELECT id, status, name FROM functions WHERE id=?",
                      (args.id,)).fetchone()
    if not row:
        err({'error': f'No function with id={args.id}'})
    if row['status'] != 'wired':
        err({'error': f'{row["name"]} is not wired (status={row["status"]})',
             'hint': 'Only wired functions can be verified.'})
    cur.execute("""
        UPDATE functions SET status='verified', assigned_agent=?, updated_at=?
        WHERE id=?
    """, (args.agent, now_utc(), args.id))
    log_transition(cur, args.id, 'wired', 'verified', args.agent)
    con.commit()
    out({'id': args.id, 'name': row['name'],
         'old_status': 'wired', 'new_status': 'verified'})
    con.close()

def cmd_pending_placement(_args):
    con = db()
    rows = con.execute("""
        SELECT id, name, source_file, category, scratch_file, assigned_agent
        FROM functions
        WHERE status = 'translated' AND output_file IS NULL
        ORDER BY source_file, name
    """).fetchall()
    out({'count': len(rows), 'pending': [dict(r) for r in rows]})
    con.close()

def cmd_translation_log_entry(args):
    con = db()
    row = con.execute("SELECT * FROM functions WHERE id=?", (args.id,)).fetchone()
    if not row:
        err({'error': f'No function with id={args.id}'})
    fn = dict(row)

    notes = {}
    if fn.get('translation_notes'):
        try:
            notes = json.loads(fn['translation_notes'])
        except (json.JSONDecodeError, TypeError):
            notes = {}

    source    = fn.get('source_file', '(unknown)')
    output    = fn.get('output_file') or fn.get('scratch_file') or '(not yet placed)'
    registers = notes.get('registers') or '(not recorded)'
    hflag     = notes.get('hflag')     or '(not recorded)'
    bug_tags  = notes.get('bug_tags')  or 'none'
    freenotes = notes.get('notes')     or '(none)'

    entry = f"""
## {fn['name']}

- **Source:** `{source}:{fn['name']}`
- **Translated:** `{output}`
- **Date:** {today()}
- **H-flag:** {hflag}
- **Bug tags:** {bug_tags}
- **Registers:** {registers}
- **Notes:** {freenotes}

---
"""
    out({'id': fn['id'], 'name': fn['name'], 'entry': entry,
         'append_to': 'docs/translation_log.md'})
    con.close()

def cmd_recategorize(args):
    con = db()
    cur = con.cursor()
    row = cur.execute(
        "SELECT id, status, name, category FROM functions WHERE id=?",
        (args.id,)
    ).fetchone()
    if not row:
        err({'error': f'No function with id={args.id}'})
    if row['category'] == args.category:
        err({'error': f'{row["name"]} is already category={args.category}'})
    old_cat    = row['category']
    old_status = row['status']
    note = args.notes or f'recategorized {old_cat}→{args.category}'
    cur.execute("""
        UPDATE functions
        SET category=?, status='needs_translation', assigned_agent=NULL,
            scratch_file=NULL, notes=?, updated_at=?
        WHERE id=?
    """, (args.category, note, now_utc(), args.id))
    log_transition(cur, args.id, old_status, 'needs_translation', args.agent,
                   f'recategorized {old_cat}→{args.category}: {note}')
    con.commit()
    out({'id': args.id, 'name': row['name'],
         'old_category': old_cat, 'new_category': args.category,
         'old_status': old_status, 'new_status': 'needs_translation',
         'notes': note})
    con.close()

def cmd_unclaim(args):
    con = db()
    cur = con.cursor()
    row = cur.execute("SELECT id, status, name, assigned_agent FROM functions WHERE id=?",
                      (args.id,)).fetchone()
    if not row:
        err({'error': f'No function with id={args.id}'})
    old = row['status']
    cur.execute("""
        UPDATE functions
        SET status='needs_translation', assigned_agent=NULL,
            scratch_file=NULL, updated_at=?
        WHERE id=?
    """, (now_utc(), args.id))
    log_transition(cur, args.id, old, 'needs_translation',
                   row['assigned_agent'], 'unclaimed / returned to queue')
    con.commit()
    out({'id': args.id, 'name': row['name'], 'returned_to_queue': True})
    con.close()

def cmd_fail(args):
    con = db()
    cur = con.cursor()
    row = cur.execute("SELECT id, status, name, assigned_agent FROM functions WHERE id=?",
                      (args.id,)).fetchone()
    if not row:
        err({'error': f'No function with id={args.id}'})
    old = row['status']
    cur.execute("""
        UPDATE functions
        SET status='needs_translation', assigned_agent=NULL,
            scratch_file=NULL, notes=?, updated_at=?
        WHERE id=?
    """, (args.notes, now_utc(), args.id))
    log_transition(cur, args.id, old, 'needs_translation',
                   row['assigned_agent'], f'failed: {args.notes}')
    con.commit()
    out({'id': args.id, 'name': row['name'],
         'returned_to_queue': True, 'notes': args.notes})
    con.close()

def cmd_log(args):
    con = db()
    fn = con.execute("SELECT * FROM functions WHERE id=?", (args.id,)).fetchone()
    if not fn:
        err({'error': f'No function with id={args.id}'})
    fn_dict = dict(fn)
    if fn_dict.get('translation_notes'):
        try:
            fn_dict['translation_notes'] = json.loads(fn_dict['translation_notes'])
        except (json.JSONDecodeError, TypeError):
            pass
    log = con.execute("""
        SELECT old_status, new_status, agent, notes, timestamp
        FROM translation_log WHERE function_id=? ORDER BY id
    """, (args.id,)).fetchall()
    out({'function': fn_dict, 'log': [dict(r) for r in log]})
    con.close()

# ── CLI wiring ───────────────────────────────────────────────────────────────────

def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = ap.add_subparsers(dest='cmd', required=True)

    sub.add_parser('status', help='Queue summary')

    p = sub.add_parser('list', help='List functions')
    p.add_argument('--category', choices=['simple', 'complex'])
    p.add_argument('--status', choices=ALL_STATUSES)
    p.add_argument('--limit', type=int)

    p = sub.add_parser('claim', help='Claim jobs for an agent')
    p.add_argument('--agent',    required=True)
    p.add_argument('--count',    type=int, default=1)
    p.add_argument('--category', choices=['simple', 'complex'], default='simple')

    p = sub.add_parser('manifest', help='Print manifest header for a worker scratch file')
    p.add_argument('--id', type=int, required=True)

    p = sub.add_parser('complete',
                       help='Worker marks job translated; parses notes from scratch file')
    p.add_argument('--id',      type=int, required=True)
    p.add_argument('--scratch', required=True,
                   help='Repo-relative path to scratch file in dos_port/scratch/')
    p.add_argument('--agent')

    p = sub.add_parser('place',
                       help='Integration Agent records final dos_port/src/ placement')
    p.add_argument('--id',     type=int, required=True)
    p.add_argument('--output', required=True,
                   help='Repo-relative path of the placed file in dos_port/src/')
    p.add_argument('--agent')

    p = sub.add_parser('reset',
                       help='Return any function to needs_translation (preserves category)')
    p.add_argument('--id', type=int, required=True)

    p = sub.add_parser('wire',
                       help='Claude Code: mark function as wired into live game loop')
    p.add_argument('--id', type=int, required=True)
    p.add_argument('--agent')

    p = sub.add_parser('verify',
                       help='Claude Code: mark function as verified working in DOSBox-X')
    p.add_argument('--id', type=int, required=True)
    p.add_argument('--agent')

    p = sub.add_parser('recategorize',
                       help='Move a job to a different category (resets to needs_translation)')
    p.add_argument('--id',       type=int, required=True)
    p.add_argument('--category', choices=['simple', 'complex'], required=True)
    p.add_argument('--agent')
    p.add_argument('--notes')

    sub.add_parser('pending-placement',
                   help='List translated jobs waiting for Integration Agent')

    p = sub.add_parser('translation-log-entry',
                       help='Emit a formatted translation_log.md entry for Docs_Commit_Agent')
    p.add_argument('--id', type=int, required=True)

    p = sub.add_parser('unclaim', help='Return a claimed job to needs_translation')
    p.add_argument('--id', type=int, required=True)

    p = sub.add_parser('fail', help='Return a job with a failure note')
    p.add_argument('--id',    type=int, required=True)
    p.add_argument('--notes', default='worker could not complete')

    p = sub.add_parser('log', help='Full audit log for one function')
    p.add_argument('--id', type=int, required=True)

    args = ap.parse_args()
    {
        'status':                cmd_status,
        'list':                  cmd_list,
        'claim':                 cmd_claim,
        'manifest':              cmd_manifest,
        'complete':              cmd_complete,
        'place':                 cmd_place,
        'reset':                 cmd_reset,
        'wire':                  cmd_wire,
        'verify':                cmd_verify,
        'pending-placement':     cmd_pending_placement,
        'translation-log-entry': cmd_translation_log_entry,
        'recategorize':          cmd_recategorize,
        'unclaim':               cmd_unclaim,
        'fail':                  cmd_fail,
        'log':                   cmd_log,
    }[args.cmd](args)

if __name__ == '__main__':
    main()