Files
openclaw-dnd-city-visualiza…/scripts/render_batch.py
T
rick 3d50023cfa Toolkit: segment, mass, render plates, batch, gate, matrices, cost, smoke test
Generalized from the render lanes' proven tools: map segmentation, 2.5D massing by
wealth zone, a CPU ray marcher with analytic landmarks and true-scale proxy people,
prompt composition from blocks, a self-calibrating batch submitter with a renderer
adapter point, the plate-vs-frame gates (drift, ECC, landmark region, skyline and
invented structures, figure brackets, corner sweep), matrix job builders, a cost
line, and an end-to-end smoke test on a synthetic town.
2026-10-14 11:39:57 +00:00

179 lines
8.7 KiB
Python

#!/usr/bin/env python3
"""Self-calibrating parallel batch submitter for an image-edit API ("re-surface IMAGE 1 with these references").
usage: render_batch.py jobs.json --out OUTDIR [--start 4] [--max 8] [--min 2] [--adapter openai_images]
jobs.json: {"model": "...", "quality": "high", "refs_common": [...], "jobs": [{"id", "prompt", "size", "refs": [...],
"prompt_alt": "..."(optional, used once on a moderation refusal)}]}
Concurrency starts at --start in flight; a 429/5xx backs off (limit -2, sleep Retry-After or exponential, requeue);
a full wave of successes at a level ratchets +2 (cap --max). Ledger: OUTDIR/results.jsonl (one line per call,
status, seconds, attempt) and events.log; summary in calibration.md. cost.py reads the ledger.
Credentials come from the environment only: IMAGE_API_KEY (required), IMAGE_API_BASE (default
https://api.openai.com/v1). Never put them in a job file or a config that goes into version control.
Adapters (--adapter): "openai_images" posts multipart to /images/edits (or /images/generations when a job has no
references). To use another renderer, add a function `submit(job, refs, spec, env) -> (status_code, png_bytes|None,
body_text)` in adapters/<name>.py; see adapters/README.md.
"""
import argparse
import base64
import importlib
import json
import os
import re
import sys
import threading
import time
import requests
def submit_openai_images(job, refs, spec, env):
key, base = env["key"], env["base"]
if not refs:
data = {"model": spec["model"], "prompt": job["prompt"], "size": job["size"],
"quality": spec.get("quality", "high"), "output_format": "png", "n": 1}
r = requests.post(base + "/images/generations", headers={"Authorization": "Bearer " + key,
"Content-Type": "application/json"}, json=data, timeout=(15, 480))
else:
files = [("image[]", (os.path.basename(p), open(p, "rb"),
"image/jpeg" if p.lower().endswith((".jpg", ".jpeg")) else "image/png")) for p in refs]
data = {"model": spec["model"], "prompt": job["prompt"], "size": job["size"],
"quality": spec.get("quality", "high"), "output_format": "png", "n": "1"}
r = requests.post(base + "/images/edits", headers={"Authorization": "Bearer " + key}, data=data, files=files,
timeout=(15, 480))
for _, f in files:
f[1].close()
png = None
if r.status_code == 200:
png = base64.b64decode(r.json()["data"][0]["b64_json"])
return r.status_code, png, (r.text[:300].replace("\n", " ") if r.status_code != 200 else ""), r.headers
def load_adapter(name):
if name == "openai_images":
return submit_openai_images
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "adapters"))
return importlib.import_module(name).submit
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("jobs")
ap.add_argument("--out", required=True)
ap.add_argument("--start", type=int, default=4)
ap.add_argument("--max", type=int, default=8)
ap.add_argument("--min", type=int, default=2)
ap.add_argument("--adapter", default="openai_images")
ap.add_argument("--done-file", default=None, help="write a timestamp here when the batch finishes")
ap.add_argument("--skip-existing", action="store_true", help="do not re-submit jobs whose PNG already exists")
a = ap.parse_args()
spec = json.load(open(a.jobs))
OUT = a.out
os.makedirs(OUT, exist_ok=True)
key = os.environ.get("IMAGE_API_KEY")
if not key:
sys.exit("IMAGE_API_KEY is not set in the environment")
env = dict(key=key, base=os.environ.get("IMAGE_API_BASE", "https://api.openai.com/v1").rstrip("/"))
submit = load_adapter(a.adapter)
lock = threading.Condition()
state = dict(limit=a.start, in_flight=0, succ_at_level=0, max_inflight_clean=0, r429=0, errors=0, done=0,
cooldown_until=0.0)
ev = open(os.path.join(OUT, "events.log"), "a")
led = open(os.path.join(OUT, "results.jsonl"), "a")
def log(msg):
line = time.strftime("%H:%M:%S ") + msg
print(line, flush=True)
ev.write(line + "\n")
ev.flush()
def ledger(**row):
led.write(json.dumps(row) + "\n")
led.flush()
def worker(job, attempt):
with lock:
while state["in_flight"] >= state["limit"] or time.time() < state["cooldown_until"]:
lock.wait(1.0)
state["in_flight"] += 1
inflight, level = state["in_flight"], state["limit"]
log("DISPATCH %s attempt=%d in_flight=%d limit=%d" % (job["id"], attempt, inflight, level))
refs = [os.path.abspath(p) for p in spec.get("refs_common", []) + job.get("refs", [])]
t0 = time.time()
try:
code, png, body, headers = submit(job, refs, spec, env)
except Exception as e: # network errors count as retryable
code, png, body, headers = -1, None, str(e)[:200], {}
log("EXC %s %s" % (job["id"], body))
dt = time.time() - t0
with lock:
state["in_flight"] -= 1
nrefs = len(refs)
if code == 200 and png:
open(os.path.join(OUT, job["id"] + ".png"), "wb").write(png)
state["done"] += 1
state["succ_at_level"] += 1
state["max_inflight_clean"] = max(state["max_inflight_clean"], inflight)
ledger(id=job["id"], status=200, secs=round(dt, 1), in_flight=inflight, limit=level, attempt=attempt,
refs=nrefs, size=job.get("size"))
log("OK %s %.0fs" % (job["id"], dt))
if state["succ_at_level"] >= state["limit"] and state["limit"] < a.max:
state["limit"] = min(a.max, state["limit"] + 2)
state["succ_at_level"] = 0
log("RATCHET limit -> %d" % state["limit"])
lock.notify_all()
return True
ledger(id=job["id"], status=code, secs=round(dt, 1), in_flight=inflight, limit=level, attempt=attempt,
refs=nrefs, body=body)
if "insufficient_quota" in body:
log("QUOTA %s -- not retrying" % job["id"])
state["errors"] += 1
lock.notify_all()
return False
if code == 429 or code >= 500 or code == -1:
state["r429"] += 1 if code == 429 else 0
ra = headers.get("Retry-After") if headers else None
wait = float(ra) if ra and ra.replace(".", "").isdigit() else min(60, 5 * (2 ** (attempt - 1)))
state["limit"] = max(a.min, state["limit"] - 2)
state["succ_at_level"] = 0
state["cooldown_until"] = time.time() + wait
log("BACKOFF %s http=%s wait=%.0fs limit -> %d %s" % (job["id"], code, wait, state["limit"], body[:120]))
lock.notify_all()
return worker(job, attempt + 1) if attempt < 6 else False
state["errors"] += 1
log("FAIL %s http=%s %s" % (job["id"], code, body[:200]))
# one rephrase on a moderation refusal, never more; after that the lane reports it and stops
if attempt == 1 and job.get("prompt_alt") and re.search(r"safety|moderat|content_policy|rejected", body, re.I):
log("REPHRASE %s" % job["id"])
lock.notify_all()
return worker(dict(job, prompt=job["prompt_alt"]), 2)
lock.notify_all()
return False
jobs = spec["jobs"]
if a.skip_existing:
jobs = [j for j in jobs if not os.path.exists(os.path.join(OUT, j["id"] + ".png"))]
t_start = time.time()
threads = [threading.Thread(target=worker, args=(j, 1), daemon=True) for j in jobs]
for t in threads:
t.start()
for t in threads:
t.join()
el = max(time.time() - t_start, 1e-6)
summary = ("# batch calibration — %s\n\n- jobs: %d, succeeded: %d, hard failures: %d, 429s: %d\n"
"- start concurrency %d, ceiling reached: %d (cap %d); max in-flight without pushback: %d\n"
"- elapsed %.1f min, %.1f images/min; model %s, quality %s\n"
% (time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime()), len(jobs), state["done"], state["errors"],
state["r429"], a.start, state["limit"], a.max, state["max_inflight_clean"], el / 60,
state["done"] / (el / 60), spec["model"], spec.get("quality", "high")))
open(os.path.join(OUT, "calibration.md"), "w").write(summary)
log("DONE\n" + summary)
if a.done_file:
open(a.done_file, "w").write(time.strftime("%FT%TZ", time.gmtime()))
if __name__ == "__main__":
main()