Generalized from the render lanes' proven tools: map segmentation, 2.5D massing by wealth zone, a CPU ray marcher with analytic landmarks and true-scale proxy people, prompt composition from blocks, a self-calibrating batch submitter with a renderer adapter point, the plate-vs-frame gates (drift, ECC, landmark region, skyline and invented structures, figure brackets, corner sweep), matrix job builders, a cost line, and an end-to-end smoke test on a synthetic town.
179 lines
8.7 KiB
Python
179 lines
8.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Self-calibrating parallel batch submitter for an image-edit API ("re-surface IMAGE 1 with these references").
|
|
|
|
usage: render_batch.py jobs.json --out OUTDIR [--start 4] [--max 8] [--min 2] [--adapter openai_images]
|
|
jobs.json: {"model": "...", "quality": "high", "refs_common": [...], "jobs": [{"id", "prompt", "size", "refs": [...],
|
|
"prompt_alt": "..."(optional, used once on a moderation refusal)}]}
|
|
|
|
Concurrency starts at --start in flight; a 429/5xx backs off (limit -2, sleep Retry-After or exponential, requeue);
|
|
a full wave of successes at a level ratchets +2 (cap --max). Ledger: OUTDIR/results.jsonl (one line per call,
|
|
status, seconds, attempt) and events.log; summary in calibration.md. cost.py reads the ledger.
|
|
|
|
Credentials come from the environment only: IMAGE_API_KEY (required), IMAGE_API_BASE (default
|
|
https://api.openai.com/v1). Never put them in a job file or a config that goes into version control.
|
|
|
|
Adapters (--adapter): "openai_images" posts multipart to /images/edits (or /images/generations when a job has no
|
|
references). To use another renderer, add a function `submit(job, refs, spec, env) -> (status_code, png_bytes|None,
|
|
body_text)` in adapters/<name>.py; see adapters/README.md.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import importlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
import requests
|
|
|
|
|
|
def submit_openai_images(job, refs, spec, env):
|
|
key, base = env["key"], env["base"]
|
|
if not refs:
|
|
data = {"model": spec["model"], "prompt": job["prompt"], "size": job["size"],
|
|
"quality": spec.get("quality", "high"), "output_format": "png", "n": 1}
|
|
r = requests.post(base + "/images/generations", headers={"Authorization": "Bearer " + key,
|
|
"Content-Type": "application/json"}, json=data, timeout=(15, 480))
|
|
else:
|
|
files = [("image[]", (os.path.basename(p), open(p, "rb"),
|
|
"image/jpeg" if p.lower().endswith((".jpg", ".jpeg")) else "image/png")) for p in refs]
|
|
data = {"model": spec["model"], "prompt": job["prompt"], "size": job["size"],
|
|
"quality": spec.get("quality", "high"), "output_format": "png", "n": "1"}
|
|
r = requests.post(base + "/images/edits", headers={"Authorization": "Bearer " + key}, data=data, files=files,
|
|
timeout=(15, 480))
|
|
for _, f in files:
|
|
f[1].close()
|
|
png = None
|
|
if r.status_code == 200:
|
|
png = base64.b64decode(r.json()["data"][0]["b64_json"])
|
|
return r.status_code, png, (r.text[:300].replace("\n", " ") if r.status_code != 200 else ""), r.headers
|
|
|
|
|
|
def load_adapter(name):
|
|
if name == "openai_images":
|
|
return submit_openai_images
|
|
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "adapters"))
|
|
return importlib.import_module(name).submit
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("jobs")
|
|
ap.add_argument("--out", required=True)
|
|
ap.add_argument("--start", type=int, default=4)
|
|
ap.add_argument("--max", type=int, default=8)
|
|
ap.add_argument("--min", type=int, default=2)
|
|
ap.add_argument("--adapter", default="openai_images")
|
|
ap.add_argument("--done-file", default=None, help="write a timestamp here when the batch finishes")
|
|
ap.add_argument("--skip-existing", action="store_true", help="do not re-submit jobs whose PNG already exists")
|
|
a = ap.parse_args()
|
|
spec = json.load(open(a.jobs))
|
|
OUT = a.out
|
|
os.makedirs(OUT, exist_ok=True)
|
|
key = os.environ.get("IMAGE_API_KEY")
|
|
if not key:
|
|
sys.exit("IMAGE_API_KEY is not set in the environment")
|
|
env = dict(key=key, base=os.environ.get("IMAGE_API_BASE", "https://api.openai.com/v1").rstrip("/"))
|
|
submit = load_adapter(a.adapter)
|
|
lock = threading.Condition()
|
|
state = dict(limit=a.start, in_flight=0, succ_at_level=0, max_inflight_clean=0, r429=0, errors=0, done=0,
|
|
cooldown_until=0.0)
|
|
ev = open(os.path.join(OUT, "events.log"), "a")
|
|
led = open(os.path.join(OUT, "results.jsonl"), "a")
|
|
|
|
def log(msg):
|
|
line = time.strftime("%H:%M:%S ") + msg
|
|
print(line, flush=True)
|
|
ev.write(line + "\n")
|
|
ev.flush()
|
|
|
|
def ledger(**row):
|
|
led.write(json.dumps(row) + "\n")
|
|
led.flush()
|
|
|
|
def worker(job, attempt):
|
|
with lock:
|
|
while state["in_flight"] >= state["limit"] or time.time() < state["cooldown_until"]:
|
|
lock.wait(1.0)
|
|
state["in_flight"] += 1
|
|
inflight, level = state["in_flight"], state["limit"]
|
|
log("DISPATCH %s attempt=%d in_flight=%d limit=%d" % (job["id"], attempt, inflight, level))
|
|
refs = [os.path.abspath(p) for p in spec.get("refs_common", []) + job.get("refs", [])]
|
|
t0 = time.time()
|
|
try:
|
|
code, png, body, headers = submit(job, refs, spec, env)
|
|
except Exception as e: # network errors count as retryable
|
|
code, png, body, headers = -1, None, str(e)[:200], {}
|
|
log("EXC %s %s" % (job["id"], body))
|
|
dt = time.time() - t0
|
|
with lock:
|
|
state["in_flight"] -= 1
|
|
nrefs = len(refs)
|
|
if code == 200 and png:
|
|
open(os.path.join(OUT, job["id"] + ".png"), "wb").write(png)
|
|
state["done"] += 1
|
|
state["succ_at_level"] += 1
|
|
state["max_inflight_clean"] = max(state["max_inflight_clean"], inflight)
|
|
ledger(id=job["id"], status=200, secs=round(dt, 1), in_flight=inflight, limit=level, attempt=attempt,
|
|
refs=nrefs, size=job.get("size"))
|
|
log("OK %s %.0fs" % (job["id"], dt))
|
|
if state["succ_at_level"] >= state["limit"] and state["limit"] < a.max:
|
|
state["limit"] = min(a.max, state["limit"] + 2)
|
|
state["succ_at_level"] = 0
|
|
log("RATCHET limit -> %d" % state["limit"])
|
|
lock.notify_all()
|
|
return True
|
|
ledger(id=job["id"], status=code, secs=round(dt, 1), in_flight=inflight, limit=level, attempt=attempt,
|
|
refs=nrefs, body=body)
|
|
if "insufficient_quota" in body:
|
|
log("QUOTA %s -- not retrying" % job["id"])
|
|
state["errors"] += 1
|
|
lock.notify_all()
|
|
return False
|
|
if code == 429 or code >= 500 or code == -1:
|
|
state["r429"] += 1 if code == 429 else 0
|
|
ra = headers.get("Retry-After") if headers else None
|
|
wait = float(ra) if ra and ra.replace(".", "").isdigit() else min(60, 5 * (2 ** (attempt - 1)))
|
|
state["limit"] = max(a.min, state["limit"] - 2)
|
|
state["succ_at_level"] = 0
|
|
state["cooldown_until"] = time.time() + wait
|
|
log("BACKOFF %s http=%s wait=%.0fs limit -> %d %s" % (job["id"], code, wait, state["limit"], body[:120]))
|
|
lock.notify_all()
|
|
return worker(job, attempt + 1) if attempt < 6 else False
|
|
state["errors"] += 1
|
|
log("FAIL %s http=%s %s" % (job["id"], code, body[:200]))
|
|
# one rephrase on a moderation refusal, never more; after that the lane reports it and stops
|
|
if attempt == 1 and job.get("prompt_alt") and re.search(r"safety|moderat|content_policy|rejected", body, re.I):
|
|
log("REPHRASE %s" % job["id"])
|
|
lock.notify_all()
|
|
return worker(dict(job, prompt=job["prompt_alt"]), 2)
|
|
lock.notify_all()
|
|
return False
|
|
|
|
jobs = spec["jobs"]
|
|
if a.skip_existing:
|
|
jobs = [j for j in jobs if not os.path.exists(os.path.join(OUT, j["id"] + ".png"))]
|
|
t_start = time.time()
|
|
threads = [threading.Thread(target=worker, args=(j, 1), daemon=True) for j in jobs]
|
|
for t in threads:
|
|
t.start()
|
|
for t in threads:
|
|
t.join()
|
|
el = max(time.time() - t_start, 1e-6)
|
|
summary = ("# batch calibration — %s\n\n- jobs: %d, succeeded: %d, hard failures: %d, 429s: %d\n"
|
|
"- start concurrency %d, ceiling reached: %d (cap %d); max in-flight without pushback: %d\n"
|
|
"- elapsed %.1f min, %.1f images/min; model %s, quality %s\n"
|
|
% (time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime()), len(jobs), state["done"], state["errors"],
|
|
state["r429"], a.start, state["limit"], a.max, state["max_inflight_clean"], el / 60,
|
|
state["done"] / (el / 60), spec["model"], spec.get("quality", "high")))
|
|
open(os.path.join(OUT, "calibration.md"), "w").write(summary)
|
|
log("DONE\n" + summary)
|
|
if a.done_file:
|
|
open(a.done_file, "w").write(time.strftime("%FT%TZ", time.gmtime()))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|