#!/usr/bin/env python3 """Gates for re-surfaced frames against their template plates. One JSON per batch + check sheets. Per frame __t.png in OUTDIR: GEOMETRY grad_corr Pearson correlation of gradient magnitude, frame vs plate (896x504, sigma 2). Structure kept. edge_rec share of plate Canny edges with a frame edge within 4 px. ecc_cc, scale, rot_deg, tx_px, ty_px affine ECC frame->plate on gradient magnitude. Reframing shows up as scale != 1 (zoom) or a shift of more than a few px. A take that zoomed 4 % fails here. landmark_grad_corr the same gradient correlation inside the landmark mask (the solids of the named groups), which is where the renderer invents palaces: a castle redrawn as a palace scored 0.16 while faithful takes scored 0.4-0.7 at 3 km. SKYLINE sky_s/sky_tx/sky_ty/sky_res similarity fit of the frame's sky/ground boundary to the plate's (landscape frames, where smooth surfaces make grad_corr run low by nature). invented_structures runs of columns where the frame skyline rises more than --bump px above the plate's outside any landmark: the "invented large building" gate for wide town views. FIGURES every proxy person visible in the plate (from _people.json and the oid buffer) drawn as a bracket on qc/_figs.jpg: foot line, true-height head line, and the +-25 % band. A dressed figure whose head is outside the band fails by eye; qc/_figzoom.jpg crops the 16 largest for the check. CORNERS four corners + top/bottom edge bands of every frame, contrast-boosted, on one sheet per position (qc/CORNERS--.jpg): the text/watermark/logo sweep. For relit frames (__t) the reference is the day keeper from --keepers JSON, not the plate. Thresholds are reported, not enforced: the numbers depend on the view (a textured facade at 20 m scores 0.5-0.7, a smooth landscape 0.15-0.3), so calibrate per view on a known-good take and a known-bad one (shift the plate 3 px, scale it 1.01) before trusting a cut-off. references/gates.md gives the measured bands. usage: gate.py OUTDIR --config citykit.json [--keepers keepers.json] [--landmark-groups castle,library] [ids...] """ import argparse import glob import json import os import re import cv2 import numpy as np from PIL import Image, ImageDraw, ImageFont from citykit import load_config, resolve try: FB = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 20) except OSError: FB = ImageFont.load_default() def view_of(fid): return re.sub(r"_[A-Za-z0-9]+_t\d+$", "", fid) def gmag(img): g = cv2.GaussianBlur(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY).astype(np.float32), (0, 0), 2) return np.hypot(cv2.Sobel(g, cv2.CV_32F, 1, 0), cv2.Sobel(g, cv2.CV_32F, 0, 1)) def edges(img): g = cv2.GaussianBlur(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), (0, 0), 1.5) v = np.median(g) return cv2.Canny(g, int(max(0, 0.66 * v)), int(min(255, 1.33 * v))) > 0 def drift(plate, frame): s = cv2.resize(plate, (896, 504), interpolation=cv2.INTER_AREA) o = cv2.resize(frame, (896, 504), interpolation=cv2.INTER_AREA) gs, go = gmag(s).ravel(), gmag(o).ravel() gc = float(np.corrcoef(gs, go)[0, 1]) if gs.std() > 1e-6 and go.std() > 1e-6 else 0.0 es, eo = edges(s), edges(o) k = np.ones((9, 9), np.uint8) rec = float((cv2.dilate(eo.astype(np.uint8), k)[es] > 0).mean()) if es.any() else 0.0 return dict(grad_corr=round(gc, 3), edge_rec=round(rec, 3)) def gm_norm(img): g = cv2.GaussianBlur(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY).astype(np.float32), (0, 0), 2.5) m = np.hypot(cv2.Sobel(g, cv2.CV_32F, 1, 0), cv2.Sobel(g, cv2.CV_32F, 0, 1)) return cv2.GaussianBlur(m / (m.mean() + 1e-6), (0, 0), 2.0) def ecc(plate, frame): a = gm_norm(cv2.resize(plate, (1024, 576), interpolation=cv2.INTER_AREA)) b = gm_norm(cv2.resize(frame, (1024, 576), interpolation=cv2.INTER_AREA)) w = np.eye(2, 3, dtype=np.float32) try: cc, w = cv2.findTransformECC(a.copy(), b.copy(), w, cv2.MOTION_AFFINE, (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 200, 1e-6), None, 5) except cv2.error: return dict(ecc_cc=None) sx = float(np.hypot(w[0, 0], w[1, 0])) sy = float(np.hypot(w[0, 1], w[1, 1])) W2 = w.copy() W2[:, 2] *= plate.shape[1] / 1024.0 return dict(_warp=W2.tolist(), ecc_cc=round(float(cc), 3), scale=round((sx + sy) / 2, 4), rot_deg=round(float(np.degrees(np.arctan2(w[1, 0], w[0, 0]))), 2), tx_px=round(float(w[0, 2]) * plate.shape[1] / 1024.0, 1), ty_px=round(float(w[1, 2]) * plate.shape[1] / 1024.0, 1)) def region_corr(plate, frame, mask): a = gmag(plate) b = gmag(frame) m = mask > 0 if m.sum() < 60 or a[m].std() < 1e-6 or b[m].std() < 1e-6: return None return round(float(np.corrcoef(a[m], b[m])[0, 1]), 3) def plate_skyline(cls): sky = cls == 0 first = np.argmax(~sky, axis=0).astype(float) first[sky.all(0)] = np.nan return first def frame_skyline(bgr): b, g, r = [bgr[..., i].astype(np.float32) for i in range(3)] lum = (b + g + r) / 3 sky = (b > r + 8) & (lum > 120) sky |= (lum > 205) & (np.abs(b - r) < 30) sky = cv2.morphologyEx(sky.astype(np.uint8), cv2.MORPH_OPEN, np.ones((5, 5), np.uint8)) > 0 H, W = sky.shape first = np.full(W, np.nan) for x in range(W): col = sky[:, x] k = np.argmin(col) if not col.all() else -1 if k > 3: first[x] = k return first def skyline_fit(ps, fs, cx, cy): xs = np.arange(len(ps)) best = (1e9, 1.0, 0.0, 0.0) for s in np.arange(0.92, 1.161, 0.004): for tx in np.arange(-150, 151, 6): xp = np.round(s * (xs - cx) + cx + tx).astype(int) ok = (xp >= 0) & (xp < len(fs)) & ~np.isnan(ps) if ok.sum() < 300: continue fy = fs[xp[ok]] py = s * (ps[ok] - cy) + cy m = ~np.isnan(fy) if m.sum() < 300: continue d = fy[m] - py[m] ty = np.median(d) res = np.median(np.abs(d - ty)) if res < best[0]: best = (res, s, tx, ty) res, s, tx, ty = best return dict(sky_s=round(float(s), 4), sky_tx=round(float(tx), 1), sky_ty=round(float(ty), 1), sky_res=round(float(res), 2)) def invented_structures(ps, fs, lm_mask, bump_px=18, min_run=12): """Columns where the frame's skyline is higher (smaller row) than the plate's by > bump_px, outside landmarks.""" W = len(ps) lm_cols = (lm_mask > 0).any(0) if lm_mask is not None else np.zeros(W, bool) lm_cols = cv2.dilate(lm_cols.astype(np.uint8)[None], np.ones((1, 15), np.uint8))[0] > 0 d = ps - fs up = np.nan_to_num(d, nan=0.0) > bump_px up &= ~lm_cols runs = [] x = 0 while x < W: if up[x]: x1 = x while x1 < W and up[x1]: x1 += 1 if x1 - x >= min_run: runs.append([int(x), int(x1), int(np.nanmax(d[x:x1]))]) x = x1 else: x += 1 return runs def boost(bgr): lab = cv2.cvtColor(bgr, cv2.COLOR_BGR2LAB) lab[..., 0] = cv2.createCLAHE(clipLimit=4.0, tileGridSize=(8, 8)).apply(lab[..., 0]) out = cv2.cvtColor(lab, cv2.COLOR_LAB2BGR) return cv2.addWeighted(out, 1.6, cv2.GaussianBlur(out, (0, 0), 3), -0.6, 0) def corners(frames, tag, qc): if not frames: return H, W = frames[0][1].shape[:2] cw, ch = W // 6, H // 6 regs = {"TL": (0, 0, cw, ch), "TR": (W - cw, 0, cw, ch), "BL": (0, H - ch, cw, ch), "BR": (W - cw, H - ch, cw, ch), "TOP": (W // 4, 0, W // 2, H // 16), "BOT": (W // 4, H - H // 16, W // 2, H // 16)} for nm, (x0, y0, CW, CH) in regs.items(): S = 1.4 if CW < W // 3 else 0.62 pw, ph = int(CW * S), int(CH * S) cols = 4 if CW < W // 3 else 2 rows = (len(frames) + cols - 1) // cols sh = Image.new("RGB", (cols * (pw + 2) + 4, rows * (ph + 4)), (0, 0, 0)) d = ImageDraw.Draw(sh) for i, (fid, im) in enumerate(frames): c = cv2.resize(boost(im[y0:y0 + CH, x0:x0 + CW]), (pw, ph), interpolation=cv2.INTER_CUBIC) xy = ((i % cols) * (pw + 2), (i // cols) * (ph + 4)) sh.paste(Image.fromarray(cv2.cvtColor(c, cv2.COLOR_BGR2RGB)), xy) d.text((xy[0] + 5, xy[1] + 3), fid, fill=(255, 255, 0), font=FB, stroke_width=3, stroke_fill=(0, 0, 0)) sh.save(os.path.join(qc, "CORNERS-%s-%s.jpg" % (tag, nm)), quality=90) def visible_people(people, oid, people_oids, W, H): out = [] for p in people: hp = p["px_height"] if hp < 4: continue x0, x1 = int(max(0, p["x"] - 0.3 * hp)), int(min(W, p["x"] + 0.3 * hp + 1)) y0, y1 = int(max(0, p["y_head"] - 2)), int(min(H, p["y_foot"] + 2)) if x1 <= x0 or y1 <= y0: continue own = np.isin(oid[y0:y1, x0:x1], list(people_oids)).sum() p = dict(p, own_px=int(own), vis=round(float(own / max(1.0, 0.22 * hp * hp)), 2)) if own >= 8 and p["vis"] >= 0.2: out.append(p) return out def fig_overlay(fid, frame, people, qc, warp=None): im = frame.copy() H, W = im.shape[:2] if warp is not None: Wp = np.array(warp) def tf(x, y): return Wp[0, 0] * x + Wp[0, 1] * y + Wp[0, 2], Wp[1, 0] * x + Wp[1, 1] * y + Wp[1, 2] pp = [] for p in people: x, yf = tf(p["x"], p["y_foot"]) _, yh = tf(p["x"], p["y_head"]) pp.append(dict(p, x=x, y_foot=yf, y_head=yh, px_height=yf - yh)) people = pp for p in people: x, yf, yh = p["x"], p["y_foot"], p["y_head"] hp = yf - yh if hp < 4: continue x, yf, yh = int(round(x)), int(round(yf)), int(round(yh)) lo, hi = int(round(yf - hp * 1.25)), int(round(yf - hp * 0.75)) c = (60, 255, 60) off = max(4, int(hp * 0.28)) cv2.line(im, (x + off, yf), (x + off, yh), c, 1) cv2.line(im, (x + off - 3, yf), (x + off + 4, yf), c, 1) cv2.line(im, (x + off - 3, yh), (x + off + 4, yh), c, 2) cv2.line(im, (x + off - 1, lo), (x + off + 3, lo), (0, 200, 255), 1) cv2.line(im, (x + off - 1, hi), (x + off + 3, hi), (0, 200, 255), 1) cv2.imwrite(os.path.join(qc, fid + "_figs.jpg"), im, [cv2.IMWRITE_JPEG_QUALITY, 90]) tiles = [] for p in sorted(people, key=lambda p: -p["px_height"])[:16]: hp = p["px_height"] x = int(p["x"]) yf = int(p["y_foot"]) w2 = int(max(40, hp * 1.2)) x0, x1 = max(0, x - w2), min(W, x + w2) y0, y1 = max(0, int(yf - hp * 1.7)), min(H, int(yf + hp * 0.3) + 4) if x1 - x0 < 16 or y1 - y0 < 16: continue a = im[y0:y1, x0:x1] s = min(6.0, 260 / (y1 - y0)) a = cv2.resize(a, None, fx=s, fy=s, interpolation=cv2.INTER_CUBIC) t = "%s %sm %.0fm" % (p["label"][:14], p["h_m"], p["dist_m"]) cv2.putText(a, t, (4, 16), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 3) cv2.putText(a, t, (4, 16), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 255), 1) tiles.append(a) if tiles: Hh = max(t.shape[0] for t in tiles) rows, row, wsum = [], [], 0 for t in tiles: t = np.vstack([t, np.zeros((Hh - t.shape[0], t.shape[1], 3), np.uint8)]) if wsum + t.shape[1] > 2400 and row: rows.append(np.hstack(row)) row, wsum = [], 0 row.append(t) wsum += t.shape[1] + 4 row.append(np.zeros((Hh, 4, 3), np.uint8)) if row: rows.append(np.hstack(row)) Wm = max(r.shape[1] for r in rows) rows = [np.hstack([r, np.zeros((r.shape[0], Wm - r.shape[1], 3), np.uint8)]) for r in rows] cv2.imwrite(os.path.join(qc, fid + "_figzoom.jpg"), np.vstack(rows), [cv2.IMWRITE_JPEG_QUALITY, 90]) def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("outdir") ap.add_argument("ids", nargs="*") ap.add_argument("--config", required=True) ap.add_argument("--keepers", default=None, help="JSON {view: keeper.png} for relit frames") ap.add_argument("--landmark-groups", default=None, help="comma list of solid groups; default from config.gates") ap.add_argument("--bump", type=int, default=18, help="skyline rise (px) that counts as an invented structure") ap.add_argument("--qc", default=None, help="output dir; default OUTDIR/qc") a = ap.parse_args() cfg = load_config(a.config) plates = resolve(cfg, cfg.get("plates_dir", "plates")) qc = a.qc or os.path.join(a.outdir, "qc") os.makedirs(qc, exist_ok=True) keepers = json.load(open(a.keepers)) if a.keepers else {} lm_groups = (a.landmark_groups or cfg.get("gates", {}).get("landmark_groups", "")).split(",") lm_groups = [g for g in lm_groups if g] ids = a.ids or sorted(os.path.splitext(os.path.basename(p))[0] for p in glob.glob(os.path.join(a.outdir, "*.png"))) import scene_solids as SC res = {} frames = [] for fid in ids: v = view_of(fid) is_relight = v in keepers ref_path = keepers[v] if is_relight else os.path.join(plates, v + "_mass.png") plate = cv2.imread(ref_path) frame = cv2.imread(os.path.join(a.outdir, fid + ".png")) if plate is None or frame is None: print("skip", fid, "(missing plate or frame)") continue H, W = plate.shape[:2] if frame.shape[:2] != (H, W): frame = cv2.resize(frame, (W, H), interpolation=cv2.INTER_LANCZOS4) r = drift(plate, frame) r.update(ecc(plate, frame)) oid_path = os.path.join(plates, v + "_oid.png") cls_path = os.path.join(plates, v + "_cls.png") oid = cv2.imread(oid_path, cv2.IMREAD_UNCHANGED) if os.path.exists(oid_path) else None lmk = None if oid is not None and lm_groups and not is_relight: # rebuild the scene to know which oids belong to the landmark groups VIEWS = json.load(open(resolve(cfg, cfg["views"]))) scene_path = resolve(cfg, VIEWS.get(v, {}).get("scene", cfg.get("scene"))) SC.reset() if scene_path and os.path.exists(scene_path): SC.build_from_json(json.load(open(scene_path))) lm_oids = set() for g in lm_groups: lm_oids |= {o + 1 for o in SC.GROUPS.get(g, [])} if lm_oids: lmk = np.isin(oid, list(lm_oids)).astype(np.uint8) lm = cv2.dilate(lmk, np.ones((5, 5), np.uint8)) r["landmark_grad_corr"] = region_corr(plate, frame, lm) ov = frame.copy() e = cv2.morphologyEx(lmk, cv2.MORPH_GRADIENT, np.ones((3, 3), np.uint8)) > 0 ov[e] = (255, 255, 0) cv2.imwrite(os.path.join(qc, fid + "_geo.jpg"), ov, [cv2.IMWRITE_JPEG_QUALITY, 90]) if os.path.exists(cls_path) and not is_relight: cls = np.asarray(Image.open(cls_path)) if (cls == 0).any(): ps = plate_skyline(cls) fs = frame_skyline(frame) if np.isfinite(ps).sum() > 300 and np.isfinite(fs).sum() > 300: r.update(skyline_fit(ps, fs, W / 2, H / 2)) runs = invented_structures(ps, fs, lmk, bump_px=a.bump) r["invented_structures"] = runs r["invented_structures_n"] = len(runs) ppl_path = os.path.join(plates, v + "_people.json") if oid is not None and os.path.exists(ppl_path): people = visible_people(json.load(open(ppl_path)), oid, set(o + 1 for o in SC.GROUPS.get("people", [])) if SC.GROUPS else set(), W, H) r["proxies_visible"] = len(people) r["proxies_px_max"] = max([p["px_height"] for p in people] or [0]) warp = r.get("_warp") if (r.get("ecc_cc") or 0) >= 0.55 and abs(r.get("scale", 1) - 1) < 0.08 else None fig_overlay(fid, frame, people, qc, warp) r.pop("_warp", None) res[fid] = r frames.append((fid, frame)) print(fid, {k: r.get(k) for k in ("grad_corr", "edge_rec", "landmark_grad_corr", "ecc_cc", "scale", "rot_deg", "tx_px", "ty_px", "sky_res", "invented_structures_n", "proxies_visible")}, flush=True) tag = os.path.basename(os.path.normpath(a.outdir)) corners(frames, tag, qc) json.dump(res, open(os.path.join(qc, "gate_%s.json" % tag), "w"), indent=1) print("wrote", os.path.join(qc, "gate_%s.json" % tag)) if __name__ == "__main__": main()