#!/usr/bin/env python3 """Prove by behaviour that forgeway keeps ONE Ollama runner config. Drives, against a live router, the exact mix that used to thrash: * HA-style short turns (native /api/chat, options.num_ctx 8192.0, keep_alive -1.0) * an 8192-sending scripted caller * the health probe * large retrieval prompts through the OpenAI dialect (~40k and ~100k tokens), needle-in-a-haystack so the answer proves the whole prompt was read and prints per-request latency. Pair it with forge's journal for the window: journalctl -u ollama --since "" | grep -oE "llama_context: n_ctx += *[0-9]+" | awk '{print $NF}' journalctl -u ollama --since "" | grep -c "load_tensors: offloaded" Healthy = one n_ctx value at most (a first load) and no further reloads. verify_numctx_pin.py http://172.19.10.90 [--big 100000] [--mid 40000] """ from __future__ import annotations import argparse import json import random import sys import time import urllib.request WORDS = ("harbour granite lantern meadow copper falcon river timber saddle ember frost " "anchor beacon cinder willow marble quarry heather tundra vellum").split() def haystack(target_tokens: int, needle: str) -> str: """~1.3 tokens per word for this vocabulary; needle buried at ~60 %.""" rng = random.Random(target_tokens) words = int(target_tokens / 1.3) body = [rng.choice(WORDS) for _ in range(words)] body.insert(int(words * 0.6), f" The secret passphrase is {needle}. ") return " ".join(body) def post(url: str, body: dict, timeout: float = 900) -> tuple[int, dict, float]: req = urllib.request.Request(url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}, method="POST") t0 = time.perf_counter() with urllib.request.urlopen(req, timeout=timeout) as r: data = json.loads(r.read().decode()) return r.status, data, time.perf_counter() - t0 def ha_turn(base: str, mode: str, text: str) -> str: st, d, took = post(f"{base}/{mode}/api/chat", { "model": "gpt-oss:120b", "stream": False, "think": False, "keep_alive": -1.0, "options": {"num_ctx": 8192.0}, "messages": [{"role": "system", "content": "You are a voice assistant. Answer tersely."}, {"role": "user", "content": text}], }) return f"{took:6.2f} s HTTP {st} {d.get('message', {}).get('content', '')[:40]!r}" def retrieval(base: str, tokens: int) -> str: needle = f"OTTER-{tokens // 1000}K" st, d, took = post(f"{base}/local-strict/v1/chat/completions", { "messages": [{"role": "user", "content": haystack(tokens, needle) + "\n\nWhat is the secret passphrase? Answer with the passphrase only."}], "max_tokens": 300, }) ans = d.get("choices", [{}])[0].get("message", {}).get("content", "") usage = d.get("usage", {}) ok = needle in ans return f"{took:6.2f} s HTTP {st} prompt_tokens={usage.get('prompt_tokens')} answer={ans.strip()[:30]!r} {'FOUND' if ok else 'MISSED'}" def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("base") ap.add_argument("--mid", type=int, default=40000) ap.add_argument("--big", type=int, default=100000) a = ap.parse_args() b = a.base.rstrip("/") print(f"start {time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime())}Z") steps = [ ("HA-style turn (num_ctx 8192.0), strict", lambda: ha_turn(b, "local-strict", "Answer with the single word: PONG")), ("health probe", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('reason')}")(*_get(f"{b}/healthz"))), (f"retrieval ~{a.mid} tokens, OpenAI dialect", lambda: retrieval(b, a.mid)), ("HA-style turn (num_ctx 8192.0), optimistic", lambda: ha_turn(b, "local-optimistic", "Answer with the single word: PONG")), (f"retrieval ~{a.big} tokens, OpenAI dialect", lambda: retrieval(b, a.big)), ("health probe", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('reason')}")(*_get(f"{b}/healthz"))), ("HA-style turn (num_ctx 8192.0), strict", lambda: ha_turn(b, "local-strict", "Answer with the single word: PONG")), ("scripted caller num_ctx 4096, generate", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('response', '')[:20]!r}")( *post(f"{b}/local-strict/api/generate", {"model": "gpt-oss:120b", "stream": False, "think": False, "options": {"num_ctx": 4096}, "prompt": "Answer with the single word: PONG"}))), ("HA-style turn (num_ctx 8192.0), optimistic", lambda: ha_turn(b, "local-optimistic", "Answer with the single word: PONG")), ] for label, fn in steps: try: print(f" {label:46s} {fn()}") except Exception as exc: # noqa: BLE001 print(f" {label:46s} ERROR {type(exc).__name__}: {str(exc)[:80]}") time.sleep(6) # let the health cache expire so the next request re-probes print(f"end {time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime())}Z") return 0 def _get(url: str) -> tuple[int, dict, float]: t0 = time.perf_counter() try: with urllib.request.urlopen(url, timeout=120) as r: return r.status, json.loads(r.read().decode()), time.perf_counter() - t0 except urllib.error.HTTPError as e: return e.code, json.loads(e.read().decode() or "{}"), time.perf_counter() - t0 if __name__ == "__main__": import urllib.error # noqa: E402 sys.exit(main())