Files
forgeway/scripts/verify_numctx_pin.py
Dolon (Agent)andClaude Fable 5.1 2d3c548542 0.2.2: pin num_ctx on every forge-bound request and on the health probe
Ollama keeps one runner config per model and reloads the 64 GB model on
every request whose num_ctx differs from the loaded runner's. The router
now sets options.num_ctx = FORGEWAY_NUM_CTX (default 131072) on the Ollama
dialect and carries the same value in the probe (native /api/chat; the
OpenAI /v1 endpoint ignores options, verified). /status exposes the pin and
the observed runner context and warns on mismatch; check 23 asserts it.

Verified by behaviour: an 8192 caller, a 4096 caller, probes and 40k/100k-
token retrieval prompts in one window -> 0 reloads, 0 timeouts, HA turns
0.3-0.6 s; before, 3 rounds of an 8192 caller = 7 loads and two probe
timeouts. 23/23 acceptance on both clusters.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-18 19:43:13 +00:00

114 lines
5.3 KiB
Python

#!/usr/bin/env python3
"""Prove by behaviour that forgeway keeps ONE Ollama runner config.
Drives, against a live router, the exact mix that used to thrash:
* HA-style short turns (native /api/chat, options.num_ctx 8192.0, keep_alive -1.0)
* an 8192-sending scripted caller
* the health probe
* large retrieval prompts through the OpenAI dialect (~40k and ~100k tokens),
needle-in-a-haystack so the answer proves the whole prompt was read
and prints per-request latency. Pair it with forge's journal for the window:
journalctl -u ollama --since "<start>" | grep -oE "llama_context: n_ctx += *[0-9]+" | awk '{print $NF}'
journalctl -u ollama --since "<start>" | grep -c "load_tensors: offloaded"
Healthy = one n_ctx value at most (a first load) and no further reloads.
verify_numctx_pin.py http://172.19.10.90 [--big 100000] [--mid 40000]
"""
from __future__ import annotations
import argparse
import json
import random
import sys
import time
import urllib.request
WORDS = ("harbour granite lantern meadow copper falcon river timber saddle ember frost "
"anchor beacon cinder willow marble quarry heather tundra vellum").split()
def haystack(target_tokens: int, needle: str) -> str:
"""~1.3 tokens per word for this vocabulary; needle buried at ~60 %."""
rng = random.Random(target_tokens)
words = int(target_tokens / 1.3)
body = [rng.choice(WORDS) for _ in range(words)]
body.insert(int(words * 0.6), f" The secret passphrase is {needle}. ")
return " ".join(body)
def post(url: str, body: dict, timeout: float = 900) -> tuple[int, dict, float]:
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}, method="POST")
t0 = time.perf_counter()
with urllib.request.urlopen(req, timeout=timeout) as r:
data = json.loads(r.read().decode())
return r.status, data, time.perf_counter() - t0
def ha_turn(base: str, mode: str, text: str) -> str:
st, d, took = post(f"{base}/{mode}/api/chat", {
"model": "gpt-oss:120b", "stream": False, "think": False, "keep_alive": -1.0,
"options": {"num_ctx": 8192.0},
"messages": [{"role": "system", "content": "You are a voice assistant. Answer tersely."},
{"role": "user", "content": text}],
})
return f"{took:6.2f} s HTTP {st} {d.get('message', {}).get('content', '')[:40]!r}"
def retrieval(base: str, tokens: int) -> str:
needle = f"OTTER-{tokens // 1000}K"
st, d, took = post(f"{base}/local-strict/v1/chat/completions", {
"messages": [{"role": "user", "content": haystack(tokens, needle) + "\n\nWhat is the secret passphrase? Answer with the passphrase only."}],
"max_tokens": 300,
})
ans = d.get("choices", [{}])[0].get("message", {}).get("content", "")
usage = d.get("usage", {})
ok = needle in ans
return f"{took:6.2f} s HTTP {st} prompt_tokens={usage.get('prompt_tokens')} answer={ans.strip()[:30]!r} {'FOUND' if ok else 'MISSED'}"
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("base")
ap.add_argument("--mid", type=int, default=40000)
ap.add_argument("--big", type=int, default=100000)
a = ap.parse_args()
b = a.base.rstrip("/")
print(f"start {time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime())}Z")
steps = [
("HA-style turn (num_ctx 8192.0), strict", lambda: ha_turn(b, "local-strict", "Answer with the single word: PONG")),
("health probe", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('reason')}")(*_get(f"{b}/healthz"))),
(f"retrieval ~{a.mid} tokens, OpenAI dialect", lambda: retrieval(b, a.mid)),
("HA-style turn (num_ctx 8192.0), optimistic", lambda: ha_turn(b, "local-optimistic", "Answer with the single word: PONG")),
(f"retrieval ~{a.big} tokens, OpenAI dialect", lambda: retrieval(b, a.big)),
("health probe", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('reason')}")(*_get(f"{b}/healthz"))),
("HA-style turn (num_ctx 8192.0), strict", lambda: ha_turn(b, "local-strict", "Answer with the single word: PONG")),
("scripted caller num_ctx 4096, generate", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('response', '')[:20]!r}")(
*post(f"{b}/local-strict/api/generate", {"model": "gpt-oss:120b", "stream": False, "think": False, "options": {"num_ctx": 4096}, "prompt": "Answer with the single word: PONG"}))),
("HA-style turn (num_ctx 8192.0), optimistic", lambda: ha_turn(b, "local-optimistic", "Answer with the single word: PONG")),
]
for label, fn in steps:
try:
print(f" {label:46s} {fn()}")
except Exception as exc: # noqa: BLE001
print(f" {label:46s} ERROR {type(exc).__name__}: {str(exc)[:80]}")
time.sleep(6) # let the health cache expire so the next request re-probes
print(f"end {time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime())}Z")
return 0
def _get(url: str) -> tuple[int, dict, float]:
t0 = time.perf_counter()
try:
with urllib.request.urlopen(url, timeout=120) as r:
return r.status, json.loads(r.read().decode()), time.perf_counter() - t0
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read().decode() or "{}"), time.perf_counter() - t0
if __name__ == "__main__":
import urllib.error # noqa: E402
sys.exit(main())