Ollama keeps one runner config per model and reloads the 64 GB model on every request whose num_ctx differs from the loaded runner's. The router now sets options.num_ctx = FORGEWAY_NUM_CTX (default 131072) on the Ollama dialect and carries the same value in the probe (native /api/chat; the OpenAI /v1 endpoint ignores options, verified). /status exposes the pin and the observed runner context and warns on mismatch; check 23 asserts it. Verified by behaviour: an 8192 caller, a 4096 caller, probes and 40k/100k- token retrieval prompts in one window -> 0 reloads, 0 timeouts, HA turns 0.3-0.6 s; before, 3 rounds of an 8192 caller = 7 loads and two probe timeouts. 23/23 acceptance on both clusters. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
114 lines
5.3 KiB
Python
114 lines
5.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Prove by behaviour that forgeway keeps ONE Ollama runner config.
|
|
|
|
Drives, against a live router, the exact mix that used to thrash:
|
|
* HA-style short turns (native /api/chat, options.num_ctx 8192.0, keep_alive -1.0)
|
|
* an 8192-sending scripted caller
|
|
* the health probe
|
|
* large retrieval prompts through the OpenAI dialect (~40k and ~100k tokens),
|
|
needle-in-a-haystack so the answer proves the whole prompt was read
|
|
|
|
and prints per-request latency. Pair it with forge's journal for the window:
|
|
|
|
journalctl -u ollama --since "<start>" | grep -oE "llama_context: n_ctx += *[0-9]+" | awk '{print $NF}'
|
|
journalctl -u ollama --since "<start>" | grep -c "load_tensors: offloaded"
|
|
|
|
Healthy = one n_ctx value at most (a first load) and no further reloads.
|
|
|
|
verify_numctx_pin.py http://172.19.10.90 [--big 100000] [--mid 40000]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import random
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
WORDS = ("harbour granite lantern meadow copper falcon river timber saddle ember frost "
|
|
"anchor beacon cinder willow marble quarry heather tundra vellum").split()
|
|
|
|
|
|
def haystack(target_tokens: int, needle: str) -> str:
|
|
"""~1.3 tokens per word for this vocabulary; needle buried at ~60 %."""
|
|
rng = random.Random(target_tokens)
|
|
words = int(target_tokens / 1.3)
|
|
body = [rng.choice(WORDS) for _ in range(words)]
|
|
body.insert(int(words * 0.6), f" The secret passphrase is {needle}. ")
|
|
return " ".join(body)
|
|
|
|
|
|
def post(url: str, body: dict, timeout: float = 900) -> tuple[int, dict, float]:
|
|
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}, method="POST")
|
|
t0 = time.perf_counter()
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
data = json.loads(r.read().decode())
|
|
return r.status, data, time.perf_counter() - t0
|
|
|
|
|
|
def ha_turn(base: str, mode: str, text: str) -> str:
|
|
st, d, took = post(f"{base}/{mode}/api/chat", {
|
|
"model": "gpt-oss:120b", "stream": False, "think": False, "keep_alive": -1.0,
|
|
"options": {"num_ctx": 8192.0},
|
|
"messages": [{"role": "system", "content": "You are a voice assistant. Answer tersely."},
|
|
{"role": "user", "content": text}],
|
|
})
|
|
return f"{took:6.2f} s HTTP {st} {d.get('message', {}).get('content', '')[:40]!r}"
|
|
|
|
|
|
def retrieval(base: str, tokens: int) -> str:
|
|
needle = f"OTTER-{tokens // 1000}K"
|
|
st, d, took = post(f"{base}/local-strict/v1/chat/completions", {
|
|
"messages": [{"role": "user", "content": haystack(tokens, needle) + "\n\nWhat is the secret passphrase? Answer with the passphrase only."}],
|
|
"max_tokens": 300,
|
|
})
|
|
ans = d.get("choices", [{}])[0].get("message", {}).get("content", "")
|
|
usage = d.get("usage", {})
|
|
ok = needle in ans
|
|
return f"{took:6.2f} s HTTP {st} prompt_tokens={usage.get('prompt_tokens')} answer={ans.strip()[:30]!r} {'FOUND' if ok else 'MISSED'}"
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("base")
|
|
ap.add_argument("--mid", type=int, default=40000)
|
|
ap.add_argument("--big", type=int, default=100000)
|
|
a = ap.parse_args()
|
|
b = a.base.rstrip("/")
|
|
print(f"start {time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime())}Z")
|
|
steps = [
|
|
("HA-style turn (num_ctx 8192.0), strict", lambda: ha_turn(b, "local-strict", "Answer with the single word: PONG")),
|
|
("health probe", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('reason')}")(*_get(f"{b}/healthz"))),
|
|
(f"retrieval ~{a.mid} tokens, OpenAI dialect", lambda: retrieval(b, a.mid)),
|
|
("HA-style turn (num_ctx 8192.0), optimistic", lambda: ha_turn(b, "local-optimistic", "Answer with the single word: PONG")),
|
|
(f"retrieval ~{a.big} tokens, OpenAI dialect", lambda: retrieval(b, a.big)),
|
|
("health probe", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('reason')}")(*_get(f"{b}/healthz"))),
|
|
("HA-style turn (num_ctx 8192.0), strict", lambda: ha_turn(b, "local-strict", "Answer with the single word: PONG")),
|
|
("scripted caller num_ctx 4096, generate", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('response', '')[:20]!r}")(
|
|
*post(f"{b}/local-strict/api/generate", {"model": "gpt-oss:120b", "stream": False, "think": False, "options": {"num_ctx": 4096}, "prompt": "Answer with the single word: PONG"}))),
|
|
("HA-style turn (num_ctx 8192.0), optimistic", lambda: ha_turn(b, "local-optimistic", "Answer with the single word: PONG")),
|
|
]
|
|
for label, fn in steps:
|
|
try:
|
|
print(f" {label:46s} {fn()}")
|
|
except Exception as exc: # noqa: BLE001
|
|
print(f" {label:46s} ERROR {type(exc).__name__}: {str(exc)[:80]}")
|
|
time.sleep(6) # let the health cache expire so the next request re-probes
|
|
print(f"end {time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime())}Z")
|
|
return 0
|
|
|
|
|
|
def _get(url: str) -> tuple[int, dict, float]:
|
|
t0 = time.perf_counter()
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=120) as r:
|
|
return r.status, json.loads(r.read().decode()), time.perf_counter() - t0
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, json.loads(e.read().decode() or "{}"), time.perf_counter() - t0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import urllib.error # noqa: E402
|
|
sys.exit(main())
|