0.2.2: pin num_ctx on every forge-bound request and on the health probe
Ollama keeps one runner config per model and reloads the 64 GB model on every request whose num_ctx differs from the loaded runner's. The router now sets options.num_ctx = FORGEWAY_NUM_CTX (default 131072) on the Ollama dialect and carries the same value in the probe (native /api/chat; the OpenAI /v1 endpoint ignores options, verified). /status exposes the pin and the observed runner context and warns on mismatch; check 23 asserts it. Verified by behaviour: an 8192 caller, a 4096 caller, probes and 40k/100k- token retrieval prompts in one window -> 0 reloads, 0 timeouts, HA turns 0.3-0.6 s; before, 3 rounds of an 8192 caller = 7 loads and two probe timeouts. 23/23 acceptance on both clusters. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
297b1c421b
commit
2d3c548542
@@ -46,3 +46,8 @@ echo "19 ollama OPTIMISTIC chat, UNAVAIL .... HTTP $(j -XPOST $BAD/local-optimis
|
||||
echo "20 ollama OPTIMISTIC STREAM, UNAVAIL .. HTTP $(j -XPOST $BAD/local-optimistic/api/chat -H 'Content-Type: application/json' -d "$OSTREAM") backend=$(backend) $(ndjson_summary)"
|
||||
echo "21 ollama /api/tags, forge UNAVAIL .... HTTP $(j $BAD/local-optimistic/api/tags) (listing, not liveness)"
|
||||
echo "22 ollama /api/pull guard (expect 400) HTTP $(j -XPOST $GOOD/local-strict/api/pull -H 'Content-Type: application/json' -d '{"model":"llama3"}')"
|
||||
|
||||
# ---- one runner config per model ---------------------------------------------
|
||||
# Every forge-bound request carries the pinned num_ctx; the runner forge reports
|
||||
# must match it, or callers are thrashing the 64 GB model on every alternation.
|
||||
echo "23 num_ctx pin == runner context ....... HTTP $(j $GOOD/status) $(sed -n 's/.*"num_ctx_pin": *\([0-9null]*\).*"runner_context": *\([0-9null]*\).*/pin=\1 runner=\2/p' /tmp/b)"
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove by behaviour that forgeway keeps ONE Ollama runner config.
|
||||
|
||||
Drives, against a live router, the exact mix that used to thrash:
|
||||
* HA-style short turns (native /api/chat, options.num_ctx 8192.0, keep_alive -1.0)
|
||||
* an 8192-sending scripted caller
|
||||
* the health probe
|
||||
* large retrieval prompts through the OpenAI dialect (~40k and ~100k tokens),
|
||||
needle-in-a-haystack so the answer proves the whole prompt was read
|
||||
|
||||
and prints per-request latency. Pair it with forge's journal for the window:
|
||||
|
||||
journalctl -u ollama --since "<start>" | grep -oE "llama_context: n_ctx += *[0-9]+" | awk '{print $NF}'
|
||||
journalctl -u ollama --since "<start>" | grep -c "load_tensors: offloaded"
|
||||
|
||||
Healthy = one n_ctx value at most (a first load) and no further reloads.
|
||||
|
||||
verify_numctx_pin.py http://172.19.10.90 [--big 100000] [--mid 40000]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
WORDS = ("harbour granite lantern meadow copper falcon river timber saddle ember frost "
|
||||
"anchor beacon cinder willow marble quarry heather tundra vellum").split()
|
||||
|
||||
|
||||
def haystack(target_tokens: int, needle: str) -> str:
|
||||
"""~1.3 tokens per word for this vocabulary; needle buried at ~60 %."""
|
||||
rng = random.Random(target_tokens)
|
||||
words = int(target_tokens / 1.3)
|
||||
body = [rng.choice(WORDS) for _ in range(words)]
|
||||
body.insert(int(words * 0.6), f" The secret passphrase is {needle}. ")
|
||||
return " ".join(body)
|
||||
|
||||
|
||||
def post(url: str, body: dict, timeout: float = 900) -> tuple[int, dict, float]:
|
||||
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}, method="POST")
|
||||
t0 = time.perf_counter()
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
data = json.loads(r.read().decode())
|
||||
return r.status, data, time.perf_counter() - t0
|
||||
|
||||
|
||||
def ha_turn(base: str, mode: str, text: str) -> str:
|
||||
st, d, took = post(f"{base}/{mode}/api/chat", {
|
||||
"model": "gpt-oss:120b", "stream": False, "think": False, "keep_alive": -1.0,
|
||||
"options": {"num_ctx": 8192.0},
|
||||
"messages": [{"role": "system", "content": "You are a voice assistant. Answer tersely."},
|
||||
{"role": "user", "content": text}],
|
||||
})
|
||||
return f"{took:6.2f} s HTTP {st} {d.get('message', {}).get('content', '')[:40]!r}"
|
||||
|
||||
|
||||
def retrieval(base: str, tokens: int) -> str:
|
||||
needle = f"OTTER-{tokens // 1000}K"
|
||||
st, d, took = post(f"{base}/local-strict/v1/chat/completions", {
|
||||
"messages": [{"role": "user", "content": haystack(tokens, needle) + "\n\nWhat is the secret passphrase? Answer with the passphrase only."}],
|
||||
"max_tokens": 300,
|
||||
})
|
||||
ans = d.get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
usage = d.get("usage", {})
|
||||
ok = needle in ans
|
||||
return f"{took:6.2f} s HTTP {st} prompt_tokens={usage.get('prompt_tokens')} answer={ans.strip()[:30]!r} {'FOUND' if ok else 'MISSED'}"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("base")
|
||||
ap.add_argument("--mid", type=int, default=40000)
|
||||
ap.add_argument("--big", type=int, default=100000)
|
||||
a = ap.parse_args()
|
||||
b = a.base.rstrip("/")
|
||||
print(f"start {time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime())}Z")
|
||||
steps = [
|
||||
("HA-style turn (num_ctx 8192.0), strict", lambda: ha_turn(b, "local-strict", "Answer with the single word: PONG")),
|
||||
("health probe", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('reason')}")(*_get(f"{b}/healthz"))),
|
||||
(f"retrieval ~{a.mid} tokens, OpenAI dialect", lambda: retrieval(b, a.mid)),
|
||||
("HA-style turn (num_ctx 8192.0), optimistic", lambda: ha_turn(b, "local-optimistic", "Answer with the single word: PONG")),
|
||||
(f"retrieval ~{a.big} tokens, OpenAI dialect", lambda: retrieval(b, a.big)),
|
||||
("health probe", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('reason')}")(*_get(f"{b}/healthz"))),
|
||||
("HA-style turn (num_ctx 8192.0), strict", lambda: ha_turn(b, "local-strict", "Answer with the single word: PONG")),
|
||||
("scripted caller num_ctx 4096, generate", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('response', '')[:20]!r}")(
|
||||
*post(f"{b}/local-strict/api/generate", {"model": "gpt-oss:120b", "stream": False, "think": False, "options": {"num_ctx": 4096}, "prompt": "Answer with the single word: PONG"}))),
|
||||
("HA-style turn (num_ctx 8192.0), optimistic", lambda: ha_turn(b, "local-optimistic", "Answer with the single word: PONG")),
|
||||
]
|
||||
for label, fn in steps:
|
||||
try:
|
||||
print(f" {label:46s} {fn()}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" {label:46s} ERROR {type(exc).__name__}: {str(exc)[:80]}")
|
||||
time.sleep(6) # let the health cache expire so the next request re-probes
|
||||
print(f"end {time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime())}Z")
|
||||
return 0
|
||||
|
||||
|
||||
def _get(url: str) -> tuple[int, dict, float]:
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=120) as r:
|
||||
return r.status, json.loads(r.read().decode()), time.perf_counter() - t0
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, json.loads(e.read().decode() or "{}"), time.perf_counter() - t0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import urllib.error # noqa: E402
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user