From 2d3c548542ce15f4ff3f8e4f755c5f1373da3dcd Mon Sep 17 00:00:00 2001 From: "Dolon (Agent)" Date: Fri, 18 Sep 2026 19:43:13 +0000 Subject: [PATCH] 0.2.2: pin num_ctx on every forge-bound request and on the health probe Ollama keeps one runner config per model and reloads the 64 GB model on every request whose num_ctx differs from the loaded runner's. The router now sets options.num_ctx = FORGEWAY_NUM_CTX (default 131072) on the Ollama dialect and carries the same value in the probe (native /api/chat; the OpenAI /v1 endpoint ignores options, verified). /status exposes the pin and the observed runner context and warns on mismatch; check 23 asserts it. Verified by behaviour: an 8192 caller, a 4096 caller, probes and 40k/100k- token retrieval prompts in one window -> 0 reloads, 0 timeouts, HA turns 0.3-0.6 s; before, 3 rounds of an 8192 caller = 7 loads and two probe timeouts. 23/23 acceptance on both clusters. Co-Authored-By: Claude Fable 5.1 --- README.md | 41 ++++++++++++- REPORT.md | 82 +++++++++++++++++++++++++ app/config.py | 11 ++++ app/main.py | 4 ++ app/ollama_api.py | 9 +++ app/upstream.py | 62 +++++++++++++++---- scripts/checks.sh | 5 ++ scripts/verify_numctx_pin.py | 113 +++++++++++++++++++++++++++++++++++ 8 files changed, 315 insertions(+), 12 deletions(-) create mode 100644 scripts/verify_numctx_pin.py diff --git a/README.md b/README.md index 0826c1a..fbe5836 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,44 @@ so entry titles carry the URL and mode, and the pre-existing optimistic subentry reads *Forge Conversation* in the integration's subentry list (rename it in the UI if it bothers you; the agent name users see is the device/entity name above). +## One runner config per model: the `num_ctx` pin + +Ollama keeps **one loaded runner configuration per model**, keyed on the request's +options. If one caller sends `options.num_ctx 8192` and the next sends nothing (→ the +model default, 131072 for `gpt-oss:120b`), Ollama evicts and fully reloads the 64 GB +model — 15–30 s — on *every* alternation. On 2026-09-18 that was Home Assistant (8192 +by default) against the router's own health probes (default): four reloads in two +minutes, 33–66 s per HA turn, and one silent cloud fallback on `local-optimistic` +because a probe queued behind a reload exceeded its 45 s limit. + +**forgeway therefore pins `num_ctx` on every forge-bound request** (`FORGEWAY_NUM_CTX`, +default **131072**), overriding whatever the caller sent, and the health probe carries +the same value via Ollama's native `/api/chat`. A caller's own `num_ctx` cannot +re-introduce the thrash; the router logs `pinning num_ctx 8192.0 -> 131072` when it +overrides one. `/status` reports `num_ctx_pin` and `runner_context` (what forge's +loaded runner was last seen with) and the router warns if they ever differ. + +Why 131072 and not something smaller: it is the model's trained window and exactly +what an unpinned request loads anyway, so it costs no extra VRAM (KV ≈ 4.6 GB inside a +61 GB resident model on a 97 GB card); the retrieval workload that motivated the box +sends 40k–100k-token prompts; and a large *allocated* context does not slow short +turns — with the pin, HA turns measure 0.3–0.6 s end to end and a 1-token probe 60–100 ms. + +One caveat, verified empirically: **Ollama's OpenAI-compatible `/v1` endpoint ignores +`options.num_ctx`**, so OpenAI-dialect callers (OpenClaw, scripts) always get the model +default. The pin must equal that default — it does today — and the `runner_context` +warning is the tripwire if an Ollama upgrade or an `OLLAMA_CONTEXT_LENGTH` setting on +forge ever changes it. Diagnose on forge in one line: + +```bash +journalctl -u ollama --since "-20 min" | grep -oE "llama_context: n_ctx += *[0-9]+" | awk '{print $NF}' | tr '\n' ' ' +# healthy: nothing, or a single value. thrash: 8192 131072 8192 131072 … +``` + +`scripts/verify_numctx_pin.py ` drives the exact mix that used to thrash (HA-style +turns, off-value scripted callers, probes, 40k and 100k-token needle-in-a-haystack +prompts) and prints latencies; pair it with the one-liner above for the window. + ## The Ollama dialect, and the cloud fallback through it forge *is* Ollama, so on the forge path the router is a raw proxy: the response — @@ -361,6 +399,7 @@ Everything is environment-driven; one image serves every site. See | `FORGEWAY_HEALTH_TIMEOUT` | `45` | must exceed a ~25–30 s cold load | | `FORGEWAY_DISCOVERY_TIMEOUT` | `4` | `6` at Serenity — the mesh hop | | `FORGEWAY_HEALTH_COLD_LOAD` | `true` | see the training conflict above | +| `FORGEWAY_NUM_CTX` | `131072` | pinned on every forge-bound request and the probe; `0` disables | | `FORGEWAY_REQUEST_TIMEOUT` | `600` | proxied completions | | `OPENAI_API_KEY` | — | from the `forgeway-cloud` Secret, applied out of band | @@ -390,6 +429,6 @@ Then bump the image tag in each cluster repo and push — ArgoCD syncs: ./scripts/verify-cluster.sh ~/.kube/nidavellir.yaml serenity ``` -`verify-cluster.sh` runs 22 checks from inside the cluster (13 OpenAI, 9 Ollama). It produces the +`verify-cluster.sh` runs 23 checks from inside the cluster (13 OpenAI, 9 Ollama, 1 pin-vs-runner). It produces the "forge unavailable" case with a throwaway Deployment pointed at a dead upstream — **it never stops Ollama on forge.** diff --git a/REPORT.md b/REPORT.md index 833c2f8..5ef0582 100644 --- a/REPORT.md +++ b/REPORT.md @@ -549,3 +549,85 @@ history by message count). Router-only, one config knob, a 0.2.2 image. The alte is setting `num_ctx` to 131072 in each HA subentry; it has the same effect but has to be repeated per agent and is invisible in git. I stopped short of either because it overrides a user-visible HA setting and the main session is tracking the thrash. + +--- + +# Addendum 6, 2026-09-18 — `num_ctx` pinned at the router (0.2.2), verified by behaviour + +**Done and shipped to both clusters.** Every forge-bound request from forgeway now carries +`options.num_ctx = 131072` (`FORGEWAY_NUM_CTX`), overriding the caller's value, and the +health probe carries the same value via Ollama's native `/api/chat`. `/status` exposes +`num_ctx_pin` and `runner_context` and the router warns if they differ. Vikunja card +#190 (), moved to Done. + +## Value: 131072, and why it is not a compromise + +| Consideration | Finding | +|---|---| +| Retrieval workload | needs 40k–100k-token prompts; gpt-oss's trained window is 131072 (`gptoss.context_length`) | +| VRAM | an *unpinned* request already loads at 131072 (forge has no `OLLAMA_CONTEXT_LENGTH`; Ollama 0.34 uses the model's trained length), so the pin adds nothing: runner 61,466 MiB resident, 66.7 GB used of 97.9 GB on card 1 | +| Short turns | a large *allocated* context does not slow them: 1-token probes 60–100 ms, HA turns 0.3–0.6 s end to end, scripted short turns 0.5 s | +| OpenAI dialect | Ollama's `/v1` endpoint **ignores `options.num_ctx`** (tested on qwen2.5vl: asked 6144, loaded 128000; native `/api/chat` asked 5120, loaded 5120), so OpenAI callers get the model default — the pin equals it, and `runner_context` is the tripwire | + +One value serves both workloads; no alternative needed. + +## Before (0.2.1, 19:29–19:33 UTC) — an 8192-sending caller alternating with the probe + +Forge's journal for three rounds: `n_ctx` sequence **`8192 131072 8192 131072 8192 131072`**, +**7 model loads**, and forge's own request log: + +``` +19:30:09 200 32.6 s /api/chat (8192 caller: reload + answer) +19:30:30 499 47.0 s /v1/chat/completions (probe timed out during the reload back) +19:30:33 499 46.9 s /v1/chat/completions (Serenity's probe, same) +19:31:33 200 27.9 s /api/chat +19:32:41 200 28.8 s /api/chat +19:33:16 200 33.2 s /v1/chat/completions (probe) +``` + +Every alternation cost a reload; two probes hit the 45 s limit — the spurious +"unavailable" that produced the 19:10:53 cloud fallback. (HA itself was already on +131072 since the main session's 19:21 subentry fix; its turns measured 0.3–0.6 s +before and after — the router pin makes that independent of what any caller sends.) + +## After (0.2.2, 19:36:59–19:38:28 UTC) — the mix that used to thrash, one window + +``` +HA-style turn (num_ctx 8192.0), strict 0.50 s +health probe 0.13 s +retrieval ~40k tokens, OpenAI dialect 7.61 s prompt_tokens=38484 needle FOUND +HA-style turn (num_ctx 8192.0), optimistic 2.03 s +retrieval ~100k tokens, OpenAI dialect 23.01 s prompt_tokens=96175 needle FOUND +health probe 0.01 s +HA-style turn (num_ctx 8192.0), strict 0.50 s +scripted caller num_ctx 4096, generate 0.58 s +HA-style turn (num_ctx 8192.0), optimistic 1.08 s +real Home Assistant turns ×5 0.6 / 0.5 / 0.4 / 0.3 / 0.4 s +``` + +Forge's journal for that window: **`n_ctx` (re)loads: none; model loads: 0; timeouts: 0**; +`ollama ps`: `gpt-oss:120b`, context 131072, 61,466 MiB. The model was never reloaded +across 8192-, 4096-, default- and 100k-token callers. (Traffic from `127.0.0.1` on forge in +the same window is the engine-evaluation lane, not mine; left alone.) + +| | before | after | +|---|---|---| +| reloads per 8192↔default alternation | 1 (15–30 s) | 0 | +| turn with an off-value `num_ctx` | 27–33 s | 0.5 s | +| probe during contention | 27–47 s, two timeouts | 0.01–0.13 s | +| HA turn end to end | 0.3–0.6 s (already fixed on HA's side) | 0.3–0.6 s, now caller-independent | + +## Acceptance + +23/23 on Lounge and 23/23 on Serenity (13 OpenAI unchanged, 9 Ollama, plus check 23: +`num_ctx_pin == runner_context`). Both instances report `pin=131072 runner=131072`. +Ollama on forge was not stopped; ComfyUI on card 0 untouched (559 MiB, idle). + +## Follow-up (not done, from the main session's note) + +The hot-path gate still runs a real completion before every proxied request (cached 5 s). +With the pin it no longer thrashes, but a probe that needs a cold load still cannot fit +in 45 s under contention and fails the optimistic request over to the cloud. The cleaner +design — treat "model resident per `/api/ps`" as available on the hot path and keep the +real completion for `/healthz` only — is a semantic change to what "available" means and +stays Rick's call. diff --git a/app/config.py b/app/config.py index 145af3f..9027010 100644 --- a/app/config.py +++ b/app/config.py @@ -58,6 +58,17 @@ class Config: ) ) local_model: str = field(default_factory=lambda: _env("FORGEWAY_LOCAL_MODEL", "gpt-oss:120b")) + # Ollama keeps ONE runner config per model and fully reloads the model (64 GB, + # 15-30 s) whenever a request's num_ctx differs from the loaded runner's. So + # every forge-bound request from here carries the same num_ctx, overriding + # whatever the caller sent (HA defaults to 8192 on a new subentry). 131072 is + # gpt-oss:120b's trained window and what an unpinned request loads anyway, so + # it costs no extra VRAM; the retrieval workload needs ~100k tokens regardless. + # 0 disables the pin. NOTE: Ollama's OpenAI-compatible /v1 endpoint ignores + # options.num_ctx (verified 2026-09-18), so OpenAI-dialect requests get the + # model default; the pin must equal that default, and /status + a WARNING + # flag it if the runner's context is ever observed to differ. + num_ctx: int = field(default_factory=lambda: _env_int("FORGEWAY_NUM_CTX", 131072)) # --- cloud fallback (local-optimistic only) ------------------------- cloud_base_url: str = field( diff --git a/app/main.py b/app/main.py index 691a5bc..04c44fa 100644 --- a/app/main.py +++ b/app/main.py @@ -231,6 +231,10 @@ async def status() -> JSONResponse: "cloud_fallback_enabled": CONFIG.cloud_enabled, "discovery_candidates": CONFIG.upstreams, "discovered_upstream": app.state.locator.sticky, + # one runner config per model: the pin every forge-bound request carries, + # and the context the loaded runner was last seen with (must match) + "num_ctx_pin": CONFIG.num_ctx or None, + "runner_context": app.state.locator.runner_context, "health": health.as_dict(), } ) diff --git a/app/ollama_api.py b/app/ollama_api.py index d0d66ee..2372a29 100644 --- a/app/ollama_api.py +++ b/app/ollama_api.py @@ -390,6 +390,15 @@ async def _route(request: Request, mode: str, path: str, body: dict[str, Any], s if health.healthy and health.upstream: payload = dict(body) payload["model"] = CONFIG.local_model + if CONFIG.num_ctx > 0: + # One runner config for everyone: a caller's own num_ctx (HA sends + # 8192.0 by default) would otherwise force a full model reload. + opts = dict(payload.get("options") or {}) + sent = opts.get("num_ctx") + if sent is not None and sent != CONFIG.num_ctx: + log.info("pinning num_ctx %s -> %s on %s (one runner config, no reload)", sent, CONFIG.num_ctx, path) + opts["num_ctx"] = CONFIG.num_ctx + payload["options"] = opts try: return await _forge_raw(request, health.upstream, path, payload, mode, stream) except UpstreamError as exc: diff --git a/app/upstream.py b/app/upstream.py index ac221d1..b33944b 100644 --- a/app/upstream.py +++ b/app/upstream.py @@ -66,6 +66,7 @@ class ForgeLocator: self._sticky: str | None = None self._health: Health | None = None self._lock = asyncio.Lock() + self.runner_context: int | None = None # last observed loaded context on forge @property def sticky(self) -> str | None: @@ -189,20 +190,36 @@ class ForgeLocator: return None async def _completion(self, base: str) -> dict: - """One real, minimal completion. Not a ping — that is the whole point.""" - payload = { + """One real, minimal completion. Not a ping — that is the whole point. + + Uses Ollama's native /api/chat so the probe can carry the pinned num_ctx: + the OpenAI-compatible /v1 endpoint ignores `options`, and an unpinned + probe was half of the runner-reload thrash. Falls back to /v1 for a + non-Ollama upstream (404 on /api/chat). + """ + options: dict = {"num_predict": 1} + if CONFIG.num_ctx > 0: + options["num_ctx"] = CONFIG.num_ctx + native = { "model": CONFIG.local_model, "messages": [{"role": "user", "content": "ping"}], - "max_tokens": 1, - "temperature": 0, "stream": False, + "options": options, } try: - resp = await self._client.post( - f"{base}/v1/chat/completions", - json=payload, - timeout=CONFIG.health_timeout, - ) + resp = await self._client.post(f"{base}/api/chat", json=native, timeout=CONFIG.health_timeout) + if resp.status_code == 404: + resp = await self._client.post( + f"{base}/v1/chat/completions", + json={ + "model": CONFIG.local_model, + "messages": [{"role": "user", "content": "ping"}], + "max_tokens": 1, + "temperature": 0, + "stream": False, + }, + timeout=CONFIG.health_timeout, + ) except httpx.HTTPError as exc: raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc @@ -216,6 +233,29 @@ class ForgeLocator: data = resp.json() except ValueError as exc: raise UpstreamError("upstream returned non-JSON") from exc - if not data.get("choices"): - raise UpstreamError("upstream returned no choices") + if not (data.get("done") or data.get("choices")): + raise UpstreamError("upstream returned no completion") + await self._observe_runner_context(base) return data + + async def _observe_runner_context(self, base: str) -> None: + """Record the loaded runner's context so a pin/default mismatch is visible + (in /status and as a WARNING) instead of silently reloading forever.""" + try: + resp = await self._client.get(f"{base}/api/ps", timeout=CONFIG.discovery_timeout) + if resp.status_code != 200: + return + for m in resp.json().get("models", []): + if m.get("model") == CONFIG.local_model or m.get("name") == CONFIG.local_model: + ctx = m.get("context_length") + if ctx != self.runner_context: + if CONFIG.num_ctx > 0 and ctx not in (None, CONFIG.num_ctx): + log.warning( + "runner context %s differs from the pinned num_ctx %s — a caller that " + "bypasses the pin (OpenAI dialect uses the model default) is thrashing", + ctx, CONFIG.num_ctx, + ) + self.runner_context = ctx + return + except (httpx.HTTPError, ValueError): + return diff --git a/scripts/checks.sh b/scripts/checks.sh index 96ef96e..8867c01 100755 --- a/scripts/checks.sh +++ b/scripts/checks.sh @@ -46,3 +46,8 @@ echo "19 ollama OPTIMISTIC chat, UNAVAIL .... HTTP $(j -XPOST $BAD/local-optimis echo "20 ollama OPTIMISTIC STREAM, UNAVAIL .. HTTP $(j -XPOST $BAD/local-optimistic/api/chat -H 'Content-Type: application/json' -d "$OSTREAM") backend=$(backend) $(ndjson_summary)" echo "21 ollama /api/tags, forge UNAVAIL .... HTTP $(j $BAD/local-optimistic/api/tags) (listing, not liveness)" echo "22 ollama /api/pull guard (expect 400) HTTP $(j -XPOST $GOOD/local-strict/api/pull -H 'Content-Type: application/json' -d '{"model":"llama3"}')" + +# ---- one runner config per model --------------------------------------------- +# Every forge-bound request carries the pinned num_ctx; the runner forge reports +# must match it, or callers are thrashing the 64 GB model on every alternation. +echo "23 num_ctx pin == runner context ....... HTTP $(j $GOOD/status) $(sed -n 's/.*"num_ctx_pin": *\([0-9null]*\).*"runner_context": *\([0-9null]*\).*/pin=\1 runner=\2/p' /tmp/b)" diff --git a/scripts/verify_numctx_pin.py b/scripts/verify_numctx_pin.py new file mode 100644 index 0000000..4f248e8 --- /dev/null +++ b/scripts/verify_numctx_pin.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Prove by behaviour that forgeway keeps ONE Ollama runner config. + +Drives, against a live router, the exact mix that used to thrash: + * HA-style short turns (native /api/chat, options.num_ctx 8192.0, keep_alive -1.0) + * an 8192-sending scripted caller + * the health probe + * large retrieval prompts through the OpenAI dialect (~40k and ~100k tokens), + needle-in-a-haystack so the answer proves the whole prompt was read + +and prints per-request latency. Pair it with forge's journal for the window: + + journalctl -u ollama --since "" | grep -oE "llama_context: n_ctx += *[0-9]+" | awk '{print $NF}' + journalctl -u ollama --since "" | grep -c "load_tensors: offloaded" + +Healthy = one n_ctx value at most (a first load) and no further reloads. + + verify_numctx_pin.py http://172.19.10.90 [--big 100000] [--mid 40000] +""" +from __future__ import annotations + +import argparse +import json +import random +import sys +import time +import urllib.request + +WORDS = ("harbour granite lantern meadow copper falcon river timber saddle ember frost " + "anchor beacon cinder willow marble quarry heather tundra vellum").split() + + +def haystack(target_tokens: int, needle: str) -> str: + """~1.3 tokens per word for this vocabulary; needle buried at ~60 %.""" + rng = random.Random(target_tokens) + words = int(target_tokens / 1.3) + body = [rng.choice(WORDS) for _ in range(words)] + body.insert(int(words * 0.6), f" The secret passphrase is {needle}. ") + return " ".join(body) + + +def post(url: str, body: dict, timeout: float = 900) -> tuple[int, dict, float]: + req = urllib.request.Request(url, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}, method="POST") + t0 = time.perf_counter() + with urllib.request.urlopen(req, timeout=timeout) as r: + data = json.loads(r.read().decode()) + return r.status, data, time.perf_counter() - t0 + + +def ha_turn(base: str, mode: str, text: str) -> str: + st, d, took = post(f"{base}/{mode}/api/chat", { + "model": "gpt-oss:120b", "stream": False, "think": False, "keep_alive": -1.0, + "options": {"num_ctx": 8192.0}, + "messages": [{"role": "system", "content": "You are a voice assistant. Answer tersely."}, + {"role": "user", "content": text}], + }) + return f"{took:6.2f} s HTTP {st} {d.get('message', {}).get('content', '')[:40]!r}" + + +def retrieval(base: str, tokens: int) -> str: + needle = f"OTTER-{tokens // 1000}K" + st, d, took = post(f"{base}/local-strict/v1/chat/completions", { + "messages": [{"role": "user", "content": haystack(tokens, needle) + "\n\nWhat is the secret passphrase? Answer with the passphrase only."}], + "max_tokens": 300, + }) + ans = d.get("choices", [{}])[0].get("message", {}).get("content", "") + usage = d.get("usage", {}) + ok = needle in ans + return f"{took:6.2f} s HTTP {st} prompt_tokens={usage.get('prompt_tokens')} answer={ans.strip()[:30]!r} {'FOUND' if ok else 'MISSED'}" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("base") + ap.add_argument("--mid", type=int, default=40000) + ap.add_argument("--big", type=int, default=100000) + a = ap.parse_args() + b = a.base.rstrip("/") + print(f"start {time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime())}Z") + steps = [ + ("HA-style turn (num_ctx 8192.0), strict", lambda: ha_turn(b, "local-strict", "Answer with the single word: PONG")), + ("health probe", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('reason')}")(*_get(f"{b}/healthz"))), + (f"retrieval ~{a.mid} tokens, OpenAI dialect", lambda: retrieval(b, a.mid)), + ("HA-style turn (num_ctx 8192.0), optimistic", lambda: ha_turn(b, "local-optimistic", "Answer with the single word: PONG")), + (f"retrieval ~{a.big} tokens, OpenAI dialect", lambda: retrieval(b, a.big)), + ("health probe", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('reason')}")(*_get(f"{b}/healthz"))), + ("HA-style turn (num_ctx 8192.0), strict", lambda: ha_turn(b, "local-strict", "Answer with the single word: PONG")), + ("scripted caller num_ctx 4096, generate", lambda: (lambda st, d, t: f"{t:6.2f} s HTTP {st} {d.get('response', '')[:20]!r}")( + *post(f"{b}/local-strict/api/generate", {"model": "gpt-oss:120b", "stream": False, "think": False, "options": {"num_ctx": 4096}, "prompt": "Answer with the single word: PONG"}))), + ("HA-style turn (num_ctx 8192.0), optimistic", lambda: ha_turn(b, "local-optimistic", "Answer with the single word: PONG")), + ] + for label, fn in steps: + try: + print(f" {label:46s} {fn()}") + except Exception as exc: # noqa: BLE001 + print(f" {label:46s} ERROR {type(exc).__name__}: {str(exc)[:80]}") + time.sleep(6) # let the health cache expire so the next request re-probes + print(f"end {time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime())}Z") + return 0 + + +def _get(url: str) -> tuple[int, dict, float]: + t0 = time.perf_counter() + try: + with urllib.request.urlopen(url, timeout=120) as r: + return r.status, json.loads(r.read().decode()), time.perf_counter() - t0 + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read().decode() or "{}"), time.perf_counter() - t0 + + +if __name__ == "__main__": + import urllib.error # noqa: E402 + sys.exit(main())