Files
Dolon (Agent)andClaude Opus 5 f7cd9b6690 0.2.3: keep the model resident — pin keep_alive on the native dialect
A cold load of gpt-oss:120b is 21-66 s and lands entirely in front of the
caller, so every idle spell put that pause on Rick's next Home Assistant turn.
The 0.3-0.6 s figures from 2026-09-18 were warm-model numbers.

Measured on Ollama 0.34.0, and NOT symmetrical with the num_ctx pin:

  native /api/chat, keep_alive:-1     -> Forever
  native /api/chat, explicit finite   -> that duration, and it BEATS the
                                         server-side OLLAMA_KEEP_ALIVE default
  native /api/chat, absent            -> does NOT reset an existing value;
                                         the duration is fixed AT LOAD TIME
  /v1/chat/completions, keep_alive    -> IGNORED, like options.num_ctx
                                         (confirmed in openai/openai.go: the
                                         request struct has no such field)

So the router alone can never carry residency: an OpenAI-dialect caller cannot
set it, and whichever request happens to trigger the cold load fixes the
duration. forge therefore also gets OLLAMA_KEEP_ALIVE=-1 in
/etc/systemd/system/ollama.service.d/30-keepalive.conf. The two are
complementary rather than redundant — the server default covers dialects and
callers we never proxy, this pin defeats a native caller that sends a short
keep_alive of its own.

The health probe asserts residency rather than inheriting it: it runs before
every proxied request, so it is the most frequent caller forge has and the one
most likely to be what triggers a cold load.

/status gains keep_alive_pin, runner_resident and runner_expires_at, and the
locator logs a WARNING if forge's runner stops being permanently resident —
the same visibility pattern as num_ctx_pin vs runner_context.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-18 20:02:03 +00:00

295 lines
12 KiB
Python

"""forge discovery and health, both gated on a real completion."""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
import httpx
from .config import CONFIG
log = logging.getLogger("forgeway.upstream")
class UpstreamError(RuntimeError):
"""An upstream refused, failed, or timed out."""
def __init__(self, message: str, status_code: int | None = None, body: str | None = None):
super().__init__(message)
self.status_code = status_code
self.body = body
@dataclass(frozen=True)
class Health:
healthy: bool
reason: str
upstream: str | None
latency_ms: float | None
checked_at: float
cached: bool = False
def as_dict(self) -> dict:
return {
"status": "ok" if self.healthy else "down",
"reason": self.reason,
"upstream": self.upstream,
"model": CONFIG.local_model,
"latency_ms": round(self.latency_ms, 1) if self.latency_ms is not None else None,
"checked_at": round(self.checked_at, 3),
"age_s": round(time.time() - self.checked_at, 3),
"cached": self.cached,
"site": CONFIG.site,
"service": CONFIG.name,
}
class ForgeLocator:
"""Finds forge among the candidate upstreams and keeps a short-lived verdict.
Discovery is two-phase so that a moved host is found quickly without paying a
long timeout per candidate:
1. reachability — probe every candidate concurrently with a short timeout
2. completion — run one real 1-token completion against the preferred
reachable candidate
The last good upstream is sticky and tried first, so the steady state costs a
single fast probe.
"""
def __init__(self, client: httpx.AsyncClient):
self._client = client
self._sticky: str | None = None
self._health: Health | None = None
self._lock = asyncio.Lock()
self.runner_context: int | None = None # last observed loaded context on forge
self.runner_expires_at: str | None = None # last observed residency deadline
self.runner_resident: bool | None = None # True once it is "forever"
@property
def sticky(self) -> str | None:
return self._sticky
async def health(self, force: bool = False) -> Health:
cached = self._health
if not force and cached is not None:
ttl = CONFIG.health_ttl if cached.healthy else CONFIG.health_fail_ttl
if (time.time() - cached.checked_at) < ttl:
return Health(**{**cached.__dict__, "cached": True})
async with self._lock:
# Another waiter may have refreshed while we queued on the lock.
cached = self._health
if not force and cached is not None:
ttl = CONFIG.health_ttl if cached.healthy else CONFIG.health_fail_ttl
if (time.time() - cached.checked_at) < ttl:
return Health(**{**cached.__dict__, "cached": True})
health = await self._probe()
self._health = health
return health
def invalidate(self) -> None:
"""Drop the cached verdict after a live request failed against it."""
self._health = None
async def _probe(self) -> Health:
started = time.perf_counter()
target = await self._discover()
if not target:
return Health(
healthy=False,
reason="no-upstream-discovered",
upstream=None,
latency_ms=None,
checked_at=time.time(),
)
if not CONFIG.health_cold_load:
loaded = await self._model_resident(target)
if loaded is False:
return Health(
healthy=False,
reason="model-not-loaded (cold load disabled)",
upstream=target,
latency_ms=None,
checked_at=time.time(),
)
try:
await self._completion(target)
except UpstreamError as exc:
self._sticky = None
return Health(
healthy=False,
reason=f"completion-failed: {exc}",
upstream=target,
latency_ms=None,
checked_at=time.time(),
)
self._sticky = target
return Health(
healthy=True,
reason="completion-ok",
upstream=target,
latency_ms=(time.perf_counter() - started) * 1000.0,
checked_at=time.time(),
)
async def _discover(self) -> str | None:
"""Find where forge currently answers.
All candidates are probed concurrently, but awaited in priority order, so
the happy path costs one fast round trip instead of waiting for dead
candidates to time out. That matters because the candidate list carries
both the Lounge and the future Serenity address of forge: whichever site
silicon lives at, the live one is found without a config change, and the
dead one never slows the probe down.
"""
candidates = list(CONFIG.upstreams)
if self._sticky and self._sticky in candidates:
candidates.remove(self._sticky)
candidates.insert(0, self._sticky)
async def reachable(base: str) -> str | None:
try:
resp = await self._client.get(f"{base}/v1/models", timeout=CONFIG.discovery_timeout)
return base if resp.status_code < 500 else None
except httpx.HTTPError:
return None
tasks = [asyncio.create_task(reachable(c)) for c in candidates]
try:
for task in tasks:
try:
found = await task
except asyncio.CancelledError:
continue
if found:
if found != self._sticky:
log.info("discovery: forge answers at %s (of %d candidates)", found, len(candidates))
return found
return None
finally:
for task in tasks:
if not task.done():
task.cancel()
async def _model_resident(self, base: str) -> bool | None:
"""Ask Ollama whether the model is already in VRAM. None = can't tell."""
try:
resp = await self._client.get(f"{base}/api/ps", timeout=CONFIG.discovery_timeout)
if resp.status_code != 200:
return None
names = {m.get("model") or m.get("name") for m in resp.json().get("models", [])}
return CONFIG.local_model in names
except (httpx.HTTPError, ValueError):
return None
async def _completion(self, base: str) -> dict:
"""One real, minimal completion. Not a ping — that is the whole point.
Uses Ollama's native /api/chat so the probe can carry the pinned num_ctx
and keep_alive: the OpenAI-compatible /v1 endpoint ignores both, and an
unpinned probe was half of the runner-reload thrash. Falls back to /v1 for
a non-Ollama upstream (404 on /api/chat).
"""
options: dict = {"num_predict": 1}
if CONFIG.num_ctx > 0:
options["num_ctx"] = CONFIG.num_ctx
native = {
"model": CONFIG.local_model,
"messages": [{"role": "user", "content": "ping"}],
"stream": False,
"options": options,
}
if CONFIG.keep_alive != 0:
# This probe runs before every proxied request, so it is the most frequent
# caller forge has. It must ASSERT residency rather than inherit it: Ollama
# fixes keep_alive at LOAD time, so if the probe is what triggers a cold
# load, the duration the probe carries is the one the model then keeps.
native["keep_alive"] = CONFIG.keep_alive
try:
resp = await self._client.post(f"{base}/api/chat", json=native, timeout=CONFIG.health_timeout)
if resp.status_code == 404:
resp = await self._client.post(
f"{base}/v1/chat/completions",
json={
"model": CONFIG.local_model,
"messages": [{"role": "user", "content": "ping"}],
"max_tokens": 1,
"temperature": 0,
"stream": False,
},
timeout=CONFIG.health_timeout,
)
except httpx.HTTPError as exc:
raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc
if resp.status_code != 200:
raise UpstreamError(
f"HTTP {resp.status_code}",
status_code=resp.status_code,
body=resp.text[:500],
)
try:
data = resp.json()
except ValueError as exc:
raise UpstreamError("upstream returned non-JSON") from exc
if not (data.get("done") or data.get("choices")):
raise UpstreamError("upstream returned no completion")
await self._observe_runner(base)
return data
async def _observe_runner(self, base: str) -> None:
"""Record the loaded runner's context and residency, so a pin/default mismatch
or a lost keep_alive is visible (in /status and as a WARNING) instead of
silently reloading forever."""
try:
resp = await self._client.get(f"{base}/api/ps", timeout=CONFIG.discovery_timeout)
if resp.status_code != 200:
return
for m in resp.json().get("models", []):
if m.get("model") == CONFIG.local_model or m.get("name") == CONFIG.local_model:
ctx = m.get("context_length")
if ctx != self.runner_context:
if CONFIG.num_ctx > 0 and ctx not in (None, CONFIG.num_ctx):
log.warning(
"runner context %s differs from the pinned num_ctx %s — a caller that "
"bypasses the pin (OpenAI dialect uses the model default) is thrashing",
ctx, CONFIG.num_ctx,
)
self.runner_context = ctx
self._observe_residency(m.get("expires_at"))
return
except (httpx.HTTPError, ValueError):
return
def _observe_residency(self, expires_at: str | None) -> None:
"""A cold load costs 21-66 s and lands in front of the caller, so residency is
a property worth watching, not assuming. Ollama renders "forever" as a year-2318
timestamp, so "more than a year out" is the portable test for it."""
resident = None
if expires_at:
try:
delta = datetime.fromisoformat(expires_at.replace("Z", "+00:00")) - datetime.now(timezone.utc)
resident = delta > timedelta(days=365)
except ValueError:
resident = None
if resident is not None and resident != self.runner_resident:
if not resident and CONFIG.keep_alive != 0:
log.warning(
"forge's runner is no longer permanently resident (expires_at %s) despite a "
"keep_alive pin of %s — a caller sending its own finite keep_alive, or forge's "
"OLLAMA_KEEP_ALIVE default, is winning; the next idle spell will cost a cold load",
expires_at, CONFIG.keep_alive,
)
self.runner_resident = resident
self.runner_expires_at = expires_at