A cold load of gpt-oss:120b is 21-66 s and lands entirely in front of the
caller, so every idle spell put that pause on Rick's next Home Assistant turn.
The 0.3-0.6 s figures from 2026-09-18 were warm-model numbers.
Measured on Ollama 0.34.0, and NOT symmetrical with the num_ctx pin:
native /api/chat, keep_alive:-1 -> Forever
native /api/chat, explicit finite -> that duration, and it BEATS the
server-side OLLAMA_KEEP_ALIVE default
native /api/chat, absent -> does NOT reset an existing value;
the duration is fixed AT LOAD TIME
/v1/chat/completions, keep_alive -> IGNORED, like options.num_ctx
(confirmed in openai/openai.go: the
request struct has no such field)
So the router alone can never carry residency: an OpenAI-dialect caller cannot
set it, and whichever request happens to trigger the cold load fixes the
duration. forge therefore also gets OLLAMA_KEEP_ALIVE=-1 in
/etc/systemd/system/ollama.service.d/30-keepalive.conf. The two are
complementary rather than redundant — the server default covers dialects and
callers we never proxy, this pin defeats a native caller that sends a short
keep_alive of its own.
The health probe asserts residency rather than inheriting it: it runs before
every proxied request, so it is the most frequent caller forge has and the one
most likely to be what triggers a cold load.
/status gains keep_alive_pin, runner_resident and runner_expires_at, and the
locator logs a WARNING if forge's runner stops being permanently resident —
the same visibility pattern as num_ctx_pin vs runner_context.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
295 lines
12 KiB
Python
295 lines
12 KiB
Python
"""forge discovery and health, both gated on a real completion."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import httpx
|
|
|
|
from .config import CONFIG
|
|
|
|
log = logging.getLogger("forgeway.upstream")
|
|
|
|
|
|
class UpstreamError(RuntimeError):
|
|
"""An upstream refused, failed, or timed out."""
|
|
|
|
def __init__(self, message: str, status_code: int | None = None, body: str | None = None):
|
|
super().__init__(message)
|
|
self.status_code = status_code
|
|
self.body = body
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Health:
|
|
healthy: bool
|
|
reason: str
|
|
upstream: str | None
|
|
latency_ms: float | None
|
|
checked_at: float
|
|
cached: bool = False
|
|
|
|
def as_dict(self) -> dict:
|
|
return {
|
|
"status": "ok" if self.healthy else "down",
|
|
"reason": self.reason,
|
|
"upstream": self.upstream,
|
|
"model": CONFIG.local_model,
|
|
"latency_ms": round(self.latency_ms, 1) if self.latency_ms is not None else None,
|
|
"checked_at": round(self.checked_at, 3),
|
|
"age_s": round(time.time() - self.checked_at, 3),
|
|
"cached": self.cached,
|
|
"site": CONFIG.site,
|
|
"service": CONFIG.name,
|
|
}
|
|
|
|
|
|
class ForgeLocator:
|
|
"""Finds forge among the candidate upstreams and keeps a short-lived verdict.
|
|
|
|
Discovery is two-phase so that a moved host is found quickly without paying a
|
|
long timeout per candidate:
|
|
|
|
1. reachability — probe every candidate concurrently with a short timeout
|
|
2. completion — run one real 1-token completion against the preferred
|
|
reachable candidate
|
|
|
|
The last good upstream is sticky and tried first, so the steady state costs a
|
|
single fast probe.
|
|
"""
|
|
|
|
def __init__(self, client: httpx.AsyncClient):
|
|
self._client = client
|
|
self._sticky: str | None = None
|
|
self._health: Health | None = None
|
|
self._lock = asyncio.Lock()
|
|
self.runner_context: int | None = None # last observed loaded context on forge
|
|
self.runner_expires_at: str | None = None # last observed residency deadline
|
|
self.runner_resident: bool | None = None # True once it is "forever"
|
|
|
|
@property
|
|
def sticky(self) -> str | None:
|
|
return self._sticky
|
|
|
|
async def health(self, force: bool = False) -> Health:
|
|
cached = self._health
|
|
if not force and cached is not None:
|
|
ttl = CONFIG.health_ttl if cached.healthy else CONFIG.health_fail_ttl
|
|
if (time.time() - cached.checked_at) < ttl:
|
|
return Health(**{**cached.__dict__, "cached": True})
|
|
|
|
async with self._lock:
|
|
# Another waiter may have refreshed while we queued on the lock.
|
|
cached = self._health
|
|
if not force and cached is not None:
|
|
ttl = CONFIG.health_ttl if cached.healthy else CONFIG.health_fail_ttl
|
|
if (time.time() - cached.checked_at) < ttl:
|
|
return Health(**{**cached.__dict__, "cached": True})
|
|
|
|
health = await self._probe()
|
|
self._health = health
|
|
return health
|
|
|
|
def invalidate(self) -> None:
|
|
"""Drop the cached verdict after a live request failed against it."""
|
|
self._health = None
|
|
|
|
async def _probe(self) -> Health:
|
|
started = time.perf_counter()
|
|
target = await self._discover()
|
|
if not target:
|
|
return Health(
|
|
healthy=False,
|
|
reason="no-upstream-discovered",
|
|
upstream=None,
|
|
latency_ms=None,
|
|
checked_at=time.time(),
|
|
)
|
|
|
|
if not CONFIG.health_cold_load:
|
|
loaded = await self._model_resident(target)
|
|
if loaded is False:
|
|
return Health(
|
|
healthy=False,
|
|
reason="model-not-loaded (cold load disabled)",
|
|
upstream=target,
|
|
latency_ms=None,
|
|
checked_at=time.time(),
|
|
)
|
|
|
|
try:
|
|
await self._completion(target)
|
|
except UpstreamError as exc:
|
|
self._sticky = None
|
|
return Health(
|
|
healthy=False,
|
|
reason=f"completion-failed: {exc}",
|
|
upstream=target,
|
|
latency_ms=None,
|
|
checked_at=time.time(),
|
|
)
|
|
|
|
self._sticky = target
|
|
return Health(
|
|
healthy=True,
|
|
reason="completion-ok",
|
|
upstream=target,
|
|
latency_ms=(time.perf_counter() - started) * 1000.0,
|
|
checked_at=time.time(),
|
|
)
|
|
|
|
async def _discover(self) -> str | None:
|
|
"""Find where forge currently answers.
|
|
|
|
All candidates are probed concurrently, but awaited in priority order, so
|
|
the happy path costs one fast round trip instead of waiting for dead
|
|
candidates to time out. That matters because the candidate list carries
|
|
both the Lounge and the future Serenity address of forge: whichever site
|
|
silicon lives at, the live one is found without a config change, and the
|
|
dead one never slows the probe down.
|
|
"""
|
|
candidates = list(CONFIG.upstreams)
|
|
if self._sticky and self._sticky in candidates:
|
|
candidates.remove(self._sticky)
|
|
candidates.insert(0, self._sticky)
|
|
|
|
async def reachable(base: str) -> str | None:
|
|
try:
|
|
resp = await self._client.get(f"{base}/v1/models", timeout=CONFIG.discovery_timeout)
|
|
return base if resp.status_code < 500 else None
|
|
except httpx.HTTPError:
|
|
return None
|
|
|
|
tasks = [asyncio.create_task(reachable(c)) for c in candidates]
|
|
try:
|
|
for task in tasks:
|
|
try:
|
|
found = await task
|
|
except asyncio.CancelledError:
|
|
continue
|
|
if found:
|
|
if found != self._sticky:
|
|
log.info("discovery: forge answers at %s (of %d candidates)", found, len(candidates))
|
|
return found
|
|
return None
|
|
finally:
|
|
for task in tasks:
|
|
if not task.done():
|
|
task.cancel()
|
|
|
|
async def _model_resident(self, base: str) -> bool | None:
|
|
"""Ask Ollama whether the model is already in VRAM. None = can't tell."""
|
|
try:
|
|
resp = await self._client.get(f"{base}/api/ps", timeout=CONFIG.discovery_timeout)
|
|
if resp.status_code != 200:
|
|
return None
|
|
names = {m.get("model") or m.get("name") for m in resp.json().get("models", [])}
|
|
return CONFIG.local_model in names
|
|
except (httpx.HTTPError, ValueError):
|
|
return None
|
|
|
|
async def _completion(self, base: str) -> dict:
|
|
"""One real, minimal completion. Not a ping — that is the whole point.
|
|
|
|
Uses Ollama's native /api/chat so the probe can carry the pinned num_ctx
|
|
and keep_alive: the OpenAI-compatible /v1 endpoint ignores both, and an
|
|
unpinned probe was half of the runner-reload thrash. Falls back to /v1 for
|
|
a non-Ollama upstream (404 on /api/chat).
|
|
"""
|
|
options: dict = {"num_predict": 1}
|
|
if CONFIG.num_ctx > 0:
|
|
options["num_ctx"] = CONFIG.num_ctx
|
|
native = {
|
|
"model": CONFIG.local_model,
|
|
"messages": [{"role": "user", "content": "ping"}],
|
|
"stream": False,
|
|
"options": options,
|
|
}
|
|
if CONFIG.keep_alive != 0:
|
|
# This probe runs before every proxied request, so it is the most frequent
|
|
# caller forge has. It must ASSERT residency rather than inherit it: Ollama
|
|
# fixes keep_alive at LOAD time, so if the probe is what triggers a cold
|
|
# load, the duration the probe carries is the one the model then keeps.
|
|
native["keep_alive"] = CONFIG.keep_alive
|
|
try:
|
|
resp = await self._client.post(f"{base}/api/chat", json=native, timeout=CONFIG.health_timeout)
|
|
if resp.status_code == 404:
|
|
resp = await self._client.post(
|
|
f"{base}/v1/chat/completions",
|
|
json={
|
|
"model": CONFIG.local_model,
|
|
"messages": [{"role": "user", "content": "ping"}],
|
|
"max_tokens": 1,
|
|
"temperature": 0,
|
|
"stream": False,
|
|
},
|
|
timeout=CONFIG.health_timeout,
|
|
)
|
|
except httpx.HTTPError as exc:
|
|
raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc
|
|
|
|
if resp.status_code != 200:
|
|
raise UpstreamError(
|
|
f"HTTP {resp.status_code}",
|
|
status_code=resp.status_code,
|
|
body=resp.text[:500],
|
|
)
|
|
try:
|
|
data = resp.json()
|
|
except ValueError as exc:
|
|
raise UpstreamError("upstream returned non-JSON") from exc
|
|
if not (data.get("done") or data.get("choices")):
|
|
raise UpstreamError("upstream returned no completion")
|
|
await self._observe_runner(base)
|
|
return data
|
|
|
|
async def _observe_runner(self, base: str) -> None:
|
|
"""Record the loaded runner's context and residency, so a pin/default mismatch
|
|
or a lost keep_alive is visible (in /status and as a WARNING) instead of
|
|
silently reloading forever."""
|
|
try:
|
|
resp = await self._client.get(f"{base}/api/ps", timeout=CONFIG.discovery_timeout)
|
|
if resp.status_code != 200:
|
|
return
|
|
for m in resp.json().get("models", []):
|
|
if m.get("model") == CONFIG.local_model or m.get("name") == CONFIG.local_model:
|
|
ctx = m.get("context_length")
|
|
if ctx != self.runner_context:
|
|
if CONFIG.num_ctx > 0 and ctx not in (None, CONFIG.num_ctx):
|
|
log.warning(
|
|
"runner context %s differs from the pinned num_ctx %s — a caller that "
|
|
"bypasses the pin (OpenAI dialect uses the model default) is thrashing",
|
|
ctx, CONFIG.num_ctx,
|
|
)
|
|
self.runner_context = ctx
|
|
self._observe_residency(m.get("expires_at"))
|
|
return
|
|
except (httpx.HTTPError, ValueError):
|
|
return
|
|
|
|
def _observe_residency(self, expires_at: str | None) -> None:
|
|
"""A cold load costs 21-66 s and lands in front of the caller, so residency is
|
|
a property worth watching, not assuming. Ollama renders "forever" as a year-2318
|
|
timestamp, so "more than a year out" is the portable test for it."""
|
|
resident = None
|
|
if expires_at:
|
|
try:
|
|
delta = datetime.fromisoformat(expires_at.replace("Z", "+00:00")) - datetime.now(timezone.utc)
|
|
resident = delta > timedelta(days=365)
|
|
except ValueError:
|
|
resident = None
|
|
if resident is not None and resident != self.runner_resident:
|
|
if not resident and CONFIG.keep_alive != 0:
|
|
log.warning(
|
|
"forge's runner is no longer permanently resident (expires_at %s) despite a "
|
|
"keep_alive pin of %s — a caller sending its own finite keep_alive, or forge's "
|
|
"OLLAMA_KEEP_ALIVE default, is winning; the next idle spell will cost a cold load",
|
|
expires_at, CONFIG.keep_alive,
|
|
)
|
|
self.runner_resident = resident
|
|
self.runner_expires_at = expires_at
|