"""forge discovery and health, both gated on a real completion.""" from __future__ import annotations import asyncio import logging import time from dataclasses import dataclass from datetime import datetime, timedelta, timezone import httpx from .config import CONFIG log = logging.getLogger("forgeway.upstream") class UpstreamError(RuntimeError): """An upstream refused, failed, or timed out.""" def __init__(self, message: str, status_code: int | None = None, body: str | None = None): super().__init__(message) self.status_code = status_code self.body = body @dataclass(frozen=True) class Health: healthy: bool reason: str upstream: str | None latency_ms: float | None checked_at: float cached: bool = False def as_dict(self) -> dict: return { "status": "ok" if self.healthy else "down", "reason": self.reason, "upstream": self.upstream, "model": CONFIG.local_model, "latency_ms": round(self.latency_ms, 1) if self.latency_ms is not None else None, "checked_at": round(self.checked_at, 3), "age_s": round(time.time() - self.checked_at, 3), "cached": self.cached, "site": CONFIG.site, "service": CONFIG.name, } class ForgeLocator: """Finds forge among the candidate upstreams and keeps a short-lived verdict. Discovery is two-phase so that a moved host is found quickly without paying a long timeout per candidate: 1. reachability — probe every candidate concurrently with a short timeout 2. completion — run one real 1-token completion against the preferred reachable candidate The last good upstream is sticky and tried first, so the steady state costs a single fast probe. """ def __init__(self, client: httpx.AsyncClient): self._client = client self._sticky: str | None = None self._health: Health | None = None self._lock = asyncio.Lock() self.runner_context: int | None = None # last observed loaded context on forge self.runner_expires_at: str | None = None # last observed residency deadline self.runner_resident: bool | None = None # True once it is "forever" @property def sticky(self) -> str | None: return self._sticky async def health(self, force: bool = False) -> Health: cached = self._health if not force and cached is not None: ttl = CONFIG.health_ttl if cached.healthy else CONFIG.health_fail_ttl if (time.time() - cached.checked_at) < ttl: return Health(**{**cached.__dict__, "cached": True}) async with self._lock: # Another waiter may have refreshed while we queued on the lock. cached = self._health if not force and cached is not None: ttl = CONFIG.health_ttl if cached.healthy else CONFIG.health_fail_ttl if (time.time() - cached.checked_at) < ttl: return Health(**{**cached.__dict__, "cached": True}) health = await self._probe() self._health = health return health def invalidate(self) -> None: """Drop the cached verdict after a live request failed against it.""" self._health = None async def _probe(self) -> Health: started = time.perf_counter() target = await self._discover() if not target: return Health( healthy=False, reason="no-upstream-discovered", upstream=None, latency_ms=None, checked_at=time.time(), ) if not CONFIG.health_cold_load: loaded = await self._model_resident(target) if loaded is False: return Health( healthy=False, reason="model-not-loaded (cold load disabled)", upstream=target, latency_ms=None, checked_at=time.time(), ) try: await self._completion(target) except UpstreamError as exc: self._sticky = None return Health( healthy=False, reason=f"completion-failed: {exc}", upstream=target, latency_ms=None, checked_at=time.time(), ) self._sticky = target return Health( healthy=True, reason="completion-ok", upstream=target, latency_ms=(time.perf_counter() - started) * 1000.0, checked_at=time.time(), ) async def _discover(self) -> str | None: """Find where forge currently answers. All candidates are probed concurrently, but awaited in priority order, so the happy path costs one fast round trip instead of waiting for dead candidates to time out. That matters because the candidate list carries both the Lounge and the future Serenity address of forge: whichever site silicon lives at, the live one is found without a config change, and the dead one never slows the probe down. """ candidates = list(CONFIG.upstreams) if self._sticky and self._sticky in candidates: candidates.remove(self._sticky) candidates.insert(0, self._sticky) async def reachable(base: str) -> str | None: try: resp = await self._client.get(f"{base}/v1/models", timeout=CONFIG.discovery_timeout) return base if resp.status_code < 500 else None except httpx.HTTPError: return None tasks = [asyncio.create_task(reachable(c)) for c in candidates] try: for task in tasks: try: found = await task except asyncio.CancelledError: continue if found: if found != self._sticky: log.info("discovery: forge answers at %s (of %d candidates)", found, len(candidates)) return found return None finally: for task in tasks: if not task.done(): task.cancel() async def _model_resident(self, base: str) -> bool | None: """Ask Ollama whether the model is already in VRAM. None = can't tell.""" try: resp = await self._client.get(f"{base}/api/ps", timeout=CONFIG.discovery_timeout) if resp.status_code != 200: return None names = {m.get("model") or m.get("name") for m in resp.json().get("models", [])} return CONFIG.local_model in names except (httpx.HTTPError, ValueError): return None async def _completion(self, base: str) -> dict: """One real, minimal completion. Not a ping — that is the whole point. Uses Ollama's native /api/chat so the probe can carry the pinned num_ctx and keep_alive: the OpenAI-compatible /v1 endpoint ignores both, and an unpinned probe was half of the runner-reload thrash. Falls back to /v1 for a non-Ollama upstream (404 on /api/chat). """ options: dict = {"num_predict": 1} if CONFIG.num_ctx > 0: options["num_ctx"] = CONFIG.num_ctx native = { "model": CONFIG.local_model, "messages": [{"role": "user", "content": "ping"}], "stream": False, "options": options, } if CONFIG.keep_alive != 0: # This probe runs before every proxied request, so it is the most frequent # caller forge has. It must ASSERT residency rather than inherit it: Ollama # fixes keep_alive at LOAD time, so if the probe is what triggers a cold # load, the duration the probe carries is the one the model then keeps. native["keep_alive"] = CONFIG.keep_alive try: resp = await self._client.post(f"{base}/api/chat", json=native, timeout=CONFIG.health_timeout) if resp.status_code == 404: resp = await self._client.post( f"{base}/v1/chat/completions", json={ "model": CONFIG.local_model, "messages": [{"role": "user", "content": "ping"}], "max_tokens": 1, "temperature": 0, "stream": False, }, timeout=CONFIG.health_timeout, ) except httpx.HTTPError as exc: raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc if resp.status_code != 200: raise UpstreamError( f"HTTP {resp.status_code}", status_code=resp.status_code, body=resp.text[:500], ) try: data = resp.json() except ValueError as exc: raise UpstreamError("upstream returned non-JSON") from exc if not (data.get("done") or data.get("choices")): raise UpstreamError("upstream returned no completion") await self._observe_runner(base) return data async def _observe_runner(self, base: str) -> None: """Record the loaded runner's context and residency, so a pin/default mismatch or a lost keep_alive is visible (in /status and as a WARNING) instead of silently reloading forever.""" try: resp = await self._client.get(f"{base}/api/ps", timeout=CONFIG.discovery_timeout) if resp.status_code != 200: return for m in resp.json().get("models", []): if m.get("model") == CONFIG.local_model or m.get("name") == CONFIG.local_model: ctx = m.get("context_length") if ctx != self.runner_context: if CONFIG.num_ctx > 0 and ctx not in (None, CONFIG.num_ctx): log.warning( "runner context %s differs from the pinned num_ctx %s — a caller that " "bypasses the pin (OpenAI dialect uses the model default) is thrashing", ctx, CONFIG.num_ctx, ) self.runner_context = ctx self._observe_residency(m.get("expires_at")) return except (httpx.HTTPError, ValueError): return def _observe_residency(self, expires_at: str | None) -> None: """A cold load costs 21-66 s and lands in front of the caller, so residency is a property worth watching, not assuming. Ollama renders "forever" as a year-2318 timestamp, so "more than a year out" is the portable test for it.""" resident = None if expires_at: try: delta = datetime.fromisoformat(expires_at.replace("Z", "+00:00")) - datetime.now(timezone.utc) resident = delta > timedelta(days=365) except ValueError: resident = None if resident is not None and resident != self.runner_resident: if not resident and CONFIG.keep_alive != 0: log.warning( "forge's runner is no longer permanently resident (expires_at %s) despite a " "keep_alive pin of %s — a caller sending its own finite keep_alive, or forge's " "OLLAMA_KEEP_ALIVE default, is winning; the next idle spell will cost a cold load", expires_at, CONFIG.keep_alive, ) self.runner_resident = resident self.runner_expires_at = expires_at