Files
Dolon (Agent)andClaude Opus 5 f7cd9b6690 0.2.3: keep the model resident — pin keep_alive on the native dialect
A cold load of gpt-oss:120b is 21-66 s and lands entirely in front of the
caller, so every idle spell put that pause on Rick's next Home Assistant turn.
The 0.3-0.6 s figures from 2026-09-18 were warm-model numbers.

Measured on Ollama 0.34.0, and NOT symmetrical with the num_ctx pin:

  native /api/chat, keep_alive:-1     -> Forever
  native /api/chat, explicit finite   -> that duration, and it BEATS the
                                         server-side OLLAMA_KEEP_ALIVE default
  native /api/chat, absent            -> does NOT reset an existing value;
                                         the duration is fixed AT LOAD TIME
  /v1/chat/completions, keep_alive    -> IGNORED, like options.num_ctx
                                         (confirmed in openai/openai.go: the
                                         request struct has no such field)

So the router alone can never carry residency: an OpenAI-dialect caller cannot
set it, and whichever request happens to trigger the cold load fixes the
duration. forge therefore also gets OLLAMA_KEEP_ALIVE=-1 in
/etc/systemd/system/ollama.service.d/30-keepalive.conf. The two are
complementary rather than redundant — the server default covers dialects and
callers we never proxy, this pin defeats a native caller that sends a short
keep_alive of its own.

The health probe asserts residency rather than inheriting it: it runs before
every proxied request, so it is the most frequent caller forge has and the one
most likely to be what triggers a cold load.

/status gains keep_alive_pin, runner_resident and runner_expires_at, and the
locator logs a WARNING if forge's runner stops being permanently resident —
the same visibility pattern as num_ctx_pin vs runner_context.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-18 20:02:03 +00:00

399 lines
14 KiB
Python

"""OpenAI-compatible router with two query endpoints.
/local-optimistic/v1/... forge if available, otherwise the cloud model
/local-strict/v1/... forge or a clean 503 — callers handle the error
One instance runs per cluster. There is deliberately no central instance: the
cross-site link to forge may itself be the thing that is down, and a central
router would then *be* the outage.
"""
from __future__ import annotations
import logging
import time
from contextlib import asynccontextmanager
from typing import Any
import httpx
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, StreamingResponse
from .common import MODE_OPTIMISTIC, MODE_STRICT, MODES, base_headers
from .config import CONFIG
from .ollama_api import router as ollama_router
from .upstream import ForgeLocator, UpstreamError
logging.basicConfig(
level=getattr(logging, CONFIG.log_level, logging.INFO),
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
log = logging.getLogger("forgeway")
@asynccontextmanager
async def lifespan(app: FastAPI):
app.state.client = httpx.AsyncClient(
timeout=httpx.Timeout(
CONFIG.request_timeout,
connect=CONFIG.connect_timeout,
read=CONFIG.request_timeout,
write=60.0,
pool=CONFIG.connect_timeout,
),
limits=httpx.Limits(max_connections=64, max_keepalive_connections=16),
follow_redirects=False,
)
app.state.locator = ForgeLocator(app.state.client)
log.info(
"%s starting: site=%s model=%s cloud=%s candidates=%s",
CONFIG.name,
CONFIG.site,
CONFIG.local_model,
CONFIG.cloud_model if CONFIG.cloud_enabled else "disabled (no key)",
", ".join(CONFIG.upstreams),
)
try:
yield
finally:
await app.state.client.aclose()
app = FastAPI(
title=f"{CONFIG.name} — OpenAI-compatible LLM router",
docs_url=None,
redoc_url=None,
lifespan=lifespan,
)
app.include_router(ollama_router) # native Ollama dialect: /{mode}/api/...
# --------------------------------------------------------------------------
# helpers
# --------------------------------------------------------------------------
_base_headers = base_headers # shared with the native-Ollama dialect
def _error(
status: int,
message: str,
code: str,
mode: str,
backend: str = "none",
upstream: str | None = None,
extra: dict[str, Any] | None = None,
) -> JSONResponse:
"""OpenAI-shaped error so existing SDK error handling keeps working."""
payload: dict[str, Any] = {
"error": {
"message": message,
"type": "forgeway_error",
"code": code,
"param": None,
}
}
if extra:
payload["error"].update(extra)
# Deliberately no Retry-After: a 503 from local-strict means the local model
# is unavailable *now*, and blind retries just hammer a busy or absent forge.
return JSONResponse(payload, status_code=status, headers=_base_headers(mode, backend, upstream))
def _adapt_payload(body: dict[str, Any], backend: str) -> dict[str, Any]:
"""Reconcile the one parameter the two backends spell differently.
gpt-5.4-mini rejects `max_tokens` outright (400, "use max_completion_tokens"),
while Ollama's OpenAI layer only understands `max_tokens`. Callers should be
able to write either spelling and have both endpoints work, so translate here
rather than making every caller care which backend served them.
"""
payload = dict(body)
if backend == "cloud":
if "max_tokens" in payload:
payload.setdefault("max_completion_tokens", payload.pop("max_tokens"))
payload.pop("max_tokens", None)
else:
if "max_completion_tokens" in payload:
payload.setdefault("max_tokens", payload.pop("max_completion_tokens"))
payload.pop("max_completion_tokens", None)
return payload
async def _proxy_chat(
base_url: str,
model: str,
body: dict[str, Any],
mode: str,
backend: str,
api_key: str | None = None,
) -> JSONResponse | StreamingResponse:
"""Forward one chat completion, honouring stream=true."""
client: httpx.AsyncClient = app.state.client
payload = _adapt_payload(body, backend)
payload["model"] = model
stream = bool(payload.get("stream"))
headers = {"content-type": "application/json"}
if api_key:
headers["authorization"] = f"Bearer {api_key}"
url = f"{base_url}/v1/chat/completions" if not base_url.endswith("/v1") else f"{base_url}/chat/completions"
if not stream:
try:
resp = await client.post(url, json=payload, headers=headers)
except httpx.HTTPError as exc:
raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc
if resp.status_code >= 400:
raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, resp.text[:500])
return JSONResponse(
resp.json(),
status_code=resp.status_code,
headers=_base_headers(mode, backend, base_url),
)
# Streaming: send first and inspect the status before committing to a
# StreamingResponse, so a failed upstream can still fall back.
request = client.build_request("POST", url, json=payload, headers=headers)
try:
resp = await client.send(request, stream=True)
except httpx.HTTPError as exc:
raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc
if resp.status_code >= 400:
detail = (await resp.aread())[:500].decode("utf-8", "replace")
await resp.aclose()
raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, detail)
async def body_iter():
try:
async for chunk in resp.aiter_raw():
yield chunk
finally:
await resp.aclose()
return StreamingResponse(
body_iter(),
status_code=resp.status_code,
media_type=resp.headers.get("content-type", "text/event-stream"),
headers=_base_headers(mode, backend, base_url),
)
# --------------------------------------------------------------------------
# health — for Uptime Kuma (active HTTP probing, not push)
# --------------------------------------------------------------------------
@app.get("/healthz")
async def healthz() -> JSONResponse:
"""Gated on a real completion against forge.
200 forge answered a real completion
503 forge did not
(no response at all means the router itself is gone)
Results are cached for a few seconds so a burst of traffic does not turn
into a burst of probe completions against forge.
"""
health = await app.state.locator.health()
return JSONResponse(
health.as_dict(),
status_code=200 if health.healthy else 503,
headers=_base_headers("healthz", "forge" if health.healthy else "none", health.upstream),
)
@app.get("/healthz/live")
async def healthz_live() -> JSONResponse:
"""Liveness only. Never gated on forge — otherwise the pod would restart-loop
whenever forge is down, and local-strict could not return its 503."""
return JSONResponse({"status": "ok", "service": CONFIG.name, "site": CONFIG.site})
@app.get("/healthz/ready")
async def healthz_ready() -> JSONResponse:
"""Readiness is also NOT gated on forge. If it were, k8s would pull the pod
out of the Service when forge dies and callers would get a connection
failure instead of the clean 503 that local-strict promises."""
return JSONResponse({"status": "ok", "service": CONFIG.name, "site": CONFIG.site})
@app.get("/status")
async def status() -> JSONResponse:
health = await app.state.locator.health()
return JSONResponse(
{
"service": CONFIG.name,
"site": CONFIG.site,
"endpoints": [f"/{m}/v1" for m in MODES],
"local_model": CONFIG.local_model,
"cloud_model": CONFIG.cloud_model if CONFIG.cloud_enabled else None,
"cloud_fallback_enabled": CONFIG.cloud_enabled,
"discovery_candidates": CONFIG.upstreams,
"discovered_upstream": app.state.locator.sticky,
# one runner config per model: the pin every forge-bound request carries,
# and the context the loaded runner was last seen with (must match)
"num_ctx_pin": CONFIG.num_ctx or None,
"runner_context": app.state.locator.runner_context,
# residency: the pin we send, and whether forge's runner is actually
# permanently loaded. False means the next idle spell costs a cold load.
"keep_alive_pin": CONFIG.keep_alive or None,
"runner_resident": app.state.locator.runner_resident,
"runner_expires_at": app.state.locator.runner_expires_at,
"health": health.as_dict(),
}
)
@app.get("/")
async def root() -> JSONResponse:
return JSONResponse(
{
"service": CONFIG.name,
"site": CONFIG.site,
"description": "OpenAI-compatible LLM router, one instance per cluster",
"endpoints": {
f"/{MODE_OPTIMISTIC}/v1": f"forge, falling back to {CONFIG.cloud_model} when forge is unavailable",
f"/{MODE_STRICT}/v1": "forge or HTTP 503 — callers are expected to handle the error",
},
"health": "/healthz (real completion against forge; 200 up, 503 down)",
"dialects": {
"openai": "/{endpoint}/v1/chat/completions, /{endpoint}/v1/models",
"ollama": "/{endpoint}/api/chat, /api/generate, /api/tags, /api/version (paste /{endpoint} as an Ollama server URL)",
},
}
)
# --------------------------------------------------------------------------
# OpenAI-compatible surface, under both base paths
# --------------------------------------------------------------------------
def _validate_mode(mode: str) -> JSONResponse | None:
if mode not in MODES:
return JSONResponse(
{
"error": {
"message": f"unknown endpoint '{mode}'. Use one of: {', '.join(MODES)}.",
"type": "forgeway_error",
"code": "unknown_endpoint",
"param": None,
}
},
status_code=404,
)
return None
@app.get("/{mode}/v1/models")
async def models(mode: str) -> JSONResponse:
invalid = _validate_mode(mode)
if invalid:
return invalid
created = int(time.time())
entries = [{"id": CONFIG.local_model, "object": "model", "created": created, "owned_by": CONFIG.name}]
if mode == MODE_OPTIMISTIC and CONFIG.cloud_enabled:
entries.append(
{"id": CONFIG.cloud_model, "object": "model", "created": created, "owned_by": f"{CONFIG.name}-fallback"}
)
return JSONResponse({"object": "list", "data": entries}, headers=_base_headers(mode, "none", None))
@app.get("/{mode}/healthz")
async def mode_healthz(mode: str) -> JSONResponse:
invalid = _validate_mode(mode)
if invalid:
return invalid
return await healthz()
@app.post("/{mode}/v1/chat/completions")
async def chat_completions(mode: str, request: Request):
invalid = _validate_mode(mode)
if invalid:
return invalid
try:
body = await request.json()
except ValueError:
return _error(400, "request body must be JSON", "invalid_request", mode)
if not isinstance(body, dict) or not body.get("messages"):
return _error(400, "'messages' is required", "invalid_request", mode)
locator = app.state.locator
health = await locator.health()
if health.healthy and health.upstream:
try:
return await _proxy_chat(health.upstream, CONFIG.local_model, body, mode, "forge")
except UpstreamError as exc:
# The cached verdict is stale — the upstream just failed a live call.
locator.invalidate()
log.warning("forge request failed on %s: %s", health.upstream, exc)
local_failure = str(exc)
else:
local_failure = health.reason
if mode == MODE_STRICT:
return _error(
503,
(
f"the local model '{CONFIG.local_model}' is unavailable at this site "
f"({CONFIG.site}). local-strict does not fall back by design; "
f"handle this error or use /{MODE_OPTIMISTIC}/v1."
),
"local_backend_unavailable",
mode,
upstream=health.upstream,
extra={"detail": local_failure, "site": CONFIG.site},
)
if not CONFIG.cloud_enabled:
return _error(
503,
"forge is unavailable and no cloud fallback credential is configured",
"no_backend_available",
mode,
extra={"detail": local_failure},
)
try:
log.info("falling back to cloud %s (%s)", CONFIG.cloud_model, local_failure)
return await _proxy_chat(
CONFIG.cloud_base_url,
CONFIG.cloud_model,
body,
mode,
"cloud",
api_key=CONFIG.cloud_api_key,
)
except UpstreamError as exc:
return _error(
502,
f"forge unavailable and the cloud fallback also failed: {exc}",
"all_backends_failed",
mode,
extra={"detail": local_failure, "cloud_detail": exc.body},
)
@app.api_route("/v1/{rest:path}", methods=["GET", "POST"])
async def bare_v1(rest: str) -> JSONResponse:
"""Guard rail: /v1/... without a mode prefix is almost always a misconfigured
base_url, and silently guessing a mode would defeat the point of having two."""
return JSONResponse(
{
"error": {
"message": (
"no endpoint selected. Set base_url to "
f"'<host>/{MODE_OPTIMISTIC}/v1' or '<host>/{MODE_STRICT}/v1'."
),
"type": "forgeway_error",
"code": "endpoint_not_selected",
"param": None,
}
},
status_code=404,
)