A cold load of gpt-oss:120b is 21-66 s and lands entirely in front of the
caller, so every idle spell put that pause on Rick's next Home Assistant turn.
The 0.3-0.6 s figures from 2026-09-18 were warm-model numbers.
Measured on Ollama 0.34.0, and NOT symmetrical with the num_ctx pin:
native /api/chat, keep_alive:-1 -> Forever
native /api/chat, explicit finite -> that duration, and it BEATS the
server-side OLLAMA_KEEP_ALIVE default
native /api/chat, absent -> does NOT reset an existing value;
the duration is fixed AT LOAD TIME
/v1/chat/completions, keep_alive -> IGNORED, like options.num_ctx
(confirmed in openai/openai.go: the
request struct has no such field)
So the router alone can never carry residency: an OpenAI-dialect caller cannot
set it, and whichever request happens to trigger the cold load fixes the
duration. forge therefore also gets OLLAMA_KEEP_ALIVE=-1 in
/etc/systemd/system/ollama.service.d/30-keepalive.conf. The two are
complementary rather than redundant — the server default covers dialects and
callers we never proxy, this pin defeats a native caller that sends a short
keep_alive of its own.
The health probe asserts residency rather than inheriting it: it runs before
every proxied request, so it is the most frequent caller forge has and the one
most likely to be what triggers a cold load.
/status gains keep_alive_pin, runner_resident and runner_expires_at, and the
locator logs a WARNING if forge's runner stops being permanently resident —
the same visibility pattern as num_ctx_pin vs runner_context.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
399 lines
14 KiB
Python
399 lines
14 KiB
Python
"""OpenAI-compatible router with two query endpoints.
|
|
|
|
/local-optimistic/v1/... forge if available, otherwise the cloud model
|
|
/local-strict/v1/... forge or a clean 503 — callers handle the error
|
|
|
|
One instance runs per cluster. There is deliberately no central instance: the
|
|
cross-site link to forge may itself be the thing that is down, and a central
|
|
router would then *be* the outage.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from contextlib import asynccontextmanager
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from fastapi import FastAPI, Request
|
|
from fastapi.responses import JSONResponse, StreamingResponse
|
|
|
|
from .common import MODE_OPTIMISTIC, MODE_STRICT, MODES, base_headers
|
|
from .config import CONFIG
|
|
from .ollama_api import router as ollama_router
|
|
from .upstream import ForgeLocator, UpstreamError
|
|
|
|
logging.basicConfig(
|
|
level=getattr(logging, CONFIG.log_level, logging.INFO),
|
|
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
|
)
|
|
log = logging.getLogger("forgeway")
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
app.state.client = httpx.AsyncClient(
|
|
timeout=httpx.Timeout(
|
|
CONFIG.request_timeout,
|
|
connect=CONFIG.connect_timeout,
|
|
read=CONFIG.request_timeout,
|
|
write=60.0,
|
|
pool=CONFIG.connect_timeout,
|
|
),
|
|
limits=httpx.Limits(max_connections=64, max_keepalive_connections=16),
|
|
follow_redirects=False,
|
|
)
|
|
app.state.locator = ForgeLocator(app.state.client)
|
|
log.info(
|
|
"%s starting: site=%s model=%s cloud=%s candidates=%s",
|
|
CONFIG.name,
|
|
CONFIG.site,
|
|
CONFIG.local_model,
|
|
CONFIG.cloud_model if CONFIG.cloud_enabled else "disabled (no key)",
|
|
", ".join(CONFIG.upstreams),
|
|
)
|
|
try:
|
|
yield
|
|
finally:
|
|
await app.state.client.aclose()
|
|
|
|
|
|
app = FastAPI(
|
|
title=f"{CONFIG.name} — OpenAI-compatible LLM router",
|
|
docs_url=None,
|
|
redoc_url=None,
|
|
lifespan=lifespan,
|
|
)
|
|
app.include_router(ollama_router) # native Ollama dialect: /{mode}/api/...
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# helpers
|
|
# --------------------------------------------------------------------------
|
|
|
|
_base_headers = base_headers # shared with the native-Ollama dialect
|
|
|
|
|
|
def _error(
|
|
status: int,
|
|
message: str,
|
|
code: str,
|
|
mode: str,
|
|
backend: str = "none",
|
|
upstream: str | None = None,
|
|
extra: dict[str, Any] | None = None,
|
|
) -> JSONResponse:
|
|
"""OpenAI-shaped error so existing SDK error handling keeps working."""
|
|
payload: dict[str, Any] = {
|
|
"error": {
|
|
"message": message,
|
|
"type": "forgeway_error",
|
|
"code": code,
|
|
"param": None,
|
|
}
|
|
}
|
|
if extra:
|
|
payload["error"].update(extra)
|
|
# Deliberately no Retry-After: a 503 from local-strict means the local model
|
|
# is unavailable *now*, and blind retries just hammer a busy or absent forge.
|
|
return JSONResponse(payload, status_code=status, headers=_base_headers(mode, backend, upstream))
|
|
|
|
|
|
def _adapt_payload(body: dict[str, Any], backend: str) -> dict[str, Any]:
|
|
"""Reconcile the one parameter the two backends spell differently.
|
|
|
|
gpt-5.4-mini rejects `max_tokens` outright (400, "use max_completion_tokens"),
|
|
while Ollama's OpenAI layer only understands `max_tokens`. Callers should be
|
|
able to write either spelling and have both endpoints work, so translate here
|
|
rather than making every caller care which backend served them.
|
|
"""
|
|
payload = dict(body)
|
|
if backend == "cloud":
|
|
if "max_tokens" in payload:
|
|
payload.setdefault("max_completion_tokens", payload.pop("max_tokens"))
|
|
payload.pop("max_tokens", None)
|
|
else:
|
|
if "max_completion_tokens" in payload:
|
|
payload.setdefault("max_tokens", payload.pop("max_completion_tokens"))
|
|
payload.pop("max_completion_tokens", None)
|
|
return payload
|
|
|
|
|
|
async def _proxy_chat(
|
|
base_url: str,
|
|
model: str,
|
|
body: dict[str, Any],
|
|
mode: str,
|
|
backend: str,
|
|
api_key: str | None = None,
|
|
) -> JSONResponse | StreamingResponse:
|
|
"""Forward one chat completion, honouring stream=true."""
|
|
client: httpx.AsyncClient = app.state.client
|
|
payload = _adapt_payload(body, backend)
|
|
payload["model"] = model
|
|
stream = bool(payload.get("stream"))
|
|
|
|
headers = {"content-type": "application/json"}
|
|
if api_key:
|
|
headers["authorization"] = f"Bearer {api_key}"
|
|
|
|
url = f"{base_url}/v1/chat/completions" if not base_url.endswith("/v1") else f"{base_url}/chat/completions"
|
|
|
|
if not stream:
|
|
try:
|
|
resp = await client.post(url, json=payload, headers=headers)
|
|
except httpx.HTTPError as exc:
|
|
raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc
|
|
if resp.status_code >= 400:
|
|
raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, resp.text[:500])
|
|
return JSONResponse(
|
|
resp.json(),
|
|
status_code=resp.status_code,
|
|
headers=_base_headers(mode, backend, base_url),
|
|
)
|
|
|
|
# Streaming: send first and inspect the status before committing to a
|
|
# StreamingResponse, so a failed upstream can still fall back.
|
|
request = client.build_request("POST", url, json=payload, headers=headers)
|
|
try:
|
|
resp = await client.send(request, stream=True)
|
|
except httpx.HTTPError as exc:
|
|
raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc
|
|
|
|
if resp.status_code >= 400:
|
|
detail = (await resp.aread())[:500].decode("utf-8", "replace")
|
|
await resp.aclose()
|
|
raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, detail)
|
|
|
|
async def body_iter():
|
|
try:
|
|
async for chunk in resp.aiter_raw():
|
|
yield chunk
|
|
finally:
|
|
await resp.aclose()
|
|
|
|
return StreamingResponse(
|
|
body_iter(),
|
|
status_code=resp.status_code,
|
|
media_type=resp.headers.get("content-type", "text/event-stream"),
|
|
headers=_base_headers(mode, backend, base_url),
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# health — for Uptime Kuma (active HTTP probing, not push)
|
|
# --------------------------------------------------------------------------
|
|
|
|
@app.get("/healthz")
|
|
async def healthz() -> JSONResponse:
|
|
"""Gated on a real completion against forge.
|
|
|
|
200 forge answered a real completion
|
|
503 forge did not
|
|
(no response at all means the router itself is gone)
|
|
|
|
Results are cached for a few seconds so a burst of traffic does not turn
|
|
into a burst of probe completions against forge.
|
|
"""
|
|
health = await app.state.locator.health()
|
|
return JSONResponse(
|
|
health.as_dict(),
|
|
status_code=200 if health.healthy else 503,
|
|
headers=_base_headers("healthz", "forge" if health.healthy else "none", health.upstream),
|
|
)
|
|
|
|
|
|
@app.get("/healthz/live")
|
|
async def healthz_live() -> JSONResponse:
|
|
"""Liveness only. Never gated on forge — otherwise the pod would restart-loop
|
|
whenever forge is down, and local-strict could not return its 503."""
|
|
return JSONResponse({"status": "ok", "service": CONFIG.name, "site": CONFIG.site})
|
|
|
|
|
|
@app.get("/healthz/ready")
|
|
async def healthz_ready() -> JSONResponse:
|
|
"""Readiness is also NOT gated on forge. If it were, k8s would pull the pod
|
|
out of the Service when forge dies and callers would get a connection
|
|
failure instead of the clean 503 that local-strict promises."""
|
|
return JSONResponse({"status": "ok", "service": CONFIG.name, "site": CONFIG.site})
|
|
|
|
|
|
@app.get("/status")
|
|
async def status() -> JSONResponse:
|
|
health = await app.state.locator.health()
|
|
return JSONResponse(
|
|
{
|
|
"service": CONFIG.name,
|
|
"site": CONFIG.site,
|
|
"endpoints": [f"/{m}/v1" for m in MODES],
|
|
"local_model": CONFIG.local_model,
|
|
"cloud_model": CONFIG.cloud_model if CONFIG.cloud_enabled else None,
|
|
"cloud_fallback_enabled": CONFIG.cloud_enabled,
|
|
"discovery_candidates": CONFIG.upstreams,
|
|
"discovered_upstream": app.state.locator.sticky,
|
|
# one runner config per model: the pin every forge-bound request carries,
|
|
# and the context the loaded runner was last seen with (must match)
|
|
"num_ctx_pin": CONFIG.num_ctx or None,
|
|
"runner_context": app.state.locator.runner_context,
|
|
# residency: the pin we send, and whether forge's runner is actually
|
|
# permanently loaded. False means the next idle spell costs a cold load.
|
|
"keep_alive_pin": CONFIG.keep_alive or None,
|
|
"runner_resident": app.state.locator.runner_resident,
|
|
"runner_expires_at": app.state.locator.runner_expires_at,
|
|
"health": health.as_dict(),
|
|
}
|
|
)
|
|
|
|
|
|
@app.get("/")
|
|
async def root() -> JSONResponse:
|
|
return JSONResponse(
|
|
{
|
|
"service": CONFIG.name,
|
|
"site": CONFIG.site,
|
|
"description": "OpenAI-compatible LLM router, one instance per cluster",
|
|
"endpoints": {
|
|
f"/{MODE_OPTIMISTIC}/v1": f"forge, falling back to {CONFIG.cloud_model} when forge is unavailable",
|
|
f"/{MODE_STRICT}/v1": "forge or HTTP 503 — callers are expected to handle the error",
|
|
},
|
|
"health": "/healthz (real completion against forge; 200 up, 503 down)",
|
|
"dialects": {
|
|
"openai": "/{endpoint}/v1/chat/completions, /{endpoint}/v1/models",
|
|
"ollama": "/{endpoint}/api/chat, /api/generate, /api/tags, /api/version (paste /{endpoint} as an Ollama server URL)",
|
|
},
|
|
}
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# OpenAI-compatible surface, under both base paths
|
|
# --------------------------------------------------------------------------
|
|
|
|
def _validate_mode(mode: str) -> JSONResponse | None:
|
|
if mode not in MODES:
|
|
return JSONResponse(
|
|
{
|
|
"error": {
|
|
"message": f"unknown endpoint '{mode}'. Use one of: {', '.join(MODES)}.",
|
|
"type": "forgeway_error",
|
|
"code": "unknown_endpoint",
|
|
"param": None,
|
|
}
|
|
},
|
|
status_code=404,
|
|
)
|
|
return None
|
|
|
|
|
|
@app.get("/{mode}/v1/models")
|
|
async def models(mode: str) -> JSONResponse:
|
|
invalid = _validate_mode(mode)
|
|
if invalid:
|
|
return invalid
|
|
|
|
created = int(time.time())
|
|
entries = [{"id": CONFIG.local_model, "object": "model", "created": created, "owned_by": CONFIG.name}]
|
|
if mode == MODE_OPTIMISTIC and CONFIG.cloud_enabled:
|
|
entries.append(
|
|
{"id": CONFIG.cloud_model, "object": "model", "created": created, "owned_by": f"{CONFIG.name}-fallback"}
|
|
)
|
|
return JSONResponse({"object": "list", "data": entries}, headers=_base_headers(mode, "none", None))
|
|
|
|
|
|
@app.get("/{mode}/healthz")
|
|
async def mode_healthz(mode: str) -> JSONResponse:
|
|
invalid = _validate_mode(mode)
|
|
if invalid:
|
|
return invalid
|
|
return await healthz()
|
|
|
|
|
|
@app.post("/{mode}/v1/chat/completions")
|
|
async def chat_completions(mode: str, request: Request):
|
|
invalid = _validate_mode(mode)
|
|
if invalid:
|
|
return invalid
|
|
|
|
try:
|
|
body = await request.json()
|
|
except ValueError:
|
|
return _error(400, "request body must be JSON", "invalid_request", mode)
|
|
if not isinstance(body, dict) or not body.get("messages"):
|
|
return _error(400, "'messages' is required", "invalid_request", mode)
|
|
|
|
locator = app.state.locator
|
|
health = await locator.health()
|
|
|
|
if health.healthy and health.upstream:
|
|
try:
|
|
return await _proxy_chat(health.upstream, CONFIG.local_model, body, mode, "forge")
|
|
except UpstreamError as exc:
|
|
# The cached verdict is stale — the upstream just failed a live call.
|
|
locator.invalidate()
|
|
log.warning("forge request failed on %s: %s", health.upstream, exc)
|
|
local_failure = str(exc)
|
|
else:
|
|
local_failure = health.reason
|
|
|
|
if mode == MODE_STRICT:
|
|
return _error(
|
|
503,
|
|
(
|
|
f"the local model '{CONFIG.local_model}' is unavailable at this site "
|
|
f"({CONFIG.site}). local-strict does not fall back by design; "
|
|
f"handle this error or use /{MODE_OPTIMISTIC}/v1."
|
|
),
|
|
"local_backend_unavailable",
|
|
mode,
|
|
upstream=health.upstream,
|
|
extra={"detail": local_failure, "site": CONFIG.site},
|
|
)
|
|
|
|
if not CONFIG.cloud_enabled:
|
|
return _error(
|
|
503,
|
|
"forge is unavailable and no cloud fallback credential is configured",
|
|
"no_backend_available",
|
|
mode,
|
|
extra={"detail": local_failure},
|
|
)
|
|
|
|
try:
|
|
log.info("falling back to cloud %s (%s)", CONFIG.cloud_model, local_failure)
|
|
return await _proxy_chat(
|
|
CONFIG.cloud_base_url,
|
|
CONFIG.cloud_model,
|
|
body,
|
|
mode,
|
|
"cloud",
|
|
api_key=CONFIG.cloud_api_key,
|
|
)
|
|
except UpstreamError as exc:
|
|
return _error(
|
|
502,
|
|
f"forge unavailable and the cloud fallback also failed: {exc}",
|
|
"all_backends_failed",
|
|
mode,
|
|
extra={"detail": local_failure, "cloud_detail": exc.body},
|
|
)
|
|
|
|
|
|
@app.api_route("/v1/{rest:path}", methods=["GET", "POST"])
|
|
async def bare_v1(rest: str) -> JSONResponse:
|
|
"""Guard rail: /v1/... without a mode prefix is almost always a misconfigured
|
|
base_url, and silently guessing a mode would defeat the point of having two."""
|
|
return JSONResponse(
|
|
{
|
|
"error": {
|
|
"message": (
|
|
"no endpoint selected. Set base_url to "
|
|
f"'<host>/{MODE_OPTIMISTIC}/v1' or '<host>/{MODE_STRICT}/v1'."
|
|
),
|
|
"type": "forgeway_error",
|
|
"code": "endpoint_not_selected",
|
|
"param": None,
|
|
}
|
|
},
|
|
status_code=404,
|
|
)
|