"""OpenAI-compatible router with two query endpoints. /local-optimistic/v1/... forge if available, otherwise the cloud model /local-strict/v1/... forge or a clean 503 — callers handle the error One instance runs per cluster. There is deliberately no central instance: the cross-site link to forge may itself be the thing that is down, and a central router would then *be* the outage. """ from __future__ import annotations import logging import time from contextlib import asynccontextmanager from typing import Any import httpx from fastapi import FastAPI, Request from fastapi.responses import JSONResponse, StreamingResponse from .common import MODE_OPTIMISTIC, MODE_STRICT, MODES, base_headers from .config import CONFIG from .ollama_api import router as ollama_router from .upstream import ForgeLocator, UpstreamError logging.basicConfig( level=getattr(logging, CONFIG.log_level, logging.INFO), format="%(asctime)s %(levelname)s %(name)s %(message)s", ) log = logging.getLogger("forgeway") @asynccontextmanager async def lifespan(app: FastAPI): app.state.client = httpx.AsyncClient( timeout=httpx.Timeout( CONFIG.request_timeout, connect=CONFIG.connect_timeout, read=CONFIG.request_timeout, write=60.0, pool=CONFIG.connect_timeout, ), limits=httpx.Limits(max_connections=64, max_keepalive_connections=16), follow_redirects=False, ) app.state.locator = ForgeLocator(app.state.client) log.info( "%s starting: site=%s model=%s cloud=%s candidates=%s", CONFIG.name, CONFIG.site, CONFIG.local_model, CONFIG.cloud_model if CONFIG.cloud_enabled else "disabled (no key)", ", ".join(CONFIG.upstreams), ) try: yield finally: await app.state.client.aclose() app = FastAPI( title=f"{CONFIG.name} — OpenAI-compatible LLM router", docs_url=None, redoc_url=None, lifespan=lifespan, ) app.include_router(ollama_router) # native Ollama dialect: /{mode}/api/... # -------------------------------------------------------------------------- # helpers # -------------------------------------------------------------------------- _base_headers = base_headers # shared with the native-Ollama dialect def _error( status: int, message: str, code: str, mode: str, backend: str = "none", upstream: str | None = None, extra: dict[str, Any] | None = None, ) -> JSONResponse: """OpenAI-shaped error so existing SDK error handling keeps working.""" payload: dict[str, Any] = { "error": { "message": message, "type": "forgeway_error", "code": code, "param": None, } } if extra: payload["error"].update(extra) # Deliberately no Retry-After: a 503 from local-strict means the local model # is unavailable *now*, and blind retries just hammer a busy or absent forge. return JSONResponse(payload, status_code=status, headers=_base_headers(mode, backend, upstream)) def _adapt_payload(body: dict[str, Any], backend: str) -> dict[str, Any]: """Reconcile the one parameter the two backends spell differently. gpt-5.4-mini rejects `max_tokens` outright (400, "use max_completion_tokens"), while Ollama's OpenAI layer only understands `max_tokens`. Callers should be able to write either spelling and have both endpoints work, so translate here rather than making every caller care which backend served them. """ payload = dict(body) if backend == "cloud": if "max_tokens" in payload: payload.setdefault("max_completion_tokens", payload.pop("max_tokens")) payload.pop("max_tokens", None) else: if "max_completion_tokens" in payload: payload.setdefault("max_tokens", payload.pop("max_completion_tokens")) payload.pop("max_completion_tokens", None) return payload async def _proxy_chat( base_url: str, model: str, body: dict[str, Any], mode: str, backend: str, api_key: str | None = None, ) -> JSONResponse | StreamingResponse: """Forward one chat completion, honouring stream=true.""" client: httpx.AsyncClient = app.state.client payload = _adapt_payload(body, backend) payload["model"] = model stream = bool(payload.get("stream")) headers = {"content-type": "application/json"} if api_key: headers["authorization"] = f"Bearer {api_key}" url = f"{base_url}/v1/chat/completions" if not base_url.endswith("/v1") else f"{base_url}/chat/completions" if not stream: try: resp = await client.post(url, json=payload, headers=headers) except httpx.HTTPError as exc: raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc if resp.status_code >= 400: raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, resp.text[:500]) return JSONResponse( resp.json(), status_code=resp.status_code, headers=_base_headers(mode, backend, base_url), ) # Streaming: send first and inspect the status before committing to a # StreamingResponse, so a failed upstream can still fall back. request = client.build_request("POST", url, json=payload, headers=headers) try: resp = await client.send(request, stream=True) except httpx.HTTPError as exc: raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc if resp.status_code >= 400: detail = (await resp.aread())[:500].decode("utf-8", "replace") await resp.aclose() raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, detail) async def body_iter(): try: async for chunk in resp.aiter_raw(): yield chunk finally: await resp.aclose() return StreamingResponse( body_iter(), status_code=resp.status_code, media_type=resp.headers.get("content-type", "text/event-stream"), headers=_base_headers(mode, backend, base_url), ) # -------------------------------------------------------------------------- # health — for Uptime Kuma (active HTTP probing, not push) # -------------------------------------------------------------------------- @app.get("/healthz") async def healthz() -> JSONResponse: """Gated on a real completion against forge. 200 forge answered a real completion 503 forge did not (no response at all means the router itself is gone) Results are cached for a few seconds so a burst of traffic does not turn into a burst of probe completions against forge. """ health = await app.state.locator.health() return JSONResponse( health.as_dict(), status_code=200 if health.healthy else 503, headers=_base_headers("healthz", "forge" if health.healthy else "none", health.upstream), ) @app.get("/healthz/live") async def healthz_live() -> JSONResponse: """Liveness only. Never gated on forge — otherwise the pod would restart-loop whenever forge is down, and local-strict could not return its 503.""" return JSONResponse({"status": "ok", "service": CONFIG.name, "site": CONFIG.site}) @app.get("/healthz/ready") async def healthz_ready() -> JSONResponse: """Readiness is also NOT gated on forge. If it were, k8s would pull the pod out of the Service when forge dies and callers would get a connection failure instead of the clean 503 that local-strict promises.""" return JSONResponse({"status": "ok", "service": CONFIG.name, "site": CONFIG.site}) @app.get("/status") async def status() -> JSONResponse: health = await app.state.locator.health() return JSONResponse( { "service": CONFIG.name, "site": CONFIG.site, "endpoints": [f"/{m}/v1" for m in MODES], "local_model": CONFIG.local_model, "cloud_model": CONFIG.cloud_model if CONFIG.cloud_enabled else None, "cloud_fallback_enabled": CONFIG.cloud_enabled, "discovery_candidates": CONFIG.upstreams, "discovered_upstream": app.state.locator.sticky, # one runner config per model: the pin every forge-bound request carries, # and the context the loaded runner was last seen with (must match) "num_ctx_pin": CONFIG.num_ctx or None, "runner_context": app.state.locator.runner_context, # residency: the pin we send, and whether forge's runner is actually # permanently loaded. False means the next idle spell costs a cold load. "keep_alive_pin": CONFIG.keep_alive or None, "runner_resident": app.state.locator.runner_resident, "runner_expires_at": app.state.locator.runner_expires_at, "health": health.as_dict(), } ) @app.get("/") async def root() -> JSONResponse: return JSONResponse( { "service": CONFIG.name, "site": CONFIG.site, "description": "OpenAI-compatible LLM router, one instance per cluster", "endpoints": { f"/{MODE_OPTIMISTIC}/v1": f"forge, falling back to {CONFIG.cloud_model} when forge is unavailable", f"/{MODE_STRICT}/v1": "forge or HTTP 503 — callers are expected to handle the error", }, "health": "/healthz (real completion against forge; 200 up, 503 down)", "dialects": { "openai": "/{endpoint}/v1/chat/completions, /{endpoint}/v1/models", "ollama": "/{endpoint}/api/chat, /api/generate, /api/tags, /api/version (paste /{endpoint} as an Ollama server URL)", }, } ) # -------------------------------------------------------------------------- # OpenAI-compatible surface, under both base paths # -------------------------------------------------------------------------- def _validate_mode(mode: str) -> JSONResponse | None: if mode not in MODES: return JSONResponse( { "error": { "message": f"unknown endpoint '{mode}'. Use one of: {', '.join(MODES)}.", "type": "forgeway_error", "code": "unknown_endpoint", "param": None, } }, status_code=404, ) return None @app.get("/{mode}/v1/models") async def models(mode: str) -> JSONResponse: invalid = _validate_mode(mode) if invalid: return invalid created = int(time.time()) entries = [{"id": CONFIG.local_model, "object": "model", "created": created, "owned_by": CONFIG.name}] if mode == MODE_OPTIMISTIC and CONFIG.cloud_enabled: entries.append( {"id": CONFIG.cloud_model, "object": "model", "created": created, "owned_by": f"{CONFIG.name}-fallback"} ) return JSONResponse({"object": "list", "data": entries}, headers=_base_headers(mode, "none", None)) @app.get("/{mode}/healthz") async def mode_healthz(mode: str) -> JSONResponse: invalid = _validate_mode(mode) if invalid: return invalid return await healthz() @app.post("/{mode}/v1/chat/completions") async def chat_completions(mode: str, request: Request): invalid = _validate_mode(mode) if invalid: return invalid try: body = await request.json() except ValueError: return _error(400, "request body must be JSON", "invalid_request", mode) if not isinstance(body, dict) or not body.get("messages"): return _error(400, "'messages' is required", "invalid_request", mode) locator = app.state.locator health = await locator.health() if health.healthy and health.upstream: try: return await _proxy_chat(health.upstream, CONFIG.local_model, body, mode, "forge") except UpstreamError as exc: # The cached verdict is stale — the upstream just failed a live call. locator.invalidate() log.warning("forge request failed on %s: %s", health.upstream, exc) local_failure = str(exc) else: local_failure = health.reason if mode == MODE_STRICT: return _error( 503, ( f"the local model '{CONFIG.local_model}' is unavailable at this site " f"({CONFIG.site}). local-strict does not fall back by design; " f"handle this error or use /{MODE_OPTIMISTIC}/v1." ), "local_backend_unavailable", mode, upstream=health.upstream, extra={"detail": local_failure, "site": CONFIG.site}, ) if not CONFIG.cloud_enabled: return _error( 503, "forge is unavailable and no cloud fallback credential is configured", "no_backend_available", mode, extra={"detail": local_failure}, ) try: log.info("falling back to cloud %s (%s)", CONFIG.cloud_model, local_failure) return await _proxy_chat( CONFIG.cloud_base_url, CONFIG.cloud_model, body, mode, "cloud", api_key=CONFIG.cloud_api_key, ) except UpstreamError as exc: return _error( 502, f"forge unavailable and the cloud fallback also failed: {exc}", "all_backends_failed", mode, extra={"detail": local_failure, "cloud_detail": exc.body}, ) @app.api_route("/v1/{rest:path}", methods=["GET", "POST"]) async def bare_v1(rest: str) -> JSONResponse: """Guard rail: /v1/... without a mode prefix is almost always a misconfigured base_url, and silently guessing a mode would defeat the point of having two.""" return JSONResponse( { "error": { "message": ( "no endpoint selected. Set base_url to " f"'/{MODE_OPTIMISTIC}/v1' or '/{MODE_STRICT}/v1'." ), "type": "forgeway_error", "code": "endpoint_not_selected", "param": None, } }, status_code=404, )