Files
forgeway/app/ollama_api.py
Dolon (Agent) 2db73ecd08 ollama: translate STREAMED generate->chat chunks back to generate shape (0.2.6)
0.2.5 rewrote /api/generate to /api/chat upstream when a format is set,
but only converted the response shape on the NON-streaming path.
Streamed chunks passed through in chat shape (message.content) while the
caller read generate shape (response) -- arriving as a run of literal
'undefined'. Karakeep tags via a streamed generate call and reported
exactly that: 'didn't respond with the expected JSON ...
undefinedundefinedun'. Summaries were unaffected because summarisation
is not streamed, which is why they started working and tags did not.

Now every NDJSON line is translated: message.content -> response,
thinking preserved unless the caller asked for none, context added on
the terminal chunk.
2026-09-19 18:23:25 +00:00

685 lines
30 KiB
Python

"""Native Ollama API dialect, served under the same two modes.
/{mode}/api/tags GET what this endpoint serves (never blocks on forge)
/{mode}/api/chat POST forge, else cloud (optimistic) / 503 (strict)
/{mode}/api/generate POST same routing, prompt-style
/{mode}/api/version GET
/{mode}/api/show POST pass-through to forge
/{mode}/api/ps GET pass-through to forge
Home Assistant's Ollama integration speaks exactly this: ``list()`` → tags with
a 5 s timeout, then ``chat(stream=True)``. Its client keeps the URL path, so
``https://<name>/local-optimistic`` is a valid "server URL".
Two things are non-obvious here:
* forge *is* Ollama, so the forge path is a raw proxy — the response, including
the streamed NDJSON frames, is forge's own. Only the cloud fallback needs
translation, in both directions, and that is where most of this file goes.
* Ollama's default for ``stream`` is **true**, the opposite of OpenAI's.
"""
from __future__ import annotations
import json
import logging
import time
from datetime import datetime, timezone
from typing import Any, AsyncIterator
import httpx
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse, Response, StreamingResponse
from .common import MODE_STRICT, MODES, base_headers, strict_unavailable_message
from .config import CONFIG
from .upstream import UpstreamError
log = logging.getLogger("forgeway.ollama")
router = APIRouter()
NDJSON = "application/x-ndjson"
QUICK_TIMEOUT = 2.5 # anything a client may call with a short timeout (HA: 5 s on tags)
STARVED_HEADER = "x-forgeway-warning"
# --------------------------------------------------------------------------
# small helpers
# --------------------------------------------------------------------------
def _now() -> str:
"""RFC 3339 with nanosecond precision, the way Ollama prints created_at."""
dt = datetime.now(timezone.utc)
return dt.strftime("%Y-%m-%dT%H:%M:%S.") + f"{dt.microsecond:06d}000Z"
def _ns(seconds: float) -> int:
return int(seconds * 1_000_000_000)
def _error(status: int, message: str, mode: str, backend: str = "none", upstream: str | None = None) -> JSONResponse:
"""Ollama's error shape: a bare {"error": "..."}; the python client raises ResponseError on it."""
return JSONResponse({"error": message}, status_code=status, headers=base_headers(mode, backend, upstream))
def _bad_mode(mode: str) -> JSONResponse | None:
if mode not in MODES:
return JSONResponse({"error": f"unknown endpoint '{mode}'. Use one of: {', '.join(MODES)}."}, status_code=404)
return None
def _done_reason(finish_reason: str | None) -> str:
return {"stop": "stop", "length": "length", "tool_calls": "stop", "content_filter": "stop"}.get(
finish_reason or "stop", "stop"
)
def _tool_call_to_ollama(index: int, call: dict[str, Any]) -> dict[str, Any]:
fn = call.get("function") or {}
args = fn.get("arguments")
if isinstance(args, str):
try:
args = json.loads(args) if args.strip() else {}
except ValueError:
args = {"_raw": args}
return {
"id": call.get("id") or f"call_fw{index}",
"function": {"index": index, "name": fn.get("name", ""), "arguments": args if isinstance(args, dict) else {}},
}
def _starved(message: dict[str, Any], done_reason: str | None) -> bool:
"""A thinking model spent its whole budget thinking and produced no answer."""
return (
not (message.get("content") or "").strip()
and not message.get("tool_calls")
and (bool(message.get("thinking")) or done_reason == "length")
)
# --------------------------------------------------------------------------
# request translation: Ollama chat → OpenAI chat completion
# --------------------------------------------------------------------------
def _messages_to_openai(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Ollama messages carry no tool-call ids; OpenAI insists on them. Synthesise
ids in order and hand them to the following tool-result messages in order."""
out: list[dict[str, Any]] = []
pending_ids: list[str] = []
counter = 0
for m in messages:
role = m.get("role") or "user"
content = m.get("content") or ""
if role == "assistant":
entry: dict[str, Any] = {"role": "assistant", "content": content or None}
calls = m.get("tool_calls") or []
if calls:
pending_ids = []
entry["tool_calls"] = []
for call in calls:
counter += 1
fn = call.get("function") or {}
cid = call.get("id") or f"call_fw{counter}"
entry["tool_calls"].append(
{
"id": cid,
"type": "function",
"function": {"name": fn.get("name", ""), "arguments": json.dumps(fn.get("arguments") or {})},
}
)
pending_ids.append(cid)
out.append(entry)
elif role == "tool":
cid = pending_ids.pop(0) if pending_ids else f"call_fw{counter or 1}"
out.append({"role": "tool", "tool_call_id": cid, "content": content})
elif role == "user" and m.get("images"):
parts: list[dict[str, Any]] = [{"type": "text", "text": content}]
for img in m["images"]:
if isinstance(img, str):
parts.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}})
out.append({"role": "user", "content": parts})
else:
out.append({"role": role, "content": content})
return out
def _chat_to_openai(body: dict[str, Any], stream: bool) -> dict[str, Any]:
req: dict[str, Any] = {
"model": CONFIG.cloud_model,
"messages": _messages_to_openai(body.get("messages") or []),
"stream": stream,
}
if stream:
req["stream_options"] = {"include_usage": True}
if body.get("tools"):
req["tools"] = body["tools"] # Ollama's tool schema is already OpenAI's
fmt = body.get("format")
if fmt == "json":
req["response_format"] = {"type": "json_object"}
elif isinstance(fmt, dict) and fmt:
req["response_format"] = {"type": "json_schema", "json_schema": {"name": "response", "schema": fmt}}
# Ollama's `think` → OpenAI's reasoning_effort. HA sends False by default;
# "low" keeps the assistant snappy without switching reasoning off outright.
# With tools present it MUST be "none": gpt-5.4-mini rejects any other value
# alongside function tools on /v1/chat/completions (verified 2026-09-18),
# and HA's Assist API always sends tools.
# 2026-09-19: `think: false` must NOT be honoured literally for the local model.
# gpt-oss:120b generates its answer AFTER the reasoning pass; at reasoning_effort
# "low" combined with a JSON `format` it returns EMPTY content and no thinking at
# all. Karakeep sends think=false + format, so every tagging job failed in <1 s with
# "the model ignored our prompt and didn't respond with the expected JSON: {}".
# Verified by curl: think=false -> content ''; think=true -> content '{"tags":["x"]}'.
# So we keep reasoning ON and instead STRIP the `thinking` field from the response,
# which is what a caller asking for no thinking actually wants. See _strip_thinking.
think = body.get("think")
if req.get("tools"):
# gpt-5.4-mini rejects any other value alongside function tools on
# /v1/chat/completions (verified 2026-09-18), and HA's Assist API always
# sends tools.
req["reasoning_effort"] = "none"
elif isinstance(think, bool):
req["reasoning_effort"] = "medium"
elif think in ("low", "medium", "high"):
req["reasoning_effort"] = think
opts = body.get("options") or {}
if isinstance(opts.get("num_predict"), int) and opts["num_predict"] > 0:
req["max_completion_tokens"] = opts["num_predict"]
for key in ("temperature", "top_p"):
if opts.get(key) is not None:
req[key] = opts[key]
return req
# --------------------------------------------------------------------------
# response translation: OpenAI → Ollama
# --------------------------------------------------------------------------
def _stats(usage: dict[str, Any] | None, started: float) -> dict[str, Any]:
usage = usage or {}
return {
"total_duration": _ns(time.perf_counter() - started),
"load_duration": 0,
"prompt_eval_count": int(usage.get("prompt_tokens") or 0),
"prompt_eval_duration": 0,
"eval_count": int(usage.get("completion_tokens") or 0),
"eval_duration": 0,
}
def _openai_to_ollama_chat(data: dict[str, Any], model: str, started: float) -> dict[str, Any]:
choice = (data.get("choices") or [{}])[0]
msg = choice.get("message") or {}
message: dict[str, Any] = {"role": "assistant", "content": msg.get("content") or ""}
if msg.get("tool_calls"):
message["tool_calls"] = [_tool_call_to_ollama(i, c) for i, c in enumerate(msg["tool_calls"])]
return {
"model": model,
"created_at": _now(),
"message": message,
"done": True,
"done_reason": _done_reason(choice.get("finish_reason")),
**_stats(data.get("usage"), started),
}
def _dumps(obj: Any) -> str:
"""Compact, the way Ollama's Go encoder prints — byte-shape parity for naive parsers."""
return json.dumps(obj, separators=(",", ":"), ensure_ascii=False)
def _chat_frame(model: str, message: dict[str, Any], done: bool = False, **extra: Any) -> bytes:
frame: dict[str, Any] = {"model": model, "created_at": _now(), "message": message, "done": done}
frame.update(extra)
return (_dumps(frame) + "\n").encode()
def _generate_frame(model: str, text: str, done: bool = False, **extra: Any) -> bytes:
frame: dict[str, Any] = {"model": model, "created_at": _now(), "response": text, "done": done}
frame.update(extra)
return (_dumps(frame) + "\n").encode()
async def _sse_to_ndjson(resp: httpx.Response, model: str, started: float, shape: str) -> AsyncIterator[bytes]:
"""OpenAI SSE deltas → Ollama NDJSON frames (chat or generate shape).
Tool calls arrive as fragments spread over many deltas; Ollama emits them
whole, so they are accumulated and flushed as one frame before `done`.
"""
usage: dict[str, Any] | None = None
finish: str | None = None
tool_acc: dict[int, dict[str, Any]] = {}
try:
async for raw in resp.aiter_lines():
line = raw.strip()
if not line.startswith("data:"):
continue
payload = line[5:].strip()
if payload == "[DONE]":
break
try:
evt = json.loads(payload)
except ValueError:
continue
if evt.get("usage"):
usage = evt["usage"]
for ch in evt.get("choices") or []:
delta = ch.get("delta") or {}
if delta.get("content"):
if shape == "chat":
yield _chat_frame(model, {"role": "assistant", "content": delta["content"]})
else:
yield _generate_frame(model, delta["content"])
for frag in delta.get("tool_calls") or []:
idx = int(frag.get("index") or 0)
acc = tool_acc.setdefault(idx, {"id": None, "name": "", "arguments": ""})
if frag.get("id"):
acc["id"] = frag["id"]
fn = frag.get("function") or {}
if fn.get("name"):
acc["name"] += fn["name"]
if fn.get("arguments"):
acc["arguments"] += fn["arguments"]
if ch.get("finish_reason"):
finish = ch["finish_reason"]
if tool_acc and shape == "chat":
calls = [
_tool_call_to_ollama(i, {"id": acc["id"], "function": {"name": acc["name"], "arguments": acc["arguments"]}})
for i, acc in sorted(tool_acc.items())
]
yield _chat_frame(model, {"role": "assistant", "content": "", "tool_calls": calls})
stats = _stats(usage, started)
if shape == "chat":
yield _chat_frame(model, {"role": "assistant", "content": ""}, done=True, done_reason=_done_reason(finish), **stats)
else:
yield _generate_frame(model, "", done=True, done_reason=_done_reason(finish), context=[], **stats)
except httpx.HTTPError as exc:
# Mid-stream failure: the client raises on an `error` frame.
yield (_dumps({"error": f"cloud stream failed: {type(exc).__name__}: {exc}"}) + "\n").encode()
finally:
await resp.aclose()
# --------------------------------------------------------------------------
# backends
# --------------------------------------------------------------------------
async def _forge_raw(
request: Request,
upstream: str,
path: str,
payload: dict[str, Any],
mode: str,
stream: bool,
strip_thinking: bool = False,
as_generate: bool = False,
) -> Response:
"""forge is Ollama: forward as-is (model pinned), return its bytes untouched."""
client: httpx.AsyncClient = request.app.state.client
url = f"{upstream}{path}"
if not stream:
try:
resp = await client.post(url, json=payload)
except httpx.HTTPError as exc:
raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc
if resp.status_code >= 400:
raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, resp.text[:500])
headers = base_headers(mode, "forge", upstream)
content = resp.content
try:
data = resp.json()
if path == "/api/chat" and _starved(data.get("message") or {}, data.get("done_reason")):
headers[STARVED_HEADER] = "content-starved-by-reasoning"
log.warning("forge answer had thinking but no content (done_reason=%s)", data.get("done_reason"))
if as_generate and isinstance(data.get("message"), dict):
# Present the chat answer in generate's shape for the caller.
msg = data.pop("message")
data["response"] = msg.get("content", "")
if msg.get("thinking") and not strip_thinking:
data["thinking"] = msg["thinking"]
data.setdefault("context", [])
content = _dumps(data).encode()
if strip_thinking and isinstance(data.get("message"), dict) and "thinking" in data["message"]:
# Caller asked for no thinking; give it none -- without having asked the
# model to stop reasoning, which would have produced no answer at all.
data["message"].pop("thinking", None)
content = _dumps(data).encode()
except ValueError:
pass
return Response(content, status_code=resp.status_code, media_type="application/json", headers=headers)
req = client.build_request("POST", url, json=payload)
try:
resp = await client.send(req, stream=True)
except httpx.HTTPError as exc:
raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc
if resp.status_code >= 400:
detail = (await resp.aread())[:500].decode("utf-8", "replace")
await resp.aclose()
raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, detail)
async def body():
try:
async for chunk in resp.aiter_raw():
yield chunk
finally:
await resp.aclose()
async def body_as_generate():
# We rewrote /api/generate -> /api/chat upstream (see the note where
# rewrote_generate is set). Streaming chunks therefore arrive in CHAT shape
# (`message.content`) while the caller is reading GENERATE shape (`response`),
# which reaches it as a run of literal "undefined". Karakeep tags via a
# STREAMED generate call and reported exactly that: "didn't respond with the
# expected JSON ... undefinedundefinedun". So translate every line.
buf = b""
try:
async for chunk in resp.aiter_raw():
buf += chunk
while b"\n" in buf:
line, buf = buf.split(b"\n", 1)
if not line.strip():
continue
try:
obj = json.loads(line)
except ValueError:
yield line + b"\n"
continue
msg = obj.pop("message", None)
if isinstance(msg, dict):
obj["response"] = msg.get("content", "")
if msg.get("thinking") and not strip_thinking:
obj["thinking"] = msg["thinking"]
if obj.get("done"):
obj.setdefault("context", [])
yield (_dumps(obj) + "\n").encode()
if buf.strip():
yield buf
finally:
await resp.aclose()
return StreamingResponse(
body_as_generate() if as_generate else body(),
media_type=NDJSON,
headers=base_headers(mode, "forge", upstream),
)
async def _cloud(request: Request, body: dict[str, Any], mode: str, stream: bool, shape: str, model: str) -> Response:
client: httpx.AsyncClient = request.app.state.client
started = time.perf_counter()
payload = _chat_to_openai(body, stream)
url = f"{CONFIG.cloud_base_url}/chat/completions"
headers = {"content-type": "application/json", "authorization": f"Bearer {CONFIG.cloud_api_key}"}
if not stream:
try:
resp = await client.post(url, json=payload, headers=headers)
except httpx.HTTPError as exc:
raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc
if resp.status_code >= 400:
raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, resp.text[:500])
data = _openai_to_ollama_chat(resp.json(), model, started)
out_headers = base_headers(mode, "cloud", CONFIG.cloud_base_url)
if shape == "generate":
msg = data.pop("message")
data["response"] = msg.get("content", "")
data["context"] = []
elif _starved(data["message"], data.get("done_reason")):
out_headers[STARVED_HEADER] = "content-starved-by-reasoning"
return Response(_dumps(data), media_type="application/json", headers=out_headers)
req = client.build_request("POST", url, json=payload, headers=headers)
try:
resp = await client.send(req, stream=True)
except httpx.HTTPError as exc:
raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc
if resp.status_code >= 400:
detail = (await resp.aread())[:500].decode("utf-8", "replace")
await resp.aclose()
raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, detail)
return StreamingResponse(
_sse_to_ndjson(resp, model, started, shape),
media_type=NDJSON,
headers=base_headers(mode, "cloud", CONFIG.cloud_base_url),
)
async def _route(request: Request, mode: str, path: str, body: dict[str, Any], shape: str) -> Response:
"""forge if it answers a real completion; else 503 (strict) or cloud (optimistic)."""
stream = bool(body.get("stream", True)) # Ollama's default, unlike OpenAI's
model = body.get("model") or CONFIG.local_model
locator = request.app.state.locator
health = await locator.health()
if health.healthy and health.upstream:
payload = dict(body)
payload["model"] = CONFIG.local_model
if CONFIG.num_ctx > 0:
# One runner config for everyone: a caller's own num_ctx (HA sends
# 8192.0 by default) would otherwise force a full model reload.
opts = dict(payload.get("options") or {})
sent = opts.get("num_ctx")
if sent is not None and sent != CONFIG.num_ctx:
log.info("pinning num_ctx %s -> %s on %s (one runner config, no reload)", sent, CONFIG.num_ctx, path)
opts["num_ctx"] = CONFIG.num_ctx
payload["options"] = opts
# 2026-09-19: a literal `think: false` makes gpt-oss:120b return EMPTY content.
# It generates its answer AFTER the reasoning pass, so switching reasoning off
# switches the answer off -- and with a JSON `format` set it returns nothing at
# all, not even thinking. Karakeep sends think=false + format and every tagging
# job failed in <1 s ("the model ignored our prompt ... {}"). Verified by curl:
# think=false -> content ''; think=true -> content '{"tags":["x"]}'.
# So: keep reasoning ON upstream, and strip `thinking` from the reply instead --
# which is what a caller asking for no thinking actually wants. HA also sends
# think=false, so this fixes a latent failure there too.
strip_thinking = payload.pop("think", None) is False
# 2026-09-19: gpt-oss:120b returns an EMPTY `response` on /api/generate whenever a
# JSON `format` is requested -- verified against forge DIRECTLY, bypassing this
# router, so it is an upstream model/runtime behaviour and not ours:
# /api/generate + format:"json" -> response "" done_reason "stop"
# /api/generate, no format -> response "OK"
# /api/chat + format:"json" -> content '{"tags":["x"]}' (works)
# Karakeep tags via /api/generate WITH format, so every job came back empty.
# Fix: when a format is set, run it upstream as /api/chat (which works) and hand
# the answer back in generate's shape. Callers need change nothing.
rewrote_generate = False
if path == "/api/generate" and payload.get("format"):
msgs: list[dict[str, Any]] = []
if payload.get("system"):
msgs.append({"role": "system", "content": payload["system"]})
u: dict[str, Any] = {"role": "user", "content": payload.get("prompt") or ""}
if payload.get("images"):
u["images"] = payload["images"]
msgs.append(u)
payload = {k: v for k, v in payload.items() if k not in ("prompt", "system", "images", "raw", "context")}
payload["messages"] = msgs
path = "/api/chat"
rewrote_generate = True
log.info("rewriting /api/generate -> /api/chat (format set; gpt-oss returns empty on generate+format)")
if CONFIG.keep_alive != 0:
# Residency: a cold load is 21-66 s and lands entirely in front of the
# caller. An explicit finite keep_alive from a caller BEATS forge's
# OLLAMA_KEEP_ALIVE default, so overriding it here is the only thing that
# stops one such caller evicting the model for everyone else.
sent = payload.get("keep_alive")
if sent is not None and sent != CONFIG.keep_alive:
log.info("pinning keep_alive %s -> %s on %s (stay resident)", sent, CONFIG.keep_alive, path)
payload["keep_alive"] = CONFIG.keep_alive
try:
return await _forge_raw(
request,
health.upstream,
path,
payload,
mode,
stream,
strip_thinking=strip_thinking,
as_generate=rewrote_generate,
)
except UpstreamError as exc:
locator.invalidate()
log.warning("forge %s failed on %s: %s", path, health.upstream, exc)
failure = str(exc)
else:
failure = health.reason
if mode == MODE_STRICT:
return _error(503, f"{strict_unavailable_message()} ({failure})", mode, upstream=health.upstream)
if not CONFIG.cloud_enabled:
return _error(503, f"forge is unavailable ({failure}) and no cloud fallback credential is configured", mode)
log.info("falling back to cloud %s for %s (%s)", CONFIG.cloud_model, path, failure)
if shape == "generate":
# generate → chat: prompt becomes the user turn, `system` the system turn
messages: list[dict[str, Any]] = []
if body.get("system"):
messages.append({"role": "system", "content": body["system"]})
user: dict[str, Any] = {"role": "user", "content": body.get("prompt") or ""}
if body.get("images"):
user["images"] = body["images"]
messages.append(user)
body = {**body, "messages": messages}
try:
return await _cloud(request, body, mode, stream, shape, model)
except UpstreamError as exc:
return _error(502, f"forge unavailable ({failure}) and the cloud fallback also failed: {exc} {exc.body or ''}".strip(), mode)
# --------------------------------------------------------------------------
# endpoints
# --------------------------------------------------------------------------
_tags_cache: dict[str, Any] = {"at": 0.0, "entry": None}
_version_cache: dict[str, Any] = {"value": None}
async def _forge_quick(request: Request, method: str, path: str, json_body: dict | None = None) -> dict | None:
"""Ask the last-known forge something cheap, never waiting on a model load."""
locator = request.app.state.locator
client: httpx.AsyncClient = request.app.state.client
base = locator.sticky or (CONFIG.upstreams[0] if CONFIG.upstreams else None)
if not base:
return None
try:
resp = await client.request(method, f"{base}{path}", json=json_body, timeout=QUICK_TIMEOUT)
return resp.json() if resp.status_code == 200 else None
except (httpx.HTTPError, ValueError):
return None
@router.get("/{mode}/api/tags")
async def api_tags(mode: str, request: Request):
"""What this endpoint serves — exactly one model, the local one, in both modes.
Cloud fallback is a behaviour of the endpoint, not a second model a client
could select: the request's `model` field is ignored either way. Fast on
purpose: HA's list() has a 5 s timeout and a cold model load takes ~25 s."""
invalid = _bad_mode(mode)
if invalid:
return invalid
now = time.time()
if _tags_cache["entry"] is None or now - _tags_cache["at"] > 60:
data = await _forge_quick(request, "GET", "/api/tags")
entry = None
for m in (data or {}).get("models", []):
if m.get("model") == CONFIG.local_model or m.get("name") == CONFIG.local_model:
entry = m
break
if entry:
_tags_cache.update(at=now, entry=entry)
entry = _tags_cache["entry"] or {
"name": CONFIG.local_model,
"model": CONFIG.local_model,
"modified_at": _now(),
"size": 0,
"digest": "",
"details": {"parent_model": "", "format": "", "family": "", "families": [], "parameter_size": "", "quantization_level": ""},
}
return JSONResponse({"models": [entry]}, headers=base_headers(mode, "none", request.app.state.locator.sticky))
@router.get("/{mode}/api/version")
async def api_version(mode: str, request: Request):
invalid = _bad_mode(mode)
if invalid:
return invalid
data = await _forge_quick(request, "GET", "/api/version")
if data and data.get("version"):
_version_cache["value"] = data["version"]
version = _version_cache["value"] or CONFIG.ollama_version_fallback
return JSONResponse({"version": version}, headers=base_headers(mode, "forge" if data else "none", None))
@router.get("/{mode}/api/ps")
async def api_ps(mode: str, request: Request):
invalid = _bad_mode(mode)
if invalid:
return invalid
data = await _forge_quick(request, "GET", "/api/ps")
if data is None:
return _error(503, "forge is not reachable", mode)
return JSONResponse(data, headers=base_headers(mode, "forge", request.app.state.locator.sticky))
@router.post("/{mode}/api/show")
async def api_show(mode: str, request: Request):
invalid = _bad_mode(mode)
if invalid:
return invalid
data = await _forge_quick(request, "POST", "/api/show", {"model": CONFIG.local_model})
if data is None:
return _error(503, "forge is not reachable", mode)
return JSONResponse(data, headers=base_headers(mode, "forge", request.app.state.locator.sticky))
@router.post("/{mode}/api/chat")
async def api_chat(mode: str, request: Request):
invalid = _bad_mode(mode)
if invalid:
return invalid
try:
body = await request.json()
except ValueError:
return _error(400, "request body must be JSON", mode)
if not isinstance(body, dict) or not body.get("messages"):
return _error(400, "'messages' is required", mode)
return await _route(request, mode, "/api/chat", body, "chat")
@router.post("/{mode}/api/generate")
async def api_generate(mode: str, request: Request):
invalid = _bad_mode(mode)
if invalid:
return invalid
try:
body = await request.json()
except ValueError:
return _error(400, "request body must be JSON", mode)
if not isinstance(body, dict):
return _error(400, "request body must be a JSON object", mode)
return await _route(request, mode, "/api/generate", body, "generate")
@router.api_route("/{mode}/api/{rest:path}", methods=["GET", "POST", "DELETE", "HEAD"])
async def api_unsupported(mode: str, rest: str):
invalid = _bad_mode(mode)
if invalid:
return invalid
if rest in ("pull", "push", "create", "copy", "delete", "blobs"):
msg = (
f"{CONFIG.name} does not manage models; this endpoint serves exactly "
f"'{CONFIG.local_model}'. Select it from /api/tags instead of pulling."
)
elif rest in ("embed", "embeddings"):
msg = f"embeddings are not served here ('{CONFIG.local_model}' is not an embedding model)"
else:
msg = f"/api/{rest} is not implemented by {CONFIG.name}"
return JSONResponse({"error": msg}, status_code=400)