ollama: run /api/generate as /api/chat when a format is set (0.2.5)
gpt-oss:120b returns an EMPTY response on /api/generate whenever a JSON
format is requested. Verified against forge directly, bypassing this
router, so it is upstream behaviour and not ours:
/api/generate + format:'json' -> response '' done_reason stop
/api/generate, no format -> response 'OK'
/api/chat + format:'json' -> content '{"tags":["x"]}'
Karakeep tags via /api/generate WITH format, so every job returned
empty and it reported 'the model ignored our prompt'.
Fix: when a format is set, run the request upstream as /api/chat and
hand the answer back in generate's shape (response/context). Callers
change nothing.
This commit is contained in:
+41
-1
@@ -316,6 +316,7 @@ async def _forge_raw(
|
||||
mode: str,
|
||||
stream: bool,
|
||||
strip_thinking: bool = False,
|
||||
as_generate: bool = False,
|
||||
) -> Response:
|
||||
"""forge is Ollama: forward as-is (model pinned), return its bytes untouched."""
|
||||
client: httpx.AsyncClient = request.app.state.client
|
||||
@@ -334,6 +335,14 @@ async def _forge_raw(
|
||||
if path == "/api/chat" and _starved(data.get("message") or {}, data.get("done_reason")):
|
||||
headers[STARVED_HEADER] = "content-starved-by-reasoning"
|
||||
log.warning("forge answer had thinking but no content (done_reason=%s)", data.get("done_reason"))
|
||||
if as_generate and isinstance(data.get("message"), dict):
|
||||
# Present the chat answer in generate's shape for the caller.
|
||||
msg = data.pop("message")
|
||||
data["response"] = msg.get("content", "")
|
||||
if msg.get("thinking") and not strip_thinking:
|
||||
data["thinking"] = msg["thinking"]
|
||||
data.setdefault("context", [])
|
||||
content = _dumps(data).encode()
|
||||
if strip_thinking and isinstance(data.get("message"), dict) and "thinking" in data["message"]:
|
||||
# Caller asked for no thinking; give it none -- without having asked the
|
||||
# model to stop reasoning, which would have produced no answer at all.
|
||||
@@ -433,6 +442,30 @@ async def _route(request: Request, mode: str, path: str, body: dict[str, Any], s
|
||||
# think=false, so this fixes a latent failure there too.
|
||||
strip_thinking = payload.pop("think", None) is False
|
||||
|
||||
# 2026-09-19: gpt-oss:120b returns an EMPTY `response` on /api/generate whenever a
|
||||
# JSON `format` is requested -- verified against forge DIRECTLY, bypassing this
|
||||
# router, so it is an upstream model/runtime behaviour and not ours:
|
||||
# /api/generate + format:"json" -> response "" done_reason "stop"
|
||||
# /api/generate, no format -> response "OK"
|
||||
# /api/chat + format:"json" -> content '{"tags":["x"]}' (works)
|
||||
# Karakeep tags via /api/generate WITH format, so every job came back empty.
|
||||
# Fix: when a format is set, run it upstream as /api/chat (which works) and hand
|
||||
# the answer back in generate's shape. Callers need change nothing.
|
||||
rewrote_generate = False
|
||||
if path == "/api/generate" and payload.get("format"):
|
||||
msgs: list[dict[str, Any]] = []
|
||||
if payload.get("system"):
|
||||
msgs.append({"role": "system", "content": payload["system"]})
|
||||
u: dict[str, Any] = {"role": "user", "content": payload.get("prompt") or ""}
|
||||
if payload.get("images"):
|
||||
u["images"] = payload["images"]
|
||||
msgs.append(u)
|
||||
payload = {k: v for k, v in payload.items() if k not in ("prompt", "system", "images", "raw", "context")}
|
||||
payload["messages"] = msgs
|
||||
path = "/api/chat"
|
||||
rewrote_generate = True
|
||||
log.info("rewriting /api/generate -> /api/chat (format set; gpt-oss returns empty on generate+format)")
|
||||
|
||||
if CONFIG.keep_alive != 0:
|
||||
# Residency: a cold load is 21-66 s and lands entirely in front of the
|
||||
# caller. An explicit finite keep_alive from a caller BEATS forge's
|
||||
@@ -444,7 +477,14 @@ async def _route(request: Request, mode: str, path: str, body: dict[str, Any], s
|
||||
payload["keep_alive"] = CONFIG.keep_alive
|
||||
try:
|
||||
return await _forge_raw(
|
||||
request, health.upstream, path, payload, mode, stream, strip_thinking=strip_thinking
|
||||
request,
|
||||
health.upstream,
|
||||
path,
|
||||
payload,
|
||||
mode,
|
||||
stream,
|
||||
strip_thinking=strip_thinking,
|
||||
as_generate=rewrote_generate,
|
||||
)
|
||||
except UpstreamError as exc:
|
||||
locator.invalidate()
|
||||
|
||||
Reference in New Issue
Block a user