diff --git a/app/ollama_api.py b/app/ollama_api.py index 2b15302..bf0b051 100644 --- a/app/ollama_api.py +++ b/app/ollama_api.py @@ -369,7 +369,44 @@ async def _forge_raw( finally: await resp.aclose() - return StreamingResponse(body(), media_type=NDJSON, headers=base_headers(mode, "forge", upstream)) + async def body_as_generate(): + # We rewrote /api/generate -> /api/chat upstream (see the note where + # rewrote_generate is set). Streaming chunks therefore arrive in CHAT shape + # (`message.content`) while the caller is reading GENERATE shape (`response`), + # which reaches it as a run of literal "undefined". Karakeep tags via a + # STREAMED generate call and reported exactly that: "didn't respond with the + # expected JSON ... undefinedundefinedun". So translate every line. + buf = b"" + try: + async for chunk in resp.aiter_raw(): + buf += chunk + while b"\n" in buf: + line, buf = buf.split(b"\n", 1) + if not line.strip(): + continue + try: + obj = json.loads(line) + except ValueError: + yield line + b"\n" + continue + msg = obj.pop("message", None) + if isinstance(msg, dict): + obj["response"] = msg.get("content", "") + if msg.get("thinking") and not strip_thinking: + obj["thinking"] = msg["thinking"] + if obj.get("done"): + obj.setdefault("context", []) + yield (_dumps(obj) + "\n").encode() + if buf.strip(): + yield buf + finally: + await resp.aclose() + + return StreamingResponse( + body_as_generate() if as_generate else body(), + media_type=NDJSON, + headers=base_headers(mode, "forge", upstream), + ) async def _cloud(request: Request, body: dict[str, Any], mode: str, stream: bool, shape: str, model: str) -> Response: