ollama: translate STREAMED generate->chat chunks back to generate shape (0.2.6)

0.2.5 rewrote /api/generate to /api/chat upstream when a format is set,
but only converted the response shape on the NON-streaming path.
Streamed chunks passed through in chat shape (message.content) while the
caller read generate shape (response) -- arriving as a run of literal
'undefined'. Karakeep tags via a streamed generate call and reported
exactly that: 'didn't respond with the expected JSON ...
undefinedundefinedun'. Summaries were unaffected because summarisation
is not streamed, which is why they started working and tags did not.

Now every NDJSON line is translated: message.content -> response,
thinking preserved unless the caller asked for none, context added on
the terminal chunk.
This commit is contained in:
Dolon (Agent)
2026-09-19 18:23:25 +00:00
parent a8b6a3ee6e
commit 2db73ecd08
+38 -1
View File
@@ -369,7 +369,44 @@ async def _forge_raw(
finally: finally:
await resp.aclose() await resp.aclose()
return StreamingResponse(body(), media_type=NDJSON, headers=base_headers(mode, "forge", upstream)) async def body_as_generate():
# We rewrote /api/generate -> /api/chat upstream (see the note where
# rewrote_generate is set). Streaming chunks therefore arrive in CHAT shape
# (`message.content`) while the caller is reading GENERATE shape (`response`),
# which reaches it as a run of literal "undefined". Karakeep tags via a
# STREAMED generate call and reported exactly that: "didn't respond with the
# expected JSON ... undefinedundefinedun". So translate every line.
buf = b""
try:
async for chunk in resp.aiter_raw():
buf += chunk
while b"\n" in buf:
line, buf = buf.split(b"\n", 1)
if not line.strip():
continue
try:
obj = json.loads(line)
except ValueError:
yield line + b"\n"
continue
msg = obj.pop("message", None)
if isinstance(msg, dict):
obj["response"] = msg.get("content", "")
if msg.get("thinking") and not strip_thinking:
obj["thinking"] = msg["thinking"]
if obj.get("done"):
obj.setdefault("context", [])
yield (_dumps(obj) + "\n").encode()
if buf.strip():
yield buf
finally:
await resp.aclose()
return StreamingResponse(
body_as_generate() if as_generate else body(),
media_type=NDJSON,
headers=base_headers(mode, "forge", upstream),
)
async def _cloud(request: Request, body: dict[str, Any], mode: str, stream: bool, shape: str, model: str) -> Response: async def _cloud(request: Request, body: dict[str, Any], mode: str, stream: bool, shape: str, model: str) -> Response: