ollama: translate STREAMED generate->chat chunks back to generate shape (0.2.6)
0.2.5 rewrote /api/generate to /api/chat upstream when a format is set, but only converted the response shape on the NON-streaming path. Streamed chunks passed through in chat shape (message.content) while the caller read generate shape (response) -- arriving as a run of literal 'undefined'. Karakeep tags via a streamed generate call and reported exactly that: 'didn't respond with the expected JSON ... undefinedundefinedun'. Summaries were unaffected because summarisation is not streamed, which is why they started working and tags did not. Now every NDJSON line is translated: message.content -> response, thinking preserved unless the caller asked for none, context added on the terminal chunk.
This commit is contained in:
+38
-1
@@ -369,7 +369,44 @@ async def _forge_raw(
|
||||
finally:
|
||||
await resp.aclose()
|
||||
|
||||
return StreamingResponse(body(), media_type=NDJSON, headers=base_headers(mode, "forge", upstream))
|
||||
async def body_as_generate():
|
||||
# We rewrote /api/generate -> /api/chat upstream (see the note where
|
||||
# rewrote_generate is set). Streaming chunks therefore arrive in CHAT shape
|
||||
# (`message.content`) while the caller is reading GENERATE shape (`response`),
|
||||
# which reaches it as a run of literal "undefined". Karakeep tags via a
|
||||
# STREAMED generate call and reported exactly that: "didn't respond with the
|
||||
# expected JSON ... undefinedundefinedun". So translate every line.
|
||||
buf = b""
|
||||
try:
|
||||
async for chunk in resp.aiter_raw():
|
||||
buf += chunk
|
||||
while b"\n" in buf:
|
||||
line, buf = buf.split(b"\n", 1)
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except ValueError:
|
||||
yield line + b"\n"
|
||||
continue
|
||||
msg = obj.pop("message", None)
|
||||
if isinstance(msg, dict):
|
||||
obj["response"] = msg.get("content", "")
|
||||
if msg.get("thinking") and not strip_thinking:
|
||||
obj["thinking"] = msg["thinking"]
|
||||
if obj.get("done"):
|
||||
obj.setdefault("context", [])
|
||||
yield (_dumps(obj) + "\n").encode()
|
||||
if buf.strip():
|
||||
yield buf
|
||||
finally:
|
||||
await resp.aclose()
|
||||
|
||||
return StreamingResponse(
|
||||
body_as_generate() if as_generate else body(),
|
||||
media_type=NDJSON,
|
||||
headers=base_headers(mode, "forge", upstream),
|
||||
)
|
||||
|
||||
|
||||
async def _cloud(request: Request, body: dict[str, Any], mode: str, stream: bool, shape: str, model: str) -> Response:
|
||||
|
||||
Reference in New Issue
Block a user