24 asserts forge's runner is permanently resident, not merely loaded at the moment of the check — a cold load is 21-66 s and lands in front of the caller, so "loaded right now" is not the property that matters. 25 is the regression that would otherwise be silent: an explicit finite keep_alive from a caller BEATS forge's OLLAMA_KEEP_ALIVE default, so the router override is the only thing stopping one such caller evicting the 64 GB model for everyone else. The check sends keep_alive:"30s" natively and asserts residency survives it. Both pass on lounge and nidavellir, 25/25. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
66 lines
6.3 KiB
Bash
Executable File
66 lines
6.3 KiB
Bash
Executable File
#!/bin/sh
|
|
# Runs INSIDE the cluster, against the live Service and against a throwaway
|
|
# instance whose upstream is deliberately dead. Plain POSIX sh + curl only.
|
|
GOOD=http://forgeway.forgeway.svc.cluster.local
|
|
BAD=http://forgeway-badtest.forgeway.svc.cluster.local
|
|
CHAT='{"messages":[{"role":"user","content":"Answer with the single word: PONG"}],"max_tokens":400}'
|
|
|
|
j() { curl -s -m 180 -o /tmp/b -D /tmp/h -w '%{http_code}' "$@"; }
|
|
backend() { grep -i '^x-forgeway-backend:' /tmp/h | tr -d '\r' | awk '{print $2}'; }
|
|
content() { sed -n 's/.*"content":"\([^"]*\)".*/\1/p' /tmp/b | head -1 | cut -c1-60; }
|
|
|
|
echo "1 healthz (forge up) .................. HTTP $(j $GOOD/healthz)"
|
|
echo " $(cut -c1-150 /tmp/b)"
|
|
echo "2 strict chat (forge up) .............. HTTP $(j -XPOST $GOOD/local-strict/v1/chat/completions -H 'Content-Type: application/json' -d "$CHAT") backend=$(backend)"
|
|
echo " content=[$(content)]"
|
|
echo "3 optimistic chat (forge up) .......... HTTP $(j -XPOST $GOOD/local-optimistic/v1/chat/completions -H 'Content-Type: application/json' -d "$CHAT") backend=$(backend)"
|
|
echo " content=[$(content)]"
|
|
echo "4 strict /v1/models .................. HTTP $(j $GOOD/local-strict/v1/models)"
|
|
echo " $(cut -c1-140 /tmp/b)"
|
|
echo "5 optimistic /v1/models .............. HTTP $(j $GOOD/local-optimistic/v1/models)"
|
|
echo " $(cut -c1-220 /tmp/b)"
|
|
echo "6 STRICT chat, forge UNAVAILABLE ..... HTTP $(j -XPOST $BAD/local-strict/v1/chat/completions -H 'Content-Type: application/json' -d "$CHAT")"
|
|
echo " $(cut -c1-260 /tmp/b)"
|
|
echo "7 OPTIMISTIC chat, forge UNAVAILABLE . HTTP $(j -XPOST $BAD/local-optimistic/v1/chat/completions -H 'Content-Type: application/json' -d "$CHAT") backend=$(backend)"
|
|
echo " content=[$(content)]"
|
|
echo "8 healthz, forge UNAVAILABLE ......... HTTP $(j $BAD/healthz)"
|
|
echo " $(cut -c1-150 /tmp/b)"
|
|
echo "9 liveness, forge UNAVAILABLE ........ HTTP $(j $BAD/healthz/live)"
|
|
echo "10 readiness, forge UNAVAILABLE ....... HTTP $(j $BAD/healthz/ready)"
|
|
echo "11 streaming through forge ............ HTTP $(j -XPOST $GOOD/local-strict/v1/chat/completions -H 'Content-Type: application/json' -d '{"messages":[{"role":"user","content":"count to three"}],"max_tokens":200,"stream":true}') sse_chunks=$(grep -c '^data:' /tmp/b)"
|
|
echo "12 bare /v1 guard (expect 404) ........ HTTP $(j $GOOD/v1/models)"
|
|
echo "13 unknown mode guard (expect 404) .... HTTP $(j $GOOD/local-wishful/v1/models)"
|
|
|
|
# ---- native Ollama dialect (what Home Assistant's integration speaks) -------
|
|
OCHAT='{"model":"gpt-oss:120b","messages":[{"role":"user","content":"Answer with the single word: PONG"}],"stream":false,"think":false,"keep_alive":"5m","options":{"num_ctx":8192}}'
|
|
OSTREAM='{"model":"gpt-oss:120b","messages":[{"role":"user","content":"Answer with the single word: PONG"}],"stream":true,"think":false}'
|
|
ocontent() { sed -n 's/.*"message": *{[^}]*"content": *"\([^"]*\)".*/\1/p' /tmp/b | head -1 | cut -c1-40; }
|
|
ndjson_summary() { printf 'frames=%s content_frames=%s last_done=%s' "$(wc -l < /tmp/b)" "$(grep -c '"content": *"[^"]' /tmp/b)" "$(tail -1 /tmp/b | grep -o '"done": *true' | head -1)"; }
|
|
|
|
echo "14 ollama /api/tags (forge up) ........ HTTP $(j $GOOD/local-strict/api/tags) $(grep -o '"model":"[^"]*"' /tmp/b | head -1)"
|
|
echo "15 ollama /api/chat non-stream (forge) HTTP $(j -XPOST $GOOD/local-strict/api/chat -H 'Content-Type: application/json' -d "$OCHAT") backend=$(backend) content=[$(ocontent)] $(grep -o '"done_reason":"[^"]*"' /tmp/b | head -1)"
|
|
echo "16 ollama /api/chat STREAM (forge) .... HTTP $(j -XPOST $GOOD/local-optimistic/api/chat -H 'Content-Type: application/json' -d "$OSTREAM") $(ndjson_summary) ctype=$(grep -i '^content-type' /tmp/h | tr -d '\r' | awk '{print $2}')"
|
|
echo "17 ollama /api/version + /api/generate HTTP $(j $GOOD/local-strict/api/version) $(cut -c1-30 /tmp/b) | generate HTTP $(j -XPOST $GOOD/local-strict/api/generate -H 'Content-Type: application/json' -d '{"model":"gpt-oss:120b","prompt":"Answer with the single word: PONG","stream":false,"think":false}') $(grep -o '"response":"[^"]*"' /tmp/b | head -1)"
|
|
echo "18 ollama STRICT chat, forge UNAVAIL .. HTTP $(j -XPOST $BAD/local-strict/api/chat -H 'Content-Type: application/json' -d "$OCHAT") $(cut -c1-90 /tmp/b)"
|
|
echo "19 ollama OPTIMISTIC chat, UNAVAIL .... HTTP $(j -XPOST $BAD/local-optimistic/api/chat -H 'Content-Type: application/json' -d "$OCHAT") backend=$(backend) content=[$(ocontent)] $(grep -o '"done_reason":"[^"]*"' /tmp/b | head -1)"
|
|
echo "20 ollama OPTIMISTIC STREAM, UNAVAIL .. HTTP $(j -XPOST $BAD/local-optimistic/api/chat -H 'Content-Type: application/json' -d "$OSTREAM") backend=$(backend) $(ndjson_summary)"
|
|
echo "21 ollama /api/tags, forge UNAVAIL .... HTTP $(j $BAD/local-optimistic/api/tags) (listing, not liveness)"
|
|
echo "22 ollama /api/pull guard (expect 400) HTTP $(j -XPOST $GOOD/local-strict/api/pull -H 'Content-Type: application/json' -d '{"model":"llama3"}')"
|
|
|
|
# ---- one runner config per model ---------------------------------------------
|
|
# Every forge-bound request carries the pinned num_ctx; the runner forge reports
|
|
# must match it, or callers are thrashing the 64 GB model on every alternation.
|
|
echo "23 num_ctx pin == runner context ....... HTTP $(j $GOOD/status) $(sed -n 's/.*"num_ctx_pin": *\([0-9null]*\).*"runner_context": *\([0-9null]*\).*/pin=\1 runner=\2/p' /tmp/b)"
|
|
|
|
# ---- residency ---------------------------------------------------------------
|
|
# A cold load is 21-66 s and lands in front of the caller, so the model must be
|
|
# permanently resident, not merely loaded right now. runner_resident=true means
|
|
# forge reports "forever"; false means the next idle spell costs a cold load.
|
|
echo "24 model is permanently resident ....... HTTP $(j $GOOD/status) $(sed -n 's/.*"keep_alive_pin": *\(-\{0,1\}[0-9null]*\).*"runner_resident": *\([a-z]*\).*/pin=\1 resident=\2/p' /tmp/b)"
|
|
|
|
# A caller that sends its own SHORT keep_alive must not be able to evict the model
|
|
# for everyone else: the router overrides it. Send 30s natively, then re-read.
|
|
j -XPOST $GOOD/local-strict/api/chat -H 'Content-Type: application/json' \
|
|
-d '{"model":"gpt-oss:120b","messages":[{"role":"user","content":"hi"}],"stream":false,"think":false,"keep_alive":"30s"}' >/dev/null
|
|
echo "25 short keep_alive caller overridden .. HTTP $(j $GOOD/status) $(sed -n 's/.*"runner_resident": *\([a-z]*\).*/resident=\1 (want true)/p' /tmp/b)"
|