commit 3155d4a230455a71babc9e19adb1f63ad6244b47 Author: Dolon (Agent) Date: Fri Sep 18 14:30:14 2026 +0000 forgeway 0.1.1 — per-cluster OpenAI-compatible LLM router Two endpoints over forge's Ollama gpt-oss:120b: /local-optimistic/v1 forge, else cloud gpt-5.4-mini /local-strict/v1 forge, else a clean 503 the caller handles Discovery, not configuration: each instance probes an ordered candidate list (stable DNS name, then the Lounge and future Serenity literals) concurrently but in priority order, so the happy path is one round trip and silicon's move to Serenity needs no rewrite. /healthz runs a real 1-token completion — forge can be up with Ollama stopped. Liveness/readiness are deliberately NOT forge-gated: otherwise k8s would evict the pods during an outage and callers would get a connection failure instead of the promised 503, and Kuma could not tell "forge down" from "router gone". max_tokens/max_completion_tokens are translated per backend: Ollama accepts only the former, gpt-5.4-mini rejects it outright. Co-Authored-By: Claude Opus 5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..77ac754 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.venv/ +__pycache__/ +*.pyc diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..38b71af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /srv + +COPY requirements.txt /srv/requirements.txt +RUN pip install --no-cache-dir -r /srv/requirements.txt + +COPY app /srv/app + +# Runs unprivileged; the manifests also pin a restricted securityContext. +RUN useradd --system --uid 10001 --no-create-home forgeway +USER 10001 + +EXPOSE 8080 + +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--no-access-log"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..c5726e3 --- /dev/null +++ b/README.md @@ -0,0 +1,232 @@ +# forgeway + +An OpenAI-compatible LLM router in front of forge's Ollama `gpt-oss:120b`. +One instance runs **per cluster**, not one central service. + +> **The name is not final.** `forgeway` is a working name (alternatives: `llm-router`, +> `askforge`; per-site `forgeway.lounge` / `forgeway.serenity`). Until Rick confirms it, +> **no DNS record and no ingress exist** — both instances are reachable by IP only. The +> name lives in exactly one place per deployment: the `FORGEWAY_NAME` key in each +> cluster's ConfigMap. + +## The two endpoints + +| Endpoint | Behaviour when forge is available | Behaviour when it is not | +|---|---|---| +| `/local-optimistic/v1` | serves from forge | falls back to OpenAI `gpt-5.4-mini` | +| `/local-strict/v1` | serves from forge | clean HTTP **503**, no fallback, by design | + +Both expose `/v1/chat/completions` (including `stream: true`) and `/v1/models`. + +Where they live: + +| Site | Cluster | Base address | +|---|---|---| +| Lounge | Rocinante | `http://172.19.10.90` | +| Serenity | Niðavellir | `http://172.24.10.21` | +| Chalet | — | when the cluster exists | + +### Calling them + +**`local-strict` — do not retry a 503 blindly.** A 503 means the local model is +unavailable *right now*: forge is off, Ollama is stopped, or the model is evicted for +a training run. None of those clear in the next second, and retrying just adds load +to something already struggling. Handle the error, or degrade to your own fallback. + +```bash +# strict: one attempt, handle the 503 yourself +curl -sS -X POST http://172.19.10.90/local-strict/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"gpt-oss:120b","messages":[{"role":"user","content":"hello"}]}' \ + -w '\n%{http_code}\n' +``` + +```python +import httpx + +# strict — no retries on 503; that status is an answer, not a hiccup +with httpx.Client(base_url="http://172.19.10.90/local-strict/v1", timeout=600) as c: + r = c.post("/chat/completions", json={ + "model": "gpt-oss:120b", + "messages": [{"role": "user", "content": "hello"}], + }) + if r.status_code == 503: + ... # local model is down: skip, queue, or use your own fallback + else: + r.raise_for_status() + print(r.json()["choices"][0]["message"]["content"]) + +# optimistic — fallback is already handled inside the router. +# Do NOT add your own retry loop: a failure here means forge AND the cloud both failed. +with httpx.Client(base_url="http://172.19.10.90/local-optimistic/v1", timeout=600) as c: + r = c.post("/chat/completions", json={ + "model": "gpt-oss:120b", + "messages": [{"role": "user", "content": "hello"}], + }) + r.raise_for_status() + print(r.headers["x-forgeway-backend"]) # "forge" or "cloud" +``` + +With the OpenAI SDK, point `base_url` at one of the two prefixes and use any +non-empty api key (the router does not authenticate callers): + +```python +from openai import OpenAI +client = OpenAI(base_url="http://172.19.10.90/local-optimistic/v1", api_key="unused") +``` + +Every response carries `x-forgeway-backend: forge|cloud`, `x-forgeway-upstream`, +`x-forgeway-site` and `x-forgeway-mode`, so a caller can always tell what answered. + +**Model selection.** The `model` field in a request is ignored — each endpoint routes +to its own backend. Send anything; `gpt-oss:120b` is conventional. `max_tokens` and +`max_completion_tokens` are interchangeable: the router translates between them, +because Ollama only accepts the first and `gpt-5.4-mini` rejects it outright. + +### From OpenClaw agents + +The same endpoints are registered as model providers, so agents and scripts use one +backend: + +``` +forgeway-strict/gpt-oss-120b # forge or an error +forgeway-optimistic/gpt-oss-120b # forge, else gpt-5.4-mini +``` + +## Why one instance per cluster + +A single central router would sit on one site. The cross-site link to forge is exactly +the kind of thing that goes down — and if the router lived on the far side of it, the +router *would be* the outage rather than the thing that reports it. So each cluster +runs its own, and each one answers for itself: when Serenity cannot reach forge, +Serenity's `local-optimistic` still serves from the cloud and its `local-strict` still +returns a clean 503, both without depending on anything at the Lounge. + +It does **not** run on forge itself, for the same reason: a router that dies with its +upstream cannot report on it. + +## Discovery — and what happens when silicon moves to Serenity + +No instance is told which site forge is at. Each one holds an ordered candidate list +(`FORGEWAY_UPSTREAMS`) and finds forge at request time: + +``` +http://forge.dmz.falkvinge.net:11434 the stable name — DNS follows a host move +http://172.19.0.130:11434 forge at the Lounge (today) +http://172.24.0.130:11434 forge at Serenity (after the move) +``` + +Candidates are probed concurrently but awaited in priority order, so the happy path +costs one fast round trip and dead candidates never slow it down. The winner is sticky +and tried first next time. + +**When silicon relocates to Serenity, nothing here needs rewriting.** Re-point +`forge.dmz.falkvinge.net` at its new address and both instances follow on their next +health refresh (seconds). If DNS is stale or broken, discovery still finds forge at the +Serenity literal already in the list. The only thing that changes materially is which +site pays the cross-site latency — and that flips automatically, because each instance +independently discovers the nearest working answer. + +Two things do need a human when that happens: + +1. **forge's ufw** — the allow rules are scoped to source ranges. Serenity-local + traffic will then arrive from `172.24.9.0/24` (already allowed); Lounge traffic + will arrive SNAT'd from the Serenity edge and will need its own rule. +2. **Kuma** — the monitors point at each local router by IP, so they keep working, but + the Lounge monitor will start reporting the slower cross-site path. + +## Health and monitoring + +`/healthz` performs a **real 1-token completion** against forge, never a ping. A ping +cannot tell the difference between a healthy forge and a forge that is up with Ollama +stopped — which was literally its state on the morning this was built. + +| Result | Meaning | +|---|---| +| `200` | forge answered a real completion | +| `503` | forge did not | +| no response at all | the router itself is gone | + +Verdicts are cached for a few seconds (`FORGEWAY_HEALTH_TTL`), so a burst of traffic +does not become a burst of probe completions against forge. + +`/healthz/live` and `/healthz/ready` are **deliberately not gated on forge**. If +readiness were, Kubernetes would pull every pod out of the Service the moment forge +died — callers would get a connection failure instead of the 503 `local-strict` +promises, and Kuma could not distinguish "forge down" from "router gone". + +Monitoring is **active HTTP probing, not push**: a push heartbeat cannot tell "router +dead" from "idle". One monitor per Kuma instance, each watching its own local router: + +| Kuma | Monitor | Target | +|---|---|---| +| Lounge `192.168.81.98:3001` | #307 `forgeway — local LLM router (Lounge)` | `http://172.19.10.90/healthz` | +| Serenity `192.168.24.98:3001` | #57 `forgeway — local LLM router (Serenity)` | `http://172.24.10.21/healthz` | + +Both: 60 s interval, 48 s timeout, 2 retries, accepted status `200-299`. The timeout is +wide because a **cold `gpt-oss:120b` load takes ~25–30 s** and is a legitimate success, +not a failure. + +### The one operational conflict: monitoring vs. LoRA training + +A 60 s probe interval keeps `gpt-oss:120b` (~67 GB) permanently resident on forge's +card 1. An ai-toolkit LoRA run needs ~88 GB on the same card, and the two do not fit — +`ollama stop gpt-oss:120b` is the documented pre-training step, but with monitoring +running, the next probe simply loads it again. + +Before a training run, do one of: + +- **pause the Kuma monitor** for that site (simplest), or +- set `FORGEWAY_HEALTH_COLD_LOAD=false` in the ConfigMap. The probe then checks + Ollama's `/api/ps` and reports DOWN instead of triggering a load. During training + that reading is *correct*: `local-strict` genuinely cannot serve, and + `local-optimistic` correctly falls through to the cloud. + +## Configuration + +Everything is environment-driven; one image serves every site. See +`apps/forgeway/manifests/configmap.yaml` in each cluster repo. + +| Variable | Default | Notes | +|---|---|---| +| `FORGEWAY_NAME` | `forgeway` | the single place the service's name lives | +| `FORGEWAY_SITE` | `unknown` | `lounge` / `serenity` | +| `FORGEWAY_UPSTREAMS` | forge DNS name | comma-separated, priority order | +| `FORGEWAY_LOCAL_MODEL` | `gpt-oss:120b` | | +| `FORGEWAY_CLOUD_MODEL` | `gpt-5.4-mini` | optimistic only | +| `FORGEWAY_HEALTH_TTL` | `5` | cache for a healthy verdict | +| `FORGEWAY_HEALTH_FAIL_TTL` | `3` | cache for an unhealthy verdict | +| `FORGEWAY_HEALTH_TIMEOUT` | `45` | must exceed a ~25–30 s cold load | +| `FORGEWAY_DISCOVERY_TIMEOUT` | `4` | `6` at Serenity — the mesh hop | +| `FORGEWAY_HEALTH_COLD_LOAD` | `true` | see the training conflict above | +| `FORGEWAY_REQUEST_TIMEOUT` | `600` | proxied completions | +| `OPENAI_API_KEY` | — | from the `forgeway-cloud` Secret, applied out of band | + +The cloud credential is **never in git**. It is applied with +`scripts/apply_cloud_secret.py`, which reads the value from the process environment and +hands it to kubectl on stdin. The `secretKeyRef` is `optional: true`, so the Deployment +still runs without it — `local-strict` is unaffected and `local-optimistic` reports +that no fallback credential is configured. + +## Deploying + +```bash +./scripts/build-push.sh 0.1.1 # multi-arch; both clusters are arm64-majority +``` + +Then bump the image tag in each cluster repo and push — ArgoCD syncs: + +- Lounge: `lounge-cluster-config/apps/forgeway/manifests/` + `base/apps/forgeway/` +- Serenity: `serenity-cluster-config/apps/forgeway/` + `base/apps/forgeway-app.yaml` + +## Verifying + +```bash +./scripts/local-smoke.sh # against a real forge, from this VM +./scripts/verify-cluster.sh ~/.kube/rocinante-admin.yaml lounge +./scripts/verify-cluster.sh ~/.kube/nidavellir.yaml serenity +``` + +`verify-cluster.sh` runs 13 checks from inside the cluster. It produces the +"forge unavailable" case with a throwaway Deployment pointed at a dead upstream — +**it never stops Ollama on forge.** diff --git a/REPORT.md b/REPORT.md new file mode 100644 index 0000000..e7a4835 --- /dev/null +++ b/REPORT.md @@ -0,0 +1,149 @@ +# forgeway — build report, 2026-09-18 + +Two OpenAI-compatible query endpoints over forge's Ollama `gpt-oss:120b`, deployed as +one instance per cluster on Lounge (Rocinante) and Serenity (Niðavellir). Working, in +GitOps, monitored, and reachable by IP. **No DNS record and no ingress** — the name is +still Rick's call. + +| | Lounge | Serenity | +|---|---|---| +| Address | `http://172.19.10.90` | `http://172.24.10.21` | +| Replicas | 2/2 | 2/2 | +| Kuma monitor | #307, `200 - OK` | #57, `200 - OK` | +| Real completion latency | 54 ms warm | 383 ms warm (mesh hop) | +| Acceptance checks | 13/13 | 13/13 | + +Image `git.falkvinge.net/rick/forgeway:0.1.1`, linux/amd64 + linux/arm64 (both clusters +are arm64-majority: Lounge 12/15, Nidavellir 9/12), anonymously pullable from the Gitea +registry, so no imagePullSecret is needed. + +--- + +## Three blockers found on forge, and what was changed + +None of these were visible from the brief; all were found by probing. + +**1. Ollama was bound to `127.0.0.1` only.** Nothing off-box could reach it. Added a +*new* systemd drop-in `/etc/systemd/system/ollama.service.d/20-listen.conf` with +`OLLAMA_HOST=0.0.0.0:11434`. The existing `10-gpu-pinning.conf` was not touched, and the +pinning was re-verified after the restart: `llama_prepare_model_devices: using device +CUDA0 ... (0000:02:00.0)` — still card 1, as intended. + +**2. forge's ufw denied both clusters.** Default deny-incoming with a single allow for +the OpenClaw VM. Added three port-scoped rules: + +``` +11434/tcp ALLOW IN 172.19.9.0/24 Lounge k8s nodes +11434/tcp ALLOW IN 172.24.9.0/24 Nidavellir k8s nodes +11434/tcp ALLOW IN 172.19.0.1 Lounge pfSense SNAT, cross-site Serenity traffic +``` + +**3. Serenity traffic arrives SNAT'd.** Serenity pods reach forge with source +`172.19.0.1` — the Lounge pfSense — not their real `172.24.9.x`. Confirmed in forge's +`UFW BLOCK` log. The `172.24.9.0/24` rule alone was therefore never going to match. + +> ⚠️ **Security note, for Rick's judgement.** Rule 3 is wider than I would like: it +> allows anything pfSense forwards out of the Tailscale mesh to reach Ollama, which has +> no authentication. It is scoped to port 11434, but it is not scoped to Serenity. +> The clean fix is a pfSense outbound-NAT exemption so tailscale→DMZ traffic keeps its +> original source (this matches the documented `NoSNAT` intent already set on the +> Serenity UDM), after which rule 3 can be deleted and `172.24.9.0/24` does the work. +> That is an edge-firewall change on your core router, so I did not make it. + +--- + +## Design decisions worth knowing + +**Discovery, not configuration.** Each instance holds an ordered candidate list — the +stable DNS name `forge.dmz.falkvinge.net`, then the Lounge literal `172.19.0.130`, then +the future Serenity literal `172.24.0.130`. Candidates are probed concurrently but +awaited in priority order, so the happy path is one fast round trip and dead candidates +never add latency. When silicon moves, re-pointing DNS is enough; if DNS is stale, the +Serenity literal already in the list catches it. Neither instance assumes a site. + +**Readiness is not gated on forge.** This looks wrong at first glance and is the single +most important choice here. If readiness depended on forge, Kubernetes would evict every +pod from the Service during a forge outage — callers would get a connection refusal +instead of the 503 `local-strict` promises, and Kuma would see "no response" (router +gone) instead of 503 (forge down). Forge health is exposed separately on `/healthz`; +liveness and readiness only answer for the router process. + +**Health is a real completion.** `/healthz` runs a 1-token completion, because forge can +be up with Ollama stopped — its state on the morning of the build. Verdicts are cached +5 s (3 s when unhealthy) behind a single-flight lock, so concurrent traffic collapses +into one probe. + +**A real bug the cloud path surfaced.** `gpt-5.4-mini` rejects `max_tokens` outright +(`400 Unsupported parameter ... use max_completion_tokens`), while Ollama accepts only +`max_tokens`. The first fallback attempt failed on exactly this. The router now +translates between the two spellings per backend, so a caller can write either and both +endpoints work. Verified against the live API: only `max_tokens` is affected — +`temperature`, `top_p` and `stream` are all accepted. + +--- + +## Verification + +`scripts/verify-cluster.sh` runs 13 checks from **inside** each cluster. All 13 passed +on both. The "forge unavailable" cases were produced with a throwaway Deployment pointed +at a dead upstream — **Ollama on forge was never stopped.** + +| # | Check | Lounge | Serenity | +|---|---|---|---| +| 1 | `/healthz`, forge up | 200 | 200 | +| 2 | strict chat, forge up | 200, `backend=forge`, `PONG` | 200, `backend=forge`, `PONG` | +| 3 | optimistic chat, forge up | 200, `backend=forge` | 200, `backend=forge` | +| 4–5 | `/v1/models` both modes | 200 | 200 | +| **6** | **strict chat, forge unavailable** | **503, clean error body** | **503, clean error body** | +| **7** | **optimistic chat, forge unavailable** | **200, `backend=cloud`, `PONG`** | **200, `backend=cloud`, `PONG`** | +| 8 | `/healthz`, forge unavailable | 503 | 503 | +| 9–10 | liveness / readiness, forge unavailable | 200 / 200 | 200 / 200 | +| 11 | streaming through forge | 200, 53 SSE chunks | 200, 46 SSE chunks | +| 12–13 | bare `/v1` and unknown-mode guards | 404 / 404 | 404 / 404 | + +Check 1 on Lounge recorded `latency_ms: 28056` — a genuine cold model load, which is why +the Kuma timeout is 48 s rather than something tighter. + +**Kuma DOWN path proven end-to-end**, not inferred: a temporary monitor pointed at a +temporary 503-returning instance produced `status: 0, "Request failed with status code +503"`, while the real monitor simultaneously read `status: 1, "200 - OK"`. Both the temp +monitor and the temp deployment were removed afterwards. + +**OpenClaw providers proven live:** + +``` +forgeway-strict/gpt-oss-120b -> PROVIDER-PROOF-OK (23.1 s) +forgeway-optimistic/gpt-oss-120b -> PROVIDER-PROOF-OK (21.1 s) +``` + +--- + +## Two corrections to the brief + +- **Serenity Kuma is `192.168.24.98`, not `172.24.24.98`.** The brief's address does not + respond; the memory note was right, and the monitor was created on the correct host. +- **The next free Lounge LB IP was `.90`, not `.89`** — `.89` is clipcascade, already in + use. `.90` was taken; `docs/ip-allocations.md` is updated (11 of 19 in the band used). + +## Open for Rick + +1. **The DNS name.** Still `forgeway` as a working name. Nothing is exposed until you + confirm; the name is a single ConfigMap value plus the record to be created. +2. **The pfSense SNAT exemption** described above, which would let me tighten forge's + firewall rule from "the mesh" to "Serenity's nodes". +3. **Monitoring vs. LoRA training.** A 60 s health probe keeps `gpt-oss:120b` resident on + card 1, which collides with an ai-toolkit run. Levers are documented in the README + (pause the monitor, or `FORGEWAY_HEALTH_COLD_LOAD=false`). If you would rather + monitoring never hold the card, say so and I will change the default — the trade is + detection latency and a cold load per probe. + +## Where everything lives + +| | | +|---|---| +| Service source | `/home/rick/repos/forgeway` (this repo) | +| Lounge manifests | `lounge-cluster-config/apps/forgeway/manifests/`, `base/apps/forgeway/` — commit `bba7898` | +| Serenity manifests | `serenity-cluster-config/apps/forgeway/`, `base/apps/forgeway-app.yaml` — commit `4a32fde` | +| Kuma ops files | `deploy/kuma/ops-{lounge,serenity}.json` | +| OpenClaw provider config | `deploy/openclaw/providers.batch.json` | +| Image | `git.falkvinge.net/rick/forgeway:0.1.1` | diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..4fab1b8 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""forgeway — OpenAI-compatible LLM router, one instance per cluster.""" diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..c1b8a2e --- /dev/null +++ b/app/config.py @@ -0,0 +1,96 @@ +"""Runtime configuration for the router. + +Everything is environment-driven so that one image serves every site. The only +value that encodes *identity* is ``FORGEWAY_NAME``; it is read here once and +referenced from everywhere else, because the final DNS name is still Rick's call. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field + + +def _env(key: str, default: str) -> str: + value = os.environ.get(key) + return default if value is None or value.strip() == "" else value.strip() + + +def _env_int(key: str, default: int) -> int: + try: + return int(_env(key, str(default))) + except ValueError: + return default + + +def _env_float(key: str, default: float) -> float: + try: + return float(_env(key, str(default))) + except ValueError: + return default + + +def _env_bool(key: str, default: bool) -> bool: + return _env(key, "true" if default else "false").lower() in {"1", "true", "yes", "on"} + + +def _env_list(key: str, default: str) -> list[str]: + raw = _env(key, default) + return [item.strip().rstrip("/") for item in raw.split(",") if item.strip()] + + +@dataclass(frozen=True) +class Config: + # --- identity ------------------------------------------------------- + # The working name. The final DNS name is NOT decided yet, so this is the + # single place the service learns what it is called. + name: str = field(default_factory=lambda: _env("FORGEWAY_NAME", "forgeway")) + site: str = field(default_factory=lambda: _env("FORGEWAY_SITE", "unknown")) + + # --- discovery ------------------------------------------------------ + # Ordered candidate list. The router probes these to find forge; it never + # assumes which site forge currently lives at. Put the stable DNS name + # first so a host move is followed automatically by DNS alone. + upstreams: list[str] = field( + default_factory=lambda: _env_list( + "FORGEWAY_UPSTREAMS", + "http://forge.dmz.falkvinge.net:11434", + ) + ) + local_model: str = field(default_factory=lambda: _env("FORGEWAY_LOCAL_MODEL", "gpt-oss:120b")) + + # --- cloud fallback (local-optimistic only) ------------------------- + cloud_base_url: str = field( + default_factory=lambda: _env("FORGEWAY_CLOUD_BASE_URL", "https://api.openai.com/v1").rstrip("/") + ) + cloud_model: str = field(default_factory=lambda: _env("FORGEWAY_CLOUD_MODEL", "gpt-5.4-mini")) + + # --- health --------------------------------------------------------- + # Health is gated on a REAL completion, never a ping: forge can be up with + # Ollama stopped, or Ollama up with the model evicted. + health_ttl: float = field(default_factory=lambda: _env_float("FORGEWAY_HEALTH_TTL", 5.0)) + health_fail_ttl: float = field(default_factory=lambda: _env_float("FORGEWAY_HEALTH_FAIL_TTL", 3.0)) + health_timeout: float = field(default_factory=lambda: _env_float("FORGEWAY_HEALTH_TIMEOUT", 45.0)) + discovery_timeout: float = field(default_factory=lambda: _env_float("FORGEWAY_DISCOVERY_TIMEOUT", 4.0)) + # When false, a probe that finds the model not resident reports DOWN instead + # of triggering a cold load. Used to keep monitoring from re-loading + # gpt-oss:120b during an ai-toolkit run (they collide on forge's card 1). + health_cold_load: bool = field(default_factory=lambda: _env_bool("FORGEWAY_HEALTH_COLD_LOAD", True)) + + # --- proxying ------------------------------------------------------- + request_timeout: float = field(default_factory=lambda: _env_float("FORGEWAY_REQUEST_TIMEOUT", 600.0)) + connect_timeout: float = field(default_factory=lambda: _env_float("FORGEWAY_CONNECT_TIMEOUT", 5.0)) + port: int = field(default_factory=lambda: _env_int("FORGEWAY_PORT", 8080)) + log_level: str = field(default_factory=lambda: _env("FORGEWAY_LOG_LEVEL", "INFO").upper()) + + @property + def cloud_api_key(self) -> str: + """Read lazily and never stored, logged, or echoed.""" + return os.environ.get("OPENAI_API_KEY", "").strip() + + @property + def cloud_enabled(self) -> bool: + return bool(self.cloud_api_key) + + +CONFIG = Config() diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..6fff3bf --- /dev/null +++ b/app/main.py @@ -0,0 +1,395 @@ +"""OpenAI-compatible router with two query endpoints. + + /local-optimistic/v1/... forge if available, otherwise the cloud model + /local-strict/v1/... forge or a clean 503 — callers handle the error + +One instance runs per cluster. There is deliberately no central instance: the +cross-site link to forge may itself be the thing that is down, and a central +router would then *be* the outage. +""" + +from __future__ import annotations + +import logging +import time +from contextlib import asynccontextmanager +from typing import Any + +import httpx +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, StreamingResponse + +from .config import CONFIG +from .upstream import ForgeLocator, UpstreamError + +logging.basicConfig( + level=getattr(logging, CONFIG.log_level, logging.INFO), + format="%(asctime)s %(levelname)s %(name)s %(message)s", +) +log = logging.getLogger("forgeway") + +MODE_STRICT = "local-strict" +MODE_OPTIMISTIC = "local-optimistic" +MODES = (MODE_OPTIMISTIC, MODE_STRICT) + +@asynccontextmanager +async def lifespan(app: FastAPI): + app.state.client = httpx.AsyncClient( + timeout=httpx.Timeout( + CONFIG.request_timeout, + connect=CONFIG.connect_timeout, + read=CONFIG.request_timeout, + write=60.0, + pool=CONFIG.connect_timeout, + ), + limits=httpx.Limits(max_connections=64, max_keepalive_connections=16), + follow_redirects=False, + ) + app.state.locator = ForgeLocator(app.state.client) + log.info( + "%s starting: site=%s model=%s cloud=%s candidates=%s", + CONFIG.name, + CONFIG.site, + CONFIG.local_model, + CONFIG.cloud_model if CONFIG.cloud_enabled else "disabled (no key)", + ", ".join(CONFIG.upstreams), + ) + try: + yield + finally: + await app.state.client.aclose() + + +app = FastAPI( + title=f"{CONFIG.name} — OpenAI-compatible LLM router", + docs_url=None, + redoc_url=None, + lifespan=lifespan, +) + + +# -------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------- + +def _base_headers(mode: str, backend: str, upstream: str | None) -> dict[str, str]: + headers = { + "x-forgeway-service": CONFIG.name, + "x-forgeway-site": CONFIG.site, + "x-forgeway-mode": mode, + "x-forgeway-backend": backend, + } + if upstream: + headers["x-forgeway-upstream"] = upstream + return headers + + +def _error( + status: int, + message: str, + code: str, + mode: str, + backend: str = "none", + upstream: str | None = None, + extra: dict[str, Any] | None = None, +) -> JSONResponse: + """OpenAI-shaped error so existing SDK error handling keeps working.""" + payload: dict[str, Any] = { + "error": { + "message": message, + "type": "forgeway_error", + "code": code, + "param": None, + } + } + if extra: + payload["error"].update(extra) + # Deliberately no Retry-After: a 503 from local-strict means the local model + # is unavailable *now*, and blind retries just hammer a busy or absent forge. + return JSONResponse(payload, status_code=status, headers=_base_headers(mode, backend, upstream)) + + +def _adapt_payload(body: dict[str, Any], backend: str) -> dict[str, Any]: + """Reconcile the one parameter the two backends spell differently. + + gpt-5.4-mini rejects `max_tokens` outright (400, "use max_completion_tokens"), + while Ollama's OpenAI layer only understands `max_tokens`. Callers should be + able to write either spelling and have both endpoints work, so translate here + rather than making every caller care which backend served them. + """ + payload = dict(body) + if backend == "cloud": + if "max_tokens" in payload: + payload.setdefault("max_completion_tokens", payload.pop("max_tokens")) + payload.pop("max_tokens", None) + else: + if "max_completion_tokens" in payload: + payload.setdefault("max_tokens", payload.pop("max_completion_tokens")) + payload.pop("max_completion_tokens", None) + return payload + + +async def _proxy_chat( + base_url: str, + model: str, + body: dict[str, Any], + mode: str, + backend: str, + api_key: str | None = None, +) -> JSONResponse | StreamingResponse: + """Forward one chat completion, honouring stream=true.""" + client: httpx.AsyncClient = app.state.client + payload = _adapt_payload(body, backend) + payload["model"] = model + stream = bool(payload.get("stream")) + + headers = {"content-type": "application/json"} + if api_key: + headers["authorization"] = f"Bearer {api_key}" + + url = f"{base_url}/v1/chat/completions" if not base_url.endswith("/v1") else f"{base_url}/chat/completions" + + if not stream: + try: + resp = await client.post(url, json=payload, headers=headers) + except httpx.HTTPError as exc: + raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc + if resp.status_code >= 400: + raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, resp.text[:500]) + return JSONResponse( + resp.json(), + status_code=resp.status_code, + headers=_base_headers(mode, backend, base_url), + ) + + # Streaming: send first and inspect the status before committing to a + # StreamingResponse, so a failed upstream can still fall back. + request = client.build_request("POST", url, json=payload, headers=headers) + try: + resp = await client.send(request, stream=True) + except httpx.HTTPError as exc: + raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc + + if resp.status_code >= 400: + detail = (await resp.aread())[:500].decode("utf-8", "replace") + await resp.aclose() + raise UpstreamError(f"HTTP {resp.status_code}", resp.status_code, detail) + + async def body_iter(): + try: + async for chunk in resp.aiter_raw(): + yield chunk + finally: + await resp.aclose() + + return StreamingResponse( + body_iter(), + status_code=resp.status_code, + media_type=resp.headers.get("content-type", "text/event-stream"), + headers=_base_headers(mode, backend, base_url), + ) + + +# -------------------------------------------------------------------------- +# health — for Uptime Kuma (active HTTP probing, not push) +# -------------------------------------------------------------------------- + +@app.get("/healthz") +async def healthz() -> JSONResponse: + """Gated on a real completion against forge. + + 200 forge answered a real completion + 503 forge did not + (no response at all means the router itself is gone) + + Results are cached for a few seconds so a burst of traffic does not turn + into a burst of probe completions against forge. + """ + health = await app.state.locator.health() + return JSONResponse( + health.as_dict(), + status_code=200 if health.healthy else 503, + headers=_base_headers("healthz", "forge" if health.healthy else "none", health.upstream), + ) + + +@app.get("/healthz/live") +async def healthz_live() -> JSONResponse: + """Liveness only. Never gated on forge — otherwise the pod would restart-loop + whenever forge is down, and local-strict could not return its 503.""" + return JSONResponse({"status": "ok", "service": CONFIG.name, "site": CONFIG.site}) + + +@app.get("/healthz/ready") +async def healthz_ready() -> JSONResponse: + """Readiness is also NOT gated on forge. If it were, k8s would pull the pod + out of the Service when forge dies and callers would get a connection + failure instead of the clean 503 that local-strict promises.""" + return JSONResponse({"status": "ok", "service": CONFIG.name, "site": CONFIG.site}) + + +@app.get("/status") +async def status() -> JSONResponse: + health = await app.state.locator.health() + return JSONResponse( + { + "service": CONFIG.name, + "site": CONFIG.site, + "endpoints": [f"/{m}/v1" for m in MODES], + "local_model": CONFIG.local_model, + "cloud_model": CONFIG.cloud_model if CONFIG.cloud_enabled else None, + "cloud_fallback_enabled": CONFIG.cloud_enabled, + "discovery_candidates": CONFIG.upstreams, + "discovered_upstream": app.state.locator.sticky, + "health": health.as_dict(), + } + ) + + +@app.get("/") +async def root() -> JSONResponse: + return JSONResponse( + { + "service": CONFIG.name, + "site": CONFIG.site, + "description": "OpenAI-compatible LLM router, one instance per cluster", + "endpoints": { + f"/{MODE_OPTIMISTIC}/v1": f"forge, falling back to {CONFIG.cloud_model} when forge is unavailable", + f"/{MODE_STRICT}/v1": "forge or HTTP 503 — callers are expected to handle the error", + }, + "health": "/healthz (real completion against forge; 200 up, 503 down)", + } + ) + + +# -------------------------------------------------------------------------- +# OpenAI-compatible surface, under both base paths +# -------------------------------------------------------------------------- + +def _validate_mode(mode: str) -> JSONResponse | None: + if mode not in MODES: + return JSONResponse( + { + "error": { + "message": f"unknown endpoint '{mode}'. Use one of: {', '.join(MODES)}.", + "type": "forgeway_error", + "code": "unknown_endpoint", + "param": None, + } + }, + status_code=404, + ) + return None + + +@app.get("/{mode}/v1/models") +async def models(mode: str) -> JSONResponse: + invalid = _validate_mode(mode) + if invalid: + return invalid + + created = int(time.time()) + entries = [{"id": CONFIG.local_model, "object": "model", "created": created, "owned_by": CONFIG.name}] + if mode == MODE_OPTIMISTIC and CONFIG.cloud_enabled: + entries.append( + {"id": CONFIG.cloud_model, "object": "model", "created": created, "owned_by": f"{CONFIG.name}-fallback"} + ) + return JSONResponse({"object": "list", "data": entries}, headers=_base_headers(mode, "none", None)) + + +@app.get("/{mode}/healthz") +async def mode_healthz(mode: str) -> JSONResponse: + invalid = _validate_mode(mode) + if invalid: + return invalid + return await healthz() + + +@app.post("/{mode}/v1/chat/completions") +async def chat_completions(mode: str, request: Request): + invalid = _validate_mode(mode) + if invalid: + return invalid + + try: + body = await request.json() + except ValueError: + return _error(400, "request body must be JSON", "invalid_request", mode) + if not isinstance(body, dict) or not body.get("messages"): + return _error(400, "'messages' is required", "invalid_request", mode) + + locator = app.state.locator + health = await locator.health() + + if health.healthy and health.upstream: + try: + return await _proxy_chat(health.upstream, CONFIG.local_model, body, mode, "forge") + except UpstreamError as exc: + # The cached verdict is stale — the upstream just failed a live call. + locator.invalidate() + log.warning("forge request failed on %s: %s", health.upstream, exc) + local_failure = str(exc) + else: + local_failure = health.reason + + if mode == MODE_STRICT: + return _error( + 503, + ( + f"the local model '{CONFIG.local_model}' is unavailable at this site " + f"({CONFIG.site}). local-strict does not fall back by design; " + f"handle this error or use /{MODE_OPTIMISTIC}/v1." + ), + "local_backend_unavailable", + mode, + upstream=health.upstream, + extra={"detail": local_failure, "site": CONFIG.site}, + ) + + if not CONFIG.cloud_enabled: + return _error( + 503, + "forge is unavailable and no cloud fallback credential is configured", + "no_backend_available", + mode, + extra={"detail": local_failure}, + ) + + try: + log.info("falling back to cloud %s (%s)", CONFIG.cloud_model, local_failure) + return await _proxy_chat( + CONFIG.cloud_base_url, + CONFIG.cloud_model, + body, + mode, + "cloud", + api_key=CONFIG.cloud_api_key, + ) + except UpstreamError as exc: + return _error( + 502, + f"forge unavailable and the cloud fallback also failed: {exc}", + "all_backends_failed", + mode, + extra={"detail": local_failure, "cloud_detail": exc.body}, + ) + + +@app.api_route("/v1/{rest:path}", methods=["GET", "POST"]) +async def bare_v1(rest: str) -> JSONResponse: + """Guard rail: /v1/... without a mode prefix is almost always a misconfigured + base_url, and silently guessing a mode would defeat the point of having two.""" + return JSONResponse( + { + "error": { + "message": ( + "no endpoint selected. Set base_url to " + f"'/{MODE_OPTIMISTIC}/v1' or '/{MODE_STRICT}/v1'." + ), + "type": "forgeway_error", + "code": "endpoint_not_selected", + "param": None, + } + }, + status_code=404, + ) diff --git a/app/upstream.py b/app/upstream.py new file mode 100644 index 0000000..ac221d1 --- /dev/null +++ b/app/upstream.py @@ -0,0 +1,221 @@ +"""forge discovery and health, both gated on a real completion.""" + +from __future__ import annotations + +import asyncio +import logging +import time +from dataclasses import dataclass + +import httpx + +from .config import CONFIG + +log = logging.getLogger("forgeway.upstream") + + +class UpstreamError(RuntimeError): + """An upstream refused, failed, or timed out.""" + + def __init__(self, message: str, status_code: int | None = None, body: str | None = None): + super().__init__(message) + self.status_code = status_code + self.body = body + + +@dataclass(frozen=True) +class Health: + healthy: bool + reason: str + upstream: str | None + latency_ms: float | None + checked_at: float + cached: bool = False + + def as_dict(self) -> dict: + return { + "status": "ok" if self.healthy else "down", + "reason": self.reason, + "upstream": self.upstream, + "model": CONFIG.local_model, + "latency_ms": round(self.latency_ms, 1) if self.latency_ms is not None else None, + "checked_at": round(self.checked_at, 3), + "age_s": round(time.time() - self.checked_at, 3), + "cached": self.cached, + "site": CONFIG.site, + "service": CONFIG.name, + } + + +class ForgeLocator: + """Finds forge among the candidate upstreams and keeps a short-lived verdict. + + Discovery is two-phase so that a moved host is found quickly without paying a + long timeout per candidate: + + 1. reachability — probe every candidate concurrently with a short timeout + 2. completion — run one real 1-token completion against the preferred + reachable candidate + + The last good upstream is sticky and tried first, so the steady state costs a + single fast probe. + """ + + def __init__(self, client: httpx.AsyncClient): + self._client = client + self._sticky: str | None = None + self._health: Health | None = None + self._lock = asyncio.Lock() + + @property + def sticky(self) -> str | None: + return self._sticky + + async def health(self, force: bool = False) -> Health: + cached = self._health + if not force and cached is not None: + ttl = CONFIG.health_ttl if cached.healthy else CONFIG.health_fail_ttl + if (time.time() - cached.checked_at) < ttl: + return Health(**{**cached.__dict__, "cached": True}) + + async with self._lock: + # Another waiter may have refreshed while we queued on the lock. + cached = self._health + if not force and cached is not None: + ttl = CONFIG.health_ttl if cached.healthy else CONFIG.health_fail_ttl + if (time.time() - cached.checked_at) < ttl: + return Health(**{**cached.__dict__, "cached": True}) + + health = await self._probe() + self._health = health + return health + + def invalidate(self) -> None: + """Drop the cached verdict after a live request failed against it.""" + self._health = None + + async def _probe(self) -> Health: + started = time.perf_counter() + target = await self._discover() + if not target: + return Health( + healthy=False, + reason="no-upstream-discovered", + upstream=None, + latency_ms=None, + checked_at=time.time(), + ) + + if not CONFIG.health_cold_load: + loaded = await self._model_resident(target) + if loaded is False: + return Health( + healthy=False, + reason="model-not-loaded (cold load disabled)", + upstream=target, + latency_ms=None, + checked_at=time.time(), + ) + + try: + await self._completion(target) + except UpstreamError as exc: + self._sticky = None + return Health( + healthy=False, + reason=f"completion-failed: {exc}", + upstream=target, + latency_ms=None, + checked_at=time.time(), + ) + + self._sticky = target + return Health( + healthy=True, + reason="completion-ok", + upstream=target, + latency_ms=(time.perf_counter() - started) * 1000.0, + checked_at=time.time(), + ) + + async def _discover(self) -> str | None: + """Find where forge currently answers. + + All candidates are probed concurrently, but awaited in priority order, so + the happy path costs one fast round trip instead of waiting for dead + candidates to time out. That matters because the candidate list carries + both the Lounge and the future Serenity address of forge: whichever site + silicon lives at, the live one is found without a config change, and the + dead one never slows the probe down. + """ + candidates = list(CONFIG.upstreams) + if self._sticky and self._sticky in candidates: + candidates.remove(self._sticky) + candidates.insert(0, self._sticky) + + async def reachable(base: str) -> str | None: + try: + resp = await self._client.get(f"{base}/v1/models", timeout=CONFIG.discovery_timeout) + return base if resp.status_code < 500 else None + except httpx.HTTPError: + return None + + tasks = [asyncio.create_task(reachable(c)) for c in candidates] + try: + for task in tasks: + try: + found = await task + except asyncio.CancelledError: + continue + if found: + if found != self._sticky: + log.info("discovery: forge answers at %s (of %d candidates)", found, len(candidates)) + return found + return None + finally: + for task in tasks: + if not task.done(): + task.cancel() + + async def _model_resident(self, base: str) -> bool | None: + """Ask Ollama whether the model is already in VRAM. None = can't tell.""" + try: + resp = await self._client.get(f"{base}/api/ps", timeout=CONFIG.discovery_timeout) + if resp.status_code != 200: + return None + names = {m.get("model") or m.get("name") for m in resp.json().get("models", [])} + return CONFIG.local_model in names + except (httpx.HTTPError, ValueError): + return None + + async def _completion(self, base: str) -> dict: + """One real, minimal completion. Not a ping — that is the whole point.""" + payload = { + "model": CONFIG.local_model, + "messages": [{"role": "user", "content": "ping"}], + "max_tokens": 1, + "temperature": 0, + "stream": False, + } + try: + resp = await self._client.post( + f"{base}/v1/chat/completions", + json=payload, + timeout=CONFIG.health_timeout, + ) + except httpx.HTTPError as exc: + raise UpstreamError(f"{type(exc).__name__}: {exc}") from exc + + if resp.status_code != 200: + raise UpstreamError( + f"HTTP {resp.status_code}", + status_code=resp.status_code, + body=resp.text[:500], + ) + try: + data = resp.json() + except ValueError as exc: + raise UpstreamError("upstream returned non-JSON") from exc + if not data.get("choices"): + raise UpstreamError("upstream returned no choices") + return data diff --git a/deploy/kuma/ops-lounge.json b/deploy/kuma/ops-lounge.json new file mode 100644 index 0000000..df2a55c --- /dev/null +++ b/deploy/kuma/ops-lounge.json @@ -0,0 +1,28 @@ +[ + { + "op": "add", + "key": "forgeway — local LLM router (Lounge)|http|http://172.19.10.90/healthz", + "monitor": { + "name": "forgeway — local LLM router (Lounge)", + "type": "http", + "url": "http://172.19.10.90/healthz", + "method": "GET", + "parent": 302, + "active": true, + "interval": 60, + "retryInterval": 60, + "resendInterval": 0, + "maxretries": 2, + "timeout": 48, + "maxredirects": 0, + "ignoreTls": false, + "upsideDown": false, + "expiryNotification": false, + "accepted_statuscodes": [ + "200-299" + ], + "description": "Active probe of the LOCAL forgeway instance (172.19.10.90). /healthz runs a real 1-token completion against forge's gpt-oss:120b, not a ping: forge can be up with Ollama stopped.\n\n200 = forge answered. 503 = forge did not (local-strict is returning 503 to callers; local-optimistic is serving from gpt-5.4-mini). No response at all = the router itself is gone.\n\nTimeout is 48 s because a cold gpt-oss:120b load takes ~25-30 s and is a legitimate success.\n\nPAUSE THIS MONITOR before an ai-toolkit LoRA run on forge: a 60 s probe interval keeps the 67 GB model resident on card 1, which collides with training.", + "conditions": [] + } + } +] \ No newline at end of file diff --git a/deploy/kuma/ops-serenity.json b/deploy/kuma/ops-serenity.json new file mode 100644 index 0000000..bd9ee75 --- /dev/null +++ b/deploy/kuma/ops-serenity.json @@ -0,0 +1,28 @@ +[ + { + "op": "add", + "key": "forgeway — local LLM router (Serenity)|http|http://172.24.10.21/healthz", + "monitor": { + "name": "forgeway — local LLM router (Serenity)", + "type": "http", + "url": "http://172.24.10.21/healthz", + "method": "GET", + "parent": 1, + "active": true, + "interval": 60, + "retryInterval": 60, + "resendInterval": 0, + "maxretries": 2, + "timeout": 48, + "maxredirects": 0, + "ignoreTls": false, + "upsideDown": false, + "expiryNotification": false, + "accepted_statuscodes": [ + "200-299" + ], + "description": "Active probe of the LOCAL forgeway instance (172.24.10.21). /healthz runs a real 1-token completion against forge's gpt-oss:120b (today that crosses the Tailscale mesh to the Lounge), not a ping: forge can be up with Ollama stopped.\n\n200 = forge answered. 503 = forge did not (local-strict is returning 503 to callers; local-optimistic is serving from gpt-5.4-mini). No response at all = the router itself is gone.\n\nTimeout is 48 s because a cold gpt-oss:120b load takes ~25-30 s and is a legitimate success.\n\nPAUSE THIS MONITOR before an ai-toolkit LoRA run on forge: a 60 s probe interval keeps the 67 GB model resident on card 1, which collides with training.", + "conditions": [] + } + } +] \ No newline at end of file diff --git a/deploy/openclaw/modelpolicy.batch.json b/deploy/openclaw/modelpolicy.batch.json new file mode 100644 index 0000000..71b384e --- /dev/null +++ b/deploy/openclaw/modelpolicy.batch.json @@ -0,0 +1,36 @@ +[ + { + "path": "agents.defaults.modelPolicy.allow", + "value": [ + "openrouter/auto", + "openrouter/moonshotai/kimi-k2.5", + "moonshotai/kimi-k2.5", + "openrouter/anthropic/claude-sonnet-4.5", + "openrouter/openai/gpt-5.2-codex", + "openrouter/anthropic/claude-opus-4.5", + "openai/gpt-5.3", + "openai/gpt-5.5", + "openai/gpt-5.4-mini", + "openai/gpt-5.3-codex", + "openai/gpt-5.4", + "anthropic/claude-opus-4-8", + "anthropic/claude-opus-4-7", + "anthropic/claude-sonnet-4-6", + "anthropic/claude-opus-4-6", + "anthropic/claude-opus-4-8", + "anthropic/claude-opus-5", + "anthropic/claude-opus-5", + "anthropic/claude-fable-5", + "anthropic/claude-fable-5", + "openrouter/x-ai/grok-4.6", + "openai/gpt-5.6-sol", + "openai/gpt-5.6-terra", + "openai/gpt-5.6-luna", + "claude-cli/claude-fable-5", + "claude-cli/claude-fable-5-1", + "claude-cli/claude-opus-5", + "forgeway-optimistic/gpt-oss-120b", + "forgeway-strict/gpt-oss-120b" + ] + } +] \ No newline at end of file diff --git a/deploy/openclaw/providers.batch.json b/deploy/openclaw/providers.batch.json new file mode 100644 index 0000000..cfa746e --- /dev/null +++ b/deploy/openclaw/providers.batch.json @@ -0,0 +1,44 @@ +[ + { + "path": "models.providers.forgeway-optimistic", + "value": { + "baseUrl": "http://172.19.10.90/local-optimistic/v1", + "api": "openai-completions", + "timeoutSeconds": 600, + "injectNumCtxForOpenAICompat": false, + "models": [ + { + "id": "gpt-oss-120b", + "name": "forgeway optimistic (forge gpt-oss:120b, cloud fallback)", + "api": "openai-completions", + "reasoning": true, + "input": ["text"], + "contextTokens": 131072, + "maxTokens": 32768, + "cost": { "input": 0, "output": 0 } + } + ] + } + }, + { + "path": "models.providers.forgeway-strict", + "value": { + "baseUrl": "http://172.19.10.90/local-strict/v1", + "api": "openai-completions", + "timeoutSeconds": 600, + "injectNumCtxForOpenAICompat": false, + "models": [ + { + "id": "gpt-oss-120b", + "name": "forgeway strict (forge gpt-oss:120b, 503 when unavailable)", + "api": "openai-completions", + "reasoning": true, + "input": ["text"], + "contextTokens": 131072, + "maxTokens": 32768, + "cost": { "input": 0, "output": 0 } + } + ] + } + } +] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2e8e844 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.118.0 +uvicorn[standard]==0.37.0 +httpx==0.28.1 diff --git a/scripts/apply_cloud_secret.py b/scripts/apply_cloud_secret.py new file mode 100755 index 0000000..21bf3b9 --- /dev/null +++ b/scripts/apply_cloud_secret.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Create/refresh the cloud-fallback Secret in a cluster, out of band. + +The credential is read from this process's environment and handed to kubectl on +stdin. It is never placed in a command line, an argument, a URL, a log line, or +a shell variable, and it is never written to the GitOps repo. + + python3 scripts/apply_cloud_secret.py ~/.kube/rocinante-admin.yaml +""" +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import sys + +NAMESPACE = "forgeway" +SECRET = "forgeway-cloud" + + +def main() -> int: + if len(sys.argv) != 2: + return print(__doc__) or 2 + + kubeconfig = os.path.expanduser(sys.argv[1]) + key = os.environ.get("OPENAI_API_KEY", "") + if not key: + print("OPENAI_API_KEY is not present in this environment", file=sys.stderr) + return 1 + + env = {**os.environ, "KUBECONFIG": kubeconfig} + + subprocess.run( + ["kubectl", "create", "namespace", NAMESPACE, "--dry-run=client", "-o", "json"], + env=env, check=True, capture_output=True, + ) + ns = subprocess.run( + ["kubectl", "get", "namespace", NAMESPACE], + env=env, capture_output=True, + ) + if ns.returncode != 0: + subprocess.run(["kubectl", "create", "namespace", NAMESPACE], env=env, check=True) + + manifest = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": SECRET, + "namespace": NAMESPACE, + "annotations": { + # Argo does not manage this object; it is applied out of band so + # the credential never reaches the repo. + "argocd.argoproj.io/sync-options": "Prune=false", + }, + }, + "type": "Opaque", + "data": {"OPENAI_API_KEY": base64.b64encode(key.encode()).decode()}, + } + + proc = subprocess.run( + ["kubectl", "apply", "-f", "-"], + env=env, input=json.dumps(manifest).encode(), capture_output=True, + ) + sys.stdout.write(proc.stdout.decode()) + sys.stderr.write(proc.stderr.decode()) + if proc.returncode == 0: + print(f"ok: {NAMESPACE}/{SECRET} applied (value not shown)") + return proc.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build-push.sh b/scripts/build-push.sh new file mode 100755 index 0000000..129f01d --- /dev/null +++ b/scripts/build-push.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Build the multi-arch image and push it to the in-estate Gitea registry. +# Both clusters are arm64-majority (Lounge 12/15, Nidavellir 9/12), so arm64 is +# not optional. The registry credential is piped from `pass` into docker login's +# stdin and never appears in a command line, an argument, or the environment. +set -euo pipefail + +REGISTRY="git.falkvinge.net" +OWNER="rick" +IMAGE="forgeway" +TAG="${1:-0.1.0}" +PLATFORMS="linux/amd64,linux/arm64" +ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +REF="${REGISTRY}/${OWNER}/${IMAGE}" + +if ! docker buildx inspect forgeway-builder >/dev/null 2>&1; then + docker buildx create --name forgeway-builder --driver docker-container --bootstrap +fi + +pass git/falkvinge-pat-rwall | docker login "${REGISTRY}" --username "${OWNER}" --password-stdin + +docker buildx build \ + --builder forgeway-builder \ + --platform "${PLATFORMS}" \ + --tag "${REF}:${TAG}" \ + --tag "${REF}:latest" \ + --push \ + "${ROOT}" + +echo +echo "pushed ${REF}:${TAG} (${PLATFORMS})" +docker buildx imagetools inspect "${REF}:${TAG}" | sed -n '1,40p' diff --git a/scripts/checks.sh b/scripts/checks.sh new file mode 100755 index 0000000..d113bea --- /dev/null +++ b/scripts/checks.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Runs INSIDE the cluster, against the live Service and against a throwaway +# instance whose upstream is deliberately dead. Plain POSIX sh + curl only. +GOOD=http://forgeway.forgeway.svc.cluster.local +BAD=http://forgeway-badtest.forgeway.svc.cluster.local +CHAT='{"messages":[{"role":"user","content":"Answer with the single word: PONG"}],"max_tokens":400}' + +j() { curl -s -m 180 -o /tmp/b -D /tmp/h -w '%{http_code}' "$@"; } +backend() { grep -i '^x-forgeway-backend:' /tmp/h | tr -d '\r' | awk '{print $2}'; } +content() { sed -n 's/.*"content":"\([^"]*\)".*/\1/p' /tmp/b | head -1 | cut -c1-60; } + +echo "1 healthz (forge up) .................. HTTP $(j $GOOD/healthz)" +echo " $(cut -c1-150 /tmp/b)" +echo "2 strict chat (forge up) .............. HTTP $(j -XPOST $GOOD/local-strict/v1/chat/completions -H 'Content-Type: application/json' -d "$CHAT") backend=$(backend)" +echo " content=[$(content)]" +echo "3 optimistic chat (forge up) .......... HTTP $(j -XPOST $GOOD/local-optimistic/v1/chat/completions -H 'Content-Type: application/json' -d "$CHAT") backend=$(backend)" +echo " content=[$(content)]" +echo "4 strict /v1/models .................. HTTP $(j $GOOD/local-strict/v1/models)" +echo " $(cut -c1-140 /tmp/b)" +echo "5 optimistic /v1/models .............. HTTP $(j $GOOD/local-optimistic/v1/models)" +echo " $(cut -c1-220 /tmp/b)" +echo "6 STRICT chat, forge UNAVAILABLE ..... HTTP $(j -XPOST $BAD/local-strict/v1/chat/completions -H 'Content-Type: application/json' -d "$CHAT")" +echo " $(cut -c1-260 /tmp/b)" +echo "7 OPTIMISTIC chat, forge UNAVAILABLE . HTTP $(j -XPOST $BAD/local-optimistic/v1/chat/completions -H 'Content-Type: application/json' -d "$CHAT") backend=$(backend)" +echo " content=[$(content)]" +echo "8 healthz, forge UNAVAILABLE ......... HTTP $(j $BAD/healthz)" +echo " $(cut -c1-150 /tmp/b)" +echo "9 liveness, forge UNAVAILABLE ........ HTTP $(j $BAD/healthz/live)" +echo "10 readiness, forge UNAVAILABLE ....... HTTP $(j $BAD/healthz/ready)" +echo "11 streaming through forge ............ HTTP $(j -XPOST $GOOD/local-strict/v1/chat/completions -H 'Content-Type: application/json' -d '{"messages":[{"role":"user","content":"count to three"}],"max_tokens":200,"stream":true}') sse_chunks=$(grep -c '^data:' /tmp/b)" +echo "12 bare /v1 guard (expect 404) ........ HTTP $(j $GOOD/v1/models)" +echo "13 unknown mode guard (expect 404) .... HTTP $(j $GOOD/local-wishful/v1/models)" diff --git a/scripts/local-smoke.sh b/scripts/local-smoke.sh new file mode 100755 index 0000000..437f445 --- /dev/null +++ b/scripts/local-smoke.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Local smoke test: proves both endpoints against a real forge, and proves the +# strict-503 path by pointing a second instance at a deliberately bad upstream. +# Never stops Ollama on forge to produce the failure. +set -uo pipefail + +BASE_GOOD="http://127.0.0.1:18080" +BASE_BAD="http://127.0.0.1:18081" +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PY="$ROOT/.venv/bin/python" + +start() { # name port upstreams + FORGEWAY_NAME=forgeway FORGEWAY_SITE="$1" FORGEWAY_UPSTREAMS="$3" \ + FORGEWAY_DISCOVERY_TIMEOUT=3 FORGEWAY_HEALTH_TIMEOUT=60 \ + "$PY" -m uvicorn app.main:app --host 127.0.0.1 --port "$2" --no-access-log \ + >"/tmp/forgeway-$1.log" 2>&1 & + echo $! +} + +cd "$ROOT" +PID_GOOD=$(start smoke-good 18080 "http://forge.dmz.falkvinge.net:11434") +PID_BAD=$(start smoke-bad 18081 "http://127.0.0.1:1") +trap 'kill $PID_GOOD $PID_BAD 2>/dev/null' EXIT +sleep 4 + +hdr() { printf '\n\033[1m== %s ==\033[0m\n' "$1"; } +probe() { curl -s -o /tmp/fw-body -m 120 -w "%{http_code}" "$@"; } + +hdr "healthz (forge reachable)" +echo "HTTP $(probe "$BASE_GOOD/healthz")"; cat /tmp/fw-body; echo + +hdr "local-strict chat (forge reachable)" +echo "HTTP $(probe -X POST "$BASE_GOOD/local-strict/v1/chat/completions" \ + -H 'Content-Type: application/json' \ + -d '{"messages":[{"role":"user","content":"Reply with exactly: PONG"}],"max_tokens":10}')" +head -c 400 /tmp/fw-body; echo + +hdr "local-optimistic chat (forge reachable -> should use forge)" +echo "HTTP $(probe -X POST "$BASE_GOOD/local-optimistic/v1/chat/completions" \ + -H 'Content-Type: application/json' \ + -d '{"messages":[{"role":"user","content":"Reply with exactly: PONG"}],"max_tokens":10}')" +head -c 300 /tmp/fw-body; echo + +hdr "models" +echo "strict: $(curl -s "$BASE_GOOD/local-strict/v1/models")" +echo "optimistic: $(curl -s "$BASE_GOOD/local-optimistic/v1/models")" + +hdr "BAD UPSTREAM: healthz must be 503" +echo "HTTP $(probe "$BASE_BAD/healthz")"; cat /tmp/fw-body; echo + +hdr "BAD UPSTREAM: local-strict must be a clean 503" +echo "HTTP $(probe -X POST "$BASE_BAD/local-strict/v1/chat/completions" \ + -H 'Content-Type: application/json' -d '{"messages":[{"role":"user","content":"hi"}]}')" +cat /tmp/fw-body; echo + +hdr "BAD UPSTREAM: local-optimistic must fall back to cloud" +echo "HTTP $(probe -X POST "$BASE_BAD/local-optimistic/v1/chat/completions" \ + -H 'Content-Type: application/json' \ + -d '{"messages":[{"role":"user","content":"Reply with exactly: PONG"}],"max_tokens":10}')" +head -c 400 /tmp/fw-body; echo + +hdr "BAD UPSTREAM: liveness/readiness must stay 200 (so 503s are deliverable)" +echo "live: $(curl -s -o /dev/null -w '%{http_code}' "$BASE_BAD/healthz/live")" +echo "ready: $(curl -s -o /dev/null -w '%{http_code}' "$BASE_BAD/healthz/ready")" diff --git a/scripts/probe_cloud_params.py b/scripts/probe_cloud_params.py new file mode 100644 index 0000000..2711952 --- /dev/null +++ b/scripts/probe_cloud_params.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Probe which OpenAI chat parameters gpt-5.4-mini accepts. + +Reads the key from the environment and never prints it. +""" +import json +import os +import sys +import urllib.error +import urllib.request + +KEY = os.environ.get("OPENAI_API_KEY", "") +MODEL = os.environ.get("FORGEWAY_CLOUD_MODEL", "gpt-5.4-mini") +if not KEY: + sys.exit("OPENAI_API_KEY not present in the environment") + +CASES = { + "max_completion_tokens only": {"max_completion_tokens": 16}, + "max_tokens only": {"max_tokens": 16}, + "temperature=0": {"max_completion_tokens": 16, "temperature": 0}, + "temperature=1": {"max_completion_tokens": 16, "temperature": 1}, + "top_p=0.9": {"max_completion_tokens": 16, "top_p": 0.9}, + "stream=true": {"max_completion_tokens": 16, "stream": True}, +} + +for label, extra in CASES.items(): + body = {"model": MODEL, "messages": [{"role": "user", "content": "say PONG"}], **extra} + req = urllib.request.Request( + "https://api.openai.com/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": f"Bearer {KEY}"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + print(f"{label:28s} -> {resp.status} OK") + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", "replace") + try: + detail = json.loads(detail)["error"]["message"] + except Exception: + detail = detail[:160] + print(f"{label:28s} -> {exc.code} {detail}") + except Exception as exc: # noqa: BLE001 + print(f"{label:28s} -> ERROR {type(exc).__name__}: {exc}") diff --git a/scripts/verify-cluster.sh b/scripts/verify-cluster.sh new file mode 100755 index 0000000..b2d200f --- /dev/null +++ b/scripts/verify-cluster.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# In-cluster acceptance test for one forgeway instance. +# +# The "forge unavailable" case is produced by a throwaway Deployment pointed at +# a dead upstream. Ollama on forge is never stopped. +# +# ./scripts/verify-cluster.sh ~/.kube/rocinante-admin.yaml lounge +set -uo pipefail + +KUBECONFIG_PATH="${1:?usage: verify-cluster.sh }" +SITE="${2:?usage: verify-cluster.sh }" +export KUBECONFIG="$KUBECONFIG_PATH" +NS=forgeway +ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +IMAGE="$(kubectl -n "$NS" get deploy forgeway -o jsonpath='{.spec.template.spec.containers[0].image}')" +echo "### $SITE — image $IMAGE" + +cleanup() { + kubectl -n "$NS" delete deploy forgeway-badtest --ignore-not-found --wait=false >/dev/null 2>&1 + kubectl -n "$NS" delete svc forgeway-badtest --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete cm forgeway-checks --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete pod forgeway-verify --ignore-not-found --wait=false >/dev/null 2>&1 +} +cleanup +trap cleanup EXIT + +kubectl -n "$NS" create configmap forgeway-checks --from-file=checks.sh="$ROOT/scripts/checks.sh" >/dev/null + +kubectl -n "$NS" apply -f - >/dev/null </dev/null || echo "WARN: badtest rollout slow" + +kubectl -n "$NS" apply -f - >/dev/null </dev/null)" + [ "$phase" = "Succeeded" ] || [ "$phase" = "Failed" ] && break + sleep 5 +done +echo +kubectl -n "$NS" logs forgeway-verify