diff --git a/hermes/gateway.yaml b/hermes/gateway.yaml index dcb98cc..5a9503c 100644 --- a/hermes/gateway.yaml +++ b/hermes/gateway.yaml @@ -34,9 +34,15 @@ custom_providers: # carry per-model context_length overrides, which are read from config # independently of discovery. models: - # These MUST track each node's --max-model-len in ai-k8s-iac - # (charts/inference/values.yaml > vllm.nodes[].extraArgs). They are a - # separate copy of that number; there is no single source across repos. + # NO LONGER HAND-MAINTAINED. mcp-config/sync.sh renders this map from the + # gateway's /v1/model/info on every sync (_mcp_sync_hermes_models), so the + # single source of truth is `model_info.max_input_tokens` in ai-k8s-iac's + # charts/inference/values.yaml. Fix a wrong window THERE, not here: an + # edit to these entries is overwritten by the next sync. + # + # What follows is only a first-boot seed, so a brand-new box has sane + # limits before its first sync. Anything you add that the gateway does + # not serve is left alone — the renderer only touches fields it set. # # Unlike opencode, Hermes needs NO safety margin here. opencode reserves # `limit.output` and targets an input of exactly context - output, landing diff --git a/mcp-config/README.md b/mcp-config/README.md index 6252b74..8f11872 100644 --- a/mcp-config/README.md +++ b/mcp-config/README.md @@ -56,6 +56,53 @@ needs PyYAML (it warns if that's missing). Nothing needs adding to Hermes' `platform_toolsets` — MCP servers are auto-enabled by name; listing *some* would turn it into an allowlist, and `no_mcp` disables them all. +### Context limits — discovered, not hand-copied + +OpenAI's `/v1/models` carries **no context window**, so a client that only +lists models cannot know how big a request it may send, and oversizing one is a +hard `ContextWindowExceededError` 400 (vLLM counts `prompt + max_tokens` against +its window). The window lives in LiteLLM's `/v1/model/info`, and `sync.sh` +renders it into every client from that one response: + +| client | reads | what `sync.sh` writes | why that number | +|---|---|---|---| +| opencode | `opencode.gateway.json` | `limit.context = window − 1024`, `limit.output = min(out, cap)` | reserves `output` and targets `context − output`, so needs headroom | +| Hermes | `~/.hermes/config.yaml` | `context_length = window` | omits `max_tokens`; vLLM bounds output by what is left, so no margin | +| pi | `~/.pi/agent/models.json` | `contextWindow = window`, `maxTokens = min(out, cap, reserveTokens − 2048)` | compacts at `window − reserveTokens` then asks for `maxTokens`, so `maxTokens` must fit inside the reserve | + +Same source, three different numbers — each client means something different by +"context". Do not "simplify" them into one. + +**The single source of truth is `model_info.max_input_tokens` in ai-k8s-iac's +`charts/inference/values.yaml`.** A wrong window is fixed there and propagates on +the next sync. Editing a rendered value in a client config is overwritten. + +Requirements and behaviour: + +- **The key needs `/v1/model/info`.** A virtual key scoped to `llm_api_routes` + gets a 403 and every client silently falls back to its own defaults. Grant it + in the LiteLLM UI (Virtual Keys → key → Allowed Routes). The match is on the + exact path: `/v1/model/info` does not also grant `/model/info`. +- **Cached** at `~/.config/mcp-sync/model-info.json` on each successful fetch, so + a throttled shell still renders. A bad fetch (403, error page, empty list) + never replaces a good cache. +- **Ownership is tracked, per field.** `hermes-models-managed.json` and + `pi-managed.json` record what was set, so a value is retracted only if `sync.sh` + set it and the gateway stopped declaring it. That includes vision: if a backend + falls back to language-only and `supports_vision` flips off, the `image` input + `sync.sh` added is taken back rather than left to send image parts the backend + rejects. Models and fields you add by hand are never touched. +- **Never invents a window.** A wildcard route like `claude-*` declares none, so + it is skipped. A gateway that suddenly declares no windows at all is treated as + broken, not empty — nothing is retracted. +- **`pi`'s `maxTokens` is derived from its reserve**, not the other way round: + `reserveTokens` is one global setting shared by every provider pi knows, so + raising it to suit this gateway would shrink everyone's context. Raise + `compaction.reserveTokens` in pi's settings to buy back output. The shared + output cap is `MCP_SYNC_OUTPUT_RESERVE_CAP` (default 32768), read by opencode + and pi alike. +- Opt out per client with `MCP_SYNC_HERMES_CONFIG=` or `MCP_SYNC_PI_MODELS=`. + Credentials are referenced as `{env:VAR}` and are **never** written into the config. Each user exports them in `~/.zshrc.local`: diff --git a/mcp-config/sync.sh b/mcp-config/sync.sh index 50100e3..0bfdbfc 100644 --- a/mcp-config/sync.sh +++ b/mcp-config/sync.sh @@ -76,6 +76,21 @@ MCP_SYNC_HERMES_CONFIG="${MCP_SYNC_HERMES_CONFIG-$HOME/.hermes/config.yaml}" # Names we rendered last time, so a server REMOVED upstream can be removed here # without touching MCP servers the user added to Hermes by hand. MCP_SYNC_HERMES_MANIFEST="${MCP_SYNC_HERMES_MANIFEST:-$MCP_SYNC_HOME/hermes-managed.json}" +# Model limits for the non-opencode clients. The gateway's /v1/model/info is +# cached here on every successful fetch, because the refresh above is throttled +# and a throttled shell never fetches — the renderers must still have something +# to render from. +MCP_SYNC_MODELINFO_CACHE="${MCP_SYNC_MODELINFO_CACHE:-$MCP_SYNC_HOME/model-info.json}" +# Per-model FIELDS we set in Hermes last time, so a field can be retracted +# without touching fields the user set by hand. Separate from the MCP manifest. +MCP_SYNC_HERMES_MODELS_MANIFEST="${MCP_SYNC_HERMES_MODELS_MANIFEST:-$MCP_SYNC_HOME/hermes-models-managed.json}" +# pi. Set MCP_SYNC_PI_MODELS to empty to opt out entirely. +MCP_SYNC_PI_MODELS="${MCP_SYNC_PI_MODELS-$HOME/.pi/agent/models.json}" +MCP_SYNC_PI_SETTINGS="${MCP_SYNC_PI_SETTINGS:-$HOME/.pi/agent/settings.json}" +MCP_SYNC_PI_MANIFEST="${MCP_SYNC_PI_MANIFEST:-$MCP_SYNC_HOME/pi-managed.json}" +# The most output any client is told it may reserve/request. ONE value, read by +# the opencode, Hermes and pi renderers alike, so they cannot drift apart on it. +MCP_SYNC_OUTPUT_RESERVE_CAP="${MCP_SYNC_OUTPUT_RESERVE_CAP:-32768}" # Throttle: only re-sync if the deployed copy is older than this many seconds # (default 1h). Keeps every new shell from hammering Gitea/the gateway. MCP_SYNC_TTL="${MCP_SYNC_TTL:-3600}" @@ -229,6 +244,33 @@ _mcp_sync_refresh() { unset _mi_path fi + # Cache it for the Hermes and pi renderers — but only a response that really + # is model info. curl -f already rejects a 403, yet a 200 carrying an error + # page or an empty list is possible, and caching that would strip every + # client's limits until the next good fetch. A bad fetch leaves the last good + # cache in place. + if [ -n "$_info" ] && command -v python3 >/dev/null 2>&1; then + MCP_SYNC_PY_INFO="$_info" MCP_SYNC_PY_CACHE="$MCP_SYNC_MODELINFO_CACHE" \ + python3 - <<'PYEOF' 2>/dev/null || true +import json, os, tempfile +try: + data = json.loads(os.environ["MCP_SYNC_PY_INFO"]) + rows = data.get("data") +except Exception: + raise SystemExit(0) +if not isinstance(rows, list) or not any( + isinstance(r, dict) and r.get("model_name") for r in rows): + raise SystemExit(0) +cache = os.environ["MCP_SYNC_PY_CACHE"] +d = os.path.dirname(cache) or "." +os.makedirs(d, exist_ok=True) +fd, tmp = tempfile.mkstemp(dir=d) +with os.fdopen(fd, "w") as f: + json.dump({"data": rows}, f) +os.replace(tmp, cache) +PYEOF + fi + # 3. Merge and deploy. python3 validates the base (don't clobber good config # with an error page), injects the discovered models, and — when discovery # failed — carries over the previous deployed models map so a gateway @@ -237,6 +279,7 @@ _mcp_sync_refresh() { if command -v python3 >/dev/null 2>&1; then MCP_SYNC_BASE="$_base" MCP_SYNC_MODELS="$_models" MCP_SYNC_MODELINFO="$_info" \ MCP_SYNC_PY_DEST="$MCP_SYNC_DEST" MCP_SYNC_PY_PROVIDER="$MCP_SYNC_PROVIDER" \ + MCP_SYNC_PY_OUTPUT_CAP="$MCP_SYNC_OUTPUT_RESERVE_CAP" \ python3 - <<'PYEOF' 2>/dev/null || true import json, os, tempfile @@ -263,7 +306,7 @@ except Exception: # separately — see charts/inference/values.yaml model_info), so copying it # verbatim made opencode reserve the entire window and target an input of # context - output. Cap the reserve at a generous agent-output budget instead. -OUTPUT_RESERVE_CAP = 32768 +OUTPUT_RESERVE_CAP = int(os.environ.get("MCP_SYNC_PY_OUTPUT_CAP") or 32768) # opencode targets input == context - output EXACTLY, and the server counts one # more token than opencode does, landing at context + 1 and getting a hard 400 # (ContextWindowExceededError). Observed on qwen3-coder-next: 196609 in + 65536 @@ -505,6 +548,302 @@ os.replace(tmp, manifest_path) PYEOF } +# --- render the gateway's MODEL LIMITS into Hermes' config (non-fatal) ------ +# +# WHY THIS EXISTS: OpenAI's /v1/models schema carries no context window, so a +# client that only lists models cannot know how big a request it may send. +# opencode gets the window from /v1/model/info above. Hermes does not consult +# that endpoint at all — it takes context_length from a static `models:` map +# under its custom provider — so the number used to be hand-copied into +# hermes/gateway.yaml and kept "in sync" by nobody. It drifted: the map named a +# model the gateway had stopped serving, while the model that replaced it (and +# the only one with a SMALLER window than its neighbours) had no entry at all. +# Short turns were fine; long tasks hit ContextWindowExceededError 400s. +# +# NO MARGIN, deliberately, and this is the one number that differs from the +# opencode renderer. opencode reserves limit.output and targets an input of +# context - output, so it needs headroom. Hermes omits max_tokens entirely, so +# vLLM bounds the output by whatever context is left: the raw window is right. +# +# OWNERSHIP IS PER FIELD, not per model. The manifest records which fields this +# set on which model. A field is retracted only if WE set it and the gateway no +# longer declares it (a model removed upstream, or supports_vision flipped off +# because a backend fell back to language-only — see the drift warning on +# supports_vision in ai-k8s-iac's values.yaml). Anything the user typed into the +# map by hand was never in the manifest and is never touched. +# +# Only fills in the provider install.sh created; never creates one. +_mcp_sync_hermes_models() { + [ -n "${MCP_SYNC_HERMES_CONFIG:-}" ] || return 0 + [ -f "$MCP_SYNC_HERMES_CONFIG" ] || return 0 + [ -f "$MCP_SYNC_MODELINFO_CACHE" ] || return 0 + command -v python3 >/dev/null 2>&1 || return 0 + python3 -c 'import yaml' >/dev/null 2>&1 || return 0 # warned about by _mcp_sync_hermes + + MCP_SYNC_PY_CACHE="$MCP_SYNC_MODELINFO_CACHE" \ + MCP_SYNC_PY_HERMES="$MCP_SYNC_HERMES_CONFIG" \ + MCP_SYNC_PY_MANIFEST="$MCP_SYNC_HERMES_MODELS_MANIFEST" \ + MCP_SYNC_PY_PROVIDER="$MCP_SYNC_PROVIDER" \ + python3 - <<'PYEOF' 2>/dev/null || true +import json, os, tempfile +import yaml + +provider = os.environ["MCP_SYNC_PY_PROVIDER"] +dest = os.environ["MCP_SYNC_PY_HERMES"] +manifest_path = os.environ["MCP_SYNC_PY_MANIFEST"] + +try: + with open(os.environ["MCP_SYNC_PY_CACHE"]) as f: + rows = json.load(f).get("data") or [] +except Exception: + raise SystemExit(0) + +rendered = {} +for r in rows: + if not isinstance(r, dict): + continue + name = r.get("model_name") + mi = r.get("model_info") or {} + window = mi.get("max_input_tokens") + # A wildcard route (claude-*) declares no window. Never invent one: a guessed + # context_length is worse than letting Hermes use its own default. + if not name or not isinstance(window, int) or window <= 0: + continue + fields = {"context_length": window} + if mi.get("supports_vision") is True: + fields["supports_vision"] = True + rendered[name] = fields +# A gateway that suddenly declares no windows at all is broken, not empty. Do +# not treat that as "retract everything". +if not rendered: + raise SystemExit(0) + +try: + with open(dest) as f: + cfg = yaml.safe_load(f) or {} +except Exception: + raise SystemExit(0) # never overwrite a config we cannot parse +if not isinstance(cfg, dict) or not isinstance(cfg.get("custom_providers"), list): + raise SystemExit(0) +target = next((p for p in cfg["custom_providers"] + if isinstance(p, dict) and p.get("name") == provider), None) +if target is None: + raise SystemExit(0) + +models = target.get("models") +if not isinstance(models, dict): + models = {} +before = json.dumps(models, sort_keys=True, default=str) + +try: + with open(manifest_path) as f: + prev = json.load(f) + prev = prev if isinstance(prev, dict) else {} +except Exception: + prev = {} + +for name, owned in prev.items(): + entry = models.get(name) + if not isinstance(owned, list) or not isinstance(entry, dict): + continue + still = rendered.get(name, {}) + for field in owned: + if field not in still: + entry.pop(field, None) + if not entry: + models.pop(name, None) + +for name, fields in rendered.items(): + entry = models.get(name) + entry = entry if isinstance(entry, dict) else {} + entry.update(fields) + models[name] = entry + +if json.dumps(models, sort_keys=True, default=str) != before: + target["models"] = models + d = os.path.dirname(dest) or "." + fd, tmp = tempfile.mkstemp(dir=d) + with os.fdopen(fd, "w") as f: + yaml.safe_dump(cfg, f, sort_keys=False, default_flow_style=False, + allow_unicode=True) + os.chmod(tmp, 0o600) + os.replace(tmp, dest) + +d = os.path.dirname(manifest_path) or "." +os.makedirs(d, exist_ok=True) +fd, tmp = tempfile.mkstemp(dir=d) +with os.fdopen(fd, "w") as f: + json.dump({n: sorted(fl) for n, fl in rendered.items()}, f, sort_keys=True) +os.replace(tmp, manifest_path) +PYEOF +} + +# --- render the gateway's MODEL LIMITS into pi's models.json (non-fatal) ---- +# +# Same reason as Hermes: pi reads a static models.json and never asks the +# gateway, so contextWindow/maxTokens were hand-copied and drifted. +# +# THE INVARIANT, because getting this wrong is a guaranteed 400 rather than a +# degraded answer. pi compacts when contextTokens > contextWindow - reserveTokens +# (compaction.js shouldCompact) and then sends max_tokens = maxTokens. vLLM +# counts prompt + max_tokens against its window. So the largest request pi can +# make is (window - reserveTokens) + maxTokens, and it fits only if +# +# maxTokens <= reserveTokens - margin +# +# This used to be violated outright: pi's default reserve is 16384 while +# maxTokens was set to 32768, so a full context sent 196608 - 16384 + 32768 = +# 212992 tokens at a 196608 window. +# +# maxTokens is DERIVED FROM the reserve, rather than the reserve being raised to +# fit maxTokens. reserveTokens is one global setting shared by every provider pi +# knows about; raising it to suit this gateway would silently shrink usable +# context for all of them. Capping maxTokens per model keeps the fix local, and +# means raising reserveTokens in settings.json is how you buy back output. +# +# OWNERSHIP: the manifest records models this ADDED and models it added "image" +# to. contextWindow and maxTokens are always managed on gateway models, because +# a stale value there is the bug. Models the user created are never removed, and +# a user's own "image" is never taken away — only one this added. +# +# Only fills in the provider the user configured; never creates one, because a +# provider needs a baseUrl and key this does not own. +_mcp_sync_pi() { + [ -n "${MCP_SYNC_PI_MODELS:-}" ] || return 0 + [ -f "$MCP_SYNC_PI_MODELS" ] || return 0 + [ -f "$MCP_SYNC_MODELINFO_CACHE" ] || return 0 + command -v python3 >/dev/null 2>&1 || return 0 + + MCP_SYNC_PY_CACHE="$MCP_SYNC_MODELINFO_CACHE" \ + MCP_SYNC_PY_PI="$MCP_SYNC_PI_MODELS" \ + MCP_SYNC_PY_PI_SETTINGS="$MCP_SYNC_PI_SETTINGS" \ + MCP_SYNC_PY_MANIFEST="$MCP_SYNC_PI_MANIFEST" \ + MCP_SYNC_PY_PROVIDER="$MCP_SYNC_PROVIDER" \ + MCP_SYNC_PY_OUTPUT_CAP="$MCP_SYNC_OUTPUT_RESERVE_CAP" \ + python3 - <<'PYEOF' 2>/dev/null || true +import json, os, tempfile + +provider = os.environ["MCP_SYNC_PY_PROVIDER"] +dest = os.environ["MCP_SYNC_PY_PI"] +manifest_path = os.environ["MCP_SYNC_PY_MANIFEST"] +cap = int(os.environ.get("MCP_SYNC_PY_OUTPUT_CAP") or 32768) +# Headroom between pi's compaction point and the server's hard limit. pi counts +# from the server's own usage.totalTokens, so this only has to absorb the new +# message appended since the last measured turn. +MARGIN = 2048 +# pi's built-in default when compaction.reserveTokens is unset. +PI_DEFAULT_RESERVE = 16384 + +try: + with open(os.environ["MCP_SYNC_PY_CACHE"]) as f: + rows = json.load(f).get("data") or [] +except Exception: + raise SystemExit(0) + +reserve = PI_DEFAULT_RESERVE +try: + with open(os.environ["MCP_SYNC_PY_PI_SETTINGS"]) as f: + r = (json.load(f).get("compaction") or {}).get("reserveTokens") + if isinstance(r, int) and r > 0: + reserve = r +except Exception: + pass +ceiling = reserve - MARGIN +if ceiling <= 0: + raise SystemExit(0) # no output is provably safe under this reserve; do not guess + +rendered = {} +for r in rows: + if not isinstance(r, dict): + continue + name = r.get("model_name") + mi = r.get("model_info") or {} + window = mi.get("max_input_tokens") + if not name or not isinstance(window, int) or window <= 0: + continue + out = mi.get("max_output_tokens") + # LiteLLM's max_output_tokens is the WHOLE window for a vLLM backend (it caps + # total context, not output), so it is an upper bound, never a value to copy. + out = out if isinstance(out, int) and out > 0 else cap + rendered[name] = { + "contextWindow": window, + "maxTokens": min(out, cap, ceiling), + "vision": mi.get("supports_vision") is True, + } +if not rendered: + raise SystemExit(0) + +try: + with open(dest) as f: + cfg = json.load(f) +except Exception: + raise SystemExit(0) +prov = (cfg.get("providers") or {}).get(provider) if isinstance(cfg, dict) else None +if not isinstance(prov, dict) or not isinstance(prov.get("models"), list): + raise SystemExit(0) +before = json.dumps(cfg, sort_keys=True) + +try: + with open(manifest_path) as f: + prev = json.load(f) + prev = prev if isinstance(prev, dict) else {} +except Exception: + prev = {} +added_prev = set(prev.get("added") or []) +image_prev = set(prev.get("image") or []) + +# Drop models WE added that the gateway no longer serves. A user-created entry +# is never removed, even if the gateway stops listing it. +models = [m for m in prov["models"] + if not (isinstance(m, dict) and m.get("id") in added_prev + and m.get("id") not in rendered)] +by_id = {m.get("id"): m for m in models if isinstance(m, dict)} + +added_now, image_now = set(), set() +for name, f in rendered.items(): + m = by_id.get(name) + if m is None: + m = {"id": name, "name": name, "reasoning": False, "input": ["text"], + "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}} + models.append(m) + by_id[name] = m + added_now.add(name) + elif name in added_prev: + added_now.add(name) + m["contextWindow"] = f["contextWindow"] + m["maxTokens"] = f["maxTokens"] + inp = m.get("input") if isinstance(m.get("input"), list) else ["text"] + if f["vision"]: + if "image" not in inp: + inp = inp + ["image"] + image_now.add(name) + elif name in image_prev: + image_now.add(name) + elif name in image_prev and "image" in inp: + # We added it and the gateway no longer claims vision: take it back, or + # pi keeps sending image parts the backend will reject. + inp = [x for x in inp if x != "image"] + m["input"] = inp + +prov["models"] = models +if json.dumps(cfg, sort_keys=True) != before: + d = os.path.dirname(dest) or "." + fd, tmp = tempfile.mkstemp(dir=d) + with os.fdopen(fd, "w") as f: + json.dump(cfg, f, indent=2) + f.write("\n") + os.chmod(tmp, 0o600) # the provider block can carry an apiKey + os.replace(tmp, dest) + +d = os.path.dirname(manifest_path) or "." +os.makedirs(d, exist_ok=True) +fd, tmp = tempfile.mkstemp(dir=d) +with os.fdopen(fd, "w") as f: + json.dump({"added": sorted(added_now), "image": sorted(image_now)}, f) +os.replace(tmp, manifest_path) +PYEOF +} + # --- deploy the shared skills tree (throttled, non-fatal) ------------------ _mcp_sync_skills() { # Same TTL as the config, tracked on the deployed directory's own mtime. @@ -565,6 +904,11 @@ _mcp_sync_export_key _mcp_sync_refresh # After refresh: renders the just-deployed MCP servers into Hermes' schema. _mcp_sync_hermes +# Then the gateway's model limits into the two clients that do not discover +# them on their own. Both run from the cache, so a throttled shell still +# converges on the last known good values. +_mcp_sync_hermes_models +_mcp_sync_pi _mcp_sync_skills # opencode only merges config from known locations; point OPENCODE_CONFIG at