dotfiles/mcp-config/sync.sh

#!/usr/bin/env sh
# mcp-sync — refresh the shared LiteLLM-gateway opencode config on the laptop,
# auto-discover the gateway's models, deploy the shared skills, and export the
# user's LiteLLM key.
#
# Sourced from the user's shell rc (installed by install.sh) so it runs at the
# start of every interactive shell ("session start"). It is intentionally:
#   - FAST and SILENT on success (no output unless something is wrong)
#   - NON-FATAL: a network/Gitea hiccup must never break the user's shell, so
#     every failure path just returns, leaving the last-good config in place.
#
# What it does:
#   1. Fetch the shared opencode.gateway.json from Gitea (the single source of
#      truth: the covert-gpt provider + the client-side MCP servers). The file
#      is identical for all users — EVERY credential (LiteLLM key, Gitea PAT,
#      Proxmox token) is injected via {env:VAR} and never baked into the file.
#   2. Auto-discover models: query the gateway's /v1/models with the user's
#      key and fill the provider's "models" map. New models exposed on the
#      gateway appear on every box at the next sync — no config edit anywhere.
#      (opencode cannot list models from a custom provider itself; the map
#      must be materialized client-side.) MCP servers need no discovery step:
#      they are declared in the config and run locally under `npx`, each with
#      the user's OWN credentials. The gateway hosts no MCP servers at all —
#      see mcp-config/README.md for why (user-scoped APIs, no shared tokens).
#   3. Deploy the repo's skills/ tree to ~/.config/mcp-sync/skills. Skills are
#      plain files, so there is nothing on the gateway to discover — the repo
#      IS the source of truth, and the tree is fetched whole (repo tarball).
#      They land in mcp-sync's OWN directory, not ~/.config/opencode/skills,
#      so the atomic replace below can never eat a skill the user wrote by
#      hand; opencode picks them up via skills.paths in the shared config.
#   4. Drop the result at ~/.config/opencode/opencode.gateway.json and export
#      OPENCODE_CONFIG pointing at it. opencode does NOT auto-load extra files
#      in its config dir — it only merges known locations, and OPENCODE_CONFIG
#      is merged between the global opencode.json and any project config. The
#      user's own opencode.json and local-only MCP servers are never touched.
#   5. Export LITELLM_KEY from ~/.config/mcp-sync/key so {env:LITELLM_KEY}
#      resolves.

# --- config (overridable via env before sourcing) -------------------------
MCP_SYNC_RAW_URL="${MCP_SYNC_RAW_URL:-https://git.nic-oconnor.com/public/dotfiles/raw/branch/main/mcp-config/opencode.gateway.json}"
MCP_SYNC_HOME="${MCP_SYNC_HOME:-$HOME/.config/mcp-sync}"
MCP_SYNC_DEST="${MCP_SYNC_DEST:-$HOME/.config/opencode/opencode.gateway.json}"
MCP_SYNC_KEYFILE="${MCP_SYNC_KEYFILE:-$MCP_SYNC_HOME/key}"
# Provider whose models are auto-discovered, and the gateway base URL.
MCP_SYNC_PROVIDER="${MCP_SYNC_PROVIDER:-covert-gpt}"
MCP_SYNC_GATEWAY="${MCP_SYNC_GATEWAY:-https://llm.covert-gpt.com}"
# Skills. A skill is a folder of files, so unlike the config there is no single
# raw URL to pull — we take the repo tarball and lift out its skills/ subtree.
# Set MCP_SYNC_SKILLS_SRC to a local directory (or file://) to sync from a
# working checkout instead; see zsh/zshrc.local.example.
MCP_SYNC_SKILLS_SRC="${MCP_SYNC_SKILLS_SRC:-https://git.nic-oconnor.com/public/dotfiles/archive/main.tar.gz}"
MCP_SYNC_SKILLS_SUBDIR="${MCP_SYNC_SKILLS_SUBDIR:-skills}"
MCP_SYNC_SKILLS_DEST="${MCP_SYNC_SKILLS_DEST:-$MCP_SYNC_HOME/skills}"
# Hermes: the SAME MCP servers, rendered into Hermes' own schema. Set
# MCP_SYNC_HERMES_CONFIG to empty to opt out entirely.
MCP_SYNC_HERMES_CONFIG="${MCP_SYNC_HERMES_CONFIG-$HOME/.hermes/config.yaml}"
# Names we rendered last time, so a server REMOVED upstream can be removed here
# without touching MCP servers the user added to Hermes by hand.
MCP_SYNC_HERMES_MANIFEST="${MCP_SYNC_HERMES_MANIFEST:-$MCP_SYNC_HOME/hermes-managed.json}"
# Throttle: only re-sync if the deployed copy is older than this many seconds
# (default 1h). Keeps every new shell from hammering Gitea/the gateway.
MCP_SYNC_TTL="${MCP_SYNC_TTL:-3600}"

# --- export the per-user key (first: model discovery needs it) -------------
_mcp_sync_export_key() {
  [ -f "$MCP_SYNC_KEYFILE" ] || return 0
  # Strip whitespace/newline; export so {env:LITELLM_KEY} resolves in opencode.
  LITELLM_KEY="$(tr -d ' \t\r\n' < "$MCP_SYNC_KEYFILE" 2>/dev/null)"
  [ -n "$LITELLM_KEY" ] && export LITELLM_KEY
}

# --- refresh config + discover models (throttled, non-fatal) ---------------
_mcp_sync_refresh() {
  mkdir -p "$(dirname "$MCP_SYNC_DEST")" 2>/dev/null || return 0

  # Skip if we synced recently (mtime within TTL). `find -mmin` is portable
  # enough; fall back to always-sync if find is unavailable.
  if [ -f "$MCP_SYNC_DEST" ] && command -v find >/dev/null 2>&1; then
    if find "$MCP_SYNC_DEST" -mmin -"$(( MCP_SYNC_TTL / 60 ))" 2>/dev/null | grep -q .; then
      return 0
    fi
  fi

  command -v curl >/dev/null 2>&1 || return 0

  # 1. Fetch the shared base config. On failure fall back to the currently
  #    deployed file so model discovery can still refresh it.
  _base="$(mktemp 2>/dev/null)" || return 0
  # -f: fail on HTTP error; -s: silent; -m: hard timeout so a hung network
  # can't stall the shell.
  if ! curl -fsS -m 8 "$MCP_SYNC_RAW_URL" -o "$_base" 2>/dev/null || [ ! -s "$_base" ]; then
    rm -f "$_base"; _base=""
    [ -f "$MCP_SYNC_DEST" ] || return 0
  fi

  # 2. Discover the gateway's models with the user's key (best-effort).
  _models=""
  if [ -n "${LITELLM_KEY:-}" ]; then
    _models="$(curl -fsS -m 8 -H "Authorization: Bearer $LITELLM_KEY" \
      "$MCP_SYNC_GATEWAY/v1/models" 2>/dev/null)" || _models=""
  fi

  # 2b. Enrich with per-model metadata. /v1/models returns ONLY ids — no context
  #     window, no cost, no capability flags — and opencode needs those or it
  #     runs every model on defaults (it cannot manage truncation without a
  #     context limit, and won't know a model can call tools). /model/info has
  #     them. Best-effort: a key still scoped to llm_api_routes 403s here, and
  #     we fall back to names-only, exactly as before.
  _info=""
  if [ -n "${LITELLM_KEY:-}" ]; then
    _info="$(curl -fsS -m 8 -H "Authorization: Bearer $LITELLM_KEY" \
      "$MCP_SYNC_GATEWAY/model/info" 2>/dev/null)" || _info=""
  fi

  # 3. Merge and deploy. python3 validates the base (don't clobber good config
  #    with an error page), injects the discovered models, and — when discovery
  #    failed — carries over the previous deployed models map so a gateway
  #    hiccup never empties the model list. Without python3, deploy the raw
  #    base as-is (no discovery).
  if command -v python3 >/dev/null 2>&1; then
    MCP_SYNC_BASE="$_base" MCP_SYNC_MODELS="$_models" MCP_SYNC_MODELINFO="$_info" \
    MCP_SYNC_PY_DEST="$MCP_SYNC_DEST" MCP_SYNC_PY_PROVIDER="$MCP_SYNC_PROVIDER" \
    python3 - <<'PYEOF' 2>/dev/null || true
import json, os, tempfile

dest = os.environ["MCP_SYNC_PY_DEST"]
provider = os.environ["MCP_SYNC_PY_PROVIDER"]

def load(path):
    with open(path) as f:
        return json.load(f)

base_path = os.environ.get("MCP_SYNC_BASE") or dest
try:
    cfg = load(base_path)
except Exception:
    raise SystemExit(0)  # invalid base: keep last-good deployed config

# LiteLLM /model/info -> opencode's per-model spec. Only fields the gateway
# actually reports are emitted; a model LiteLLM knows nothing about (a
# self-hosted vLLM entry with no model_info in the gateway's model_list) simply
# stays {}, which is what every model got before this existed.
# opencode's limit.output is a RESERVE it subtracts from limit.context to decide
# when to compact — NOT a server-side cap. LiteLLM reports max_output_tokens as
# the whole window for self-hosted vLLM (vLLM caps TOTAL context, not output
# separately — see charts/inference/values.yaml model_info), so copying it
# verbatim made opencode reserve the entire window and target an input of
# context - output. Cap the reserve at a generous agent-output budget instead.
OUTPUT_RESERVE_CAP = 32768
# opencode targets input == context - output EXACTLY, and the server counts one
# more token than opencode does, landing at context + 1 and getting a hard 400
# (ContextWindowExceededError). Observed on qwen3-coder-next: 196609 in + 65536
# out = 262145 vs a 262144 window, one token over, every time. Hold a margin
# back so opencode's target can never sit on the boundary.
CONTEXT_SAFETY_MARGIN = 1024


def spec_for(mi):
    spec = {}
    limit = {}
    if mi.get("max_input_tokens"):
        limit["context"] = max(1, mi["max_input_tokens"] - CONTEXT_SAFETY_MARGIN)
    if mi.get("max_output_tokens"):
        limit["output"] = min(mi["max_output_tokens"], OUTPUT_RESERVE_CAP)
    if limit:
        spec["limit"] = limit
    # LiteLLM prices PER TOKEN; opencode (like models.dev) uses $ per 1M tokens.
    cost = {}
    for src, dst in (("input_cost_per_token", "input"),
                     ("output_cost_per_token", "output")):
        if mi.get(src) is not None:
            cost[dst] = mi[src] * 1_000_000
    if cost:
        spec["cost"] = cost
    for src, dst in (("supports_function_calling", "tool_call"),
                     ("supports_reasoning", "reasoning"),
                     ("supports_vision", "attachment")):
        if mi.get(src) is not None:
            spec[dst] = bool(mi[src])
    return spec

info = {}
try:
    for entry in (json.loads(os.environ.get("MCP_SYNC_MODELINFO") or "{}")
                  .get("data", [])):
        name = entry.get("model_name")
        if name:
            info[name] = spec_for(entry.get("model_info") or {})
except Exception:
    pass

models = {}
try:
    data = json.loads(os.environ.get("MCP_SYNC_MODELS") or "{}")
    models = {m["id"]: dict(info.get(m["id"], {}))
              for m in data.get("data", []) if m.get("id")}
except Exception:
    pass
if not models:
    try:  # discovery failed: carry over the previous deployed map
        models = load(dest)["provider"][provider]["models"]
    except Exception:
        pass

prov = cfg.get("provider", {}).get(provider)
if prov is not None and models:
    prov["models"] = models

fd, tmp = tempfile.mkstemp(dir=os.path.dirname(dest))
with os.fdopen(fd, "w") as f:
    json.dump(cfg, f, indent=2)
os.replace(tmp, dest)
PYEOF
  elif [ -n "$_base" ]; then
    # No python3: the base config is deployed verbatim, so its "models" map
    # stays EMPTY and opencode shows the provider with zero models. That looks
    # like a broken gateway rather than a missing dependency, so say so — this
    # is the one case worth breaking the silence for.
    mv "$_base" "$MCP_SYNC_DEST" 2>/dev/null || true
    _base=""
    printf 'mcp-sync: python3 not found — models cannot be discovered.\n' >&2
    printf 'mcp-sync: install python3 (bootstrap.sh does), then open a new shell.\n' >&2
  fi
  [ -n "$_base" ] && rm -f "$_base"
}

# --- render the SAME MCP servers into Hermes' config (non-fatal) -----------
#
# WHY THIS EXISTS: the MCP servers are defined ONCE, in opencode.gateway.json's
# `mcp` block (see README: "all MCP servers are defined here"). Hermes speaks a
# different schema, so without this it would need a second copy of every server
# — two definitions to drift apart, one pinned to gitea-mcp@0.0.10 and the other
# to whatever got typed. Instead we translate the one definition into Hermes'
# shape, the same way this file already translates LiteLLM's /model/info into
# opencode's. Add a server to Gitea once and BOTH clients get it.
#
# It is NOT a shared server process: MCP stdio servers are subprocesses, spawned
# per client over stdin/stdout and dying with it. Each client running its own
# `npx -y <pkg>` is the design — same package, same pinned version, same
# per-user credential, same identity. There is no daemon to share.
#
# Reads the DEPLOYED opencode config (post-refresh), so it always reflects what
# Gitea just served. No-ops silently when Hermes isn't installed.
_mcp_sync_hermes() {
  [ -n "${MCP_SYNC_HERMES_CONFIG:-}" ] || return 0
  # Hermes not installed (or never run) — nothing to configure, stay silent.
  [ -f "$MCP_SYNC_HERMES_CONFIG" ] || return 0
  [ -f "$MCP_SYNC_DEST" ] || return 0
  command -v python3 >/dev/null 2>&1 || return 0
  # PyYAML is needed to touch config.yaml at all. Unlike models, a missing MCP
  # block is invisible rather than obviously broken, so say something.
  if ! python3 -c 'import yaml' >/dev/null 2>&1; then
    printf 'mcp-sync: python3 has no PyYAML — Hermes MCP servers not synced.\n' >&2
    return 0
  fi

  MCP_SYNC_PY_SRC="$MCP_SYNC_DEST" \
  MCP_SYNC_PY_HERMES="$MCP_SYNC_HERMES_CONFIG" \
  MCP_SYNC_PY_MANIFEST="$MCP_SYNC_HERMES_MANIFEST" \
  MCP_SYNC_PY_PROVIDER="$MCP_SYNC_PROVIDER" \
  python3 - <<'PYEOF' 2>/dev/null || true
import json, os, re, tempfile
import yaml

src = os.environ["MCP_SYNC_PY_SRC"]
dest = os.environ["MCP_SYNC_PY_HERMES"]
manifest_path = os.environ["MCP_SYNC_PY_MANIFEST"]

try:
    with open(src) as f:
        oc = json.load(f)
except Exception:
    raise SystemExit(0)  # bad/incomplete opencode config: leave Hermes alone

# opencode's "{env:VAR}" -> Hermes' "${VAR}". Both are by-NAME references
# resolved at runtime, so no secret is ever written to either file.
_ENV_REF = re.compile(r"\{env:([A-Za-z_][A-Za-z0-9_]*)\}")


def _refs(v):
    if isinstance(v, str):
        return _ENV_REF.sub(r"${\1}", v)
    if isinstance(v, dict):
        return {k: _refs(x) for k, x in v.items()}
    if isinstance(v, list):
        return [_refs(x) for x in v]
    return v


def spec_for(name, s):
    if not isinstance(s, dict):
        return None
    # opencode defaults enabled to true; so does Hermes. Honour an explicit off.
    if s.get("enabled") is False:
        return None
    kind = s.get("type", "local")
    if kind == "remote":
        url = s.get("url")
        if not url:
            return None
        out = {"url": url}
        headers = s.get("headers")
        if isinstance(headers, dict) and headers:
            out["headers"] = _refs(headers)
        return out
    # local/stdio. opencode: command is a LIST (argv). Hermes: command is a
    # STRING and args is a separate list -- a list here stringifies to
    # "['npx', '-y']" and fails at exec, so the split is mandatory.
    cmd = s.get("command")
    if not isinstance(cmd, list) or not cmd:
        return None
    out = {"command": str(cmd[0])}
    if len(cmd) > 1:
        out["args"] = [str(a) for a in cmd[1:]]
    env = s.get("environment")
    if isinstance(env, dict) and env:
        # Hermes hands the child only a filtered baseline env, so every secret
        # the server needs MUST be passed explicitly here.
        out["env"] = _refs(env)
    return out


rendered = {}
for name, s in (oc.get("mcp") or {}).items():
    spec = spec_for(name, s)
    if spec is not None:
        rendered[name] = spec
if not rendered:
    raise SystemExit(0)

try:
    with open(dest) as f:
        cfg = yaml.safe_load(f) or {}
except Exception:
    raise SystemExit(0)  # never overwrite a config we cannot parse
if not isinstance(cfg, dict):
    raise SystemExit(0)

current = cfg.get("mcp_servers")
if not isinstance(current, dict):
    current = {}
merged = dict(current)

# Drop servers we rendered before that upstream has since removed. Anything the
# user added to Hermes themselves was never in the manifest, so it survives.
try:
    with open(manifest_path) as f:
        prev = json.load(f)
    prev = prev if isinstance(prev, list) else []
except Exception:
    prev = []
for gone in set(prev) - set(rendered):
    merged.pop(gone, None)

merged.update(rendered)

if merged != current:
    cfg["mcp_servers"] = merged
    d = os.path.dirname(dest) or "."
    fd, tmp = tempfile.mkstemp(dir=d)
    with os.fdopen(fd, "w") as f:
        yaml.safe_dump(cfg, f, sort_keys=False, default_flow_style=False,
                       allow_unicode=True)
    os.chmod(tmp, 0o600)
    os.replace(tmp, dest)

os.makedirs(os.path.dirname(manifest_path) or ".", exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(manifest_path) or ".")
with os.fdopen(fd, "w") as f:
    json.dump(sorted(rendered), f)
os.replace(tmp, manifest_path)
PYEOF
}

# --- deploy the shared skills tree (throttled, non-fatal) ------------------
_mcp_sync_skills() {
  # Same TTL as the config, tracked on the deployed directory's own mtime.
  if [ -d "$MCP_SYNC_SKILLS_DEST" ] && command -v find >/dev/null 2>&1; then
    if find "$MCP_SYNC_SKILLS_DEST" -maxdepth 0 -mmin -"$(( MCP_SYNC_TTL / 60 ))" 2>/dev/null | grep -q .; then
      return 0
    fi
  fi

  mkdir -p "$MCP_SYNC_HOME" 2>/dev/null || return 0
  # Stage inside MCP_SYNC_HOME (not /tmp) so the publish below is a same-
  # filesystem rename rather than a copy that can be seen half-finished.
  _work="$(mktemp -d "$MCP_SYNC_HOME/.skills.XXXXXX" 2>/dev/null)" || return 0

  _tree=""
  case "$MCP_SYNC_SKILLS_SRC" in
    /*|file://*)
      # Working checkout: the source is the skills/ dir itself.
      _tree="${MCP_SYNC_SKILLS_SRC#file://}"
      ;;
    *)
      if command -v curl >/dev/null 2>&1 && command -v tar >/dev/null 2>&1 &&
         curl -fsS -m 15 "$MCP_SYNC_SKILLS_SRC" -o "$_work/repo.tgz" 2>/dev/null &&
         [ -s "$_work/repo.tgz" ] &&
         mkdir -p "$_work/x" && tar -xzf "$_work/repo.tgz" -C "$_work/x" 2>/dev/null
      then
        # Gitea wraps the archive in a single top-level dir whose name we don't
        # want to hardcode; find the skills/ subtree instead of assuming depth.
        _tree="$(find "$_work/x" -maxdepth 2 -type d -name "$MCP_SYNC_SKILLS_SUBDIR" 2>/dev/null | head -1)"
      fi
      ;;
  esac
  if [ -z "$_tree" ] || [ ! -d "$_tree" ]; then rm -rf "$_work"; return 0; fi

  mkdir -p "$_work/stage" 2>/dev/null || { rm -rf "$_work"; return 0; }
  cp -R "$_tree/." "$_work/stage/" 2>/dev/null || { rm -rf "$_work"; return 0; }
  # Never replace a good tree with an empty one (truncated fetch, empty subdir).
  if [ -z "$(ls -A "$_work/stage" 2>/dev/null)" ]; then rm -rf "$_work"; return 0; fi

  # Publish: swap the new tree in, keeping the old one until the move lands so a
  # failure leaves the previous skills intact rather than no skills at all.
  rm -rf "$MCP_SYNC_SKILLS_DEST.old" 2>/dev/null
  if [ -d "$MCP_SYNC_SKILLS_DEST" ]; then
    mv "$MCP_SYNC_SKILLS_DEST" "$MCP_SYNC_SKILLS_DEST.old" 2>/dev/null || { rm -rf "$_work"; return 0; }
  fi
  if ! mv "$_work/stage" "$MCP_SYNC_SKILLS_DEST" 2>/dev/null; then
    mv "$MCP_SYNC_SKILLS_DEST.old" "$MCP_SYNC_SKILLS_DEST" 2>/dev/null
  fi
  rm -rf "$MCP_SYNC_SKILLS_DEST.old" "$_work" 2>/dev/null
}

_mcp_sync_export_key
_mcp_sync_refresh
# After refresh: renders the just-deployed MCP servers into Hermes' schema.
_mcp_sync_hermes
_mcp_sync_skills

# opencode only merges config from known locations; point OPENCODE_CONFIG at
# the synced file so it is layered on top of the user's global opencode.json.
[ -f "$MCP_SYNC_DEST" ] && export OPENCODE_CONFIG="$MCP_SYNC_DEST"

# --- mcp-sync: force a refresh now, ignoring the TTL ------------------------
# The automatic sync above runs ONLY at shell start and is throttled to
# MCP_SYNC_TTL (1h). So a model or MCP server added on the gateway can take up
# to an hour AND a new terminal before it shows up in opencode -- which reads
# as "my new model isn't there" when in fact the gateway is fine and the local
# config is just stale. Run `mcp-sync` to pull it immediately.
#
# Defined for the interactive shell (zsh/bash), which is the only place sync.sh
# is sourced; the hyphen in the name is fine there.
MCP_SYNC_SELF="${MCP_SYNC_SELF:-$HOME/.config/mcp-sync/sync.sh}"
mcp-sync() {
  [ -f "$MCP_SYNC_SELF" ] || {
    printf 'mcp-sync: %s not found\n' "$MCP_SYNC_SELF" >&2; return 1
  }
  # Re-source with the throttle disabled; this redefines and re-runs everything.
  # shellcheck disable=SC1090
  MCP_SYNC_TTL=0 . "$MCP_SYNC_SELF" || return 1
  command -v python3 >/dev/null 2>&1 || { printf 'mcp-sync: refreshed.\n' >&2; return 0; }
  MCP_SYNC_PY_DEST="$MCP_SYNC_DEST" MCP_SYNC_PY_PROVIDER="$MCP_SYNC_PROVIDER" \
  python3 - <<'PYEOF' >&2
import json, os
cfg = json.load(open(os.environ["MCP_SYNC_PY_DEST"]))
models = cfg.get("provider", {}).get(os.environ["MCP_SYNC_PY_PROVIDER"], {}).get("models", {})
print(f"mcp-sync: {len(models)} models -> {', '.join(models) or '(none)'}")
PYEOF
}

unset -f _mcp_sync_export_key _mcp_sync_refresh _mcp_sync_hermes _mcp_sync_skills 2>/dev/null || true