Skip to content

flurdy/agent-skills / skills/model-update-check

model-update-check

Verified

Audit Pi startup defaults, model routing, discovery and coordinated migration previews across second-opinion and pi-spend; read-only by default, with separately confirmed native catalog refresh.

tests/test-model-update-check.sh

View on GitHub
#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
HELPER="$(cd "$SCRIPT_DIR/.." && pwd)/scripts/model-update-check.sh"
ORIGINAL_PATH="$PATH"

fail() {
  printf 'FAIL: %s\n' "$*" >&2
  exit 1
}

TMP_DIR="$(mktemp -d)"
trap 'rm -rf "$TMP_DIR"' EXIT
mkdir -p "$TMP_DIR/bin" "$TMP_DIR/home/.pi/agent"
printf '{}\n' > "$TMP_DIR/home/.pi/agent/auth.json"
PI_LOG="$TMP_DIR/pi.log"
CURL_LOG="$TMP_DIR/curl.log"
BREW_LOG="$TMP_DIR/brew.log"
export PI_LOG CURL_LOG BREW_LOG

cat > "$TMP_DIR/bin/pi" <<'FAKE_PI'
#!/usr/bin/env bash
set -euo pipefail
printf '%s\n' "$*" >> "$PI_LOG"
case "$*" in
  --version)
    printf '%s\n' '0.80.0'
    ;;
  update\ --help)
    [[ "${REFRESH_CASE:-ok}" != unsupported ]] || exit 0
    printf '  --models  Refresh model catalogs only\n'
    ;;
  update\ --models)
    [[ "${REFRESH_CASE:-ok}" != fail ]] || exit 1
    if [[ "${REFRESH_CASE:-ok}" == timeout ]]; then sleep 5; fi
    printf 'native-output-must-not-be-exposed\n'
    ;;
  --offline\ --no-extensions\ --no-skills\ --no-prompt-templates\ --no-themes\ --no-context-files\ --no-approve\ --list-models)
    cat <<'MODELS'
provider      model                 context  max-out  thinking  images
anthropic     claude-sonnet-5       1M       128K     yes       yes
openai-codex  gpt-5.6-terra         372K     128K     yes       yes
google        gemini-3.5-flash      1M       64K      yes       yes
openrouter    qwen/current-reasoner 262K     32K      yes       no
openrouter    ~google/gemini-flash-latest 1M 64K      yes       yes
MODELS
    ;;
  *)
    printf 'unexpected fake pi arguments: %s\n' "$*" >&2
    exit 2
    ;;
esac
FAKE_PI
chmod +x "$TMP_DIR/bin/pi"

cat > "$TMP_DIR/bin/claude" <<'FAKE_CLAUDE'
#!/usr/bin/env bash
set -euo pipefail
[[ "$*" == --version ]] || exit 2
printf '2.1.499 (Claude Code)\n'
FAKE_CLAUDE
chmod +x "$TMP_DIR/bin/claude"

cat > "$TMP_DIR/bin/brew" <<'FAKE_BREW'
#!/usr/bin/env bash
set -euo pipefail
printf '%s\n' "$*" >> "$BREW_LOG"
[[ "$1" == "info" && "$2" == "--json=v2" && "$3" == "pi-coding-agent" ]] || exit 2
outdated="${BREW_OUTDATED:-true}"
cat <<JSON
{"formulae":[{"name":"pi-coding-agent","installed":[{"version":"0.80.0"}],"versions":{"stable":"0.81.0"},"outdated":$outdated}]}
JSON
FAKE_BREW
chmod +x "$TMP_DIR/bin/brew"

cat > "$TMP_DIR/models-dev.json" <<'JSON'
{
  "anthropic": {
    "models": {
      "claude-sonnet-5": {
        "name": "Claude Sonnet 5",
        "release_date": "2026-06-29",
        "reasoning": true,
        "limit": {"context": 1000000, "output": 128000}
      }
    }
  },
  "openai": {
    "models": {
      "gpt-5.6-terra": {
        "name": "GPT-5.6 Terra",
        "release_date": "2026-07-09",
        "reasoning": true,
        "limit": {"context": 372000, "output": 128000}
      }
    }
  },
  "google": {
    "models": {
      "gemini-3.5-flash": {
        "name": "Gemini 3.5 Flash",
        "release_date": "2026-05-19",
        "reasoning": true
      }
    }
  },
  "openrouter": {
    "models": {
      "qwen/new-reasoner": {
        "name": "New Qwen Reasoner",
        "release_date": "2026-07-01",
        "reasoning": true
      },
      "x-ai/current-critic": {
        "name": "Current xAI Critic",
        "release_date": "2026-07-08",
        "reasoning": true
      },
      "~google/gemini-flash-latest": {
        "name": "Gemini Flash Latest",
        "release_date": "2026-04-27",
        "reasoning": true
      }
    }
  }
}
JSON

cat > "$TMP_DIR/bin/curl" <<'FAKE_CURL'
#!/usr/bin/env bash
set -euo pipefail
url=""
output=""
original_args="$*"
while [[ $# -gt 0 ]]; do
  case "$1" in
    --output)
      output="$2"
      shift 2
      ;;
    http://*|https://*)
      url="$1"
      shift
      ;;
    *)
      shift
      ;;
  esac
done
[[ -n "$url" && -n "$output" ]] || exit 2
printf '%s\n' "$url" >> "$CURL_LOG"
case "$url" in
  https://fixture.test/models)
    cp "$FIXTURE_MODELS_DEV" "$output"
    ;;
  https://fixture.test/openrouter)
    cat > "$output" <<'JSON'
{"data":[
  {"id":"qwen/current-reasoner","name":"Current Qwen","created":1,"context_length":262144,"pricing":{},"expiration_date":null},
  {"id":"x-ai/current-critic","name":"Current xAI","created":2,"context_length":500000,"pricing":{},"expiration_date":null},
  {"id":"~google/gemini-flash-latest","name":"Gemini Flash Latest","created":3,"context_length":1048576,"pricing":{},"expiration_date":null}
]}
JSON
    ;;
  https://code.claude.com/docs/en/model-config.md)
    [[ "$original_args" != *--location* && "$original_args" == *"--proto =https"* ]] || exit 2
    cat > "$output" <<'DOCS'
### Model aliases
| Model alias | Behavior |
| **`sonnet`** | Latest Sonnet |
| **`opus`** | Latest Opus |
| **`fable`** | Provider-dependent Fable |
### Versions
Sonnet 9.1 requires Claude Code v2.1.500 or later.
DOCS
    ;;
  https://fixture.test/pi-latest)
    printf '%s\n' '{"version":"0.81.0"}' > "$output"
    ;;
  *)
    exit 22
    ;;
esac
FAKE_CURL
chmod +x "$TMP_DIR/bin/curl"

ROUTER_CONFIG="$TMP_DIR/router.json"
cat > "$ROUTER_CONFIG" <<'JSON'
{
  "enabled": true,
  "tiers": {
    "standard": {
      "candidates": [
        {"model": "openai-codex/gpt-5.6-terra", "metered": false},
        {"model": "anthropic/claude-sonnet-5", "metered": true}
      ]
    },
    "economy": {
      "candidates": [
        {"model": "google/gemini-3.5-flash", "metered": true},
        {"model": "openrouter/~google/gemini-flash-latest", "metered": true},
        {"model": "custom/example-model", "metered": true}
      ]
    }
  }
}
JSON

CONSENSUS_CONFIG="$TMP_DIR/consensus.json"
cat > "$CONSENSUS_CONFIG" <<'JSON'
{
  "version": 1,
  "profiles": {
    "test": {
      "models": [
        {"model": "openrouter/qwen/current-reasoner", "vendor": "Qwen", "role": "reasoning"},
        {"model": "openrouter/x-ai/current-critic", "vendor": "xAI", "role": "critique"}
      ],
      "limits": {
        "maxParallel": 2,
        "maxPromptBytes": 1024,
        "maxOutputTokensPerModel": 100,
        "defaultTimeoutSeconds": 5
      }
    },
    "hybrid": {
      "quorum": 3,
      "consensusQuorum": 4,
      "routes": [
        {"id": "claude", "kind": "local", "agent": "claude", "model": "fable", "effort": "high", "role": "reasoning"},
        {"id": "codex", "kind": "local", "agent": "codex", "role": "critique"},
        {"id": "qwen", "kind": "openrouter", "model": "openrouter/qwen/current-reasoner", "vendor": "Qwen", "role": "verification", "effort": "high"},
        {"id": "xai", "kind": "openrouter", "model": "openrouter/x-ai/current-critic", "vendor": "xAI", "role": "adversarial review", "maxOutputTokens": 2000},
        {"id": "gemini", "kind": "openrouter", "model": "openrouter/~google/gemini-flash-latest", "vendor": "Google", "role": "fast critique"}
      ],
      "limits": {
        "maxParallel": 4,
        "maxPromptBytes": 65536,
        "maxOutputTokensPerModel": 16000,
        "defaultTimeoutSeconds": 1800
      }
    }
  }
}
JSON

export FIXTURE_MODELS_DEV="$TMP_DIR/models-dev.json"
RUN_ENV=(env \
  "PATH=$TMP_DIR/bin:$ORIGINAL_PATH" \
  "HOME=$TMP_DIR/home" \
  "PI_CODING_AGENT_DIR=$TMP_DIR/home/.pi/agent" \
  "MODELS_DEV_URL=https://fixture.test/models" \
  "OPENROUTER_MODELS_URL=https://fixture.test/openrouter" \
  "PI_PACKAGE_URL=https://fixture.test/pi-latest")
COMMON_ARGS=(--router-config "$ROUTER_CONFIG" --consensus-config "$CONSENSUS_CONFIG")

result_json="$("${RUN_ENV[@]}" "$HELPER" "${COMMON_ARGS[@]}")"
jq -e '
  .mode == "hybrid" and
  .readOnly == true and
  .schemaVersion == 2 and
  .refresh.status == "not-requested" and
  any(.configurationInventory[];
    .identity == "local/claude/fable" and .resolution == "native-alias") and
  any(.configurationInventory[];
    .identity == "openai-codex/gpt-5.6-terra" and .metered == false) and
  .sources.routerConfig.status == "ok" and
  .sources.consensusConfig.status == "ok" and
  .sources.piCatalog.installedVersion == "0.80.0" and
  .sources.piRelease.latestVersion == "0.81.0" and
  .sources.homebrew.status == "ok" and
  .sources.homebrew.formula == "pi-coding-agent" and
  .sources.homebrew.installedVersion == "0.80.0" and
  .sources.homebrew.latestVersion == "0.81.0" and
  .piUpdateAvailable == true and
  .piNpmUpdateAvailable == true and
  .piHomebrewUpdateAvailable == true and
  (.configuredModels | length == 10) and
  any(.configuredModels[];
    .model == "openai-codex/gpt-5.6-terra" and
    .catalogProvider == "openai" and .piAvailable == true and .liveFound == true) and
  any(.configuredModels[];
    .model == "custom/example-model" and
    .catalogProviderFound == false and .liveFound == null) and
  any(.configuredModels[];
    .model == "openrouter/x-ai/current-critic" and
    .piAvailable == false and .liveFound == true and .openRouterFound == true) and
  any(.configuredModels[];
    .usage == "test" and .model == "openrouter/qwen/current-reasoner" and
    .piAvailable == true and .liveFound == false) and
  any(.configuredModels[];
    .usage == "hybrid" and .model == "openrouter/x-ai/current-critic" and
    .piAvailable == false and .liveFound == true and .openRouterFound == true) and
  any(.configuredModels[];
    .model == "openrouter/~google/gemini-flash-latest" and
    .catalogModel == "~google/gemini-flash-latest" and
    .piAvailable == true and .liveFound == true and .openRouterFound == true) and
  all(.configuredModels[]; .model != "fable" and .model != "native-default") and
  any(.findings[]; .kind == "pi-update" and .manager == "homebrew") and
  any(.findings[]; .kind == "pi-unavailable" and .model == "openrouter/x-ai/current-critic") and
  any(.findings[]; .kind == "live-missing" and .model == "openrouter/qwen/current-reasoner") and
  all(.findings[]; .kind != "openrouter-missing" or .model != "openrouter/~google/gemini-flash-latest") and
  (.recentOpenRouterByNamespace.qwen[0].model == "qwen/new-reasoner") and
  (.recentOpenRouterByNamespace | has("google") and (has("~google") | not))
' <<< "$result_json" >/dev/null || fail "hybrid audit output was incorrect"
[[ "$(wc -l < "$CURL_LOG" | tr -d '[:space:]')" -eq 4 ]] || \
  fail "hybrid mode did not make exactly four public metadata requests"
jq -e '.sources.claudeClient.installedVersion == "2.1.499" and
  .sources.claudeDocumentation.status == "ok" and
  .claudeDiscovery.inventory.status == "not-assessed" and
  .claudeDiscovery.accountAccess == "unverified"' <<< "$result_json" >/dev/null || fail "Claude fallback provenance"
grep -Fqx -- 'info --json=v2 pi-coding-agent' "$BREW_LOG" || \
  fail "hybrid mode did not inspect Homebrew formula metadata"

result_json="$(BREW_OUTDATED=false "${RUN_ENV[@]}" "$HELPER" "${COMMON_ARGS[@]}")"
jq -e '
  .sources.homebrew.status == "ok" and
  .piNpmUpdateAvailable == true and
  .piHomebrewUpdateAvailable == false and
  .piUpdateAvailable == false and
  any(.findings[]; .kind == "pi-npm-ahead-of-homebrew") and
  all(.findings[]; .kind != "pi-update")
' <<< "$result_json" >/dev/null || fail "Homebrew state did not override npm update availability"

CLAUDE_ROUTER="$TMP_DIR/claude-router.json"
printf '%s\n' '{"tiers":{"standard":{"candidates":[
  {"model":"pi-claude-cli/claude-sonnet-9-1","metered":false},
  {"model":"pi-claude-cli/opus","metered":false},
  {"model":"pi-claude-cli/claude-haiku-9-1","metered":false,"enabled":false}
]}}}' > "$CLAUDE_ROUTER"
result_json="$(BREW_OUTDATED=false "${RUN_ENV[@]}" "$HELPER" --router-config "$CLAUDE_ROUTER" --consensus-config "$CONSENSUS_CONFIG")"
jq -e '
  all(.configuredModels[] | select(.model | startswith("pi-claude-cli/"));
    .piAvailable == null and .piAssessment == "out-of-scope") and
  all(.findings[]; ((.kind != "pi-unavailable" and .kind != "live-missing") or (.model | startswith("pi-claude-cli/") | not))) and
  any(.configurationInventory[]; .identity == "pi-claude-cli/claude-sonnet-9-1" and
    .claudeClientSupport.status == "minimum-version-mismatch") and
  any(.configurationInventory[]; .identity == "pi-claude-cli/opus" and .resolution == "native-alias") and
  any(.configurationInventory[]; .identity == "pi-claude-cli/claude-haiku-9-1" and .enabled == false) and
  .verdict == "INCOMPLETE EVIDENCE"
' <<< "$result_json" >/dev/null || fail "extension-disabled catalogue produced false model availability"

: > "$CURL_LOG"
: > "$PI_LOG"
result_json="$("${RUN_ENV[@]}" "$HELPER" --offline "${COMMON_ARGS[@]}")"
jq -e '
  .mode == "offline" and
  .sources.modelsDev.status == "skipped" and
  .sources.openRouter.status == "skipped" and
  .sources.piRelease.status == "skipped" and
  .sources.claudeDocumentation.status == "skipped" and
  .sources.homebrew.status == "ok" and
  .piUpdateAvailable == true and
  .piNpmUpdateAvailable == false and
  .piHomebrewUpdateAvailable == true and
  all(.configuredModels[]; .liveFound == null)
' <<< "$result_json" >/dev/null || fail "offline audit output was incorrect"
[[ ! -s "$CURL_LOG" ]] || fail "offline mode invoked curl"
grep -Fqx -- '--offline --no-extensions --no-skills --no-prompt-templates --no-themes --no-context-files --no-approve --list-models' "$PI_LOG" || \
  fail "offline mode did not constrain Pi startup"

# Startup sources are optional by default, explicit missing paths are actionable.
jq -e '.sources.launcherConfig.status == "missing" and .sources.piSettings.status == "missing" and
  .startupAudit.auditOnly and .startupAudit.findings == []' <<< "$result_json" >/dev/null || fail "missing optional startup sources"
LAUNCHER="$TMP_DIR/home/.pi/agent/pl-launcher.json"
SETTINGS="$TMP_DIR/home/.pi/agent/settings.json"
printf '%s\n' '{"defaultModel":"openai-codex/example-old","defaultThinking":"high"}' > "$LAUNCHER"
printf '%s\n' '{"defaultProvider":"openai-codex","defaultModel":"example-other","enabledModels":["openai-codex/gpt-5.6-terra"],"apiKey":"PRIVATE_SENTINEL"}' > "$SETTINGS"
startup_before="$(sha256sum "$LAUNCHER" "$SETTINGS")"
result_json="$("${RUN_ENV[@]}" "$HELPER" --offline "${COMMON_ARGS[@]}")"
jq -e --arg launcher "$LAUNCHER" --arg settings "$SETTINGS" '
  .sources.launcherConfig.path == $launcher and .sources.piSettings.path == $settings and
  .sources.launcherConfig.status == "ok" and .sources.piSettings.status == "ok" and
  (.sources.piSettings.sha256 | length == 64) and
  .startupAudit.defaults.piSettings.enabledModelsMembership == "not-exactly-listed" and
  any(.startupAudit.findings[]; .code == "startup-defaults-disagree") and
  .interaction.primaryAction.kind == "inspect-startup" and
  ([.startupAudit.launchPaths[].path] == ["fresh-pl","plain-pi","resume-continue","orca"]) and
  .recommendations == []' <<< "$result_json" >/dev/null || fail "startup defaults or provenance"
[[ "$result_json" != *PRIVATE_SENTINEL* ]] || fail "unrelated settings leaked"
[[ "$startup_before" == "$(sha256sum "$LAUNCHER" "$SETTINGS")" ]] || fail "startup sources mutated"
# The Pi agent-dir override affects settings, not the launcher's fixed home path.
mkdir -p "$TMP_DIR/alternate-agent"
printf '{}\n' > "$TMP_DIR/alternate-agent/settings.json"
result_json="$("${RUN_ENV[@]}" PI_CODING_AGENT_DIR="$TMP_DIR/alternate-agent" "$HELPER" --offline "${COMMON_ARGS[@]}")"
jq -e --arg path "$TMP_DIR/alternate-agent/settings.json" --arg launcher "$LAUNCHER" '
  .sources.piSettings.path == $path and .sources.launcherConfig.path == $launcher
' <<< "$result_json" >/dev/null || fail "agent-dir settings path"
result_json="$("${RUN_ENV[@]}" "$HELPER" --offline "${COMMON_ARGS[@]}" --launcher-config "$TMP_DIR/missing-launcher" --settings-config "$TMP_DIR/missing-settings")"
jq -e '.sources.launcherConfig.required and .sources.piSettings.required and
  .sources.piSettings.status == "missing" and
  ([.startupAudit.findings[] | select(.code == "startup-config-unavailable")] | length == 2)
' <<< "$result_json" >/dev/null || fail "explicit missing startup sources"
rm -f "$LAUNCHER" "$SETTINGS"

INVALID_ROUTER="$TMP_DIR/invalid-router.json"
printf '%s\n' '{}' > "$INVALID_ROUTER"
result_json="$("${RUN_ENV[@]}" "$HELPER" --offline \
  --router-config "$INVALID_ROUTER" --consensus-config "$CONSENSUS_CONFIG")"
jq -e '
  .sources.routerConfig.status == "invalid" and
  (.configuredModels | length == 5) and
  any(.findings[]; .kind == "config" and .source == "model-tier-router")
' <<< "$result_json" >/dev/null || fail "invalid config did not degrade independently"

assert_invalid_panel() {
  local fixture="$1"
  local label="$2"
  result_json="$("${RUN_ENV[@]}" "$HELPER" --offline \
    --router-config "$ROUTER_CONFIG" --consensus-config "$fixture")"
  jq -e . <<< "$result_json" >/dev/null || fail "$label did not emit JSON"
  jq -e '
    .sources.routerConfig.status == "ok" and
    .sources.consensusConfig.status == "invalid" and
    (.configuredModels | length == 5) and
    any(.findings[]; .kind == "config" and .source == "second-opinion" and
      .message == "panel config is invalid")
  ' <<< "$result_json" >/dev/null || fail "$label did not degrade independently"
}

for budget in 64000 128000; do
  LARGE_OUTPUT_PANEL="$TMP_DIR/large-output-$budget.json"
  jq --argjson budget "$budget" '
    .profiles.test.limits.maxOutputTokensPerModel = $budget |
    .profiles.test.models[0].maxOutputTokens = $budget |
    .profiles.hybrid.limits.maxOutputTokensPerModel = $budget |
    .profiles.hybrid.routes[3].maxOutputTokens = $budget
  ' "$CONSENSUS_CONFIG" > "$LARGE_OUTPUT_PANEL"
  result_json="$("${RUN_ENV[@]}" "$HELPER" --offline \
    --router-config "$ROUTER_CONFIG" --consensus-config "$LARGE_OUTPUT_PANEL")"
  jq -e '
    .sources.consensusConfig.status == "ok" and
    (.configuredModels | length == 10) and
    all(.findings[]; .kind != "config" or .source != "second-opinion")
  ' <<< "$result_json" >/dev/null || fail "supported $budget output budget was rejected"
done

for budget in '128001' '0' '-1' '1.5' '"64000"' 'null' 'true'; do
  INVALID_OUTPUT_PANEL="$TMP_DIR/invalid-output.json"
  jq --argjson budget "$budget" '.profiles.hybrid.limits.maxOutputTokensPerModel = $budget' \
    "$CONSENSUS_CONFIG" > "$INVALID_OUTPUT_PANEL"
  assert_invalid_panel "$INVALID_OUTPUT_PANEL" "invalid profile output budget $budget"
done

INVALID_ROUTE_OUTPUT="$TMP_DIR/invalid-route-output.json"
jq '.profiles.hybrid.limits.maxOutputTokensPerModel = 64000 |
  .profiles.hybrid.routes[3].maxOutputTokens = 64001' \
  "$CONSENSUS_CONFIG" > "$INVALID_ROUTE_OUTPUT"
assert_invalid_panel "$INVALID_ROUTE_OUTPUT" "route output budget above profile ceiling"
jq '.profiles.test.limits.maxOutputTokensPerModel = 64000 |
  .profiles.test.models[0].maxOutputTokens = 64001' \
  "$CONSENSUS_CONFIG" > "$INVALID_ROUTE_OUTPUT"
assert_invalid_panel "$INVALID_ROUTE_OUTPUT" "legacy model output budget above profile ceiling"

INVALID_PANEL="$TMP_DIR/invalid-panel.json"
jq '.profiles.hybrid.models = .profiles.test.models' "$CONSENSUS_CONFIG" > "$INVALID_PANEL"
assert_invalid_panel "$INVALID_PANEL" "models/routes conflict"

INVALID_GEMINI_EFFORT="$TMP_DIR/invalid-gemini-effort.json"
jq '.profiles.hybrid.routes[1] = {id:"gemini",kind:"local",agent:"gemini",effort:"high",role:"critique"}' \
  "$CONSENSUS_CONFIG" > "$INVALID_GEMINI_EFFORT"
assert_invalid_panel "$INVALID_GEMINI_EFFORT" "unsupported Gemini effort"

INVALID_OPENROUTER_ID="$TMP_DIR/invalid-openrouter-id.json"
jq '.profiles.hybrid.routes[2].model = "openrouter//missing-provider"' \
  "$CONSENSUS_CONFIG" > "$INVALID_OPENROUTER_ID"
assert_invalid_panel "$INVALID_OPENROUTER_ID" "malformed OpenRouter identity"

INVALID_QUORUM="$TMP_DIR/invalid-quorum.json"
jq '.profiles.hybrid.quorum = 6' "$CONSENSUS_CONFIG" > "$INVALID_QUORUM"
assert_invalid_panel "$INVALID_QUORUM" "quorum above enabled route count"

# Conversational evidence uses the same validation/digests without a user-authored file.
result_json="$(printf '%s' '{"schemaVersion":1,"recommendations":[]}' | "${RUN_ENV[@]}" "$HELPER" --offline "${COMMON_ARGS[@]}" --evidence -)"
jq -e '.sources.releaseEvidence.status == "ok" and .sources.releaseEvidence.path == "stdin" and
  (.sources.releaseEvidence.sha256 | length == 64) and .readOnly' <<< "$result_json" >/dev/null || fail "stdin evidence contract"
result_json="$(printf '%s' '{}' | "${RUN_ENV[@]}" "$HELPER" --offline "${COMMON_ARGS[@]}" --evidence -)"
jq -e '.sources.releaseEvidence.status == "invalid-or-stale" and .recommendations == []' <<< "$result_json" >/dev/null || fail "stdin bypassed validation"
if python3 -c 'print("x" * 65537)' | "${RUN_ENV[@]}" "$HELPER" --offline "${COMMON_ARGS[@]}" --evidence - >/dev/null 2>&1; then
  fail "oversized stdin evidence accepted"
fi

# Refresh never runs implicitly, and failed authorization never reaches native update.
if grep -q '^update ' "$PI_LOG"; then fail "default/offline invoked update"; fi
for extra in denied conflict unsupported fail timeout ok; do
  : > "$PI_LOG"
  refresh_flags=(--refresh-models --confirm-refresh)
  expected="$extra"
  case "$extra" in
    denied) refresh_flags=(--refresh-models); expected=authorization-required ;;
    conflict) refresh_flags+=(--offline); expected=offline-conflict ;;
    timeout) refresh_flags+=(--refresh-timeout 1) ;;
    fail) expected=failed ;;
  esac
  result_json="$(REFRESH_CASE="$extra" "${RUN_ENV[@]}" "$HELPER" "${COMMON_ARGS[@]}" "${refresh_flags[@]}")"
  jq -e --arg expected "$expected" '.refresh.status == $expected' <<< "$result_json" >/dev/null || fail "wrong refresh status: $extra"
  if [[ "$extra" == ok ]]; then
    jq -e '.refresh.fresh and (.readOnly == false)' <<< "$result_json" >/dev/null || fail "refresh success contract"
    # Reacquisition must follow the models-only update, never precede it.
    first="$(head -n 1 "$PI_LOG")"
    [[ "$first" == 'update --help' ]] || fail "refresh capability probe was not first"
    [[ "$(grep -n '^update --models$' "$PI_LOG" | cut -d: -f1)" -lt "$(grep -n '^--offline ' "$PI_LOG" | cut -d: -f1)" ]] || fail "catalog was not reacquired"
  else
    jq -e '.refresh.fresh == false and .verdict != "CURRENT"' <<< "$result_json" >/dev/null || fail "false freshness: $extra"
  fi
  if [[ "$extra" == denied || "$extra" == conflict ]]; then
    if grep -q '^update ' "$PI_LOG"; then fail "unauthorized native command"; fi
  fi
  if grep -Eq '^update($| --(self|all|extensions)$)' "$PI_LOG"; then fail "package update fallback"; fi
  [[ "$result_json" != *native-output-must-not-be-exposed* ]] || fail "native output leaked"
done

printf '%s\n' 'model-update-check tests passed'
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall