flurdy/agent-skills / skills/model-update-check
model-update-check
VerifiedAudit Pi startup defaults, model routing, discovery and coordinated migration previews across second-opinion and pi-spend; read-only by default, with separately confirmed native catalog refresh.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 21.6 KB |
| references/companion-apply.md | markdown | 5.8 KB |
| references/migration-preview.md | markdown | 10.9 KB |
| scripts/apply_migration.py | script | 25.6 KB |
| scripts/catalog_refresh.py | script | 5.8 KB |
| scripts/claude_discovery.py | script | 4.9 KB |
| scripts/migration_audit.py | script | 43.9 KB |
| scripts/model-update-check.shexecutable | script | 20.6 KB |
| scripts/startup_audit.py | script | 8.2 KB |
| tests/test-model-update-check.shexecutable | script | 21.8 KB |
| tests/test_apply_migration.py | script | 19.1 KB |
| tests/test_claude_discovery.py | script | 5.9 KB |
| tests/test_migration_audit.py | script | 39.6 KB |
| tests/test_startup_audit.py | script | 12.6 KB |
tests/test_migration_audit.py
View on GitHub"""Synthetic audit/preview and refresh contracts; no provider or credential access."""
import copy
import importlib.util
import json
import os
import sys
import tempfile
import unittest
from datetime import datetime, timezone
from pathlib import Path
from unittest.mock import patch
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
def load(name):
spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py")
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
spec.loader.exec_module(module)
return module
AUDIT = load("migration_audit")
REFRESH = load("catalog_refresh")
NOW = datetime(2026, 1, 10, tzinfo=timezone.utc)
OLD = "openai-codex/example-sol-1"
NEW = "openai-codex/example-sol-2"
class AuditTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.router = {"enabled": True, "tiers": {"standard": {"selection": "weighted-random", "thinking": "high", "candidates": [
{"model": OLD, "metered": True, "weight": 25, "enabled": False},
{"model": OLD, "weight": 75, "enabled": True}]}}, "modelPolicies": {OLD: {"metered": False, "consent": "ask"}}, "unrelated": 123}
self.panel = {"version": 1, "profiles": {"premium": {"quorum": 2, "consensusQuorum": 2, "routes": [
{"id": "codex", "kind": "local", "agent": "codex", "model": "example-sol-1", "role": "code"},
{"id": "claude", "kind": "local", "agent": "claude", "model": "opus", "role": "review"},
{"id": "native", "kind": "local", "agent": "gemini", "role": "critique"}]}},
"subscriptionRoutes": {"codex": ["example-sol-1"], "claude": ["opus"]}}
self.spend = {"schemaVersion": 1, "models": {OLD: [{"billing": "subscription", "effectiveFrom": "2026-01-01T00:00:00Z", "effectiveUntil": None}]}}
self.dev = {"openai": {"models": {"example-sol-1": {"release_date": "2025-01-01"}, "example-sol-2": {"release_date": "2025-02-01"}}}}
self.opened = {"openai/example-sol-2": {"id": "openai/example-sol-2", "created": 1}}
self.pi = {OLD, NEW}
self.paths = [self.root / name for name in ("router.json", "panel.json", "spend.json")]
self.base = {"sources": {name: {"status": "ok"} for name in ("routerConfig", "consensusConfig", "piCatalog", "modelsDev", "openRouter")},
"configuredModels": [{"model": OLD, "source": "model-tier-router", "metered": None}],
"findings": [], "piUpdateAvailable": False, "mode": "hybrid"}
for name, path in zip(("routerConfig", "consensusConfig"), self.paths, strict=False):
self.base["sources"][name]["path"] = str(path)
self.refresh = REFRESH.refresh(False, False, False)
def evidence(self, before=OLD, after=NEW):
return {"schemaVersion": 1, "recommendations": [{"from": before, "to": after, "checkedAt": "2026-01-09T00:00:00Z",
"citations": [{"url": "https://vendor.example/release", "quote": "Explicit successor suitable for this role."}],
"compatibility": {key: f"Reviewed {key} evidence" for key in AUDIT.COMPATIBILITY},
"routerPolicy": {"metered": False, "consent": "ask", "evidence": "Confirmed subscription routing"},
"billing": {"billing": "subscription", "effectiveFrom": "2026-01-11T00:00:00Z", "evidence": "Confirmed activation date"}}]}
def audit(self, evidence=None, claude_result=None):
for path, payload in zip(self.paths, (self.router, self.panel, self.spend), strict=True):
path.write_text(json.dumps(payload))
evidence_path = None
if evidence is not None:
evidence_path = self.root / "evidence.json"
evidence_path.write_text(json.dumps(evidence))
before = {path: path.read_bytes() for path in self.paths}
result = AUDIT.enrich(copy.deepcopy(self.base), *map(str, self.paths), evidence_path,
(self.dev, self.opened, self.pi), self.refresh, NOW, claude_result=claude_result)
self.assertEqual(before, {path: path.read_bytes() for path in self.paths})
return result
def test_inventory_preserves_disabled_weights_false_policy_and_native_aliases(self):
result = self.audit()
rows = result["configurationInventory"]
candidates = [r for r in rows if r["source"] == "routerConfig"]
self.assertEqual([r["weight"] for r in candidates], [25, 75])
self.assertEqual([r["enabled"] for r in candidates], [False, True])
self.assertTrue(all(r["metered"] is False for r in candidates))
self.assertIs(result["configuredModels"][0]["metered"], False)
aliases = [r for r in rows if r["resolution"] in {"native-alias", "native-default"}]
self.assertEqual(len(aliases), 3)
self.assertTrue(all(r["facts"] is None for r in aliases))
self.assertEqual(result["recommendations"], [])
def test_extension_scope_is_not_native_absence_or_public_unavailability(self):
self.dev["openai"]["models"].pop("example-sol-2")
self.opened = {}
self.pi = {OLD}
pins = ["claude-sonnet-9-1", "opus", "fable", "claude-haiku-9-1"]
self.router = {"tiers": {"standard": {"candidates": [
{"model": f"pi-claude-cli/{pin}", "enabled": pin != pins[-1], "metered": False}
for pin in pins]}}}
self.base["configuredModels"] = [
{"source": "model-tier-router", "model": f"pi-claude-cli/{pin}"} for pin in pins]
self.base["sources"]["piCatalog"]["extensionsEnabled"] = False
self.dev["anthropic"] = {"models": {pins[0]: {"release_date": "2025-01-01"}}}
self.pi.add(f"anthropic/{pins[0]}")
result = self.audit()
for row in result["configuredModels"]:
self.assertIsNone(row["piAvailable"])
self.assertEqual(row["piAssessment"], "out-of-scope")
self.assertEqual(row["runtimeAvailability"], "not-assessed")
configured = {r["model"]: r for r in result["configuredModels"]}
self.assertIs(configured[f"pi-claude-cli/{pins[0]}"]["liveFound"], True)
self.assertIsNone(configured["pi-claude-cli/opus"]["liveFound"])
extension_rows = [r for r in result["configurationInventory"] if r["source"] == "routerConfig"]
self.assertEqual([r["enabled"] for r in extension_rows], [True, True, True, False])
self.assertEqual(extension_rows[1]["resolution"], "native-alias")
self.assertFalse(any(f["kind"] in {"pi-unavailable", "live-missing"} for f in result["findings"]))
self.assertEqual(result["verdict"], "INCOMPLETE EVIDENCE")
self.assertTrue(any("extension" in reason.lower() for reason in result["incompleteReasons"]))
candidate = next(r for r in result["catalogCandidates"] if r["identity"] == f"pi-claude-cli/{pins[0]}")
self.assertIsNone(candidate["piAvailable"])
self.assertEqual(candidate["publicCatalogProvider"], "anthropic")
self.assertEqual(candidate["extensionCompatibility"], "unverified")
self.assertEqual(result["interaction"]["primaryAction"]["kind"], "inspect-availability")
def test_public_extension_successor_cannot_authorize_a_runtime_preview(self):
before, after = "pi-claude-cli/claude-sonnet-9-1", "pi-claude-cli/claude-sonnet-9-2"
self.router["tiers"]["standard"]["candidates"][1]["model"] = before
self.dev["anthropic"] = {"models": {
"claude-sonnet-9-1": {"release_date": "2025-01-01"},
"claude-sonnet-9-2": {"release_date": "2025-02-01"}}}
proposal = self.audit(self.evidence(before, after))["recommendations"][0]
self.assertEqual(proposal["status"], "incomplete")
self.assertEqual(proposal["changes"], [])
self.assertTrue(any("runtime route" in reason for reason in proposal["unresolved"]))
self.assertIsNone(proposal["availability"]["piAvailable"])
def test_failed_claude_metadata_is_not_missing_models_or_current(self):
# Keep only the current native model so an unrelated upgrade lead cannot hide the gap.
self.dev["openai"]["models"].pop("example-sol-2")
self.opened = {}
self.pi = {OLD}
metadata = {"client": {"status": "timeout", "installedVersion": None},
"documentation": {"status": "error", "aliases": [], "versionRequirements": []}}
result = self.audit(claude_result=metadata)
self.assertTrue({"claudeClient", "claudeDocumentation"} <= set(result["incompleteSources"]))
self.assertEqual(result["verdict"], "INCOMPLETE EVIDENCE")
self.assertFalse(any(f["kind"] == "live-missing" and f.get("model", "").startswith("local/claude") for f in result["findings"]))
def test_exact_claude_pin_is_not_current_from_public_or_minimum_version_evidence(self):
self.dev["openai"]["models"].pop("example-sol-2")
self.pi = {OLD}
self.opened = {}
self.dev["anthropic"] = {"models": {"claude-sonnet-9-1": {"release_date": "2025-01-01"}}}
self.panel["profiles"]["premium"]["routes"][1]["model"] = "claude-sonnet-9-1"
docs = {"status": "ok", "url": "https://code.claude.com/docs/en/model-config.md", "aliases": [],
"versionRequirements": [{"modelLabel": "Sonnet 9.1", "minimumVersion": "2.1.500", "quote": "Synthetic requirement"}]}
for version in ("2.1.500", "2.1.499"):
with self.subTest(version=version):
result = self.audit(claude_result={"client": {"status": "ok", "installedVersion": version}, "documentation": docs})
self.assertEqual(result["verdict"], "INCOMPLETE EVIDENCE")
self.assertTrue(any("exact-model acceptance" in reason for reason in result["incompleteReasons"]))
self.assertEqual(any(f["kind"] == "claude-client-version" for f in result["findings"]), version == "2.1.499")
def test_extension_only_metadata_failure_respects_enabled_and_router_gates(self):
self.panel = {"version": 1, "profiles": {}}
metadata = {"client": {"status": "timeout", "installedVersion": None},
"documentation": {"status": "error", "aliases": [], "versionRequirements": []}}
for candidate_enabled, router_enabled in ((True, True), (False, True), (True, False)):
with self.subTest(candidate_enabled=candidate_enabled, router_enabled=router_enabled):
self.router = {"enabled": router_enabled, "tiers": {"standard": {"candidates": [
{"model": "pi-claude-cli/claude-sonnet-9-1", "enabled": candidate_enabled, "metered": False}]}}}
self.base["configuredModels"] = [{"source": "model-tier-router", "model": "pi-claude-cli/claude-sonnet-9-1"}]
result = self.audit(claude_result=metadata)
failures = {"claudeClient", "claudeDocumentation"} & set(result["incompleteSources"])
self.assertEqual(bool(failures), candidate_enabled and router_enabled)
if not candidate_enabled or not router_enabled:
self.assertFalse(any("extension compatibility" in reason.lower() for reason in result["incompleteReasons"]))
self.router["enabled"] = False
metadata["client"] = {"status": "ok", "installedVersion": "2.1.499"}
metadata["documentation"] = {"status": "ok", "url": "https://code.claude.com/docs/en/model-config.md", "aliases": [],
"versionRequirements": [{"modelLabel": "Sonnet 9.1", "minimumVersion": "2.1.500", "quote": "Synthetic requirement"}]}
self.assertFalse(any(f["kind"] == "claude-client-version" for f in self.audit(claude_result=metadata)["findings"]))
def test_missing_native_model_and_failed_catalog_remain_distinct(self):
self.pi.remove(OLD)
result = self.audit()
self.assertIs(result["configuredModels"][0]["piAvailable"], False)
self.assertTrue(any(f["kind"] == "pi-unavailable" for f in result["findings"]))
self.base["sources"]["piCatalog"]["status"] = "invalid-output"
result = self.audit()
self.assertIsNone(result["configuredModels"][0]["piAvailable"])
self.assertEqual(result["configuredModels"][0]["piAssessment"], "unknown")
self.assertFalse(any(f["kind"] == "pi-unavailable" for f in result["findings"]))
self.assertIn("piCatalog", result["incompleteSources"])
def test_complete_catalog_not_top_eight_and_no_release_name_upgrade(self):
for n in range(12):
self.dev["openai"]["models"][f"unrelated-{n}"] = {"release_date": "2026-01-10"}
self.opened["openai/example-sol-2-pro"] = {"id": "openai/example-sol-2-pro"}
self.opened["openai/example-sol-2-batch"] = {"id": "openai/example-sol-2-batch"}
result = self.audit()
self.assertIn(NEW, {r["identity"] for r in result["catalogCandidates"]})
self.assertTrue(all(r["status"] == "discovered-not-successor" for r in result["catalogCandidates"]))
self.assertEqual(result["recommendations"], [])
self.assertEqual(result["verdict"], "INCOMPLETE EVIDENCE")
leads = result["discoveryLeads"]
self.assertIn(NEW, {item["to"] for item in leads})
self.assertFalse(any(item["to"].endswith(("-pro", "-batch")) for item in leads))
def test_offline_undated_models_are_not_upgrade_opportunities(self):
self.base["mode"] = "offline"
self.base["sources"]["modelsDev"]["status"] = "skipped"
self.base["sources"]["openRouter"]["status"] = "skipped"
self.dev, self.opened = {}, {}
self.router["tiers"]["standard"]["candidates"][1]["model"] = NEW
self.router["modelPolicies"][NEW] = {"metered": False, "consent": "ask"}
self.panel["profiles"]["premium"]["routes"][0]["model"] = "example-sol-2"
self.panel["subscriptionRoutes"]["codex"].append("example-sol-2")
result = self.audit()
self.assertEqual(result["discoveryLeads"], [])
self.assertEqual(result["interaction"]["opportunities"], [])
self.assertEqual(result["interaction"]["primaryAction"]["kind"], "inspect-availability")
self.assertEqual(result["verdict"], "INCOMPLETE EVIDENCE")
self.assertEqual(result["incompleteSources"], ["modelsDev", "openRouter"])
self.assertTrue({OLD, NEW} <= {item["identity"] for item in result["catalogCandidates"]})
self.assertTrue(result["discoveryUncertainties"])
self.assertEqual(result["recommendations"], [])
def test_missing_release_dates_stay_uncertain_without_a_prompt_even_online(self):
self.opened = {}
for current, candidate in ((None, None), ("2025-01-01", None), (None, "2025-02-01")):
with self.subTest(current=current, candidate=candidate):
self.dev["openai"]["models"] = {
"example-sol-1": {"release_date": current},
"example-sol-2": {"release_date": candidate},
}
result = self.audit()
self.assertEqual(result["discoveryLeads"], [])
self.assertEqual(result["interaction"]["opportunities"], [])
self.assertEqual(result["interaction"]["primaryAction"]["kind"], "none")
self.assertFalse(result["interaction"]["primaryAction"]["requiresReply"])
self.assertEqual(result["verdict"], "INCOMPLETE EVIDENCE")
self.assertTrue(result["discoveryUncertainties"])
self.assertTrue(any("release" in reason.casefold() for reason in result["incompleteReasons"]))
def test_release_order_requires_provably_later_calendar_metadata(self):
self.opened = {}
cases = (
("2025-02-01", "2025-01-01", False, False),
("2025-02-01", "2025-02-01", False, False),
("2025-01-01", "2025-02-01", True, False),
("2025-03", "2025-03-15", False, True),
("2025-03-15", "2025-03", False, True),
("2025", "2025-12-31", False, True),
("2025-02", "2025-03-01", True, False),
("2024", "2025-01-01", True, False),
("2025-01-01", "2025-13-01", False, True),
("2025-02-30", "2025-03-01", False, True),
("2025-01-01", "0000", False, True),
)
for current, candidate, newer, uncertain in cases:
with self.subTest(current=current, candidate=candidate):
self.dev["openai"]["models"] = {
"example-sol-1": {"release_date": current},
"example-sol-2": {"release_date": candidate},
}
result = self.audit()
self.assertEqual(bool(result["discoveryLeads"]), newer)
self.assertEqual(bool(result["discoveryUncertainties"]), uncertain)
self.assertEqual(result["interaction"]["primaryAction"]["kind"], "review-upgrade" if newer else "none")
self.assertEqual(result["verdict"], "INCOMPLETE EVIDENCE" if newer or uncertain else "CURRENT")
if newer:
self.assertTrue(all(item["currentRelease"] == current and item["candidateRelease"] == candidate
for item in result["discoveryLeads"]))
self.assertEqual(result["recommendations"], [])
def test_dated_catalog_fallback_still_produces_a_research_lead(self):
self.dev["openai"]["models"]["example-sol-2"] = {}
self.opened["openai/example-sol-2"]["created"] = datetime(2025, 2, 1, tzinfo=timezone.utc).timestamp()
result = self.audit()
self.assertEqual(result["interaction"]["primaryAction"]["kind"], "review-upgrade")
self.assertIn(NEW, {item["to"] for item in result["discoveryLeads"]})
self.assertEqual(result["recommendations"], [])
self.assertEqual(result["discoveryUncertainties"], [])
def test_invalid_listing_timestamp_is_not_release_evidence(self):
for created in (True, False, float("inf"), float("nan"), -1, "2025-02-01"):
with self.subTest(created=created):
self.assertIsNone(AUDIT.release_date({"openRouter": {"created": created}}))
def test_reviewed_evidence_remains_available_without_catalog_release_dates(self):
self.dev["openai"]["models"] = {"example-sol-1": {}, "example-sol-2": {}}
self.opened = {}
result = self.audit(self.evidence())
self.assertEqual(result["discoveryLeads"], [])
self.assertEqual(result["interaction"]["primaryAction"]["kind"], "review-preview")
self.assertEqual(result["recommendations"][0]["status"], "optional-upgrade")
self.assertNotIn((OLD, NEW), {(item["from"], item["to"]) for item in result["discoveryUncertainties"]})
def test_source_failure_retains_other_candidates_without_false_current(self):
self.dev = {}
self.base["sources"]["modelsDev"]["status"] = "error"
result = self.audit()
self.assertEqual(result["verdict"], "INCOMPLETE EVIDENCE")
item = next(r for r in result["catalogCandidates"] if r["identity"] == NEW)
self.assertIs(item["piAvailable"], True)
self.assertIsNone(item["liveFound"])
self.assertIsNotNone(item["crossRouteDiscoveryOnly"])
def test_cross_route_listing_does_not_prove_pi_or_cli_availability(self):
self.pi.remove(NEW)
result = self.audit(self.evidence())
proposal = result["recommendations"][0]
self.assertEqual(proposal["status"], "incomplete")
self.assertEqual(proposal["changes"], [])
row = next(r for r in result["configurationInventory"] if r["identity"] == "local/codex/example-sol-1")
self.assertIsNone(row["facts"]["piAvailable"])
self.assertEqual(row["facts"]["runtimeAvailability"], "unknown")
def test_preview_exact_paths_keeps_history_and_never_copies_consent(self):
result = self.audit(self.evidence())
proposal = result["recommendations"][0]
self.assertEqual(proposal["status"], "optional-upgrade")
changes = proposal["changes"]
self.assertEqual(len(changes), 4)
self.assertEqual(changes[0]["path"], "/tiers/standard/candidates/0/model")
self.assertEqual(changes[1]["path"], "/tiers/standard/candidates/1/model")
self.assertTrue(all(len(c["sourceSha256"]) == 64 for c in changes))
policy = next(c for c in changes if c["source"] == "billingPolicy")
self.assertEqual(policy["operation"], "add")
self.assertEqual(policy["path"], "/models/openai-codex~1example-sol-2")
self.assertTrue(any(c["consentSensitive"] for c in changes))
self.assertTrue(result["handoff"]["companionAvailable"])
self.assertEqual(result["handoff"]["companion"], "scripts/apply_migration.py")
def test_missing_billing_and_policy_evidence_remains_unknown(self):
evidence = self.evidence()
del evidence["recommendations"][0]["billing"]
del evidence["recommendations"][0]["routerPolicy"]
proposal = self.audit(evidence)["recommendations"][0]
self.assertEqual(proposal["status"], "incomplete")
self.assertEqual(len(proposal["changes"]), 2)
self.assertEqual(len(proposal["unresolved"]), 2)
def test_local_pin_and_allowlist_need_native_evidence(self):
evidence = self.evidence("local/codex/example-sol-1", "local/codex/example-sol-2")
self.assertEqual(self.audit(evidence)["recommendations"][0]["changes"], [])
evidence["recommendations"][0]["nativeAvailability"] = "Verified exact CLI model through native catalog documentation"
proposal = self.audit(evidence)["recommendations"][0]
self.assertEqual(proposal["status"], "optional-upgrade")
self.assertEqual(len(proposal["changes"]), 2)
allow = next(c for c in proposal["changes"] if c["consentSensitive"])
self.assertEqual(allow["after"], ["example-sol-1", "example-sol-2"])
self.assertEqual(allow["path"], "/subscriptionRoutes/codex")
def test_aliases_are_not_replaced_even_with_evidence(self):
evidence = self.evidence("local/claude/opus", "local/claude/new-pin")
evidence["recommendations"][0]["nativeAvailability"] = "verified"
result = self.audit(evidence)
self.assertEqual(result["recommendations"][0]["changes"], [])
self.assertIn("Native aliases", " ".join(result["recommendations"][0]["unresolved"]))
def test_spend_missing_invalid_uncovered_and_overlap_fail_closed(self):
self.spend = {"schemaVersion": 1, "models": {OLD: [{"billing": "subscription", "effectiveFrom": "2026-01-20T00:00:00Z", "effectiveUntil": None}]}}
result = self.audit()
self.assertTrue(any(f["kind"] == "spend-uncovered" for f in result["findings"]))
self.spend["models"][OLD].append(self.spend["models"][OLD][0])
result = self.audit(self.evidence())
self.assertEqual(result["sources"]["billingPolicy"]["status"], "partial")
self.assertTrue(any("Repair" in u for u in result["recommendations"][0]["unresolved"]))
self.spend = {}
self.assertEqual(self.audit()["sources"]["billingPolicy"]["status"], "invalid")
payload, metadata = AUDIT.snapshot(self.root / "does-not-exist")
self.assertEqual(metadata["status"], "missing")
self.assertEqual(payload, {})
def test_new_interval_never_changes_existing_intervals(self):
self.spend["models"][NEW] = [{"billing": "metered", "effectiveFrom": "2025-12-01T00:00:00Z", "effectiveUntil": "2026-01-01T00:00:00Z"}]
proposal = self.audit(self.evidence())["recommendations"][0]
change = next(c for c in proposal["changes"] if c["source"] == "billingPolicy")
self.assertEqual(change["after"][:-1], change["before"])
self.spend["models"][NEW][0]["effectiveUntil"] = None
proposal = self.audit(self.evidence())["recommendations"][0]
self.assertFalse(any(c["source"] == "billingPolicy" for c in proposal["changes"]))
self.assertTrue(any("overlap" in message for message in proposal["unresolved"]))
def test_preview_refuses_historical_backfill_even_with_billing_claim(self):
evidence = self.evidence()
evidence["recommendations"][0]["billing"]["effectiveFrom"] = "2026-01-01T00:00:00Z"
proposal = self.audit(evidence)["recommendations"][0]
self.assertEqual(proposal["status"], "incomplete")
self.assertFalse(any(c["source"] == "billingPolicy" for c in proposal["changes"]))
self.assertTrue(any("past" in msg for msg in proposal["unresolved"]))
def test_duplicate_interval_proposes_no_duplicate(self):
evidence = self.evidence()
interval = evidence["recommendations"][0]["billing"]
self.spend["models"][NEW] = [{"billing": interval["billing"], "effectiveFrom": interval["effectiveFrom"], "effectiveUntil": None}]
result = self.audit(evidence)
self.assertFalse(any(c["source"] == "billingPolicy" for c in result["recommendations"][0]["changes"]))
def test_evidence_stale_malformed_oversized_and_cross_route_rejected(self):
for field, value in (("checkedAt", "2025-01-01T00:00:00Z"), ("checkedAt", "2027-01-01T00:00:00Z"),
("to", "local/codex/example-sol-2"), ("to", []), ("compatibility", {}), ("citations", []),
("billing", None), ("routerPolicy", None), ("nativeAvailability", None)):
with self.subTest(field=field, value=value):
evidence = self.evidence()
evidence["recommendations"][0][field] = value
self.assertFalse(AUDIT.validate_evidence(evidence, NOW))
path = self.root / "oversize.json"
path.write_text(" " * (AUDIT.MAX_EVIDENCE_BYTES + 1))
self.assertEqual(AUDIT.snapshot(path, AUDIT.MAX_EVIDENCE_BYTES)[1]["status"], "invalid")
path.write_text('{"schemaVersion":1,"schemaVersion":2}')
self.assertEqual(AUDIT.snapshot(path)[1]["status"], "invalid")
def test_invalid_subscription_allowlist_blocks_preview(self):
self.panel["subscriptionRoutes"] = {"codex": "not-an-array"}
result = self.audit(self.evidence())
self.assertEqual(result["sources"]["consensusConfig"]["status"], "invalid")
self.assertEqual(result["recommendations"], [])
self.assertEqual(result["verdict"], "REVIEW CONFIG")
def test_failed_refresh_keeps_incomplete_verdict_even_after_catalog_read(self):
self.refresh = {**self.refresh, "status": "failed", "attempted": True}
result = self.audit()
self.assertFalse(result["readOnly"])
self.assertFalse(result["refresh"]["fresh"])
self.assertEqual(result["verdict"], "INCOMPLETE EVIDENCE")
def test_native_success_followed_by_bad_enumeration_is_not_fresh(self):
self.refresh = {**self.refresh, "status": "ok", "attempted": True, "fresh": True}
self.base["sources"]["piCatalog"]["status"] = "invalid-output"
result = self.audit()
self.assertTrue(result["refresh"]["nativeCompleted"])
self.assertFalse(result["refresh"]["fresh"])
def test_invalid_policy_type_is_unknown_not_free(self):
self.router["modelPolicies"][OLD]["consent"] = []
result = self.audit()
row = result["configurationInventory"][0]
self.assertIsNone(row["metered"])
self.assertEqual(row["policyBasis"], "invalid")
self.assertTrue(any(f["kind"] == "router-policy-unknown" for f in result["findings"]))
def test_current_identity_with_unmapped_live_provider_is_incomplete(self):
self.dev = {}
self.opened = {}
self.pi = {OLD}
result = self.audit()
self.assertEqual(result["verdict"], "INCOMPLETE EVIDENCE")
self.assertIn(OLD, " ".join(result["incompleteReasons"]))
def test_malformed_config_identity_and_policy_fail_closed(self):
for bad_model in ("/broken", "broken/", "bad model/name"):
with self.subTest(model=bad_model):
self.router["tiers"]["standard"]["candidates"][0]["model"] = bad_model
result = self.audit(self.evidence())
self.assertEqual(result["sources"]["routerConfig"]["status"], "invalid")
self.assertFalse(any(c["source"] == "routerConfig" for p in result["recommendations"] for c in p["changes"]))
self.router["tiers"]["standard"]["candidates"][0]["model"] = OLD
self.router["modelPolicies"] = None
proposal = self.audit(self.evidence())["recommendations"][0]
self.assertEqual(proposal["status"], "incomplete")
self.assertTrue(any("invalid" in msg for msg in proposal["unresolved"]))
def test_review_action_leads_over_spend_housekeeping_without_authorizing_apply(self):
self.spend["models"] = {}
result = self.audit()
interaction = result["interaction"]
action = interaction["primaryAction"]
self.assertEqual(action["kind"], "review-upgrade")
self.assertEqual(action["scope"], "read-only")
self.assertEqual([o["id"] for o in action["options"]], ["review-preview", "leave-unchanged"])
self.assertTrue(action["requiresReply"])
self.assertIn("apply", action["doesNotAuthorize"])
self.assertTrue(interaction["opportunities"])
self.assertTrue(all(o["locations"] for o in interaction["opportunities"]))
self.assertTrue(any(f["kind"] == "spend-uncovered" for f in interaction["housekeeping"]))
self.assertTrue(all(o["cliAvailability"] == "unverified" for o in interaction["opportunities"] if o["from"].startswith("local/")))
def test_retained_old_allowlist_is_not_a_pending_upgrade(self):
# The old ID remains authorized for rollback, but the active pin and allowlist
# already include its successor on the same Codex route.
self.router["tiers"]["standard"]["candidates"][1]["model"] = NEW
self.router["modelPolicies"][NEW] = {"metered": False, "consent": "ask"}
self.spend["models"][NEW] = copy.deepcopy(self.spend["models"][OLD])
self.panel["profiles"]["premium"]["routes"][0]["model"] = "example-sol-2"
self.panel["subscriptionRoutes"]["codex"].append("example-sol-2")
result = self.audit()
self.assertFalse(any(lead["from"] == "local/codex/example-sol-1" for lead in result["discoveryLeads"]))
self.assertNotEqual(result["interaction"]["primaryAction"]["kind"], "review-upgrade")
# A still-old profile pin needs review even when the replacement is allowed.
self.panel["profiles"]["premium"]["routes"][0]["model"] = "example-sol-1"
result = self.audit()
leads = [lead for lead in result["discoveryLeads"] if lead["from"] == "local/codex/example-sol-1"]
self.assertEqual(len(leads), 1)
opportunity = next(item for item in result["interaction"]["opportunities"]
if item["from"] == "local/codex/example-sol-1")
self.assertEqual([location["path"] for location in opportunity["locations"]],
["/profiles/premium/routes/0/model"])
def test_reviewed_preview_does_not_restart_candidate_review(self):
result = self.audit(self.evidence())
self.assertTrue(result["discoveryLeads"])
self.assertEqual(result["interaction"]["primaryAction"]["kind"], "review-preview")
self.assertIn("apply", result["interaction"]["primaryAction"]["doesNotAuthorize"])
def test_review_first_skill_contract_preserves_separate_approval(self):
skill = (SCRIPTS.parent / "SKILL.md").read_text()
self.assertIn("Review and preview (Recommended)", skill)
self.assertIn("Leave unchanged", skill)
self.assertIn("using AskUserQuestion", skill)
self.assertIn("Do not ask the user to author JSON", skill)
self.assertIn("--evidence -", skill)
self.assertIn("not application", skill)
self.assertIn("Never offer an `Apply` option until the full preview has passed", skill)
self.assertIn("Never say an absent rule expired", skill)
def test_broken_core_config_is_not_hidden_behind_review_prompt(self):
self.base["sources"]["routerConfig"]["status"] = "invalid"
result = self.audit()
self.assertEqual(result["interaction"]["primaryAction"]["kind"], "inspect-config")
self.assertIn("routerConfig", result["interaction"]["blockingSources"])
def test_no_candidates_does_not_generate_upgrade_or_apply_prompt(self):
self.dev["openai"]["models"].pop("example-sol-2")
self.pi.remove(NEW)
self.opened = {}
self.assertEqual(self.audit()["interaction"]["primaryAction"]["kind"], "none")
self.base["sources"]["modelsDev"]["status"] = "error"
self.assertEqual(self.audit()["interaction"]["primaryAction"]["kind"], "inspect-availability")
def test_spend_explanations_distinguish_missing_invalid_and_date_gaps(self):
cases = [
({}, "missing-model-rule"),
({OLD: [{"billing": "invalid"}]}, "invalid-model-rule"),
({OLD: [{"billing": "subscription", "effectiveFrom": "2026-01-11T00:00:00Z", "effectiveUntil": None}]}, "not-started"),
({OLD: [{"billing": "subscription", "effectiveFrom": "2026-01-01T00:00:00Z", "effectiveUntil": "2026-01-10T00:00:00Z"}]}, "ended"),
({OLD: [{"billing": "subscription", "effectiveFrom": "2026-01-01T00:00:00Z", "effectiveUntil": "2026-01-09T00:00:00Z"},
{"billing": "metered", "effectiveFrom": "2026-01-11T00:00:00Z", "effectiveUntil": None}]}, "gap"),
]
for models, reason in cases:
with self.subTest(reason=reason):
self.spend["models"] = models
result = self.audit()
coverage = result["configurationInventory"][0]["spendCoverage"]
self.assertEqual(coverage["billing"], "unknown")
self.assertEqual(coverage["reason"], reason)
self.assertIn("report", coverage["explanation"])
self.assertEqual(result["interaction"]["primaryAction"]["kind"], "review-upgrade")
if reason == "missing-model-rule":
self.assertIn("no spend-reporting rule", coverage["explanation"])
self.assertNotIn("expired", coverage["explanation"])
self.spend = {}
self.assertEqual(self.audit()["configurationInventory"][0]["spendCoverage"]["reason"], "invalid-policy")
missing = AUDIT.SPEND.missing_policy(AUDIT.SPEND.EFFECTIVE_POLICY, "missing")
self.assertEqual(AUDIT.spend_coverage(OLD, missing, NOW, {}, "missing")["reason"], "missing-policy")
def test_valid_entry_in_partial_policy_and_start_boundary_stay_covered(self):
self.spend["models"][OLD][0]["effectiveFrom"] = "2026-01-10T00:00:00Z"
self.spend["models"][NEW] = [{"invalid": True}]
result = self.audit()
coverage = result["configurationInventory"][0]["spendCoverage"]
self.assertEqual(coverage["billing"], "subscription")
self.assertEqual(coverage["reason"], "covered")
self.assertEqual(result["sources"]["billingPolicy"]["status"], "partial")
def test_router_projection_inline_false_conflict_explicit_and_unknown(self):
router = copy.deepcopy(self.router)
del router["modelPolicies"]
router["tiers"]["standard"]["candidates"][0]["metered"] = False
self.assertEqual(AUDIT.router_policy(router, OLD), (False, "inline", "ask"))
router["tiers"]["standard"]["candidates"][1]["metered"] = True
self.assertEqual(AUDIT.router_policy(router, OLD), (True, "inline-conflict", "ask"))
router["modelPolicies"] = {OLD: {"metered": False}}
self.assertEqual(AUDIT.router_policy(router, OLD), (False, "modelPolicies", "ask"))
router["modelPolicies"][OLD]["metered"] = "false"
self.assertEqual(AUDIT.router_policy(router, OLD), (None, "invalid", None))
self.assertEqual(AUDIT.router_policy(router, NEW), (None, "unknown", None))
class RefreshTests(unittest.TestCase):
@patch.object(REFRESH, "run_native")
def test_default_denied_offline_and_invalid_confirmation_never_run(self, native):
for requested, confirmed, offline, expected in ((False, False, False, "not-requested"),
(False, False, True, "not-requested"), (True, False, False, "authorization-required"),
(True, True, True, "offline-conflict"), (False, True, False, "invalid-confirmation")):
result = REFRESH.refresh(requested, confirmed, offline)
self.assertEqual(result["status"], expected)
self.assertFalse(result["attempted"])
native.assert_not_called()
@patch.object(REFRESH, "run_native")
def test_only_exact_models_command_after_capability_check(self, native):
native.side_effect = [("ok", " --models Refresh model catalogs only\n"), ("ok", "secret must never be shown")]
result = REFRESH.refresh(True, True, False)
self.assertTrue(result["fresh"])
self.assertTrue(result["attempted"])
self.assertNotIn("secret", json.dumps(result))
self.assertEqual([call.args[0] for call in native.call_args_list], [["pi", "update", "--help"], ["pi", "update", "--models"]])
@patch.object(REFRESH, "run_native")
def test_unsupported_timeout_failure_do_not_claim_freshness_or_fallback(self, native):
native.return_value = ("ok", "pi update --all")
self.assertEqual(REFRESH.refresh(True, True, False)["status"], "unsupported")
self.assertEqual(native.call_count, 1)
for status in ("timeout", "failed", "unavailable"):
native.reset_mock()
native.side_effect = [("ok", " --models Refresh model catalogs only"), (status, "")]
result = REFRESH.refresh(True, True, False)
self.assertFalse(result["fresh"])
self.assertEqual(result["status"], status)
self.assertEqual(native.call_count, 2)
def test_real_subprocess_timeout_and_missing_executable(self):
status, output = REFRESH.run_native([sys.executable, "-c", "import time; time.sleep(5)"], 0.05)
self.assertEqual((status, output), ("timeout", ""))
self.assertEqual(REFRESH.run_native(["/nonexistent/audit-command"], 1), ("unavailable", ""))
@patch.object(REFRESH.Path, "is_file", return_value=True)
@patch.object(REFRESH, "run_native")
def test_enumeration_is_offline_and_disables_executable_resources(self, native, _exists):
native.side_effect = [("ok", "0.87.1"), ("ok", "provider model context max-out thinking images\nexample model 1M 32K yes yes\n")]
result = REFRESH.collect_catalog()
self.assertEqual(result["models"], [{"provider": "example", "model": "model"}])
command = native.call_args_list[1].args[0]
self.assertIn("--offline", command)
self.assertIn("--no-extensions", command)
self.assertIn("--no-approve", command)
self.assertNotIn("update", command)
@patch.object(REFRESH.Path, "is_file", return_value=True)
@patch.object(REFRESH, "run_native")
def test_malformed_enumeration_does_not_become_empty_success(self, native, _exists):
native.side_effect = [("ok", "0.87.1"), ("ok", "unexpected output")]
self.assertEqual(REFRESH.collect_catalog()["status"], "invalid-output")
@patch.object(REFRESH, "run_native", return_value=("ok", "unexpected-native-output"))
def test_unexpected_version_is_not_echoed(self, native):
result = REFRESH.collect_catalog()
self.assertEqual(result["status"], "invalid-version")
self.assertNotIn("unexpected-native-output", json.dumps(result))
self.assertEqual(native.call_count, 1)
@patch.object(REFRESH.Path, "is_file", return_value=False)
@patch.object(REFRESH, "run_native", return_value=("ok", "0.87.1"))
def test_missing_native_auth_store_prevents_implicit_creation(self, native, _exists):
self.assertEqual(REFRESH.collect_catalog()["status"], "auth-store-absent")
self.assertEqual(native.call_count, 1)
if __name__ == "__main__":
os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
unittest.main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |