akiiwa-games/AgentCompass / skills/agent-compass
agent-compass
FlaggedChoose owner, model and effort, outbound authorization, advisor use, and continuation for delegated work. Reads local usage and effective account policy, profiles the task with the configured System-One scorer, then allocates a compatible worker. Use for independent worker or advisor work; ordinary local execution needs no routing.
scripts/test_host_policy.py
View on GitHub"""Host exclusions apply at dispatch boundaries; fixtures never read real host state."""
from contextlib import contextmanager, nullcontext
from datetime import datetime, timedelta, timezone
import json
from pathlib import Path
import unittest
from unittest.mock import patch
import allocator
import account_fleet
import host_policy
import route_judge as router
import decision_ledger
import dispatch_registry
import subscription_cli
import usage_reading
from router_test_support import isolated_router_state, profile, temporary_root
def setUpModule():
isolated_router_state(unittest.addModuleCleanup, policy=False)
@contextmanager
def override_file(path, accounts):
path.write_text(json.dumps({'schema_version': 2, 'accounts': accounts}), encoding='utf-8')
try:
yield
finally:
path.unlink(missing_ok=True)
@contextmanager
def cooling(launch_failures, now):
bundle = {'account': 'claude', 'provider': 'claude', 'model': 'claude-opus-5-5'}
with patch.object(launch_failures, 'enabled', return_value=True):
launch_failures.record(bundle, 'model_unavailable', now=now)
try:
yield
finally:
launch_failures.clear('claude')
LS = {'allowed_accounts': ['codex', 'antigravity', 'dot', 'chatgpt-web'],
'judges': ['codex'], 'usage_providers': ['codex', 'antigravity']}
class HostPolicyTests(unittest.TestCase):
def setUp(self):
self.directory = temporary_root(self.addCleanup)
registry_env = patch.dict(router.os.environ, {'ROUTER_DISPATCH_DB': str(self.directory / 'dispatch.sqlite3')})
registry_env.start()
self.addCleanup(registry_env.stop)
self.path = self.directory/'host-policy.json'
self.write(LS)
patcher = patch.object(host_policy, 'policy_path', return_value=(self.path, False))
patcher.start()
self.addCleanup(patcher.stop)
self.accounts = account_fleet.load_accounts(account_fleet.reference_fleet_path())
self.now = datetime(2026, 10, 11, tzinfo=timezone.utc)
def write(self, data):
self.path.write_text(json.dumps(data), encoding='utf-8')
def plan(self, task=None, **kwargs):
return allocator.plan(task or {'work_kind': 'code_implementation', 'difficulty': 'hard'},
{'accounts': {}}, now=self.now, accounts=self.accounts, **kwargs)
def test_missing_default_preserves_defaults(self):
self.path.unlink()
self.assertIsNone(host_policy.load_policy())
self.assertTrue(host_policy.account_allowed(None, 'claude'))
def test_missing_explicit_override_is_error(self):
self.path.unlink()
with patch.object(host_policy, 'policy_path', return_value=(self.path, True)):
with self.assertRaisesRegex(host_policy.HostPolicyError, 'missing'):
self.plan()
def test_environment_override(self):
# Exercise resolution, independently of this suite's isolated loader path.
with patch.dict(host_policy.os.environ, {'ROUTER_HOST_POLICY': str(self.path)}):
self.assertEqual(self.original_policy_path(), (self.path, True))
original_policy_path = staticmethod(host_policy.policy_path)
def test_invalid_profiles_fail_closed(self):
for value in ('{', '[]', '{}', '{"allowed_accounts":"codex"}',
'{"allowed_accounts":[1]}', '{"allowed_accounts":[],"judge":[]}',
'{"allowed_accounts":[],"usage_providers":null}'):
with self.subTest(value=value):
self.path.write_text(value, encoding='utf-8')
with self.assertRaises(host_policy.HostPolicyError):
self.plan()
self.path.write_bytes(b'\xff')
with self.assertRaisesRegex(host_policy.HostPolicyError, 'UTF-8'):
host_policy.load_policy()
def test_whitelist_applies_to_all_plan_calls_and_cannot_be_overridden(self):
for unavailable in ([], ['sol_xhigh']):
result = self.plan({'work_kind': 'code_implementation', 'difficulty': 'hard',
'allowed_accounts': ['claude'], 'host_policy': None}, unavailable=unavailable)
for row in result['ranking']:
if row['bundle']['account'] not in LS['allowed_accounts']:
self.assertIn('excluded by host policy', row['excluded'])
self.assertIn(result['selected_bundle']['account'], LS['allowed_accounts'])
if unavailable:
row = next(r for r in result['ranking'] if r['id'] == 'sol_xhigh')
self.assertIn('marked unavailable by the parent', row['excluded'])
def test_empty_whitelist_is_not_default(self):
self.write({'allowed_accounts': []})
self.assertEqual(self.plan()['choice'], 'none')
def test_a_policy_a_released_version_wrote_with_judges_loads_plans_and_syncs_untouched(self):
"""AGENTS.md, what a released version wrote for the user is kept: `judges` (read by the removed select and
delegate stages) is accepted, left in the file as written, and changes nothing a plan decides."""
import state_upgrade
without = {'allowed_accounts': ['codex', 'claude'], 'known_accounts': ['codex', 'claude']}
self.write(without)
plain = self.plan()
for judges in (['codex'], ['sonnet'], ['codex', 'sonnet'], [], ['a-name-no-release-wrote']):
with self.subTest(judges=judges):
document = {**without, 'judges': judges}
self.write(document)
written = self.path.read_bytes()
self.assertEqual(host_policy.load_policy(), document)
self.assertIsNone(state_upgrade.check_host_policy(self.path))
self.assertFalse([i for i in state_upgrade.collect_issues(environ={'ROUTER_HOST_POLICY': str(self.path)})
if 'host-policy' in i]) # what `agentcompass doctor` reports
self.assertEqual(self.plan(), plain)
self.assertEqual(host_policy.sync_discovered(self.registered(codex='ready', claude='ready'))['added'], [])
self.assertEqual(self.path.read_bytes(), written)
host_policy.sync_discovered(self.registered(codex='ready', claude='ready', **{'codex-work': 'ready'}))
self.assertEqual(host_policy.load_policy()['judges'], judges)
def test_excluded_coupled_owner_is_blocked(self):
result = self.plan({'work_kind': 'code_implementation', 'difficulty': 'coupled',
'chain': {'id': 'a', 'coupling': 'high', 'worker': {'bundle': 'opus55_high'}}})
self.assertIn('excluded by host policy', result['decision']['resume_blocked_by'])
self.assertTrue(result['decision']['needs_user_decision'])
def record(self, constraint='user_override', model='claude-opus-5-5', usage=None):
"""record_constraint for a route on `claude`, an account outside LS (out of automatic routing)."""
task = self.directory/'task.json'
task.write_text(json.dumps({'task': {'work_kind': 'code_implementation', 'difficulty': 'hard', 'profile': profile()}}))
args = router.argparse.Namespace(basis='User: run this on Opus on claude', provider='claude', account='claude',
model=model, effort='high', input=task, usage_cache=None, accounts=None,
constraint=constraint, specialty=None)
with patch.object(decision_ledger, 'ledger_path', return_value=self.directory/'ledger.jsonl'), \
patch.object(usage_reading, 'usage_snapshot', return_value=usage or {'accounts': {}}), \
patch.object(subscription_cli, 'cli_prefix', side_effect=AssertionError('model call forbidden')):
return router.record_constraint(args), decision_ledger.read_ledger()
def test_a_route_the_user_named_outside_automatic_routing_dispatches_with_a_warning(self):
# Rule (router-internals.md, Pins): allowed_accounts governs automatic routing; the user's own naming
# (user_override) reaches an account outside it, says so in the decision, and is launch authority.
result, ledger = self.record()
self.assertNotIn('host_policy_violation', result)
self.assertNotIn('host_policy_violation', ledger[-1])
# The one field the audit filters on, written by the ledger owner.
self.assertEqual(ledger[-1]['outside_automatic_routing'], ['account'])
self.assertTrue(any(w.startswith('account claude is out of automatic routing on this host')
and w.endswith('launched because the user named it in this request')
for w in result['warnings']), result.get('warnings'))
self.assertTrue(dispatch_registry.decision_record(result['decision_id'])['dispatchable'])
# The router by itself would still not choose it.
self.assertIn('excluded by host policy', result['override_cost']['excluded'])
def test_a_project_rule_does_not_override_this_hosts_switch(self):
# Rule: a hard_constraint (or any constraint but user_override) is shared between machines and does not override
# one machine's switch; on an account in automatic routing it is launch authority as before.
for constraint in ('hard_constraint', 'dot_closed_loop'):
with self.subTest(constraint=constraint):
result, ledger = self.record(constraint)
violation = result['host_policy_violation']
self.assertFalse(violation['dispatch_allowed'])
self.assertEqual(violation['account'], 'claude')
self.assertIn('--constraint user_override', violation['reason'])
self.assertEqual(ledger[-1]['host_policy_violation'], violation)
self.assertNotIn('outside_automatic_routing', ledger[-1]) # refused, so nothing reached past it
self.assertFalse(dispatch_registry.decision_record(result['decision_id'])['dispatchable'])
self.write({**LS, 'allowed_accounts': [*LS['allowed_accounts'], 'claude']})
result, ledger = self.record('hard_constraint')
self.assertNotIn('outside_automatic_routing', ledger[-1])
result, ledger = self.record()
self.assertNotIn('outside_automatic_routing', ledger[-1]) # in automatic routing: nothing to mark
result, _ = self.record('hard_constraint')
self.assertNotIn('host_policy_violation', result)
self.assertFalse(any('out of automatic routing' in w for w in result.get('warnings', ())))
def test_a_named_model_unchecked_in_model_management_launches_and_a_project_rule_does_not(self):
# Amendment 2 (router-internals.md, Pins): the checkbox says which models take part in automatic routing, like
# allowed_accounts for accounts. The user's naming reaches an unchecked model with a warning and the ledger
# marker; a project rule does not override the user's checkbox.
self.write({**LS, 'allowed_accounts': [*LS['allowed_accounts'], 'claude']})
override = Path(router.os.environ['AGENTCOMPASS_ACCOUNTS_OVERRIDE'])
unchecked = {'claude': {'bundles': {b: {'disabled': True} for b in (
'opus55_high', 'opus55_medium', 'opus55_low', 'opus55_xhigh', 'opus55_max')}}}
with override_file(override, unchecked):
result, ledger = self.record()
self.assertTrue(any(w.startswith('claude/claude-opus-5-5 is out of automatic routing on this host '
'(unchecked in model management)')
and w.endswith('launched because the user named it in this request')
for w in result['warnings']), result.get('warnings'))
self.assertEqual(ledger[-1]['outside_automatic_routing'], ['model'])
self.assertTrue(dispatch_registry.decision_record(result['decision_id'])['dispatchable'])
for constraint in ('hard_constraint', 'dot_closed_loop'):
with self.subTest(constraint=constraint), \
self.assertRaisesRegex(ValueError, 'unchecked in model management.*--constraint user_override'):
self.record(constraint)
result, ledger = self.record()
self.assertNotIn('outside_automatic_routing', ledger[-1])
self.assertFalse(any('out of automatic routing' in w for w in result.get('warnings', ())))
def test_every_launch_row_is_marked_by_the_ledger_owner_and_nothing_else_is(self):
# Rule (router-internals.md, Pins; review N3): the ledger owner derives `outside_automatic_routing` for every row
# that carries a launch on a bundle, from the switches in effect when the row is written; rows that launch nothing
# carry none, and a marker a writer hands in is never kept. Stated over the row shape of every writer.
bundle = {'provider': 'claude', 'account': 'claude', 'model': 'claude-opus-5-5', 'effort': 'high',
'bundle_id': 'opus55_high'}
launch_rows = { # writer -> row it hands append_ledger
**{f'finalize:{stage}': {'stage': stage, 'choice': 'opus55_high', 'bundle': bundle,
'constraint': constraint}
for stage, constraint in (('plan', None), ('record', 'user_override'), ('record', 'explicit_resume'))},
**{f'fallback_retry:{stage}': {'stage': stage, 'source': 'fallback_retry', 'choice': 'opus55_high',
'bundle': bundle, 'action': 'start'} for stage in ('plan', 'record')},
}
quiet_rows = {
'fallback': {'stage': 'fallback', 'action': 'relaunch', 'failed': bundle},
'split': {'stage': 'split', 'choice': 'split'},
'select': {'stage': 'select', 'choice': 'opus55_high', 'bundle': bundle}, # a stage removed 2026-11-21
'scorer_unavailable': {'stage': 'plan', 'choice': 'none', 'error': 'scorer_unavailable'},
'self': {'stage': 'plan', 'choice': 'self', 'bundle': bundle},
'override': {'stage': 'override', 'action': 'upsert', 'account': 'claude', 'model': 'claude-opus-5-5'},
'refused': {'stage': 'record', 'choice': 'opus55_high', 'bundle': bundle,
'host_policy_violation': {'dispatch_allowed': False}},
}
override = Path(router.os.environ['AGENTCOMPASS_ACCOUNTS_OVERRIDE'])
model_off = {'claude': {'bundles': {b: {'disabled': True} for b in (
'opus55_high', 'opus55_medium', 'opus55_low', 'opus55_xhigh', 'opus55_max')}}}
ledger = self.directory/'marker-ledger.jsonl'
for account_on in (False, True):
for model_checked in (True, False):
self.write({**LS, 'allowed_accounts': [*LS['allowed_accounts'], *(['claude'] if account_on else [])]})
expected = [*([] if account_on else ['account']), *([] if model_checked else ['model'])] or None
with override_file(override, {} if model_checked else model_off), \
patch.object(decision_ledger, 'ledger_path', return_value=ledger):
for writer, row in {**launch_rows, **quiet_rows}.items():
with self.subTest(account_on=account_on, model_checked=model_checked, writer=writer):
ledger.unlink(missing_ok=True)
# A writer's own marker is dropped, whatever it says.
self.assertTrue(decision_ledger.append_ledger(dict(row, outside_automatic_routing=['x'])))
written = decision_ledger.read_ledger()[-1]
want = expected if writer in launch_rows else None
self.assertEqual(written.get('outside_automatic_routing'), want)
# Every ledger write goes through `append_ledger`: no other module appends to the ledger file.
scripts = Path(decision_ledger.__file__).parent
root = scripts.parents[2]
appending = [p for p in [*scripts.glob('*.py'), *(root/'setup').glob('*.py')]
if not p.name.startswith('test_') and p.name != 'decision_ledger.py'
and 'ledger_path' in (text := p.read_text(encoding='utf-8'))
and ("open('a'" in text or 'open("a"' in text)]
self.assertEqual(appending, [])
def test_facts_still_refuse_a_named_route_outside_automatic_routing(self):
# Rule 3: naming lifts the preference switches only; no sign-in, a model the account does not offer, an
# exhausted window and a launch cooldown still refuse.
now = datetime.now(timezone.utc)
exhausted = {'accounts': {'claude': {'status': 'fresh', 'fetched_at': now.isoformat(), 'available': True,
'windows': [{'id': 'five_hour', 'used_pct': 100,
'resets_at': (now + timedelta(hours=2)).isoformat()}]}}}
import launch_failures
import provider_inventory
cases = (
('no sign-in', lambda: patch.object(provider_inventory, 'current', return_value={
'accounts': {'claude': {'family': 'claude', 'status': 'auth_failed', 'reason': 'credentials_rejected'}},
'launchers': {}}),
{}, 'credentials_rejected'),
('model not offered', None, {'model': 'gpt-6-astra'}, 'named_model_unavailable|no known provider/account/model'),
('exhausted window', None, {'usage': exhausted}, 'five_hour'),
('cooldown', lambda: cooling(launch_failures, now), {}, 'cooling down'),
)
for label, context, kwargs, cause in cases:
with self.subTest(fact=label), (context() if context else nullcontext()):
with self.assertRaisesRegex(ValueError, cause):
self.record(**kwargs)
def test_automatic_resume_never_picks_a_worker_out_of_automatic_routing_or_on_an_unchecked_model(self):
# Rule 1: resuming an idle worker without being told to is the router's choice, so both preference switches
# apply; an explicit --resume reads only the gate's facts (roll._auto_resumable / _worker_gate_refusal).
import roll
accounts = account_fleet.load_accounts()
policy = host_policy.load_policy()
worker = {'agent_id': 'w1', 'dispatch_id': 'd1', 'bundle_id': 'opus55_high',
'bundle': {'account': 'claude', 'provider': 'claude', 'model': 'claude-opus-5-5', 'effort': 'high',
'bundle_id': 'opus55_high'}}
with patch.object(roll, '_worker_gate_refusal', return_value=None):
self.assertFalse(roll._auto_resumable(worker, policy, accounts=accounts))
self.assertTrue(roll._auto_resumable(worker, {'allowed_accounts': ['claude']}, accounts=accounts))
for row in accounts['accounts']['claude']['bundles'].values():
if row.get('model') == 'claude-opus-5-5':
row['disabled'] = True
self.assertFalse(roll._auto_resumable(worker, {'allowed_accounts': ['claude']}, accounts=accounts))
with patch.object(roll, '_worker_gate_refusal', return_value='quota window five_hour is unavailable'):
self.assertFalse(roll._auto_resumable(worker, None, accounts=account_fleet.load_accounts()))
def test_same_policy_at_different_paths_has_identical_allocation(self):
first = self.plan()
other = self.directory/'other-user'/'profile.json'
other.parent.mkdir()
other.write_bytes(self.path.read_bytes())
with patch.object(host_policy, 'policy_path', return_value=(other, True)):
second = self.plan()
self.assertEqual(first, second)
self.assertNotIn(str(self.directory), json.dumps(first))
def test_usage_selection_ignores_dispatch_exclusions_and_respects_explicit_usage_providers(self):
defaults = ['claude', 'codex', 'codex-work', 'grok', 'antigravity']
self.assertEqual(host_policy.usage_providers(host_policy.load_policy(), defaults), ['codex', 'antigravity'])
self.assertEqual(host_policy.usage_providers(None, defaults), defaults)
self.assertEqual(host_policy.usage_providers({'allowed_accounts': ['codex']}, defaults), defaults)
self.assertEqual(host_policy.usage_providers({'allowed_accounts': []}, defaults), defaults)
self.assertEqual(host_policy.usage_providers({'allowed_accounts': ['codex'], 'usage_providers': ['grok']}, defaults), ['grok'])
self.assertEqual(host_policy.usage_providers({'allowed_accounts': ['codex'], 'usage_providers': []}, defaults), [])
# Dispatch exclusions still strictly govern routing/allocation even when usage is collected for all accounts.
self.write({'allowed_accounts': ['codex']})
planned = self.plan({'work_kind': 'code_implementation', 'difficulty': 'hard', 'profile': profile()})
# The fixture has no quota reading, so codex itself may be out (mode B); what matters is that nothing else is chosen.
self.assertIn((planned.get('selected_bundle') or {}).get('account'), (None, 'codex'))
for row in planned['ranking']:
if row['bundle']['account'] != 'codex':
self.assertIn('excluded by host policy', row['excluded'])
def registered(self, **states):
return {'_discovery': {'accounts': {name: {'status': state} for name, state in states.items()}},
'accounts': {name: {'billing': 'subscription', 'verified': state == 'ready', 'bundles': {
name: {'provider': name, 'model': 'service'}}} for name, state in states.items()}}
def test_known_accounts_preserves_exclusions_and_new_ready_accounts_enroll_once(self):
baseline = self.registered(codex='ready', claude='ready')
self.write({'allowed_accounts': ['codex'], 'known_accounts': ['codex', 'claude'],
'judges': ['codex'], 'usage_providers': ['codex']})
result = host_policy.sync_discovered(baseline)
self.assertEqual(result['added'], [])
self.assertEqual(host_policy.load_policy()['allowed_accounts'], ['codex'])
next_accounts = self.registered(codex='ready', claude='ready', **{'grok-bot': 'missing_configuration', 'codex-work': 'ready'})
# First observation need not be ready: it must not become a permanent manual deny.
self.assertEqual(host_policy.sync_discovered(next_accounts)['added'], ['codex-work'])
self.assertNotIn('grok-bot', host_policy.load_policy()['known_accounts'])
next_accounts = self.registered(codex='ready', claude='ready', **{'grok-bot': 'ready', 'codex-work': 'ready'})
self.assertEqual(host_policy.sync_discovered(next_accounts)['added'], ['grok-bot'])
document = host_policy.load_policy()
self.assertEqual(document['judges'], ['codex'])
self.assertEqual(document['usage_providers'], ['codex'])
after = self.path.read_bytes()
self.assertEqual(host_policy.sync_discovered(next_accounts)['added'], [])
self.assertEqual(self.path.read_bytes(), after)
# Manually disable an enrolled account; neither refresh nor temporary disappearance resurrects it.
document['allowed_accounts'].remove('grok-bot')
self.write(document)
host_policy.sync_discovered(baseline)
host_policy.sync_discovered(next_accounts)
self.assertNotIn('grok-bot', host_policy.load_policy()['allowed_accounts'])
self.assertNotIn('claude', host_policy.load_policy()['allowed_accounts'])
def test_missing_contract_or_expired_disabled_removed_rejected_account_is_not_enrolled(self):
self.write({'allowed_accounts': ['codex'], 'known_accounts': ['codex']})
accounts = self.registered(codex='ready', removed='removed', disabled='disabled', rejected='paused',
expired='ready', unregistered='ready', no_bundles='ready', banned_bundle='ready')
del accounts['accounts']['unregistered']
accounts['accounts']['no_bundles']['bundles'] = {}
accounts['accounts']['banned_bundle']['bundles']['banned_bundle']['disabled'] = True
accounts['accounts']['expired']['expires_at'] = '2000-01-01T00:00:00Z'
self.assertEqual(host_policy.sync_discovered(accounts)['added'], [])
self.assertEqual(host_policy.load_policy()['known_accounts'], ['codex'])
# Obtaining the missing registered contract later is sufficient; no second consent flag needed.
accounts['accounts']['unregistered'] = {'billing': 'subscription', 'verified': True, 'bundles': {'real': {'provider': 'unregistered', 'model': 'real'}}}
self.assertEqual(host_policy.sync_discovered(accounts)['added'], ['unregistered'])
self.assertFalse(host_policy.account_allowed(host_policy.load_policy(), 'arbitrary-request-name'))
if __name__ == '__main__':
unittest.main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |