Skip to content

anyshift-io/sre-skills / skills/iam-deceptive-escalation-auditor

iam-deceptive-escalation-auditor

Verified

FileKindSize
FAILURE_MODES.mdmarkdown3.6 KB
SKILL.mdskill md19.6 KB
fixtures/01-orphaned-passrole-deny/meta.jsonconfig349 B
fixtures/01-orphaned-passrole-deny/policy.jsonconfig573 B
fixtures/02-action-star-blanket-deny/meta.jsonconfig408 B
fixtures/02-action-star-blanket-deny/policy.jsonconfig604 B
fixtures/03-assumerole-broken-trust/meta.jsonconfig492 B
fixtures/03-assumerole-broken-trust/policy.jsonconfig492 B
fixtures/03-assumerole-broken-trust/trust-policy.jsonconfig473 B
fixtures/05-iam-mutation-boundary-capped/meta.jsonconfig696 B
fixtures/05-iam-mutation-boundary-capped/policy-1.jsonconfig550 B
fixtures/05-iam-mutation-boundary-capped/policy-2.jsonconfig945 B
fixtures/06-cross-account-assume-condition-gated/meta.jsonconfig700 B
fixtures/06-cross-account-assume-condition-gated/policy-1.jsonconfig690 B
fixtures/06-cross-account-assume-condition-gated/policy-2.jsonconfig512 B
fixtures/06-cross-account-assume-condition-gated/trust-policy.jsonconfig400 B
fixtures/07-passrole-sandboxed-role-orphaned/meta.jsonconfig997 B
fixtures/07-passrole-sandboxed-role-orphaned/policy-1.jsonconfig508 B
fixtures/07-passrole-sandboxed-role-orphaned/policy-2.jsonconfig571 B
fixtures/07-passrole-sandboxed-role-orphaned/policy-3.jsonconfig640 B
fixtures/08-ml-platform-passrole-launch-needle/meta.jsonconfig1.0 KB
fixtures/08-ml-platform-passrole-launch-needle/policy-1.jsonconfig713 B
fixtures/08-ml-platform-passrole-launch-needle/policy-2.jsonconfig670 B
fixtures/08-ml-platform-passrole-launch-needle/policy-3.jsonconfig745 B
fixtures/08-ml-platform-passrole-launch-needle/policy-4.jsonconfig647 B
fixtures/08-ml-platform-passrole-launch-needle/policy-5.jsonconfig959 B
fixtures/08-ml-platform-passrole-launch-needle/policy-6.jsonconfig568 B
tests/README.mdmarkdown3.8 KB
tests/_audit.pyscript32.5 KB
tests/_replay.pyscript720 B
tests/eval/README.mdmarkdown5.5 KB
tests/eval/eval_results.jsonconfig493.4 KB
tests/eval/judge_prompt.mdmarkdown3.0 KB
tests/eval/rubric.mdmarkdown3.3 KB
tests/eval/run_eval.pyscript16.1 KB
tests/eval/scenarios.pyscript11.2 KB
tests/replay_01_orphaned_passrole_deny.pyscript1.5 KB
tests/replay_02_action_star_blanket_deny.pyscript1.4 KB
tests/replay_03_assumerole_broken_trust.pyscript1.5 KB
tests/replay_05_iam_mutation_boundary_capped.pyscript2.1 KB
tests/replay_06_cross_account_assume_condition_gated.pyscript1.8 KB
tests/replay_07_passrole_sandboxed_role_orphaned.pyscript2.0 KB
tests/replay_08_ml_platform_passrole_launch_needle.pyscript2.3 KB

tests/eval/scenarios.py

View on GitHub
"""
Per-fixture principal contexts and expected answers, used by run_eval.py.

The ground-truth findings are NOT hand-written. The "expected_audit" function runs the
copied reference engine (_audit.py) against each fixture, so the codes / severity / clean
flag the judge is anchored to are exactly what the engine computes (see tests/replay_*.py).
The "expected_headline / fix / boundary" strings are human-readable framing for the judge
prompt; they describe the SAME verdict the engine grounds, never a different one.

This screening harness is CONTROL-ONLY: there is no SKILL.md and no treatment arm. The set
targets the cold agent's proven weak region, OVER-FLAGGING neutralised policies: six
deceptive-clean fixtures that look like critical escalation but are capped (the engine finds
nothing), each with a distinct neutralisation mechanism (explicit Deny on PassRole; Action '*'
scoped + service Deny; broken trust; a permission-boundary Deny over a full mutation kit; a
cross-account AssumeRole sealed by an unsatisfiable Condition; a PassRole whose only passable
role is read-only and whose compute verbs bind no role). One buried-hard needle is kept where a
real escalation (E1) only emerges from the union of six attached policies / ~16 statements.

Stdlib only. No external dependencies.
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

TESTS_DIR = Path(__file__).resolve().parent.parent
FIXTURES_DIR = TESTS_DIR.parent / "fixtures"

sys.path.insert(0, str(TESTS_DIR))
from _audit import run_audit  # noqa: E402

# Each entry pairs a fixture with the human-readable context the eval feeds the agent,
# plus the headline / fix / boundary that match the deterministic audit's verdict (the
# judge's anchor). Keep this list aligned with the replay_*.py files under tests/.
SCENARIOS = [
    {
        "id": "01-orphaned-passrole-deny",
        "principal": "role/build-fleet-runner",
        "role": "Runs an EC2 build fleet. Granted ec2:RunInstances and, in a separate statement, iam:PassRole; a third statement is an explicit Deny on iam:PassRole.",
        "expected_headline": "No real escalation. PassRole + RunInstances looks like the classic E1 combo, but an explicit Deny on iam:PassRole (Resource '*') overrides the scoped Allow, so the PassRole half is dead and the combo cannot complete.",
        "expected_top_fix": "None on the escalation: it is already neutralised by the Deny. Optionally remove the now-inert PassRole Allow and the retired instance-profile reference to reduce confusion.",
        "expected_boundary_join": "the privileges of the roles PassRole could pass IF it were allowed (moot here), and the principal's other attached policies / permissions boundary / org SCPs.",
    },
    {
        "id": "02-action-star-blanket-deny",
        "principal": "role/sandbox-experimenter",
        "role": "An experimentation role. Granted Action '*' on a single sandbox S3 bucket, with a Deny on every escalation-bearing service (iam, sts, kms, lambda, ec2:RunInstances, ssm, secretsmanager) on Resource '*'.",
        "expected_headline": "No real escalation. Action '*' reads like AdministratorAccess, but it is pinned to one sandbox bucket (never Resource '*'), and the Deny removes every dangerous service. The star expands to nothing useful outside one scratch bucket.",
        "expected_top_fix": "None required for security: the star is already scoped and the Deny caps it. Optionally replace Action '*' with the concrete S3 actions in use for clarity.",
        "expected_boundary_join": "whether anything outside this document re-grants the denied services (other attached policies could not, since an explicit Deny wins), and the org SCPs.",
    },
    {
        "id": "03-assumerole-broken-trust",
        "principal": "role/deploy-orchestrator",
        "role": "Granted sts:AssumeRole on role/org-admin-break-glass (an admin-sounding target) and read access to a deploy-config bucket. The target role's trust policy is supplied.",
        "expected_headline": "No real escalation. sts:AssumeRole on an admin role sounds like a lateral move, but the target's trust policy only trusts two specific operator roles behind an ExternalId, not this principal, and this principal has no iam:UpdateAssumeRolePolicy to rewrite it. The path is broken.",
        "expected_top_fix": "None: the AssumeRole grant is inert because the trust does not point back. Optionally remove the unused AssumeRole grant.",
        "expected_boundary_join": "the actual contents of the target role's trust policy and whether any OTHER principal this role can reach closes the loop (confirmed broken here from the supplied trust).",
    },
    {
        "id": "05-iam-mutation-boundary-capped",
        "principal": "role/identity-platform-operator",
        "role": "An identity-platform role across two attached policies. policy-1 grants a full mutation kit (iam:PutRolePolicy, AttachRolePolicy, CreatePolicyVersion, SetDefaultPolicyVersion, UpdateAssumeRolePolicy, PassRole, CreateAccessKey) scoped to one break-glass role/policy ARN; policy-2 is a permission-boundary-style explicit Deny on every one of those actions (plus sts:AssumeRole and the credential-minting set) across Resource '*'.",
        "expected_headline": "No real escalation. policy-1 looks like a full identity-takeover kit (every E2/E4/E5/E6 primitive), but policy-2's explicit Deny on all of them across Resource '*' wins over the Allow, so the effective permission set collapses to read-only IAM inventory. The mutation kit is fully capped.",
        "expected_top_fix": "None for security: the Deny boundary already neutralises the kit. Optionally remove the now-inert mutation Allow so the policy reads honestly, and keep the Deny as the boundary.",
        "expected_boundary_join": "whether any other attached policy or the org SCPs re-grant the denied actions (they cannot, since an explicit Deny wins), and what the break-glass role/policy ARN would have permitted if the Deny were removed.",
    },
    {
        "id": "06-cross-account-assume-condition-gated",
        "principal": "role/cost-reporting-collector",
        "role": "A cost-reporting role. policy-1 grants sts:AssumeRole on a role in a DIFFERENT account (905512347781) behind an sts:ExternalId + aws:PrincipalOrgID Condition; the target role's trust policy is supplied and uses a wildcard Principal narrowed by the same org-id + ExternalId condition. The rest is read-only billing/cost access.",
        "expected_headline": "No real escalation. The cross-account sts:AssumeRole reads like a pivot into a foreign account, but it is gated by an sts:ExternalId + aws:PrincipalOrgID Condition this principal cannot satisfy, and the target's trust wildcard Principal is fully narrowed by the same condition (so no open trust). The principal has no iam:UpdateAssumeRolePolicy to relax either side. The path is condition-sealed at both ends.",
        "expected_top_fix": "None: the AssumeRole grant is inert because the condition cannot be met and the trust does not open. Optionally remove the unused cross-account AssumeRole grant.",
        "expected_boundary_join": "whether the principal can ever present the required org-id / ExternalId (it cannot from this identity), and what the foreign-account target role can do if the path were ever opened.",
    },
    {
        "id": "07-passrole-sandboxed-role-orphaned",
        "principal": "role/sandbox-compute-operator",
        "role": "A sandbox compute-operator role across three attached policies. policy-1 grants iam:PassRole scoped to one role, sandbox-readonly-compute; policy-2 grants compute verbs (ec2:StartInstances, ecs:StartTask, lambda:InvokeFunction) on existing compute; policy-3 is the passed role's OWN policy, which is strictly read-only.",
        "expected_headline": "No real escalation. iam:PassRole plus compute verbs reads like the E1 launch combo, but Start/Invoke operate on EXISTING compute and accept no PassRole argument, so there is no role-binding launch action (RunInstances/CreateFunction/RunTask) to pair PassRole with. And the one passable role is read-only, no more privileged than the caller. An orphaned escalation: the shape is there, the gain is not.",
        "expected_top_fix": "None for security: the combo cannot complete and the passed role grants nothing extra. Optionally remove the unused PassRole grant if no role-binding launch action will be added later.",
        "expected_boundary_join": "the actual privileges of sandbox-readonly-compute (confirmed read-only here from the attached policy), and whether any future policy adds a role-binding launch action that would re-arm the combo.",
    },
    {
        "id": "08-ml-platform-passrole-launch-needle",
        "principal": "role/ml-training-platform",
        "role": "A SageMaker training platform across six attached policies (~16 statements). iam:PassRole on Resource '*' is granted in one policy (framed as passing the training execution role); sagemaker:CreateTrainingJob is granted in another; the rest are routine read/queue/experiment/output permissions.",
        "expected_headline": "Real privilege escalation (critical). iam:PassRole on Resource '*' plus sagemaker:CreateTrainingJob is the E1 combo: launch a training job with ANY role in the account attached, then use that job's credentials. The two halves sit four policies apart behind heavy benign bait, so a per-statement read clears every statement; only the union is critical.",
        "expected_top_fix": "Scope iam:PassRole to the exact execution-role ARNs the training platform must pass (never '*'), with an iam:PassedToService condition pinning it to sagemaker, or remove CreateTrainingJob from this role.",
        "expected_boundary_join": "the privileges of the roles iam:PassRole can pass (not in this policy): the escalation's blast radius is whatever the most-privileged passable role can do.",
    },
]


def fixture_dir(scenario: dict) -> Path:
    return FIXTURES_DIR / scenario["id"]


def load_fixture_text(scenario: dict) -> str:
    """The raw policy JSON the agent is given: the permissions policy, plus trust / boundary if present."""
    d = fixture_dir(scenario)
    parts = []
    for path in sorted(d.glob("policy*.json")):
        doc = json.loads(path.read_text())
        parts.append(f"PERMISSIONS POLICY ({path.name}):\n" + json.dumps(doc, indent=2))
    trust = d / "trust-policy.json"
    if trust.exists():
        parts.append("TRUST POLICY (AssumeRolePolicyDocument):\n" + json.dumps(json.loads(trust.read_text()), indent=2))
    boundary = d / "boundary.json"
    if boundary.exists():
        parts.append("PERMISSIONS BOUNDARY (boundary.json):\n" + json.dumps(json.loads(boundary.read_text()), indent=2))
    return "\n\n".join(parts)


def expected_audit(scenario: dict) -> dict:
    """Run the deterministic reference audit to get the ground-truth findings for the judge."""
    audit = run_audit(fixture_dir(scenario))
    return {
        "codes": sorted(audit.codes()),
        "top_severity": audit.top_severity,
        "clean": audit.clean,
        "boundary_count": len(audit.boundary),
    }


if __name__ == "__main__":
    # `python tests/eval/scenarios.py` prints the ground-truth answers, no API needed.
    for s in SCENARIOS:
        exp = expected_audit(s)
        print(f"{s['id']:<40} codes={str(exp['codes']):<10} top={str(exp['top_severity']):<9} clean={exp['clean']}")
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall