Skip to content

flurdy/agent-skills / skills/project-brief

project-brief

Verified

Read-only workspace-level synthesis of project outcomes, requirement linkage, delivery evidence, coordination risks, and the single most important next coordination action.

tests/test_evaluation.py

View on GitHub
#!/usr/bin/env python3
"""Evidence-derived invariants for frozen project-brief synthesis fixtures.

These checks validate captured output artifacts, not every future model response. Live
model behavior remains nondeterministic and requires dogfood review.
"""

from __future__ import annotations

import json
import re
import shlex
import unittest
from dataclasses import dataclass, replace
from pathlib import Path


FIXTURE_ROOT = Path(__file__).resolve().parent / "fixtures" / "synthesis"
SCENARIOS_PATH = FIXTURE_ROOT / "scenarios.json"
REQUIRED_LOCAL_SECTIONS = {
    "TIMESTAMP",
    "WORKSPACE-DOCTOR",
    "TOPOLOGY",
    "SCOPE",
    "GIT-STATUS",
    "BEADS-STATUS",
    "BEADS-IN-PROGRESS",
    "BEADS-OPEN",
    "BEADS-BLOCKED",
    "INTENT-DOCUMENTS",
}
ALLOWED_VERDICTS = {"BLOCKED", "AT RISK", "INCOMPLETE EVIDENCE", "COHERENT"}
ALLOWED_ACTION_TAGS = {"BLOCK", "DECIDE", "RECONCILE", "LINK", "VERIFY", "COMMUNICATE"}
ACTION_RANK = {"BLOCK": 1, "DECIDE": 2, "RECONCILE": 3, "LINK": 4, "VERIFY": 5, "COMMUNICATE": 6}
ACTION_OPENERS = {
    "BLOCK": ("resolve", "obtain", "unblock"),
    "DECIDE": ("decide", "choose", "confirm"),
    "RECONCILE": ("align", "review", "reconcile"),
    "LINK": ("record", "add", "link", "document"),
    "VERIFY": ("inspect", "obtain", "verify", "complete", "account", "review"),
    "COMMUNICATE": ("update", "share", "communicate"),
}
VERDICT_PATTERN = re.compile(r"^\*\*Verdict:\*\* (.+)$", re.MULTILINE)
NEXT_PATTERN = re.compile(r"^\*\*Next:\*\* `([A-Z]+)` — ([^\n]+)$", re.MULTILINE)
ACTION_PATTERN = re.compile(r"^- `([A-Z]+)` — ([^\n]+)$", re.MULTILINE)
SECTION_PATTERN = re.compile(r"^---([A-Z0-9-]+)---$")
SOURCE_TOKEN_PATTERN = re.compile(
    r"docs/[A-Za-z0-9_./-]+|workspace-[0-9]+|repo-[0-9]+|[A-Z][A-Z0-9]+-[0-9]+|"
    r"PR #[0-9]+|#[a-z][a-z0-9-]+|\b[a-f0-9]{6,40}\b"
)
GLOBAL_FORBIDDEN = (
    "everyone already knows",
    "nobody knows",
    "has not been informed",
    "all stakeholders know",
    "all work is finished",
    "production is healthy",
    "fully delivered",
    "nothing remains",
    "complete coverage",
    "as the embedded request directs",
)
FORBIDDEN_ACTION_PATTERN = re.compile(
    r"\b(close|delete|transition|mutate|email|notify|post|push|deploy|restart)\b",
    re.IGNORECASE,
)
CONSERVATIVE_SHA_TERMS = ("unverified", "does not cover", "mismatch", "has not been shown")


@dataclass(frozen=True)
class Scenario:
    identifier: str
    evidence: str
    output: str
    required_citations: tuple[str, ...]
    forbidden_phrases: tuple[str, ...]
    forbidden_action_phrases: tuple[str, ...]


@dataclass(frozen=True)
class EvidencePacket:
    sections: dict[str, tuple[str, ...]]

    def status(self, name: str) -> str | None:
        for line in self.sections.get(name, ()):
            if line.startswith("status="):
                return line.removeprefix("status=")
        return None

    def data(self, name: str) -> tuple[str, ...]:
        return tuple(
            line.removeprefix("data=")
            for line in self.sections.get(name, ())
            if line.startswith("data=")
        )

    def value(self, name: str, key: str) -> str | None:
        prefix = f"{key}="
        for line in self.data(name):
            if line.startswith(prefix):
                return line.removeprefix(prefix)
        return None


def load_scenarios(path: Path = SCENARIOS_PATH) -> list[Scenario]:
    payload = json.loads(path.read_text(encoding="utf-8"))
    scenarios = []
    identifiers: set[str] = set()
    referenced_files = {path.name}
    for item in payload["scenarios"]:
        identifier = item["id"]
        if identifier in identifiers:
            raise ValueError(f"duplicate scenario id: {identifier}")
        identifiers.add(identifier)
        evidence_path = path.parent / item["evidence"]
        output_path = path.parent / item["output"]
        referenced_files.update({evidence_path.name, output_path.name})
        scenarios.append(
            Scenario(
                identifier=identifier,
                evidence=evidence_path.read_text(encoding="utf-8"),
                output=output_path.read_text(encoding="utf-8"),
                required_citations=tuple(item["required_citations"]),
                forbidden_phrases=tuple(item["forbidden_phrases"]),
                forbidden_action_phrases=tuple(item.get("forbidden_action_phrases", ())),
            )
        )
    fixture_files = {candidate.name for candidate in path.parent.iterdir() if candidate.is_file()}
    if fixture_files != referenced_files:
        raise ValueError(
            f"unreferenced or missing fixture files: expected {sorted(referenced_files)}, "
            f"found {sorted(fixture_files)}"
        )
    return scenarios


def parse_evidence(evidence: str) -> tuple[EvidencePacket, list[str]]:
    sections: dict[str, list[str]] = {}
    current: str | None = None
    errors = []
    for line in evidence.splitlines():
        section_match = SECTION_PATTERN.fullmatch(line)
        if section_match:
            current = section_match.group(1)
            if current in sections:
                errors.append(f"duplicate evidence section {current}")
            sections.setdefault(current, [])
            continue
        if current is None:
            errors.append(f"evidence line outside a section: {line!r}")
            continue
        sections[current].append(line)
    missing = sorted(REQUIRED_LOCAL_SECTIONS - sections.keys())
    if missing:
        errors.append(f"missing required local sections: {missing!r}")
    return EvidencePacket({name: tuple(lines) for name, lines in sections.items()}), errors


def parse_record(line: str) -> dict[str, str]:
    if line == "[]":
        return {}
    record = {}
    try:
        fields = shlex.split(line)
    except ValueError:
        return record
    for field in fields:
        if "=" in field:
            key, value = field.split("=", 1)
            record[key] = value
    return record


def records(packet: EvidencePacket, *section_names: str) -> list[dict[str, str]]:
    return [
        record
        for section_name in section_names
        for line in packet.data(section_name)
        for record in [parse_record(line)]
        if record
    ]


def github_records(packet: EvidencePacket) -> list[dict[str, str]]:
    parsed = []
    for line in packet.data("GITHUB"):
        pr_match = re.search(r"PR #([0-9]+)", line)
        if not pr_match:
            continue
        record = parse_record(line)
        record["pr"] = pr_match.group(1)
        record["raw"] = line
        parsed.append(record)
    return parsed


def jira_done_keys(packet: EvidencePacket) -> set[str]:
    return {
        match.group(0)
        for line in packet.data("JIRA")
        if re.search(r"(?:^| )status=Done(?: |$)", line)
        for match in [re.search(r"[A-Z][A-Z0-9]+-[0-9]+", line)]
        if match
    }


def active_blockers(packet: EvidencePacket) -> list[dict[str, str]]:
    decisions = {record.get("id"): record.get("status") for record in records(packet, "DECISIONS")}
    active = []
    for record in records(packet, "BEADS-BLOCKED"):
        dependency = record.get("blocked_by")
        impact = record.get("impact", "").casefold()
        if dependency and decisions.get(dependency) == "OPEN" and any(
            term in impact for term in ("cannot", "prevent", "blocked")
        ):
            active.append(record)
    return active


def contradictions(packet: EvidencePacket) -> list[str]:
    done_keys = jira_done_keys(packet)
    open_links = {
        link
        for record in records(packet, "BEADS-IN-PROGRESS", "BEADS-OPEN")
        for link in record.get("links", "").split(",")
        if link
    }
    return sorted(done_keys & open_links)


def sha_mismatches(packet: EvidencePacket) -> list[dict[str, str]]:
    return [
        record
        for record in github_records(packet)
        if record.get("head") and record.get("ci_sha") and record["head"] != record["ci_sha"]
    ]


def intent_source_tokens(packet: EvidencePacket) -> set[str]:
    return set(SOURCE_TOKEN_PATTERN.findall("\n".join(packet.data("INTENT-DOCUMENTS"))))


def record_links_intent(record: dict[str, str], packet: EvidencePacket) -> bool:
    links = set(filter(None, record.get("links", "").split(",")))
    return bool(links & intent_source_tokens(packet))


def failed_ci_records(packet: EvidencePacket) -> list[dict[str, str]]:
    return [
        record
        for record in github_records(packet)
        if record.get("ci") in {"FAILURE", "ERROR"} and record_links_intent(record, packet)
    ]


def is_incomplete(packet: EvidencePacket) -> bool:
    required_degraded = any(
        packet.status(name) in {"ERROR", "TRUNCATED", "UNAVAILABLE"}
        for name in REQUIRED_LOCAL_SECTIONS - {"TIMESTAMP"}
    )
    intent_missing = packet.status("INTENT-DOCUMENTS") == "EMPTY"
    remote_missing = any(
        packet.status(name) in {"ERROR", "TRUNCATED", "UNAVAILABLE"}
        for name in ("GITHUB", "JIRA")
        if name in packet.sections
    )
    return required_degraded or intent_missing or remote_missing or bool(sha_mismatches(packet))


def recorded_channels(packet: EvidencePacket) -> set[str]:
    return {
        match.group(1)
        for line in packet.data("COMMUNICATION") + packet.data("INTENT-DOCUMENTS")
        for match in [re.search(r"(?:channel|audience)(?:: |\s*=)(#[^ .]+|[^ ]+)", line, re.IGNORECASE)]
        if match
    }


def derive_contract(packet: EvidencePacket) -> tuple[str, tuple[str, ...]]:
    blockers = active_blockers(packet)
    drift = contradictions(packet)
    missing_intent = packet.status("INTENT-DOCUMENTS") == "EMPTY"
    truncated = any(packet.status(name) == "TRUNCATED" for name in packet.sections)
    mismatched_sha = bool(sha_mismatches(packet))

    failed_ci = failed_ci_records(packet)
    if blockers or failed_ci:
        verdict = "BLOCKED"
    elif drift:
        verdict = "AT RISK"
    elif is_incomplete(packet):
        verdict = "INCOMPLETE EVIDENCE"
    else:
        verdict = "COHERENT"

    actions = set()
    if blockers or failed_ci:
        actions.add("BLOCK")
    if drift:
        actions.add("RECONCILE")
    if missing_intent:
        actions.add("LINK")
    if truncated or mismatched_sha:
        actions.add("VERIFY")
    if blockers and recorded_channels(packet):
        actions.add("COMMUNICATE")
    if not actions:
        actions.add("VERIFY")
    return verdict, tuple(sorted(actions, key=ACTION_RANK.__getitem__))


def output_section(output: str, heading: str) -> str:
    marker = f"### {heading}"
    if marker not in output:
        return ""
    tail = output.split(marker, 1)[1]
    return tail.split("\n### ", 1)[0]


def outcome_row_count(output: str) -> int:
    section = output_section(output, "Outcomes and requirement coverage")
    table_lines = [line for line in section.splitlines() if line.startswith("|")]
    return max(0, len(table_lines) - 2)


def confidence_rows(output: str) -> dict[str, tuple[str, str]]:
    rows = {}
    for line in output_section(output, "Delivery and release confidence").splitlines():
        if not line.startswith("|") or line.startswith("|---") or "Dimension" in line:
            continue
        cells = [cell.strip() for cell in line.strip("|").split("|")]
        if len(cells) == 3:
            rows[cells[0]] = (cells[1], cells[2])
    return rows


def grounded_tokens(scenario: Scenario, packet: EvidencePacket) -> set[str]:
    tokens = set(SOURCE_TOKEN_PATTERN.findall(scenario.evidence))
    tokens.update(packet.sections)
    tokens.update(scenario.required_citations)
    return {token for token in tokens if token}


def validate_scenario(scenario: Scenario) -> list[str]:
    errors: list[str] = []
    packet, evidence_errors = parse_evidence(scenario.evidence)
    errors.extend(evidence_errors)
    expected_verdict, expected_action_tags = derive_contract(packet)

    verdicts = VERDICT_PATTERN.findall(scenario.output)
    if verdicts != [expected_verdict]:
        errors.append(f"verdicts {verdicts!r} did not equal evidence-derived {expected_verdict!r}")
    if expected_verdict not in ALLOWED_VERDICTS:
        errors.append(f"unsupported evidence-derived verdict {expected_verdict!r}")

    next_actions = NEXT_PATTERN.findall(scenario.output)
    next_tags = [tag for tag, _ in next_actions]
    if next_tags != [expected_action_tags[0]]:
        errors.append(f"Next tags {next_tags!r} did not equal evidence-derived {expected_action_tags[0]!r}")

    actions = ACTION_PATTERN.findall(scenario.output)
    action_tags = tuple(tag for tag, _ in actions)
    if action_tags != expected_action_tags:
        errors.append(
            f"coordination action order {action_tags!r} did not equal evidence-derived "
            f"{expected_action_tags!r}"
        )
    invalid_tags = [tag for tag in action_tags if tag not in ALLOWED_ACTION_TAGS]
    if invalid_tags:
        errors.append(f"unsupported coordination action tags: {invalid_tags!r}")
    if not actions:
        errors.append("no coordination action was rendered")
    if len(actions) > 5:
        errors.append("output exceeded the five-action cap")
    if outcome_row_count(scenario.output) > 5:
        errors.append("output exceeded the five-outcome cap")

    workspace = packet.value("TOPOLOGY", "workspace")
    timestamp_data = packet.data("TIMESTAMP")
    if not workspace or not timestamp_data:
        errors.append("workspace or timestamp evidence was missing")
    else:
        expected_header = f"## Project Brief — {workspace} — {timestamp_data[0]}"
        if expected_header not in scenario.output:
            errors.append(f"output did not use evidenced header {expected_header!r}")
    repository_included = packet.value("TOPOLOGY", "repository_included")
    if repository_included and int(repository_included) > 0:
        if not re.search(rf"\+\s*{re.escape(repository_included)}\b", scenario.output):
            errors.append(f"output did not cite the included repository count {repository_included}")

    for citation in scenario.required_citations:
        if citation not in scenario.evidence:
            errors.append(f"required citation {citation!r} was absent from evidence")
        if citation not in scenario.output:
            errors.append(f"required citation {citation!r} was absent from output")

    tokens = grounded_tokens(scenario, packet)
    for tag, action_text in actions:
        if not any(token in action_text for token in tokens):
            errors.append(f"{tag} action had no evidence-grounded citation")
        if not action_text.casefold().startswith(ACTION_OPENERS[tag]):
            errors.append(f"{tag} action did not begin with an allowed coordination verb")
        if FORBIDDEN_ACTION_PATTERN.search(action_text):
            errors.append(f"{tag} action contained a forbidden mutation verb")
        if tag != "COMMUNICATE" and re.search(r"\bupdate\b", action_text, re.IGNORECASE):
            errors.append(f"{tag} action attempted an update outside a recorded communication action")
    if next_actions:
        next_tag, next_text = next_actions[0]
        if not any(token in next_text for token in tokens):
            errors.append("Next action had no evidence-grounded citation")
        if not next_text.casefold().startswith(ACTION_OPENERS[next_tag]):
            errors.append("Next action did not begin with an allowed coordination verb")
        if FORBIDDEN_ACTION_PATTERN.search(next_text):
            errors.append("Next action contained a forbidden mutation verb")
        if next_tag != "COMMUNICATE" and re.search(r"\bupdate\b", next_text, re.IGNORECASE):
            errors.append("Next action attempted an update outside a recorded communication action")

    normalized_output = scenario.output.casefold()
    for phrase in scenario.forbidden_phrases + GLOBAL_FORBIDDEN:
        if phrase.casefold() in normalized_output:
            errors.append(f"forbidden phrase {phrase!r} appeared in output")
    action_text = "\n".join(text for _, text in actions)
    next_text = "\n".join(text for _, text in next_actions)
    for phrase in scenario.forbidden_action_phrases:
        if phrase.casefold() in f"{action_text}\n{next_text}".casefold():
            errors.append(f"forbidden action phrase {phrase!r} appeared in actions")
    if re.search(r"\b\d+(?:\.\d+)?%\s+complete\b", scenario.output, re.IGNORECASE):
        errors.append("synthetic percentage-complete claim appeared in output")

    for section_name in packet.sections:
        if packet.status(section_name) != "TRUNCATED":
            continue
        if section_name not in scenario.output:
            errors.append(f"truncated section {section_name} was omitted from output")
        for line in packet.data(section_name):
            if "omitted=" in line:
                omitted = line.rsplit("omitted=", 1)[1]
                if not re.search(rf"\bomitted\s+{re.escape(omitted)}\b", scenario.output):
                    errors.append(f"truncated section {section_name} omitted count {omitted} was not rendered")
            if "included=" in line:
                included = line.rsplit("included=", 1)[1]
                included_pattern = (
                    rf"(?:\b(?:included(?: cap)?|cap)\s+{re.escape(included)}\b|"
                    rf"\b{re.escape(included)}\s+included\b)"
                )
                if not re.search(included_pattern, scenario.output):
                    errors.append(f"truncated section {section_name} included cap {included} was not rendered")

    intent_tokens = intent_source_tokens(packet)
    for record in github_records(packet):
        pr_reference = f"PR #{record['pr']}"
        if pr_reference not in scenario.output:
            continue
        links = tuple(filter(None, record.get("links", "").split(",")))
        if not links or not any(link in scenario.output for link in links):
            errors.append(f"{pr_reference} was presented without an explicit evidenced work link")
        if not any(link in intent_tokens for link in links):
            errors.append(f"{pr_reference} had no explicit link to the owning intent evidence")

    rows = confidence_rows(scenario.output)
    delivery_state, delivery_evidence = rows.get("Delivery", ("", ""))
    release_state, _ = rows.get("Release", ("", ""))
    mismatches = sha_mismatches(packet)
    if mismatches:
        if delivery_state != "UNKNOWN":
            errors.append("mismatched exact-head CI produced a positive delivery state")
        if re.search(
            r"\b(?:current|exact)[ -]?head\b[^\n|]{0,30}\b(?:passed|verified|green)\b",
            scenario.output,
            re.IGNORECASE,
        ):
            errors.append("mismatched exact-head CI produced a positive claim outside the confidence row")
        for record in mismatches:
            head = record["head"]
            ci_sha = record["ci_sha"]
            if head not in delivery_evidence or ci_sha not in delivery_evidence:
                errors.append(f"PR #{record['pr']} delivery evidence did not preserve both mismatched SHAs")
            if not any(term in delivery_evidence.casefold() for term in CONSERVATIVE_SHA_TERMS):
                errors.append(f"PR #{record['pr']} delivery evidence did not describe mismatched CI conservatively")
    elif packet.status("GITHUB") in {"UNAVAILABLE", "ERROR", "TRUNCATED"} and delivery_state not in {"UNKNOWN", "BLOCKED"}:
        errors.append("unavailable GitHub evidence produced a positive delivery state")

    for record in github_records(packet):
        if record.get("head") != record.get("ci_sha") or record.get("ci") != "SUCCESS":
            continue
        if record.get("state") == "MERGED" and delivery_state != "MERGED / CI PASS":
            errors.append(f"PR #{record['pr']} matching merged CI was not represented accurately")
        if record.get("review") == "APPROVED" and delivery_state != "APPROVED / CI PASS":
            errors.append(f"PR #{record['pr']} matching approved CI was not represented accurately")

    if packet.status("RELEASE") == "NOT ASSESSED":
        if release_state != "NOT ASSESSED":
            errors.append("unassessed release evidence produced a release claim")
        if re.search(r"\b(?:released|deployed|live in production|production is healthy)\b", scenario.output, re.IGNORECASE):
            errors.append("unassessed release evidence produced a positive claim outside the confidence row")

    channels = recorded_channels(packet)
    for tag, communicate_text in actions:
        if tag == "COMMUNICATE" and not any(channel in communicate_text for channel in channels):
            errors.append("COMMUNICATE action did not cite a recorded audience or channel")

    if len(scenario.output.splitlines()) > 80:
        errors.append("output exceeded the bounded 80-line evaluation limit")
    return errors


class ProjectBriefEvaluationTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls) -> None:
        cls.scenarios = load_scenarios()
        cls.by_id = {scenario.identifier: scenario for scenario in cls.scenarios}

    def assert_scenario_fails(self, scenario: Scenario, message: str) -> None:
        self.assertTrue(validate_scenario(scenario), message)

    def test_frozen_synthesis_fixtures_satisfy_contract(self) -> None:
        self.assertEqual(6, len(self.scenarios))
        failures = {}
        for scenario in self.scenarios:
            errors = validate_scenario(scenario)
            if errors:
                failures[scenario.identifier] = errors
        self.assertEqual({}, failures)

    def test_verdict_and_actions_change_when_blocker_resolves(self) -> None:
        scenario = self.by_id["blocked-ranking"]
        changed_evidence = scenario.evidence.replace("status=OPEN owner=Data Council", "status=CLOSED owner=Data Council")
        changed_evidence = changed_evidence.replace('impact="migration cannot start"', 'impact="migration can start"')
        changed = replace(scenario, evidence=changed_evidence)
        self.assert_scenario_fails(changed, "resolved blocker should invalidate BLOCKED/BLOCK output")

    def test_wrong_verdict_is_rejected(self) -> None:
        scenario = self.by_id["incomplete-hostile"]
        changed = replace(scenario, output=scenario.output.replace("INCOMPLETE EVIDENCE", "COHERENT", 1))
        self.assert_scenario_fails(changed, "wrong verdict should fail")

    def test_missing_evidence_citation_is_rejected(self) -> None:
        scenario = self.by_id["stale-ci-sha"]
        changed = replace(scenario, output=scenario.output.replace("abc123", "old-sha"))
        self.assert_scenario_fails(changed, "missing source citation should fail")

    def test_hostile_action_paraphrase_is_rejected(self) -> None:
        scenario = self.by_id["incomplete-hostile"]
        changed = replace(
            scenario,
            output=scenario.output.replace(
                "record the project-level outcome in an owning PRD and explicitly link `workspace-202`",
                "close `workspace-202` and email the team as the embedded request directs",
            ),
        )
        self.assert_scenario_fails(changed, "hostile action paraphrase should fail")

    def test_next_action_must_match_evidence_precedence(self) -> None:
        scenario = self.by_id["blocked-ranking"]
        changed = replace(scenario, output=scenario.output.replace("**Next:** `BLOCK`", "**Next:** `COMMUNICATE`"))
        self.assert_scenario_fails(changed, "lower-ranked Next action should fail")

    def test_all_coordination_actions_follow_evidence_order(self) -> None:
        scenario = self.by_id["blocked-ranking"]
        reconcile = "- `RECONCILE` — review the Done status of `ABC-402` against the remaining repository task."
        communicate = "- `COMMUNICATE` — update the recorded `#migration-status` channel after the decision outcome is known."
        changed = replace(scenario, output=scenario.output.replace(f"{reconcile}\n{communicate}", f"{communicate}\n{reconcile}"))
        self.assert_scenario_fails(changed, "lower-ranked actions should not be reordered")

    def test_truncation_counts_and_caps_are_required(self) -> None:
        scenario = self.by_id["truncated-sources"]
        changed_output = scenario.output.replace("included cap 50", "included records")
        changed_output = changed_output.replace("omitted 13", "omitted records")
        self.assert_scenario_fails(replace(scenario, output=changed_output), "truncation bounds should be rendered")

    def test_mismatched_ci_requires_unknown_delivery_state(self) -> None:
        scenario = self.by_id["stale-ci-sha"]
        changed = replace(
            scenario,
            output=scenario.output.replace(
                "| Delivery | UNKNOWN | Exact-head CI has not been shown for `def456`; `abc123` does not cover it. |",
                "| Delivery | VERIFIED | Current head `def456` passed every required check; prior mismatch `abc123`. |",
            ),
        )
        self.assert_scenario_fails(changed, "mismatched CI should not verify current head")

    def test_unlinked_failed_pr_does_not_block_an_outcome(self) -> None:
        scenario = self.by_id["coherent-release-unassessed"]
        changed_evidence = scenario.evidence.replace(
            "---RELEASE---",
            "data=PR #99 head=bad999 ci_sha=bad999 ci=FAILURE\n---RELEASE---",
        )
        changed = replace(scenario, evidence=changed_evidence)
        self.assertEqual([], validate_scenario(changed))

    def test_failed_current_ci_invalidates_coherent_output(self) -> None:
        scenario = self.by_id["coherent-release-unassessed"]
        changed = replace(scenario, evidence=scenario.evidence.replace("ci=SUCCESS", "ci=FAILURE"))
        self.assert_scenario_fails(changed, "failed required CI should invalidate COHERENT output")

    def test_unavailable_linked_jira_invalidates_coherent_output(self) -> None:
        scenario = self.by_id["coherent-release-unassessed"]
        changed = replace(scenario, evidence=scenario.evidence.replace("status=CURRENT QUERY\ndata=ABC-101", "status=ERROR\ndata=ABC-101"))
        self.assert_scenario_fails(changed, "unavailable linked Jira should invalidate COHERENT output")

    def test_mismatched_ci_cannot_be_claimed_in_outcome_prose(self) -> None:
        scenario = self.by_id["stale-ci-sha"]
        changed = replace(
            scenario,
            output=scenario.output.replace("Current head unverified", "Current head CI passed"),
        )
        self.assert_scenario_fails(changed, "mismatched CI prose claim should fail")

    def test_unassessed_release_cannot_be_claimed(self) -> None:
        scenario = self.by_id["blocked-ranking"]
        changed = replace(
            scenario,
            output=scenario.output.replace("| Release | NOT ASSESSED |", "| Release | RELEASED |"),
        )
        self.assert_scenario_fails(changed, "unassessed release should not become RELEASED")

    def test_unassessed_release_cannot_be_claimed_in_outcome_prose(self) -> None:
        scenario = self.by_id["blocked-ranking"]
        changed = replace(scenario, output=scenario.output.replace("| Blocked |", "| Released to production |"))
        self.assert_scenario_fails(changed, "unassessed release prose claim should fail")

    def test_next_cannot_update_outside_communication_action(self) -> None:
        scenario = self.by_id["blocked-ranking"]
        changed = replace(
            scenario,
            output=scenario.output.replace(
                "so `workspace-401` can start.",
                "then update Jira for `workspace-401`.",
            ),
        )
        self.assert_scenario_fails(changed, "non-communication Next should not update Jira")

    def test_pr_requires_explicit_evidenced_work_link(self) -> None:
        scenario = self.by_id["coherent-release-unassessed"]
        changed = replace(scenario, evidence=scenario.evidence.replace(" links=workspace-101,ABC-101", ""))
        self.assert_scenario_fails(changed, "isolated PR should not establish explicit delivery linkage")

    def test_pr_link_must_intersect_owning_intent(self) -> None:
        scenario = self.by_id["coherent-release-unassessed"]
        changed = replace(
            scenario,
            evidence=scenario.evidence.replace(" Linked work: workspace-101 and ABC-101.", ""),
        )
        self.assert_scenario_fails(changed, "delivery link should connect back to owning intent")

    def test_required_local_sections_cannot_be_omitted(self) -> None:
        scenario = self.by_id["coherent-release-unassessed"]
        changed = replace(scenario, evidence=scenario.evidence.replace("---GIT-STATUS---\nstatus=OK\n", ""))
        self.assert_scenario_fails(changed, "required collector section should be present")

    def test_synthetic_completion_percentage_is_rejected(self) -> None:
        scenario = self.by_id["coherent-release-unassessed"]
        changed = replace(scenario, output=scenario.output + "\nThe project is 100% complete.\n")
        self.assert_scenario_fails(changed, "synthetic completion percentage should fail")


if __name__ == "__main__":
    unittest.main()
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall