oberconcejo/VS-CODE / .agents/skills/skill-comply
skill-comply
VerifiedVisualize whether skills, rules, and agent definitions are actually followed — auto-generates scenarios at 3 prompt strictness levels, runs agents, classifies behavioral sequences, and reports compliance rates with full tool call timelines. Use when checking whether agents actually follow the skills, rules, and definitions they were given, rather than assuming they do.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 2.4 KB |
| fixtures/compliant_trace.jsonl | other | 1.0 KB |
| fixtures/noncompliant_trace.jsonl | other | 629 B |
| fixtures/tdd_spec.yaml | config | 1.2 KB |
| prompts/classifier.md | markdown | 1.0 KB |
| prompts/scenario_generator.md | markdown | 1.9 KB |
| prompts/spec_generator.md | markdown | 1.6 KB |
| pyproject.toml | manifest | 360 B |
| scripts/__init__.py | script | 0 B |
| scripts/classifier.py | script | 2.9 KB |
| scripts/grader.py | script | 7.6 KB |
| scripts/parser.py | script | 2.7 KB |
| scripts/report.py | script | 6.3 KB |
| scripts/run.py | script | 4.2 KB |
| scripts/runner.py | script | 9.9 KB |
| scripts/scenario_generator.py | script | 1.8 KB |
| scripts/spec_generator.py | script | 2.2 KB |
| scripts/utils.py | script | 411 B |
| tests/test_classifier.py | script | 3.4 KB |
| tests/test_grader.py | script | 19.5 KB |
| tests/test_parser.py | script | 3.1 KB |
| tests/test_runner.py | script | 21.8 KB |
| tests/test_utf8.py | script | 10.1 KB |
scripts/grader.py
View on GitHub"""Grade observation traces against compliance specs using LLM classification."""
from __future__ import annotations
from dataclasses import dataclass, replace
from scripts.classifier import classify_events
from scripts.parser import ComplianceSpec, ObservationEvent, Step
@dataclass(frozen=True)
class StepResult:
step_id: str
detected: bool
evidence: tuple[ObservationEvent, ...]
failure_reason: str | None
@dataclass(frozen=True)
class ComplianceResult:
spec_id: str
steps: tuple[StepResult, ...]
compliance_rate: float
recommend_hook_promotion: bool
classification: dict[str, list[int]]
def _check_temporal_order(
step: Step,
event: ObservationEvent,
resolved: dict[str, list[ObservationEvent]],
classified: dict[str, list[ObservationEvent]],
graded: set[str],
) -> str | None:
"""Check before_step/after_step constraints. Returns failure reason or None."""
if step.detector.after_step is not None:
after_step = step.detector.after_step
after_events = resolved.get(after_step)
if after_events is None:
if after_step in graded:
# Graded and missing from `resolved` means it failed its own checks.
# Its classified events are not evidence for anything: reusing them
# here let a dependant pass on the strength of a failed prerequisite,
# and the overstatement carried down the whole chain.
return f"after_step '{after_step}' did not pass its own checks"
# Not graded yet, so this is a forward reference to a step declared later.
# The classifier's own output is the only thing available, and using it is
# what makes an out-of-order declaration work.
after_events = classified.get(after_step, [])
if not after_events:
return f"after_step '{after_step}' not yet detected"
latest_after = max(e.timestamp for e in after_events)
if event.timestamp <= latest_after:
return (
f"must occur after '{after_step}' "
f"(last at {latest_after}), but found at {event.timestamp}"
)
if step.detector.before_step is not None:
# Look ahead using LLM classification results. A failed reference step is NOT
# excluded here the way it is above: the two fall in opposite directions. An
# `after_step` fallback can only turn a failure into a pass, while a
# `before_step` one can only turn a pass into a failure, so dropping it would
# relax a constraint because some other step failed.
before_events = resolved.get(step.detector.before_step)
if before_events is None:
before_events = classified.get(step.detector.before_step, [])
if before_events:
earliest_before = min(e.timestamp for e in before_events)
if event.timestamp >= earliest_before:
return (
f"must occur before '{step.detector.before_step}' "
f"(first at {earliest_before}), but found at {event.timestamp}"
)
return None
def _demote_steps_resting_on_failures(
step_results: tuple[StepResult, ...],
after_steps: dict[str, str],
) -> tuple[StepResult, ...]:
"""Undo passes that rest on an `after_step` which ended up failing.
A step declared before its prerequisite is graded against the classifier's raw
events for that step, because `resolved` has nothing for it yet. That fallback is
what makes an out-of-order declaration work, but the prerequisite can go on to
fail its own checks, and then the dependant is left passing on evidence that
never held. Repeat until nothing changes: demoting one step can invalidate
whatever depended on it, whichever order the two were declared in. Demotion only
ever removes passes, so the loop terminates.
"""
results = step_results
while True:
failed = {result.step_id for result in results if not result.detected}
demote = {
result.step_id
for result in results
if result.detected and after_steps.get(result.step_id) in failed
}
if not demote:
return results
results = tuple(
replace(
result,
detected=False,
evidence=(),
failure_reason=(
f"after_step '{after_steps[result.step_id]}' "
"did not pass its own checks"
),
)
if result.step_id in demote
else result
for result in results
)
def grade(
spec: ComplianceSpec,
trace: list[ObservationEvent],
classifier_model: str = "haiku",
) -> ComplianceResult:
"""Grade a trace against a compliance spec using LLM classification."""
sorted_trace = sorted(trace, key=lambda e: e.timestamp)
# Step 1: LLM classifies all events in one batch call
classification = classify_events(spec, sorted_trace, model=classifier_model)
# Convert indices to events
classified: dict[str, list[ObservationEvent]] = {
step_id: [sorted_trace[i] for i in indices if 0 <= i < len(sorted_trace)]
for step_id, indices in classification.items()
}
# Step 2: Check temporal ordering (deterministic)
resolved: dict[str, list[ObservationEvent]] = {}
# Steps already graded, pass or fail. `resolved` alone cannot tell "failed" from
# "not reached yet", and those need opposite answers in `_check_temporal_order`.
graded: set[str] = set()
step_results: list[StepResult] = []
for step in spec.steps:
candidates = classified.get(step.id, [])
matched: list[ObservationEvent] = []
failure_reason: str | None = None
for event in candidates:
temporal_fail = _check_temporal_order(step, event, resolved, classified, graded)
if temporal_fail is None:
matched.append(event)
break
else:
failure_reason = temporal_fail
detected = len(matched) > 0
if detected:
resolved[step.id] = matched
elif failure_reason is None:
failure_reason = f"no matching event classified for step '{step.id}'"
graded = graded | {step.id}
step_results.append(StepResult(
step_id=step.id,
detected=detected,
evidence=tuple(matched),
failure_reason=failure_reason if not detected else None,
))
# `graded` catches a prerequisite that had already failed when its dependant was
# graded. The other direction needs a second pass: a dependant declared first is
# graded against the classifier's raw events for a prerequisite that has not run
# yet, and only later does that prerequisite fail.
after_steps = {
step.id: step.detector.after_step
for step in spec.steps
if step.detector.after_step is not None
}
resolved_results = _demote_steps_resting_on_failures(tuple(step_results), after_steps)
required_ids = {s.id for s in spec.steps if s.required}
required_steps = [s for s in resolved_results if s.step_id in required_ids]
detected_required = sum(1 for s in required_steps if s.detected)
total_required = len(required_steps)
compliance_rate = detected_required / total_required if total_required > 0 else 1.0
return ComplianceResult(
spec_id=spec.id,
steps=resolved_results,
compliance_rate=compliance_rate,
recommend_hook_promotion=compliance_rate < spec.threshold_promote_to_hook,
classification=classification,
)
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 467k | Updated | Claude CodeCodex | Verified | Install |
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 467k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 319k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 319k | Updated | Claude CodeCodex | Verified | Install |