hashgraph-online/awesome-codex-plugins / plugins/boshu2/agentops/skills/validate
validate
VerifiedFreshly judge whether a finished change and its claims meet original acceptance: PASS, FAIL or NOT_PROVEN. Use when: asked for a go/no-go, sign-off or independent verdict.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 11.4 KB |
| references/judgment-receipts.md | markdown | 6.5 KB |
| references/mechanics.md | markdown | 9.0 KB |
| references/model-dispatch.md | markdown | 10.4 KB |
| references/validate.feature | other | 1.6 KB |
| tests/check_contract_corpus.py | script | 5.4 KB |
| tests/test_evidence_cli.py | script | 14.0 KB |
| tests/test_validate.pyexecutable | script | 15.8 KB |
| tests/validate.pyexecutable | script | 29.0 KB |
| tests/validate.shexecutable | script | 745 B |
tests/test_validate.py
View on GitHubfrom __future__ import annotations
import importlib.util
import json
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
import jsonschema
SPEC = importlib.util.spec_from_file_location("validate_tool", Path(__file__).with_name("validate.py"))
tool = importlib.util.module_from_spec(SPEC)
assert SPEC.loader
SPEC.loader.exec_module(tool)
class ValidateV2Tests(unittest.TestCase):
def draft(self):
return {
"acceptance_digest": "a" * 64,
"subject_manifest_digest": "b" * 64,
"author_context_id": "author",
"validator_context_id": "validator",
"freshness_attestation": {"source": "runtime", "attester_identity": "runtime-1"},
"verdict": "PASS",
"criteria": [{"id": "c1", "result": "PASS", "evidence_refs": ["e1"]}],
"findings": [],
"evidence_refs": ["e1"],
"checked": ["c1"],
"not_checked": [],
"validated_at": "2026-07-14T00:00:00Z",
}
def assert_schema_valid(self, artifact):
schema = json.loads((Path(__file__).parents[3] / "schemas" / "verdict.v2.schema.json").read_text())
jsonschema.Draft202012Validator(schema).validate(artifact)
def runtime_facts(self):
manifest = {
"schema_version": "subject-manifest.v1",
"declared_roots": ["src"],
"exclusions": [],
# One real file entry: build_manifest never emits an entry-less
# manifest for an implementation subject, and the store-verdict
# CLI refuses one outright.
"entries": [
{
"path": "src/app.py",
"kind": "file",
"executable": False,
"digest": "0" * 64,
}
],
}
manifest["canonical_manifest_digest"] = tool.digest_value(tool.manifest_identity(manifest))
return b"bead:agentops-test\nacceptance: works\n", manifest
def store_bound(
self,
draft,
destination,
*,
scope="PASS",
author="author",
validator="validator",
freshness_source="runtime",
freshness_attester="validator",
):
intent, manifest = self.runtime_facts()
return tool.store_verdict(
draft,
destination,
intent,
manifest,
author,
scope,
validator,
freshness_source,
freshness_attester,
)
def test_manifest_is_content_addressed_and_detects_mutation(self):
with tempfile.TemporaryDirectory() as raw:
root = Path(raw)
(root / "bin").mkdir()
subject = root / "bin" / "tool"
subject.write_text("one", encoding="utf-8")
subject.chmod(0o755)
manifest = tool.build_manifest(root, ["bin"], [])
self.assertTrue(tool.verify_manifest(manifest, root, None)[0])
subject.write_text("two", encoding="utf-8")
self.assertFalse(tool.verify_manifest(manifest, root, None)[0])
def test_git_metadata_is_not_identity_bearing(self):
with tempfile.TemporaryDirectory() as raw:
root = Path(raw)
(root / "value").write_text("same", encoding="utf-8")
first = tool.build_manifest(root, ["."], [], git_metadata={"commit": "one"})
second = tool.build_manifest(root, ["."], [], git_metadata={"commit": "two"})
self.assertEqual(first["canonical_manifest_digest"], second["canonical_manifest_digest"])
self.assertNotEqual(first["git_metadata"], second["git_metadata"])
self.assertTrue(tool.verify_manifest(first, root, None)[0])
self.assertTrue(tool.verify_manifest(second, root, None)[0])
def test_symlink_and_deletion_identity(self):
with tempfile.TemporaryDirectory() as raw:
root = Path(raw)
(root / "target").write_text("x", encoding="utf-8")
(root / "link").symlink_to("target")
base = tool.build_manifest(root, ["."], [])
(root / "target").unlink()
current = tool.build_manifest(root, ["."], [], base)
kinds = {entry["path"]: entry["kind"] for entry in current["entries"]}
self.assertEqual(kinds["link"], "symlink")
self.assertEqual(kinds["target"], "deletion")
def test_verdict_identity_floor_and_idempotence(self):
with tempfile.TemporaryDirectory() as raw:
draft = self.draft()
draft["author_context_id"] = "same"
draft["validator_context_id"] = "same"
first, path, existed = self.store_bound(draft, Path(raw), author="same", validator="same")
self.assertEqual(first["verdict"], "NOT_PROVEN")
self.assert_schema_valid(first)
self.assertFalse(existed)
second, second_path, existed = self.store_bound(draft, Path(raw), author="same", validator="same")
self.assertTrue(existed)
self.assertEqual(path, second_path)
self.assertEqual(json.loads(path.read_text())["artifact_digest"], first["artifact_digest"])
def test_runtime_identity_and_attestation_replace_missing_model_fields(self):
for missing in ("author_context_id", "validator_context_id", "freshness_attestation"):
with self.subTest(missing=missing), tempfile.TemporaryDirectory() as raw:
draft = self.draft()
draft.pop(missing)
artifact, _path, _existed = self.store_bound(draft, Path(raw))
self.assertEqual(artifact["verdict"], "PASS")
self.assert_schema_valid(artifact)
def test_runtime_validator_and_freshness_override_model_claims(self):
with tempfile.TemporaryDirectory() as raw:
draft = self.draft()
draft["validator_context_id"] = "model-claimed-validator"
draft["freshness_attestation"] = {"source": "caller", "attester_identity": "model-claimed-attester"}
artifact, _path, _existed = self.store_bound(draft, Path(raw))
self.assertEqual(artifact["validator_context_id"], "validator")
self.assertEqual(
artifact["freshness_attestation"],
{"source": "runtime", "attester_identity": "validator"},
)
self.assertEqual(artifact["verdict"], "PASS")
def test_pass_with_failed_criterion_is_downgraded(self):
with tempfile.TemporaryDirectory() as raw:
draft = self.draft()
draft["criteria"][0]["result"] = "FAIL"
artifact, _path, _existed = self.store_bound(draft, Path(raw))
self.assertEqual(artifact["verdict"], "NOT_PROVEN")
self.assert_schema_valid(artifact)
def test_pass_without_evidence_is_downgraded(self):
mutations = (
lambda draft: draft.__setitem__("evidence_refs", []),
lambda draft: draft.__setitem__("checked", []),
lambda draft: draft["criteria"][0].__setitem__("evidence_refs", []),
)
for mutate in mutations:
with self.subTest(mutate=mutate), tempfile.TemporaryDirectory() as raw:
draft = self.draft()
mutate(draft)
artifact, _path, _existed = self.store_bound(draft, Path(raw))
self.assertEqual(artifact["verdict"], "NOT_PROVEN")
self.assertIn("PASS requires evidence", artifact["findings"][-1]["summary"])
self.assert_schema_valid(artifact)
def test_intent_snapshot_is_content_addressed_and_idempotent(self):
with tempfile.TemporaryDirectory() as raw:
destination = Path(raw)
payload = b"caller intent\nacceptance: works\n"
first, existed = tool.snapshot_intent(payload, destination)
self.assertFalse(existed)
self.assertEqual(first.name, f"{tool.hashlib.sha256(payload).hexdigest()}.intent")
self.assertEqual(first.read_bytes(), payload)
second, existed = tool.snapshot_intent(payload, destination)
self.assertTrue(existed)
self.assertEqual(first, second)
def test_store_verdict_cli_snapshots_intent_before_persistence(self):
with tempfile.TemporaryDirectory() as raw:
workspace = Path(raw)
intent, manifest = self.runtime_facts()
intent_path = workspace / "intent.txt"
manifest_path = workspace / "manifest.json"
draft_path = workspace / "draft.json"
intent_path.write_bytes(intent)
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
draft_path.write_text(json.dumps(self.draft()), encoding="utf-8")
result = subprocess.run(
[
sys.executable,
str(Path(__file__).with_name("validate.py")),
"store-verdict",
"--draft",
str(draft_path),
"--intent-source",
str(intent_path),
"--subject-manifest",
str(manifest_path),
"--author-context-id",
"author",
"--validator-context-id",
"validator",
"--freshness-source",
"runtime",
"--freshness-attester-id",
"validator",
"--scope-result",
"PASS",
"--workspace",
str(workspace),
],
check=False,
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0, result.stderr)
response = json.loads(result.stdout)
snapshot = Path(response["intent_ref"])
self.assertEqual(snapshot.read_bytes(), intent)
self.assertEqual(response["acceptance_digest"], tool.hashlib.sha256(intent).hexdigest())
def test_corrupt_existing_digest_yields_new_not_proven_artifact(self):
with tempfile.TemporaryDirectory() as raw:
destination = Path(raw)
draft = self.draft()
artifact, path, _ = self.store_bound(draft, destination)
path.write_text("corrupt\n", encoding="utf-8")
replacement, replacement_path, existed = self.store_bound(draft, destination)
self.assertEqual(replacement["verdict"], "NOT_PROVEN")
self.assertNotEqual(replacement["artifact_digest"], artifact["artifact_digest"])
self.assertNotEqual(replacement_path, path)
self.assertFalse(existed)
self.assert_schema_valid(replacement)
def test_incomplete_draft_is_rejected_without_writing(self):
with tempfile.TemporaryDirectory() as raw:
with self.assertRaisesRegex(tool.ContractError, "missing required fields"):
tool.store_verdict({"verdict": "FAIL"}, Path(raw))
self.assertEqual(list(Path(raw).iterdir()), [])
def test_unknown_field_is_rejected_without_writing(self):
with tempfile.TemporaryDirectory() as raw:
draft = self.draft()
draft["next_action"] = "repair"
with self.assertRaisesRegex(tool.ContractError, "unknown fields"):
self.store_bound(draft, Path(raw))
self.assertEqual(list(Path(raw).iterdir()), [])
def test_pass_without_runtime_facts_is_not_proven(self):
with tempfile.TemporaryDirectory() as raw:
artifact, _path, _existed = tool.store_verdict(self.draft(), Path(raw))
self.assertEqual(artifact["verdict"], "NOT_PROVEN")
self.assertIn("runtime intent source is missing", artifact["findings"][-1]["summary"])
self.assert_schema_valid(artifact)
def test_runtime_facts_override_model_authored_digests(self):
with tempfile.TemporaryDirectory() as raw:
draft = self.draft()
draft["acceptance_digest"] = "c" * 64
draft["subject_manifest_digest"] = "d" * 64
artifact, _path, _existed = self.store_bound(draft, Path(raw))
intent, manifest = self.runtime_facts()
self.assertEqual(artifact["acceptance_digest"], tool.hashlib.sha256(intent).hexdigest())
self.assertEqual(artifact["subject_manifest_digest"], manifest["canonical_manifest_digest"])
self.assertEqual(artifact["verdict"], "PASS")
def test_honest_scoped_pass_round_trips_through_documented_homes(self):
"""An honest draft with declared non-goals is representable as PASS.
Both drafts below carry the same honest content. Draft A parks the
declared non-goals in ``not_checked``, which is reserved for unverified
in-scope acceptance: the result is NOT_PROVEN and the finding names
where each caveat belongs. Draft B moves the same caveats into the
documented homes and stores PASS with every caveat still readable in
the persisted artifact. Nothing is deleted to earn the PASS.
"""
bounded = "proven by the unit suite; the full integration matrix was not replayed"
boundary = "declared non-goal; the diff proves cli/** untouched"
with tempfile.TemporaryDirectory() as raw:
draft_a = self.draft()
draft_a["not_checked"] = [
"cli/** (declared non-goal)",
"Windows runners (declared non-goal)",
]
artifact_a, _path, _existed = self.store_bound(draft_a, Path(raw))
self.assertEqual(artifact_a["verdict"], "NOT_PROVEN")
summary = artifact_a["findings"][-1]["summary"]
self.assertIn("PASS cannot contain not_checked items", summary)
for home in ("criteria[].reason", "non-goal", "report"):
self.assertIn(home, summary)
self.assert_schema_valid(artifact_a)
with tempfile.TemporaryDirectory() as raw:
draft_b = self.draft()
draft_b["criteria"][0]["reason"] = bounded
draft_b["criteria"].append(
{
"id": "non-goal:cli-untouched",
"result": "PASS",
"evidence_refs": ["git-diff:cli"],
"reason": boundary,
}
)
draft_b["evidence_refs"] = ["e1", "git-diff:cli"]
draft_b["not_checked"] = []
artifact_b, path, _existed = self.store_bound(draft_b, Path(raw))
self.assertEqual(artifact_b["verdict"], "PASS")
self.assert_schema_valid(artifact_b)
# Round-trip: the caveats survive in the persisted PASS artifact.
stored = json.loads(path.read_text(encoding="utf-8"))
reasons = [criterion.get("reason") for criterion in stored["criteria"]]
self.assertIn(bounded, reasons)
self.assertIn(boundary, reasons)
self.assertEqual(stored["not_checked"], [])
self.assertEqual(stored["verdict"], "PASS")
def test_criteria_field_error_names_the_allowed_set(self):
with tempfile.TemporaryDirectory() as raw:
draft = self.draft()
draft["criteria"][0]["confidence"] = "high"
with self.assertRaisesRegex(
tool.ContractError,
r"unknown confidence.*allowed fields are \{id, result, evidence_refs, reason\}",
):
self.store_bound(draft, Path(raw))
self.assertEqual(list(Path(raw).iterdir()), [])
def test_runtime_scope_failure_forces_fail(self):
with tempfile.TemporaryDirectory() as raw:
artifact, _path, _existed = self.store_bound(self.draft(), Path(raw), scope="FAIL")
self.assertEqual(artifact["verdict"], "FAIL")
self.assertEqual(artifact["findings"][-1]["id"], "validate.scope")
self.assert_schema_valid(artifact)
if __name__ == "__main__":
unittest.main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 193k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 193k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 193k | Updated | Claude CodeCodex | Verified | Install |