hashgraph-online/awesome-codex-plugins / plugins/boshu2/agentops/skills/skill-builder
skill-builder
VerifiedCreate, repair, audit or consolidate agent skills (SKILL.md packages). Use when: writing or fixing a skill, its description or structure. Not for one-off lessons; use Memory.
scripts/conformance_profile.py
View on GitHub#!/usr/bin/env python3
"""Load and evaluate the canonical AgentOps skill-conformance profile."""
from __future__ import annotations
import argparse
import re
import sys
from pathlib import Path
from typing import Any
import yaml
PROFILE_RELATIVE_PATH = Path(
"skills/skill-builder/references/skill-conformance-profiles.yaml"
)
KNOWN_SEVERITIES = {"WARN", "FAIL"}
KNOWN_TRIGGER_FORMS = {"inline-marker", "block-marker", "metadata-list"}
REQUIRED_RULE_IDS = (
"description-has-triggers",
"constraints-frontloaded",
"rationale-present",
"verification-checkpoints",
"output-spec-explicit",
"quality-rubric",
"references-modularization",
"trigger-clarity",
)
KNOWN_EXTERNAL_CONTENT_POLICIES = {"observe-structure-only"}
REQUIRED_PROHIBITED_COPY_CATEGORIES = {
"prose",
"prompts",
"scripts",
"examples",
"names",
}
REQUIRED_PROTECTED_FRONTMATTER_FIELDS = {"description"}
class ProfileError(ValueError):
"""Raised when the conformance profile is missing or malformed."""
def _mapping(value: Any, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise ProfileError(f"profile configuration error: {label} must be a mapping")
return value
def _string_list(value: Any, label: str) -> list[str]:
if not isinstance(value, list) or not value or not all(isinstance(v, str) for v in value):
raise ProfileError(
f"profile configuration error: {label} must be a non-empty string list"
)
return value
def _positive_int(value: Any, label: str) -> int:
if not isinstance(value, int) or value < 1:
raise ProfileError(f"profile configuration error: {label} must be positive")
return value
def load_profile(repo_root: Path, profile_id: str | None = None) -> dict[str, Any]:
"""Load and fully validate one profile from the authoritative YAML file."""
path = repo_root / PROFILE_RELATIVE_PATH
if not path.is_file():
raise ProfileError(f"profile configuration missing: {path}")
try:
document = yaml.safe_load(path.read_text(encoding="utf-8"))
except (OSError, yaml.YAMLError) as exc:
raise ProfileError(f"profile configuration error in {path}: {exc}") from exc
document = _mapping(document, "document")
profiles = _mapping(document.get("profiles"), "profiles")
selected = profile_id or document.get("default_profile")
if not isinstance(selected, str) or not selected:
raise ProfileError("profile configuration error: default_profile must be a string")
if selected not in profiles:
raise ProfileError(f"profile configuration error: unknown profile {selected!r}")
profile = _mapping(profiles[selected], f"profiles.{selected}")
if profile.get("id") != selected:
raise ProfileError(
f"profile configuration error: profile id must equal selected id {selected!r}"
)
limit = profile.get("kernel_max_lines")
if not isinstance(limit, int) or limit < 1:
raise ProfileError("profile configuration error: kernel_max_lines must be positive")
trigger = _mapping(profile.get("trigger_forms"), "trigger_forms")
accepted = _string_list(trigger.get("accepted"), "trigger_forms.accepted")
unknown_forms = sorted(set(accepted) - KNOWN_TRIGGER_FORMS)
if unknown_forms:
raise ProfileError(
f"profile configuration error: unknown trigger form(s): {', '.join(unknown_forms)}"
)
_string_list(trigger.get("description_markers"), "trigger_forms.description_markers")
minimum = trigger.get("metadata_list_min_items")
if not isinstance(minimum, int) or minimum < 1:
raise ProfileError(
"profile configuration error: metadata_list_min_items must be positive"
)
output = _mapping(profile.get("output_contract"), "output_contract")
_string_list(output.get("section_headings"), "output_contract.section_headings")
components = _mapping(
output.get("required_components"), "output_contract.required_components"
)
if not components:
raise ProfileError(
"profile configuration error: output_contract.required_components is empty"
)
for component_id, component in components.items():
if not isinstance(component_id, str):
raise ProfileError("profile configuration error: component id must be a string")
component = _mapping(component, f"output component {component_id}")
_string_list(component.get("markers"), f"output component {component_id}.markers")
clean_room = _mapping(profile.get("clean_room"), "clean_room")
if clean_room.get("enabled") is not True:
raise ProfileError("profile configuration error: clean_room.enabled must be true")
policy = clean_room.get("external_content_policy")
if policy not in KNOWN_EXTERNAL_CONTENT_POLICIES:
raise ProfileError(
"profile configuration error: clean_room.external_content_policy "
f"must be one of {sorted(KNOWN_EXTERNAL_CONTENT_POLICIES)}, got {policy!r}"
)
categories = _string_list(
clean_room.get("prohibited_copy_categories"),
"clean_room.prohibited_copy_categories",
)
if len(categories) != len(set(categories)):
raise ProfileError(
"profile configuration error: clean_room.prohibited_copy_categories "
"contains duplicates"
)
if set(categories) != REQUIRED_PROHIBITED_COPY_CATEGORIES:
missing = sorted(REQUIRED_PROHIBITED_COPY_CATEGORIES - set(categories))
unknown = sorted(set(categories) - REQUIRED_PROHIBITED_COPY_CATEGORIES)
raise ProfileError(
"profile configuration error: clean_room.prohibited_copy_categories "
f"must name the exact known categories (missing={missing}, unknown={unknown})"
)
copy_detection = _mapping(clean_room.get("copy_detection"), "clean_room.copy_detection")
_positive_int(
copy_detection.get("minimum_fragment_characters"),
"clean_room.copy_detection.minimum_fragment_characters",
)
_positive_int(
copy_detection.get("minimum_name_characters"),
"clean_room.copy_detection.minimum_name_characters",
)
protected_fields = _string_list(
copy_detection.get("protected_frontmatter_fields"),
"clean_room.copy_detection.protected_frontmatter_fields",
)
if len(protected_fields) != len(set(protected_fields)):
raise ProfileError(
"profile configuration error: "
"clean_room.copy_detection.protected_frontmatter_fields contains duplicates"
)
if set(protected_fields) != REQUIRED_PROTECTED_FRONTMATTER_FIELDS:
missing = sorted(REQUIRED_PROTECTED_FRONTMATTER_FIELDS - set(protected_fields))
unknown = sorted(set(protected_fields) - REQUIRED_PROTECTED_FRONTMATTER_FIELDS)
raise ProfileError(
"profile configuration error: "
"clean_room.copy_detection.protected_frontmatter_fields must name the "
f"exact known fields (missing={missing}, unknown={unknown})"
)
_string_list(
copy_detection.get("ignored_exact_lines"),
"clean_room.copy_detection.ignored_exact_lines",
)
_string_list(
copy_detection.get("ignored_line_prefixes"),
"clean_room.copy_detection.ignored_line_prefixes",
)
rule_order = _string_list(profile.get("rule_order"), "rule_order")
if len(rule_order) != len(set(rule_order)):
raise ProfileError("profile configuration error: rule_order contains duplicates")
rules = _mapping(profile.get("rules"), "rules")
if set(rule_order) != set(rules):
missing = sorted(set(rule_order) - set(rules))
extra = sorted(set(rules) - set(rule_order))
raise ProfileError(
"profile configuration error: rule_order/rules mismatch "
f"(missing={missing}, extra={extra})"
)
known_rules = set(REQUIRED_RULE_IDS)
actual_rules = set(rules)
if actual_rules != known_rules:
missing = sorted(known_rules - actual_rules)
unknown = sorted(actual_rules - known_rules)
raise ProfileError(
"profile configuration error: profile must declare the exact known rule IDs "
f"(missing={missing}, unknown={unknown})"
)
if rule_order != list(REQUIRED_RULE_IDS):
raise ProfileError(
"profile configuration error: rule_order must use the canonical known rule "
f"order {list(REQUIRED_RULE_IDS)}"
)
for rule_id in rule_order:
rule = _mapping(rules[rule_id], f"rules.{rule_id}")
severity = rule.get("severity")
if severity not in KNOWN_SEVERITIES:
raise ProfileError(
f"profile configuration error: rule {rule_id} has unknown severity {severity!r}"
)
if "accepted_forms" in rule:
forms = _string_list(
rule["accepted_forms"], f"rules.{rule_id}.accepted_forms"
)
unknown = sorted(set(forms) - set(accepted))
if unknown:
raise ProfileError(
f"profile configuration error: rule {rule_id} names unknown forms {unknown}"
)
return profile
def split_frontmatter(text: str) -> tuple[str, str]:
"""Return raw YAML frontmatter and Markdown body."""
if not text.startswith("---\n"):
return "", text
parts = text.split("\n---", 1)
if len(parts) != 2:
return "", text
return parts[0][4:], parts[1].lstrip("-\n")
def _frontmatter_mapping(text: str) -> tuple[str, dict[str, Any]]:
raw, _ = split_frontmatter(text)
try:
payload = yaml.safe_load(raw) if raw else {}
except yaml.YAMLError as exc:
raise ProfileError(f"skill frontmatter configuration error: {exc}") from exc
return raw, _mapping(payload, "skill frontmatter")
def trigger_forms(text: str, profile: dict[str, Any]) -> list[str]:
"""Return accepted trigger form IDs found only in frontmatter semantics."""
raw, frontmatter = _frontmatter_mapping(text)
trigger = profile["trigger_forms"]
markers = trigger["description_markers"]
description = frontmatter.get("description", "")
description = description if isinstance(description, str) else ""
has_marker = any(marker.casefold() in description.casefold() for marker in markers)
scalar = re.search(r"^description:\s*([|>])", raw, re.MULTILINE)
found: set[str] = set()
if has_marker:
found.add("block-marker" if scalar else "inline-marker")
metadata = frontmatter.get("metadata")
trigger_list = metadata.get("triggers") if isinstance(metadata, dict) else None
if isinstance(trigger_list, list) and len(trigger_list) >= trigger["metadata_list_min_items"]:
found.add("metadata-list")
return [form for form in trigger["accepted"] if form in found]
def output_component_results(text: str, profile: dict[str, Any]) -> dict[str, bool]:
"""Evaluate every required executable-handoff component in one output section."""
_, body = split_frontmatter(text)
headings = {heading.casefold() for heading in profile["output_contract"]["section_headings"]}
section_lines: list[str] = []
capturing = False
for line in body.splitlines():
heading = re.match(r"^##\s+(.+?)\s*$", line)
if heading:
normalized = heading.group(1).strip().casefold()
if capturing:
break
capturing = normalized in headings
continue
if capturing:
section_lines.append(line)
section = "\n".join(section_lines).casefold()
results: dict[str, bool] = {}
for component_id, component in profile["output_contract"]["required_components"].items():
results[component_id] = any(
marker.casefold() in section for marker in component["markers"]
)
return results
def evaluation(skill_md: Path, profile: dict[str, Any]) -> dict[str, Any]:
"""Evaluate shared trigger, boundary, and output semantics for one skill."""
try:
text = skill_md.read_text(encoding="utf-8")
except OSError as exc:
raise ProfileError(f"skill read error: {skill_md}: {exc}") from exc
_, frontmatter = _frontmatter_mapping(text)
forms = trigger_forms(text, profile)
components = output_component_results(text, profile)
declared_output = frontmatter.get("output_contract")
has_declared_output = (
isinstance(declared_output, str) and bool(declared_output.strip())
) or (isinstance(declared_output, (dict, list)) and bool(declared_output))
return {
"profile_id": profile["id"],
"kernel_max_lines": profile["kernel_max_lines"],
"line_count": len(text.splitlines()),
"trigger_forms": forms,
"output_components": components,
"output_complete": has_declared_output or all(components.values()),
}
def clean_room_copies(
external_source: Path, generated_dirs: list[Path], profile: dict[str, Any]
) -> list[str]:
"""Return external content copied into generated output under profile policy."""
try:
source = external_source.read_text(encoding="utf-8")
generated = "\n".join(
path.read_text(encoding="utf-8", errors="replace")
for root in generated_dirs
for path in root.rglob("*")
if path.is_file()
)
except OSError as exc:
raise ProfileError(f"clean-room verification read error: {exc}") from exc
raw_frontmatter, body = split_frontmatter(source)
try:
source_metadata = yaml.safe_load(raw_frontmatter) if raw_frontmatter else {}
except yaml.YAMLError as exc:
raise ProfileError(f"external skill frontmatter error: {exc}") from exc
source_metadata = _mapping(source_metadata, "external skill frontmatter")
clean_room = profile["clean_room"]
policy = clean_room["external_content_policy"]
if policy != "observe-structure-only":
raise ProfileError(
f"profile configuration error: unsupported external_content_policy {policy!r}"
)
categories = set(clean_room["prohibited_copy_categories"])
detection = clean_room["copy_detection"]
minimum_fragment = detection["minimum_fragment_characters"]
ignored_exact = set(detection["ignored_exact_lines"])
ignored_prefixes = tuple(detection["ignored_line_prefixes"])
generated_folded = generated.casefold()
generated_normalized = " ".join(generated.split()).casefold()
copied: list[str] = []
if categories & {"prose", "prompts", "scripts", "examples"}:
protected = {
line.strip()
for line in body.splitlines()
if len(line.strip()) >= minimum_fragment
and line.strip() not in ignored_exact
and not line.lstrip().startswith(ignored_prefixes)
}
copied.extend(line for line in protected if line.casefold() in generated_folded)
for field in detection["protected_frontmatter_fields"]:
value = source_metadata.get(field)
if not isinstance(value, str):
continue
normalized = " ".join(value.split())
if (
len(normalized) >= minimum_fragment
and normalized.casefold() in generated_normalized
):
copied.append(normalized)
if "names" in categories:
external_name = source_metadata.get("name", "")
if (
isinstance(external_name, str)
and len(external_name.strip()) >= detection["minimum_name_characters"]
and external_name.strip().casefold() in generated_folded
):
copied.append(external_name.strip())
return sorted(set(copied))
def emit_audit_tsv(skill_md: Path, profile: dict[str, Any]) -> None:
"""Emit shell-safe, validated profile/evaluation data for audit.sh."""
result = evaluation(skill_md, profile)
print(f"profile_id\t{result['profile_id']}")
print(f"kernel_max_lines\t{result['kernel_max_lines']}")
print(f"line_count\t{result['line_count']}")
print(f"trigger_forms\t{','.join(result['trigger_forms'])}")
print(f"output_complete\t{str(result['output_complete']).lower()}")
for component_id, present in result["output_components"].items():
print(f"output_component\t{component_id}\t{str(present).lower()}")
for rule_id in profile["rule_order"]:
rule = profile["rules"][rule_id]
accepted = ",".join(rule.get("accepted_forms", []))
print(f"rule\t{rule_id}\t{rule['severity']}\t{accepted}")
def main(argv: list[str] | None = None) -> int:
"""Validate a profile and optionally emit evaluation data for audit.sh."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo-root", type=Path, required=True)
parser.add_argument("--profile-id", default=None)
parser.add_argument("--audit-tsv", type=Path)
parser.add_argument("--verify-clean-room", type=Path, metavar="EXTERNAL_SKILL_MD")
parser.add_argument("--generated-dir", type=Path, action="append", default=[])
args = parser.parse_args(argv)
try:
profile = load_profile(args.repo_root, args.profile_id)
if args.audit_tsv and args.verify_clean_room:
raise ProfileError(
"profile configuration error: choose one of --audit-tsv or --verify-clean-room"
)
if args.verify_clean_room:
if not args.generated_dir:
raise ProfileError(
"profile configuration error: --verify-clean-room requires --generated-dir"
)
copied = clean_room_copies(args.verify_clean_room, args.generated_dir, profile)
if copied:
raise ProfileError(
f"clean-room violation under profile {profile['id']}: copied external "
f"content: {copied[0]}"
)
print(profile["id"])
elif args.audit_tsv:
emit_audit_tsv(args.audit_tsv, profile)
else:
print(profile["id"])
except ProfileError as exc:
print(str(exc), file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 467k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 467k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 467k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 193k | Updated | Claude CodeCodex | Verified | Install |