Skip to content

hashgraph-online/awesome-codex-plugins / plugins/boshu2/agentops/skills/skill-builder

skill-builder

Verified

Create, repair, audit or consolidate agent skills (SKILL.md packages). Use when: writing or fixing a skill, its description or structure. Not for one-off lessons; use Memory.

scripts/audit-legacy.sh

View on GitHub
#!/usr/bin/env bash
# audit.sh — two-pass skill audit (skill-builder deep audit mode; absorbed from /skill-auditor)
# Pass 1 gates through heal.sh --check --strict; Pass 2 adds 8 NEW content-discipline checks.
# Canonical SKILL.md template: skills/skill-builder/references/skill-template.md
#
# Usage:
#   audit.sh [--strict] [--json <path>] <skills/path>
#
# Exit codes:
#   0  — PASS or WARN (success)
#   1  — FAIL (or WARN under --strict)
#   2  — usage error or missing target

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
HEAL_SH="$SCRIPT_DIR/heal.sh"
SCORE_PY="$SCRIPT_DIR/score_agentops_skill.py"
CRAFT_PY="$SCRIPT_DIR/craft_score.py"
AUTHORING_PY="$SCRIPT_DIR/authoring_scan.py"
PROFILE_TOOL="$REPO_ROOT/skills/skill-builder/scripts/conformance_profile.py"

STRICT=0
JSON_OUT=""
TARGET=""

usage() {
  echo "Usage: $0 [--strict] [--json <path>] <skills/path>" >&2
  exit 2
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --strict) STRICT=1; shift ;;
    --json)   JSON_OUT="${2:-}"; shift 2 ;;
    --help|-h) usage ;;
    --*)      echo "Unknown flag: $1" >&2; usage ;;
    *)        TARGET="$1"; shift ;;
  esac
done

[[ -n "$TARGET" ]] || usage
[[ -d "$TARGET" ]] || { echo "audit.sh: target $TARGET is not a directory" >&2; exit 2; }

SKILL_MD="$TARGET/SKILL.md"
[[ -f "$SKILL_MD" ]] || { echo "audit.sh: no SKILL.md at $SKILL_MD" >&2; exit 2; }

# Load profile identity, rule order/severities, and shared evaluations before
# emitting any verdict. Missing or malformed configuration fails closed.
TARGET_ABS="$(cd "$TARGET" && pwd)"
CANONICAL_TARGET=0
case "$TARGET_ABS" in
  "$REPO_ROOT"/skills/*) CANONICAL_TARGET=1 ;;
esac
SELECTED_PROFILE="${SKILL_CONFORMANCE_PROFILE_ID:-}"
if [[ -z "$SELECTED_PROFILE" && "$CANONICAL_TARGET" -eq 0 ]] \
  && ! grep -q '^skill_api_version:' "$SKILL_MD"; then
  SELECTED_PROFILE="external-observation"
fi
profile_args=(--repo-root "$REPO_ROOT" --audit-tsv "$SKILL_MD")
if [[ -n "$SELECTED_PROFILE" ]]; then
  profile_args+=(--profile-id "$SELECTED_PROFILE")
fi
if [[ ! -f "$PROFILE_TOOL" ]]; then
  echo "profile configuration missing: $PROFILE_TOOL" >&2
  exit 2
fi
if ! PROFILE_DATA="$(python3 "$PROFILE_TOOL" "${profile_args[@]}")"; then
  exit 2
fi

PROFILE_ID=""
KERNEL_MAX_LINES=""
PROFILE_LINE_COUNT=""
TRIGGER_FORMS=""
OUTPUT_COMPLETE="false"
RULE_IDS=()
declare -A CHECK_SEVERITY=()
declare -A PROFILE_RULE_FORMS=()
while IFS=$'\t' read -r kind value extra forms; do
  case "$kind" in
    profile_id) PROFILE_ID="$value" ;;
    kernel_max_lines) KERNEL_MAX_LINES="$value" ;;
    line_count) PROFILE_LINE_COUNT="$value" ;;
    trigger_forms) TRIGGER_FORMS="$value" ;;
    output_complete) OUTPUT_COMPLETE="$value" ;;
    rule)
      RULE_IDS+=("$value")
      CHECK_SEVERITY[$value]="$extra"
      PROFILE_RULE_FORMS[$value]="$forms"
      ;;
  esac
done <<<"$PROFILE_DATA"

if [[ -z "$PROFILE_ID" || -z "$KERNEL_MAX_LINES" || ${#RULE_IDS[@]} -eq 0 ]]; then
  echo "profile configuration error: incomplete evaluated profile data" >&2
  exit 2
fi

# Audit-only migration: lexical authoring checks are advisory for canonical source.
# Shared trigger/conformance profiles and their other consumers are unchanged.
if [[ "$PROFILE_ID" == "repo-runtime" ]]; then
  for id in "${RULE_IDS[@]}"; do CHECK_SEVERITY[$id]=WARN; done
fi

# --- Pass 1: heal structural ----------------------------------------------
PASS1_OUT=""
PASS1_FINDINGS_JSON="[]"
PASS1_AUTOFIXABLE=0
PASS1_STATUS="pass"
PASS1_EXIT_CODE=0
PASS1_FINDING_COUNT=0

if [[ -x "$HEAL_SH" ]]; then
  if PASS1_OUT="$(bash "$HEAL_SH" --check --strict "$TARGET" 2>&1)"; then
    PASS1_STATUS="pass"
    PASS1_EXIT_CODE=0
  else
    PASS1_EXIT_CODE=$?
    PASS1_STATUS="fail"
  fi
  # Parse [CODE] path: msg lines into JSON. Use Python here because BSD awk
  # lacks gawk's match(..., array) extension.
  PASS1_FINDINGS_JSON=$(PASS1_OUT="$PASS1_OUT" python3 - <<'PY'
import json
import os
import re

findings = []
pattern = re.compile(r"^\[([A-Z_]+)\] ([^:]+): (.*)$")
for line in os.environ.get("PASS1_OUT", "").splitlines():
    match = pattern.match(line)
    if match:
        code, path, msg = match.groups()
        findings.append({"code": code, "path": path, "msg": msg})
print(json.dumps(findings))
PY
)
  # Count the complete heal.sh auto-fix allowlist.
  PASS1_AUTOFIXABLE=$(echo "$PASS1_OUT" | grep -cE '^\[(MISSING_NAME|MISSING_DESC|NAME_MISMATCH|UNLINKED_REF|EMPTY_DIR|MISSING_API_VERSION)\]' || true)
else
  PASS1_STATUS="fail"
  PASS1_EXIT_CODE=2
  PASS1_OUT="heal delegate missing or not executable: $HEAL_SH"
  PASS1_FINDINGS_JSON='[{"code":"HEAL_SKILL_MISSING","path":"skills/skill-builder/scripts/heal.sh","msg":"heal delegate missing or not executable"}]'
fi
PASS1_FINDING_COUNT=$(PASS1_FINDINGS_JSON="$PASS1_FINDINGS_JSON" python3 - <<'PY'
import json
import os

try:
    print(len(json.loads(os.environ.get("PASS1_FINDINGS_JSON", "[]"))))
except Exception:
    print(0)
PY
)

# --- Pass 2: 8 NEW checks ------------------------------------------------

# Check 1: description-has-triggers (WARN on miss; run_check registers severity)
check_description_has_triggers() {
  profile_rule_has_form description-has-triggers
}

profile_rule_has_form() {
  local rule_id="$1" accepted form
  accepted=",${PROFILE_RULE_FORMS[$rule_id]},"
  IFS=',' read -r -a found_forms <<<"$TRIGGER_FORMS"
  for form in "${found_forms[@]}"; do
    [[ -n "$form" && "$accepted" == *",$form,"* ]] && return 0
  done
  return 1
}

# Check 2: constraints-frontloaded (WARN on miss)
check_constraints_frontloaded() {
  local skill_md="$1"
  if (( PROFILE_LINE_COUNT <= 100 )); then return 0; fi
  awk '
    BEGIN{n=0; i=0; found=0}
    /^---$/{n++; next}
    n==2 {
      i++
      if (i > 80) { exit 1 }
      if (/^## .*[Cc]onstraints/ || /^## .*⚠️/) { found=1; exit 0 }
    }
    END{ exit (found ? 0 : 1) }
  ' "$skill_md"
}

# Check 3: rationale-present (WARN on miss)
check_rationale_present() {
  local skill_md="$1"
  awk '
    function flush_bullet() {
      if (!bullet_open) return
      bullets++
      if (bullet_text ~ /[Ww][Hh][Yy]|[Bb]ecause|[Tt]his matters|[Tt]o prevent|[Rr]ationale:|[Mm]otivation:/) with_why++
      bullet_open=0
      bullet_text=""
    }
    BEGIN{in_constraints=0; bullets=0; with_why=0; bullet_open=0}
    /^## .*([Cc]onstraints|⚠️)/{in_constraints=1; next}
    in_constraints && /^## /{flush_bullet(); exit}
    in_constraints && /^[ ]*[-*] /{
      flush_bullet()
      bullet_open=1
      bullet_text=$0
      next
    }
    in_constraints && bullet_open{bullet_text=bullet_text " " $0}
    END{
      flush_bullet()
      if (bullets == 0) exit 0
      exit (with_why * 2 >= bullets ? 0 : 1)
    }
  ' "$skill_md"
}

# Check 4: verification-checkpoints (WARN on miss, conditional)
check_verification_checkpoints() {
  local skill_md="$1"
  local phases checkpoints
  phases=$(awk '/^## (Workflow|Methodology|Process|Execution)/{in_w=1; next} in_w && /^## /{exit} in_w && /^### /{n++} END{print n+0}' "$skill_md")
  if (( phases < 2 )); then return 0; fi
  checkpoints=$(grep -cE '\*\*Checkpoint:|confirm before|Wait for|verify before' "$skill_md" 2>/dev/null || echo 0)
  (( checkpoints >= 1 ))
}

# Check 5: output-spec-explicit (FAIL on miss)
check_output_spec_explicit() {
  [[ "$OUTPUT_COMPLETE" == "true" ]]
}

# Check 6: quality-rubric (WARN on miss)
check_quality_rubric() {
  local skill_md="$1"
  if (( PROFILE_LINE_COUNT <= 100 )); then return 0; fi
  awk '
    BEGIN{in_q=0; bullets=0}
    /^## (Quality|Checks|Checklist|Rubric|Best Practices|Acceptance)/{in_q=1; next}
    in_q && /^## /{exit}
    in_q && /^[ ]*[-*] /{bullets++}
    END{exit (bullets >= 3 ? 0 : 1)}
  ' "$skill_md"
}

# Check 7: references-modularization (WARN on miss, conditional)
check_references_modularization() {
  (( PROFILE_LINE_COUNT <= KERNEL_MAX_LINES ))
}

# Check 8: trigger-clarity (WARN on miss; run_check registers severity)
check_trigger_clarity() {
  profile_rule_has_form trigger-clarity
}

# --- Run all 8 checks ----------------------------------------------------
declare -A CHECK_STATUS=()
declare -A CHECK_EVIDENCE=()

run_check() {
  local id="$1"
  local fn="$2"
  local severity="${CHECK_SEVERITY[$id]}"
  if "$fn" "$SKILL_MD"; then
    CHECK_STATUS[$id]="pass"
    CHECK_EVIDENCE[$id]="check passed"
  else
    CHECK_STATUS[$id]="${severity,,}"
    CHECK_EVIDENCE[$id]="check failed"
  fi
}

run_check description-has-triggers   check_description_has_triggers
run_check constraints-frontloaded    check_constraints_frontloaded
run_check rationale-present          check_rationale_present
run_check verification-checkpoints   check_verification_checkpoints
run_check output-spec-explicit       check_output_spec_explicit
run_check quality-rubric             check_quality_rubric
run_check references-modularization  check_references_modularization
run_check trigger-clarity            check_trigger_clarity

# --- Advisory density report ---------------------------------------------
# This is deliberately not part of the PASS/WARN/FAIL verdict. Packet-boundary
# enforcement belongs to the execution-packet schema; this block helps reviewers
# find low-signal skill prose before fresh-context dispatch.
declare -A DENSITY_PRESENT=()
declare -A DENSITY_EVIDENCE=()

check_density_field() {
  local id="$1"
  local pattern="$2"
  if grep -Eiq -- "$pattern" "$SKILL_MD"; then
    DENSITY_PRESENT[$id]="true"
    DENSITY_EVIDENCE[$id]="matched advisory pattern"
  else
    DENSITY_PRESENT[$id]="false"
    DENSITY_EVIDENCE[$id]="missing advisory pattern"
  fi
}

check_density_field intent 'intent|goal|behavior|capability'
check_density_field boundary 'boundary|bounded context|write scope|non-goal|non-goals'
check_density_field evidence 'evidence|test|tests|verdict|validation|acceptance'
check_density_field decision 'decision|rationale|why|because|chosen'
check_density_field constraint 'constraint|constraints|guardrail|guardrails|limit|limits|scope'
check_density_field next_action 'next_action|next action|next steps|completion marker|report completion'

density_present_count=0
for id in intent boundary evidence decision constraint next_action; do
  if [[ "${DENSITY_PRESENT[$id]}" == "true" ]]; then
    density_present_count=$((density_present_count + 1))
  fi
done
if (( density_present_count == 6 )); then
  DENSITY_STATUS="pass"
else
  DENSITY_STATUS="warn"
fi

# --- Pass 3: static package-readiness scoring (advisory) -----------------
# Folds the 10-category Skill Quality Rubric (docs/reference/skill-quality-rubric.md)
# into the report via score_agentops_skill.py --audit-block. Each category gets a
# deterministic 0-3 score plus an explainable reason; total is 0-30 with a C/B/A/S
# readiness band. Advisory-only: it never changes the PASS/WARN/FAIL verdict and
# explicitly evaluates neither the safety gate nor behavioral effectiveness.
# Reason: a low score on a structurally clean skill is a triage signal, while a
# high score still cannot prove that the skill is safe or improves outcomes.
RUBRIC_JSON="null"
RUBRIC_SUMMARY=""
RUBRIC_SCORE="n/a"
RUBRIC_MAX="n/a"
RUBRIC_RATING="?"
if [[ -f "$SCORE_PY" ]] && command -v python3 >/dev/null 2>&1; then
  if rubric_out="$(python3 "$SCORE_PY" "$TARGET" --audit-block 2>/dev/null)"; then
    RUBRIC_JSON="$rubric_out"
    RUBRIC_SCORE="$(printf '%s' "$rubric_out" | awk -F': ' '/"total_score"/{gsub(/[, ]/,"",$2); print $2; exit}')"
    RUBRIC_MAX="$(printf '%s' "$rubric_out" | awk -F': ' '/"max_score"/{gsub(/[, ]/,"",$2); print $2; exit}')"
    RUBRIC_RATING="$(printf '%s' "$rubric_out" | awk -F'"' '/"rating"/{print $4; exit}')"
    RUBRIC_SUMMARY=" Static readiness: ${RUBRIC_SCORE}/${RUBRIC_MAX} (${RUBRIC_RATING}) [advisory; safety/effectiveness not evaluated]."
  fi
fi

# --- Pass 4: craft instrumentation (advisory) -----------------------------
# 12-element craft score, provenance resolution, and loop-safety findings via
# craft_score.py. Advisory-only by design: it never changes the PASS/WARN/FAIL
# verdict or the exit code — presence of craft elements is cheaply detectable,
# but craft quality stays the fresh validator's judgment. Fail-open to null
# like the Pass 3 rubric.
CRAFT_JSON="null"
CRAFT_LINES=""
if [[ -f "$CRAFT_PY" ]] && command -v python3 >/dev/null 2>&1; then
  if craft_out="$(python3 "$CRAFT_PY" "$TARGET" --audit-block --repo-root "$REPO_ROOT" 2>/dev/null)"; then
    CRAFT_JSON="$craft_out"
    CRAFT_LINES="$(CRAFT_OUT="$craft_out" python3 - 2>/dev/null <<'PY' || true
import json
import os

report = json.loads(os.environ["CRAFT_OUT"])
lines = [f"Pass 4 craft (advisory): {report['summary']}"]
prov = report["provenance"]
lines.append(f"Provenance (advisory): {prov['resolved']}/{prov['citations']} citations resolve")
for dead in prov["dead"]:
    lines.append(f"  dead citation: {dead['citation']} ({dead['kind']})")
loop = report["loop_safety"]["findings"]
lines.append(f"Loop-safety (advisory): {len(loop)} finding(s)")
for finding in loop:
    lines.append(f"  {finding['type']} in section '{finding['section']}'")
print("\n".join(lines))
PY
)"
  fi
fi

# --- Pass 5: authoring prose-quality scan (advisory) ----------------------
# Advisory suspects for the failure modes named in
# references/authoring-doctrine.md (noop-phrase, negation-without-positive,
# step-missing-done-condition). Advisory-only: the doctrine's no-op test is
# model-relative and prohibitions are sometimes correct guardrails, so these
# findings never change the PASS/WARN/FAIL verdict or exit code. Fail-open to
# null like the Pass 3 rubric and Pass 4 craft blocks.
AUTHORING_JSON="null"
AUTHORING_LINES=""
if [[ -f "$AUTHORING_PY" ]] && command -v python3 >/dev/null 2>&1; then
  if authoring_out="$(python3 "$AUTHORING_PY" "$TARGET" --audit-block 2>/dev/null)"; then
    AUTHORING_JSON="$authoring_out"
    AUTHORING_LINES="$(AUTHORING_OUT="$authoring_out" python3 - 2>/dev/null <<'PY' || true
import json
import os

report = json.loads(os.environ["AUTHORING_OUT"])
lines = [f"Pass 5 authoring (advisory): {report['summary']}"]
for finding in report["findings"]:
    lines.append(f"  {finding['id']} line {finding['line']}: {finding['evidence']}")
print("\n".join(lines))
PY
)"
  fi
fi

# --- Aggregate verdict ---------------------------------------------------
fails=0
warns=0
for id in "${RULE_IDS[@]}"; do
  case "${CHECK_STATUS[$id]}" in
    fail) fails=$((fails+1)) ;;
    warn) warns=$((warns+1)) ;;
  esac
done

if [[ "$PASS1_STATUS" == "fail" && "$CANONICAL_TARGET" -eq 1 ]]; then
  VERDICT="FAIL"
elif (( fails > 0 )); then
  VERDICT="FAIL"
elif (( warns > 0 )); then
  VERDICT="WARN"
else
  VERDICT="PASS"
fi

# --- Emit report ---------------------------------------------------------
emit_json() {
  printf '{\n'
  printf '  "target": "%s",\n' "$TARGET"
  printf '  "profile_id": "%s",\n' "$PROFILE_ID"
  printf '  "verdict": "%s",\n' "$VERDICT"
  printf '  "pass1": {\n'
  printf '    "status": "%s",\n' "$PASS1_STATUS"
  printf '    "exit_code": %s,\n' "$PASS1_EXIT_CODE"
  printf '    "strict": true,\n'
  printf '    "findings": %s,\n' "$PASS1_FINDINGS_JSON"
  printf '    "autofixable": %s\n' "$PASS1_AUTOFIXABLE"
  printf '  },\n'
  printf '  "pass2": {\n'
  printf '    "checks": [\n'
  local first=1
  for id in "${RULE_IDS[@]}"; do
    if (( ! first )); then printf ',\n'; fi
    first=0
  printf '      {"id":"%s","status":"%s","severity":"%s","evidence":"%s"' \
    "$id" "${CHECK_STATUS[$id]}" "${CHECK_SEVERITY[$id]}" "${CHECK_EVIDENCE[$id]}"
  if [[ "$id" == "description-has-triggers" || "$id" == "trigger-clarity" ]]; then
    forms_json="$(python3 - "$TRIGGER_FORMS" <<'PY'
import json
import sys

print(json.dumps([item for item in sys.argv[1].split(",") if item]))
PY
)"
    printf ',"forms":%s' "$forms_json"
  fi
  printf '}'
  done
  printf '\n    ]\n'
  printf '  },\n'
  printf '  "density": {\n'
  printf '    "status": "%s",\n' "$DENSITY_STATUS"
  printf '    "advisory": true,\n'
  printf '    "fields": [\n'
  first=1
  for id in intent boundary evidence decision constraint next_action; do
    if (( ! first )); then printf ',\n'; fi
    first=0
    printf '      {"id":"%s","present":%s,"evidence":"%s"}' "$id" "${DENSITY_PRESENT[$id]}" "${DENSITY_EVIDENCE[$id]}"
  done
  printf '\n    ],\n'
  printf '    "summary": "%d/6 density signals present; advisory-only and not execution-packet enforcement."\n' "$density_present_count"
  printf '  },\n'
  printf '  "rubric": %s,\n' "$RUBRIC_JSON"
  printf '  "craft": %s,\n' "$CRAFT_JSON"
  printf '  "authoring": %s,\n' "$AUTHORING_JSON"
  printf '  "summary": "Pass1: %s via heal --strict (exit %d, %d findings, %d autofixable). Pass2: %d fails, %d warns.%s Verdict: %s."\n' \
    "$PASS1_STATUS" "$PASS1_EXIT_CODE" "$PASS1_FINDING_COUNT" "$PASS1_AUTOFIXABLE" "$fails" "$warns" "$RUBRIC_SUMMARY" "$VERDICT"
  printf '}\n'
}

if [[ -n "$JSON_OUT" ]]; then
  emit_json > "$JSON_OUT"
fi

# Always print human-readable summary to stderr
{
  echo "=== Skill Audit: $TARGET ==="
  echo "Profile: $PROFILE_ID"
  echo "Pass 1 (heal --strict): $PASS1_STATUS (exit $PASS1_EXIT_CODE), $PASS1_FINDING_COUNT findings ($PASS1_AUTOFIXABLE autofixable)"
  echo "Pass 2 (8 NEW checks):"
  for id in "${RULE_IDS[@]}"; do
    printf "  [%-4s] %s\n" "${CHECK_STATUS[$id]}" "$id"
  done
  echo "Density advisory: $density_present_count/6 fields present ($DENSITY_STATUS)"
  echo "Pass 3 static readiness (advisory): ${RUBRIC_SCORE}/${RUBRIC_MAX} (${RUBRIC_RATING}); safety/effectiveness not evaluated"
  if [[ -n "$CRAFT_LINES" ]]; then
    echo "$CRAFT_LINES"
  fi
  if [[ -n "$AUTHORING_LINES" ]]; then
    echo "$AUTHORING_LINES"
  fi
  echo "VERDICT: $VERDICT"
  echo "Static authoring signals only; semantic completeness, safety and effectiveness are not evaluated."
} >&2

# Always print JSON to stdout (unless --json file was supplied)
if [[ -z "$JSON_OUT" ]]; then
  emit_json
fi

# --- Exit code -----------------------------------------------------------
case "$VERDICT" in
  PASS) exit 0 ;;
  WARN) [[ "$STRICT" -eq 1 && "$PROFILE_ID" != "repo-runtime" ]] && exit 1 || exit 0 ;;
  FAIL) exit 1 ;;
esac
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 469kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 469kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 469kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall