ZenUml/conf-app / .claude/skills/ai-repair-benchmark
ai-repair-benchmark
VerifiedCompare how OpenRouter models repair the same damaged DSL; run only L1 raw-model and L2 Diagramly Prompt trials, measuring repair quality, latency, tokens, and actual cost. Use ai-repair-case-validation to investigate and regress specific cases.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 4.6 KB |
| agents/openai.yaml | config | 275 B |
| references/methodology.md | markdown | 5.6 KB |
| scripts/build_diagramly_prompt.tsexecutable | script | 4.3 KB |
| scripts/fixtures/smoke-candidate.mmd | other | 118 B |
| scripts/fixtures/smoke-invalid.mmd | other | 117 B |
| scripts/fixtures/smoke-manifest.json | config | 343 B |
| scripts/fixtures/smoke-results.jsonl | other | 755 B |
| scripts/run_openrouter_matrix.pyexecutable | script | 16.2 KB |
| scripts/score_candidates.pyexecutable | script | 10.6 KB |
| scripts/summarize_benchmark.pyexecutable | script | 23.4 KB |
| scripts/validate_candidates.tsexecutable | script | 2.9 KB |
scripts/score_candidates.py
View on GitHub#!/usr/bin/env python3
"""Add deterministic validity and preservation metrics to benchmark records."""
from __future__ import annotations
import argparse
import difflib
import json
import re
import shutil
import subprocess
import tempfile
from pathlib import Path
from typing import Any
KEYWORDS = {
"graph",
"flowchart",
"sequencediagram",
"participant",
"note",
"over",
"class",
"interface",
"package",
"actor",
"entity",
"component",
"startuml",
"enduml",
"alt",
"else",
"end",
"loop",
"opt",
"par",
"and",
"rect",
"title",
"skinparam",
"legend",
"left",
"right",
"top",
"bottom",
"of",
"true",
"false",
"null",
"openapi",
"info",
"paths",
"responses",
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--manifest", required=True)
parser.add_argument("--results", action="append", required=True)
parser.add_argument("--diagramly-path", required=True)
parser.add_argument("--output", required=True)
return parser.parse_args()
def read_jsonl(path: Path) -> list[dict[str, Any]]:
records = [
json.loads(line)
for line in path.read_text(encoding="utf-8").splitlines()
if line.strip()
]
for record in records:
candidate = record.get("candidatePath")
if candidate and not Path(candidate).is_absolute():
record["candidatePath"] = str((path.parent / candidate).resolve())
return records
def manifest_relative(manifest_path: Path, value: str) -> Path:
path = Path(value).expanduser()
return (
path.resolve()
if path.is_absolute()
else (manifest_path.parent / path).resolve()
)
def normalized(value: str) -> str:
return value.replace("\r\n", "\n").strip()
def identifiers(code: str) -> set[str]:
found: set[str] = set()
patterns = [
r"\bparticipant\s+([A-Za-z_][\w.-]*)",
r"\b(?:class|interface|entity|actor|component|node)\s+\"?([A-Za-z_][\w.-]*)",
r"\b([A-Za-z_][\w.-]*)(?=\s*(?:--?>|--?>>|--|->>|\[|\(|\{|\.))",
]
for pattern in patterns:
found.update(re.findall(pattern, code, flags=re.IGNORECASE | re.MULTILINE))
for token in re.findall(r"\b[A-Za-z_][A-Za-z0-9_.:-]*\b", code):
if token.lower() in KEYWORDS:
continue
if any(char.isdigit() for char in token) or "_" in token or token.isupper():
found.add(token)
if found:
return found
return {
token
for token in re.findall(r"\b[A-Za-z_][A-Za-z0-9_.:-]{2,}\b", code)
if token.lower() not in KEYWORDS
}
def changed_line_ratio(before: str, after: str) -> float:
left, right = before.splitlines(), after.splitlines()
denominator = max(len(left), len(right), 1)
changed = 0
for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(a=left, b=right).get_opcodes():
if tag != "equal":
changed += max(i2 - i1, j2 - j1)
return min(1.0, changed / denominator)
def validate_plantuml(candidate: Path) -> tuple[bool | None, str | None, bool]:
if shutil.which("plantuml"):
process = subprocess.run(
["plantuml", "-checkonly", str(candidate)],
text=True,
capture_output=True,
check=False,
)
return (
process.returncode == 0,
(process.stdout + process.stderr)[-1000:] or None,
False,
)
if not shutil.which("docker"):
return None, "No plantuml command or Docker runtime", True
inspect = subprocess.run(
["docker", "image", "inspect", "yuzutech/kroki:latest"],
text=True,
capture_output=True,
check=False,
)
if inspect.returncode != 0:
return None, "Local yuzutech/kroki:latest image is unavailable", True
parent = candidate.parent.resolve()
process = subprocess.run(
[
"docker",
"run",
"--rm",
"--network",
"none",
"--mount",
f"type=bind,source={parent},target=/data,readonly",
"--entrypoint",
"/usr/bin/plantuml",
"yuzutech/kroki:latest",
"-checkonly",
f"/data/{candidate.name}",
],
text=True,
capture_output=True,
check=False,
)
return (
process.returncode == 0,
(process.stdout + process.stderr)[-1000:] or None,
False,
)
def node_validation(
records: list[dict[str, Any]], language_key: str, diagramly_path: Path
) -> dict[str, dict[str, Any]]:
items = [
{"id": record["id"], "candidatePath": record["candidatePath"]}
for record in records
if record.get("candidatePath")
]
if not items:
return {}
helper = Path(__file__).with_name("validate_candidates.ts").resolve()
with tempfile.TemporaryDirectory(prefix="ai-repair-validate-") as temporary:
manifest = Path(temporary) / "manifest.json"
output = Path(temporary) / "output.json"
manifest.write_text(
json.dumps({"languageKey": language_key, "items": items}), encoding="utf-8"
)
process = subprocess.run(
[
"pnpm",
"exec",
"tsx",
"--tsconfig",
"scripts/tsconfig.json",
str(helper),
"--manifest",
str(manifest),
"--output",
str(output),
"--diagramly-path",
str(diagramly_path),
],
cwd=diagramly_path,
text=True,
capture_output=True,
check=False,
)
if process.returncode != 0:
raise RuntimeError(
f"Diagramly validator failed: {(process.stdout + process.stderr)[-1500:]}"
)
return {
item["id"]: item for item in json.loads(output.read_text(encoding="utf-8"))
}
def main() -> int:
args = parse_args()
manifest_path = Path(args.manifest).resolve()
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
original = manifest_relative(manifest_path, manifest["dslPath"]).read_text(
encoding="utf-8"
)
records: list[dict[str, Any]] = []
for result_path in args.results:
records.extend(read_jsonl(Path(result_path).resolve()))
candidates = [
Path(record["candidatePath"]).read_text(encoding="utf-8")
if record.get("candidatePath") and Path(record["candidatePath"]).exists()
else ""
for record in records
]
no_ops = [
bool(candidate) and normalized(candidate) == normalized(original)
for candidate in candidates
]
language_key = str(manifest["languageKey"])
parsed = (
{}
if language_key == "LANG_PLANTUML"
else node_validation(
[record for record, no_op in zip(records, no_ops) if not no_op],
language_key,
Path(args.diagramly_path).resolve(),
)
)
original_identifiers = identifiers(original)
required_anchors = [str(item) for item in manifest.get("requiredAnchors", [])]
output_path = Path(args.output).resolve()
output_path.parent.mkdir(parents=True, exist_ok=True)
with output_path.open("w", encoding="utf-8") as output:
for record, candidate, no_op in zip(records, candidates, no_ops):
candidate_path_value = record.get("candidatePath")
candidate_identifiers = identifiers(candidate)
missing_identifiers = sorted(original_identifiers - candidate_identifiers)
retention = (
len(original_identifiers & candidate_identifiers)
/ len(original_identifiers)
if original_identifiers
else 1.0
)
missing_anchors = [
anchor for anchor in required_anchors if anchor not in candidate
]
if no_op:
syntax_valid, syntax_error, validation_skipped = (
None,
"Unchanged candidate",
True,
)
elif not candidate:
syntax_valid, syntax_error, validation_skipped = (
False,
"Empty candidate",
False,
)
elif language_key == "LANG_PLANTUML":
syntax_valid, syntax_error, validation_skipped = validate_plantuml(
Path(candidate_path_value).resolve()
)
else:
validation = parsed.get(record["id"], {})
validation_skipped = bool(validation.get("skipped"))
syntax_error = validation.get("error")
syntax_valid = None if validation_skipped else syntax_error is None
hard_pass = bool(candidate) and not no_op and syntax_valid is True
if no_op:
quality = "FAIL"
repair_status = "FAIL"
elif validation_skipped:
quality = "UNVERIFIED"
repair_status = "SKIPPED"
elif not hard_pass:
quality = "FAIL"
repair_status = "FAIL"
elif missing_anchors or retention < 0.8:
quality = "C"
repair_status = "PASS"
elif retention < 0.95:
quality = "B"
repair_status = "PASS"
else:
quality = "A"
repair_status = "PASS"
scored = {
**record,
"repairStatus": repair_status,
"syntaxValid": syntax_valid,
"syntaxError": syntax_error,
"validationSkipped": validation_skipped,
"noOp": no_op,
"originalIdentifierCount": len(original_identifiers),
"candidateIdentifierCount": len(candidate_identifiers),
"missingIdentifierCount": len(missing_identifiers),
"missingIdentifierSample": missing_identifiers[:20],
"identifierRetention": round(retention, 4),
"changedLineRatio": round(changed_line_ratio(original, candidate), 4)
if candidate
else None,
"missingRequiredAnchors": missing_anchors,
"qualityGrade": quality,
}
output.write(json.dumps(scored, ensure_ascii=False) + "\n")
print(json.dumps({"output": str(output_path), "records": len(records)}))
return 0
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |