Skip to content

bjwanneng/veriflow-cc / src/claude_skills/vf-rtl

vf-rtl

Verified

FileKindSize
SELF_IMPROVE.mdmarkdown3.8 KB
SKILL.mdskill md11.2 KB
SKILL_INDEX.mdmarkdown5.5 KB
analysis/bug_pattern_match.pyscript26.9 KB
analysis/corner_case_generator.pyscript4.9 KB
analysis/coverage_analyzer.pyscript5.3 KB
analysis/design_graph.pyscript8.9 KB
analysis/expected_trace_gen.pyscript12.5 KB
analysis/timing_contract_checker.pyscript25.4 KB
analysis/timing_diagnostic.pyscript21.1 KB
analysis/vcd2table.pyscript31.3 KB
bug_patterns.mdmarkdown23.2 KB
coding_style.mdmarkdown9.4 KB
core/init.pyscript13.8 KB
core/rtl_utils.pyscript7.5 KB
core/state.pyscript38.9 KB
design_rules.mdmarkdown2.9 KB
error_recovery.mdmarkdown7.0 KB
kb/knowledge_base.pyscript11.7 KB
kb/reference_kb.pyscript5.6 KB
kb/self_improve.pyscript24.4 KB
references/arbiter_round_robin.vother1.4 KB
references/counter.vother545 B
references/fifo.vother1.3 KB
references/fsm_moore.vother847 B
references/handshake_valid_ready.vother1.1 KB
references/pipeline_aligner.vother760 B
runners/benchmark_runner.pyscript23.6 KB
runners/cocotb_runner.pyscript26.3 KB
runners/iverilog_runner.pyscript26.3 KB
runners/yosys_equiv.pyscript14.2 KB
stage1_spec_golden.mdmarkdown6.1 KB
stage2_codegen.mdmarkdown5.9 KB
stage3_verify_fix.mdmarkdown17.0 KB
stage4_lint_synth.mdmarkdown6.3 KB
templates/cocotb_template.pyscript42.4 KB
templates/golden_model_template.pyscript8.7 KB
templates/spec_template.jsonconfig8.5 KB
templates/tb_integration_template.vother12.4 KB
verify/candidate_selector.pyscript7.0 KB
verify/formal_prove.pyscript12.1 KB
verify/synth_score.pyscript4.4 KB

runners/benchmark_runner.py

View on GitHub
#!/usr/bin/env python3
"""Benchmark runner for VeriFlow-CC — batch evaluation and reporting.

Usage:
    python benchmark_runner.py --project example_test/sm3/deepseek
    python benchmark_runner.py --compare example_test/sm3 --variants deepseek,glm4.7,glm5.1
    python benchmark_runner.py --all-projects example_test/
    python benchmark_runner.py --realbench /path/to/realbench.jsonl --output results.json

Outputs structured JSON with pass/fail breakdown, per-stage stats,
budget consumption, and LLM variant comparison.
"""

from __future__ import annotations

import argparse
import csv
import json
import sys
from dataclasses import dataclass, asdict, field
from pathlib import Path
import contextlib


@dataclass
class ProjectResult:
    project_dir: str
    project_name: str
    stages_completed: list[str]
    sim_passed: bool | None = None
    lint_passed: bool | None = None
    synth_passed: bool | None = None
    equiv_passed: bool | None = None
    retry_count: int = 0
    fix_loops_detected: int = 0
    rtl_lines: int = 0
    rtl_modules: int = 0
    sim_log_path: str = ""
    lint_log_path: str = ""
    synth_report_path: str = ""
    timing_diagnostic_path: str = ""
    stage_timings: dict = field(default_factory=dict)
    total_duration_s: float | None = None
    retries_per_stage: dict = field(default_factory=dict)
    notes: list[str] = None

    def __post_init__(self):
        if self.notes is None:
            self.notes = []

    def overall_pass(self) -> bool:
        # A benchmark pass requires sim PASS plus lint/synth/equiv not
        # explicitly failed. Unknown (None — tool absent or check not run)
        # does not block, but a recorded FAIL is disqualifying: previously
        # equivalence and synthesis results were ignored entirely, so a
        # design that failed SAT equivalence still counted as a pass — and
        # this verdict is the self-improvement referee's baseline metric.
        return (
            self.sim_passed is True
            and self.lint_passed is not False
            and self.synth_passed is not False
            and self.equiv_passed is not False
        )


class BenchmarkRunner:
    """Evaluate one or more VeriFlow projects and aggregate results."""

    def __init__(self, skill_dir: str | Path | None = None):
        self.skill_dir = Path(skill_dir) if skill_dir else None

    # ------------------------------------------------------------------
    # Per-project analysis
    # ------------------------------------------------------------------

    def analyze_project(self, project_dir: str | Path) -> ProjectResult:
        """Parse a single project's state and logs into a result record."""
        p = Path(project_dir).resolve()
        name = p.name

        result = ProjectResult(
            project_dir=str(p),
            project_name=name,
            stages_completed=[],
            notes=[],
        )

        # --- pipeline_state.json --------------------------------------
        state_file = p / ".veriflow" / "pipeline_state.json"
        if state_file.exists():
            try:
                state = json.loads(state_file.read_text(encoding="utf-8"))
                result.stages_completed = state.get("stages_completed", [])
                _rc = state.get("retry_count", {}) or {}
                result.retries_per_stage = dict(_rc) if isinstance(_rc, dict) else {}
                result.retry_count = sum(_rc.values()) if isinstance(_rc, dict) else 0
                # Per-stage durations (keep duration_s; drop raw timestamps).
                _st = state.get("stage_timings", {}) or {}
                _inferred = []
                for _stage, _t in _st.items():
                    if isinstance(_t, dict):
                        if "duration_s" in _t:
                            result.stage_timings[_stage] = _t["duration_s"]
                        if _t.get("duration_inferred"):
                            _inferred.append(_stage)
                if _inferred:
                    result.notes.append(
                        f"timing inferred (no --start) for: {', '.join(_inferred)}"
                    )
                # Total duration: prefer start_time/last_updated; else sum stages.
                _start = state.get("start_time")
                _last = state.get("last_updated")
                if isinstance(_start, (int, float)) and isinstance(_last, (int, float)):
                    result.total_duration_s = round(_last - _start, 1)
                elif result.stage_timings:
                    result.total_duration_s = round(sum(result.stage_timings.values()), 1)
            except (json.JSONDecodeError, OSError) as e:
                result.notes.append(f"state parse error: {e}")
        else:
            result.notes.append("no pipeline_state.json found")

        # --- simulation log -------------------------------------------
        sim_log = p / "logs" / "sim.log"
        result.sim_log_path = str(sim_log) if sim_log.exists() else ""
        if sim_log.exists():
            content = sim_log.read_text(encoding="utf-8", errors="ignore")
            result.sim_passed = (
                "ALL TESTS PASSED" in content
                and "[FAIL]" not in content
                and "FAILED:" not in content
            )
        else:
            result.sim_passed = None

        # --- lint log -------------------------------------------------
        lint_log = p / "logs" / "lint.log"
        result.lint_log_path = str(lint_log) if lint_log.exists() else ""
        if lint_log.exists():
            content = lint_log.read_text(encoding="utf-8", errors="ignore")
            # Line-based check: a summary line like "0 errors" must not mask a
            # real "error:" diagnostic elsewhere in the log (the old
            # whole-text rule did exactly that). "syntax error" contains
            # "error" so it is covered by the same test.
            def _is_error_line(ln: str) -> bool:
                s = ln.strip().lower()
                if not s:
                    return False
                if "no errors" in s or "0 errors" in s or "errors: 0" in s:
                    return False  # summary line, not a diagnostic
                return "error" in s

            result.lint_passed = not any(
                _is_error_line(ln) for ln in content.splitlines()
            )
        else:
            result.lint_passed = None

        # --- synthesis report -----------------------------------------
        synth_report = p / "workspace" / "synth" / "synth_report.txt"
        result.synth_report_path = str(synth_report) if synth_report.exists() else ""
        if synth_report.exists():
            content = synth_report.read_text(encoding="utf-8", errors="ignore")
            # Tight heuristic: require a positive completion marker (yosys prints
            # "Number of cells:") AND no error markers. The old rule
            # `"error" not in content or "synth" in content` was always True
            # (every report contains "synth"), masking real synthesis errors.
            lower = content.lower()
            has_synth_output = "number of cells:" in lower
            has_synth_error = "error" in lower or "synthesis failed" in lower
            result.synth_passed = has_synth_output and not has_synth_error
            # Extract cell count
            for line in content.splitlines():
                if "Number of cells:" in line:
                    with contextlib.suppress(Exception):
                        result.notes.append(line.strip())
                    break
        else:
            result.synth_passed = None

        # --- equivalence check ----------------------------------------
        equiv_file = p / "logs" / "yosys_equiv_synth.json"
        if equiv_file.exists():
            try:
                equiv = json.loads(equiv_file.read_text(encoding="utf-8"))
                result.equiv_passed = equiv.get("equivalent")
            except Exception:
                result.equiv_passed = None

        # --- RTL stats ------------------------------------------------
        rtl_dir = p / "workspace" / "rtl"
        if rtl_dir.exists():
            v_files = list(rtl_dir.glob("*.v"))
            result.rtl_modules = len(v_files)
            result.rtl_lines = sum(
                len(f.read_text(encoding="utf-8", errors="ignore").splitlines())
                for f in v_files
            )

        # --- timing diagnostic ----------------------------------------
        diag_file = p / "logs" / "timing_diagnostic.json"
        result.timing_diagnostic_path = str(diag_file) if diag_file.exists() else ""
        if diag_file.exists():
            try:
                diag = json.loads(diag_file.read_text(encoding="utf-8"))
                bug_class = diag.get("bug_class", "?")
                result.notes.append(f"timing_diagnostic: {bug_class}")
            except Exception:
                pass

        return result

    # ------------------------------------------------------------------
    # Batch operations
    # ------------------------------------------------------------------

    def run_projects(self, project_dirs: list[str | Path]) -> list[ProjectResult]:
        """Analyze multiple projects and return results."""
        return [self.analyze_project(d) for d in project_dirs]

    def compare_variants(
        self,
        base_dir: str | Path,
        variants: list[str],
    ) -> dict:
        """Compare multiple LLM variants on the same base design."""
        base = Path(base_dir)
        results = []
        for variant in variants:
            variant_dir = base / variant
            if variant_dir.exists():
                results.append(self.analyze_project(variant_dir))
            else:
                results.append(ProjectResult(
                    project_dir=str(variant_dir),
                    project_name=variant,
                    stages_completed=[],
                    notes=["variant directory not found"],
                ))

        # Build comparison table
        return {
            "base_design": base.name,
            "variants": [r.project_name for r in results],
            "overall_pass": {r.project_name: r.overall_pass() for r in results},
            "rtl_lines": {r.project_name: r.rtl_lines for r in results},
            "rtl_modules": {r.project_name: r.rtl_modules for r in results},
            "retry_count": {r.project_name: r.retry_count for r in results},
            "per_variant": [asdict(r) for r in results],
        }

    def run_all_in_dir(self, root_dir: str | Path) -> list[ProjectResult]:
        """Find all project directories under root and analyze them."""
        root = Path(root_dir)
        # A project directory has .veriflow/pipeline_state.json
        project_dirs = [
            d.parent.parent
            for d in root.rglob(".veriflow/pipeline_state.json")
        ]
        # Also include immediate subdirs that look like projects
        for subdir in root.iterdir():
            if subdir.is_dir():
                state_file = subdir / ".veriflow" / "pipeline_state.json"
                if state_file.exists() and subdir not in project_dirs:
                    project_dirs.append(subdir)
        return self.run_projects(sorted(set(str(d) for d in project_dirs)))

    # ------------------------------------------------------------------
    # Reporting
    # ------------------------------------------------------------------

    @staticmethod
    def aggregate(results: list[ProjectResult]) -> dict:
        """Compute aggregate statistics across all results."""
        total = len(results)
        passed = sum(1 for r in results if r.overall_pass())
        failed = total - passed
        sim_passed = sum(1 for r in results if r.sim_passed is True)
        lint_passed = sum(1 for r in results if r.lint_passed is not False)
        synth_done = sum(1 for r in results if r.synth_passed is not None)
        total_retries = sum(r.retry_count for r in results)
        total_lines = sum(r.rtl_lines for r in results)

        durations = [r.total_duration_s for r in results if r.total_duration_s is not None]
        avg_duration_s = round(sum(durations) / len(durations), 1) if durations else None
        per_stage_avg = {}
        for stage in ("spec_golden", "codegen", "verify_fix", "lint_synth"):
            vals = [r.stage_timings[stage] for r in results
                    if isinstance(r.stage_timings.get(stage), (int, float))]
            if vals:
                per_stage_avg[stage] = round(sum(vals) / len(vals), 1)

        return {
            "total_projects": total,
            "overall_pass": passed,
            "overall_fail": failed,
            "pass_rate": round(passed / total, 2) if total else 0,
            "sim_passed": sim_passed,
            "lint_passed": lint_passed,
            "synth_completed": synth_done,
            "total_retries": total_retries,
            "total_rtl_lines": total_lines,
            "total_duration_s_sum": sum(durations),
            "avg_duration_s": avg_duration_s,
            "per_stage_avg": per_stage_avg,
            "per_project": [asdict(r) for r in results],
        }

    @staticmethod
    def to_markdown(aggregate: dict) -> str:
        """Render aggregate results as Markdown."""
        lines = [
            "# VeriFlow Benchmark Report",
            "",
            f"**Projects evaluated**: {aggregate['total_projects']}",
            f"**Overall pass**: {aggregate['overall_pass']} / {aggregate['total_projects']} "
            f"({aggregate['pass_rate']:.0%})",
            f"**Simulation passed**: {aggregate['sim_passed']}",
            f"**Lint passed**: {aggregate['lint_passed']}",
            f"**Synthesis completed**: {aggregate['synth_completed']}",
            f"**Total retries**: {aggregate['total_retries']}",
            f"**Total RTL lines**: {aggregate['total_rtl_lines']}",
            "",
            "| Project | Stages | Sim | Lint | Synth | Lines | Notes |",
            "|---------|--------|-----|------|-------|-------|-------|",
        ]
        for r in aggregate["per_project"]:
            stages = ", ".join(r["stages_completed"]) if r["stages_completed"] else "—"
            sim = "PASS" if r["sim_passed"] else ("FAIL" if r["sim_passed"] is False else "—")
            lint = "PASS" if r["lint_passed"] else ("FAIL" if r["lint_passed"] is False else "—")
            synth = "PASS" if r["synth_passed"] else ("FAIL" if r["synth_passed"] is False else "—")
            notes = "; ".join(r["notes"][:2]) if r["notes"] else ""
            lines.append(
                f"| {r['project_name']} | {stages} | {sim} | {lint} | {synth} | "
                f"{r['rtl_lines']} | {notes} |"
            )

        # Timing section
        per_stage_avg = aggregate.get("per_stage_avg", {}) or {}
        total_dur = aggregate.get("total_duration_s_sum")
        avg = aggregate.get("avg_duration_s")
        lines.append("")
        lines.append("## Timing")
        if total_dur is not None:
            lines.append(f"**Total duration (sum)**: {round(total_dur, 1)}s")
        if avg is not None:
            lines.append(f"**Average per project**: {avg}s")
        if per_stage_avg:
            lines.append("")
            lines.append("| Stage | Avg (s) |")
            lines.append("|-------|---------|")
            for stage in ("spec_golden", "codegen", "verify_fix", "lint_synth"):
                if stage in per_stage_avg:
                    lines.append(f"| {stage} | {per_stage_avg[stage]} |")
        return "\n".join(lines)


# ------------------------------------------------------------------
# RealBench support
# ------------------------------------------------------------------

def parse_realbench_jsonl(path: str) -> list[dict]:
    """Parse RealBench JSONL format into task descriptors."""
    tasks = []
    with open(path, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                tasks.append(json.loads(line))
            except json.JSONDecodeError:
                continue
    return tasks


def realbench_to_project(task: dict, output_dir: Path) -> Path:
    """Convert a RealBench task into a VeriFlow project directory."""
    task_id = task.get("task_id", "unknown")
    proj = output_dir / task_id
    proj.mkdir(parents=True, exist_ok=True)

    # Write requirement.md
    requirement = task.get("description", task.get("prompt", ""))
    (proj / "requirement.md").write_text(requirement, encoding="utf-8")

    # Write constraints.md if available
    constraints = task.get("constraints", {})
    if constraints:
        lines = ["# Constraints", ""]
        for k, v in constraints.items():
            lines.append(f"- **{k}**: {v}")
        (proj / "constraints.md").write_text("\n".join(lines), encoding="utf-8")

    return proj


# ------------------------------------------------------------------
# CLI
# ------------------------------------------------------------------

def generate_pipeline_report(project_dir: str | Path) -> str:
    """Build a markdown pipeline_report.md for one project from its logs + state.

    Reads logs/sim.log, logs/lint.log, workspace/synth/synth_report.txt and
    .veriflow/pipeline_state.json. Missing artifacts render as N/A — never raises.
    Writes <project_dir>/pipeline_report.md and returns the body.
    """
    p = Path(project_dir).resolve()
    state = {}
    state_file = p / ".veriflow" / "pipeline_state.json"
    if state_file.exists():
        try:
            state = json.loads(state_file.read_text(encoding="utf-8"))
        except (json.JSONDecodeError, OSError):
            pass

    stages = state.get("stages_completed", []) or []
    st = state.get("stage_timings", {}) or {}
    retries = state.get("retry_count", {}) or {}
    start = state.get("start_time")
    last = state.get("last_updated")
    total = (round(last - start, 1)
             if isinstance(start, (int, float)) and isinstance(last, (int, float))
             else None)

    def _read(rel):
        f = p / rel
        return f.read_text(encoding="utf-8", errors="ignore") if f.exists() else ""

    sim = _read("logs/sim.log")
    lint = _read("logs/lint.log")
    synth = _read("workspace/synth/synth_report.txt")

    if sim and "ALL TESTS PASSED" in sim and "[FAIL]" not in sim and "FAILED:" not in sim:
        sim_verdict = "PASS"
    elif sim:
        sim_verdict = "FAIL"
    else:
        sim_verdict = "N/A"
    fail_line = next((ln.strip() for ln in sim.splitlines()
                      if "[FAIL]" in ln or "FAILED:" in ln), "")

    low = lint.lower()
    has_err = (("error" in low and "no errors" not in low and "0 errors" not in low)
               or "syntax error" in low)
    lint_verdict = "PASS" if (lint and not has_err) else ("FAIL" if lint else "N/A")
    cells = next((ln.strip() for ln in synth.splitlines() if "Number of cells:" in ln), "")

    lines = [
        f"# Pipeline Report — {p.name}",
        "",
        "## Summary",
        f"- Stages completed: {', '.join(stages) if stages else 'none'}",
        f"- Simulation: **{sim_verdict}**",
        f"- Lint: **{lint_verdict}**",
        f"- Synthesis: {cells or 'N/A'}",
        "",
        "## Timing",
    ]
    for stage in ("spec_golden", "codegen", "verify_fix", "lint_synth"):
        t = st.get(stage, {})
        if isinstance(t, dict):
            d = t.get("duration_s", "—")
            mark = " (inferred)" if t.get("duration_inferred") else ""
            lines.append(f"- {stage}: {d}s{mark}")
    lines.append(f"- **Total**: {total if total is not None else '—'}s")
    lines += [
        "",
        "## Simulation",
        ("ALL TESTS PASSED." if sim_verdict == "PASS"
         else (f"First failure: `{fail_line}`" if fail_line else "No simulation log.")),
        "",
        "## Retries",
    ]
    if retries:
        for k, v in retries.items():
            lines.append(f"- {k}: {v}")
    else:
        lines.append("None.")

    body = "\n".join(lines) + "\n"
    with contextlib.suppress(OSError):
        (p / "pipeline_report.md").write_text(body, encoding="utf-8")
    return body


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description="VeriFlow-CC benchmark runner")
    parser.add_argument("--project", help="Single project directory to evaluate")
    parser.add_argument("--compare", help="Base directory containing variant subdirs")
    parser.add_argument("--variants", help="Comma-separated variant names for --compare")
    parser.add_argument("--all-projects", help="Root directory — evaluate all projects found")
    parser.add_argument("--realbench", help="Path to RealBench JSONL file")
    parser.add_argument("--realbench-output", default="realbench_projects",
                        help="Directory to write RealBench project dirs (default: realbench_projects)")
    parser.add_argument("--output", "-o", help="Output JSON file path")
    parser.add_argument("--csv", help="Output CSV file path")
    parser.add_argument("--markdown", "-m", action="store_true",
                        help="Print Markdown report to stdout")
    parser.add_argument("--skill-dir", help="Path to installed skill directory")
    parser.add_argument("--report", help="Generate pipeline_report.md for a project dir and exit")
    args = parser.parse_args(argv)

    if args.report:
        generate_pipeline_report(args.report)
        print(f"[Benchmark] pipeline_report.md written to "
              f"{Path(args.report) / 'pipeline_report.md'}")
        return 0

    runner = BenchmarkRunner(skill_dir=args.skill_dir)

    # --- Single project ------------------------------------------------
    if args.project:
        result = runner.analyze_project(args.project)
        agg = BenchmarkRunner.aggregate([result])

    # --- Variant comparison --------------------------------------------
    elif args.compare and args.variants:
        variants = [v.strip() for v in args.variants.split(",")]
        comparison = runner.compare_variants(args.compare, variants)
        agg = comparison  # Already structured

    # --- All projects in directory -------------------------------------
    elif args.all_projects:
        results = runner.run_all_in_dir(args.all_projects)
        agg = BenchmarkRunner.aggregate(results)

    # --- RealBench -----------------------------------------------------
    elif args.realbench:
        tasks = parse_realbench_jsonl(args.realbench)
        output_dir = Path(args.realbench_output)
        output_dir.mkdir(parents=True, exist_ok=True)
        created = []
        for task in tasks:
            proj = realbench_to_project(task, output_dir)
            created.append(str(proj))
        agg = {
            "realbench_tasks": len(tasks),
            "projects_created": len(created),
            "project_dirs": created,
        }
        print(f"[RealBench] Created {len(created)} project directories in {output_dir}")

    else:
        parser.error("Specify one of: --project, --compare, --all-projects, --realbench")
        return 1

    # --- Output --------------------------------------------------------
    if args.output:
        Path(args.output).write_text(json.dumps(agg, indent=2), encoding="utf-8")
        print(f"[Benchmark] JSON report written to {args.output}")

    if args.csv and "per_project" in agg:
        csv_path = Path(args.csv)
        with open(csv_path, "w", newline="", encoding="utf-8") as f:
            if agg["per_project"]:
                writer = csv.DictWriter(f, fieldnames=agg["per_project"][0].keys())
                writer.writeheader()
                writer.writerows(agg["per_project"])
        print(f"[Benchmark] CSV report written to {args.csv}")

    if args.markdown and "per_project" in agg:
        print(BenchmarkRunner.to_markdown(agg))
    elif not args.output and not args.csv:
        print(json.dumps(agg, indent=2))

    return 0


if __name__ == "__main__":
    sys.exit(main())
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall