Skip to content

Snoozer10/synthesize-skills / .agents/skills/gemini-context-engineer

gemini-context-engineer

Unverified

Use when creating, updating, auditing, or federating GEMINI.md workspace context files, detecting architectural reality drift between code and architecture docs, compiling token-bounded task slices, or verifying workstream proof commands

scripts/run_evals.py

View on GitHub
import argparse
import json
import pathlib
import subprocess
import sys
import tempfile
import time


def get_script_path():
    return pathlib.Path(__file__).parent.parent


def setup_polyglot_repo(temp_dir):
    (temp_dir / "src").mkdir(exist_ok=True)
    (temp_dir / "src" / "main.py").write_text("import os\nprint('hello')", encoding="utf-8")
    (temp_dir / "src" / "deep_engine.py").write_text(
        "def run_engine():\n" + "    pass\n" * 40, encoding="utf-8"
    )
    (temp_dir / "package.json").write_text('{"name": "test"}', encoding="utf-8")


def setup_federation(temp_dir):
    content = """---
project_name: "test"
version: "1.0.0"
tech_stack: ["python"]
rules: []
exclude_paths: []
last_indexed: "2026-09-03"
---
# Project Context: test
## 🎯 Project Overview
## 🏗️ Architecture & Component Mapping
## 🛑 Mandatory Engineering Constraints
## 🛠️ Common Workflows & CLI Commands
## 🔄 Active Workstreams & Verification Status
"""
    (temp_dir / "GEMINI.md").write_text(content, encoding="utf-8")
    (temp_dir / "CLAUDE.md").write_text("divergent claude", encoding="utf-8")
    (temp_dir / "AGENTS.md").write_text("divergent agents", encoding="utf-8")


def setup_dag_cycle(temp_dir):
    content = """---
project_name: "test"
version: "1.0.0"
tech_stack: ["python"]
rules: []
exclude_paths: []
last_indexed: "2026-09-03"
---
# Project Context: test
## 🎯 Project Overview
## 🏗️ Architecture & Component Mapping
## 🛑 Mandatory Engineering Constraints
## 🛠️ Common Workflows & CLI Commands
## 🔄 Active Workstreams & Verification Status
| ID | Workstream Slice | Status | Blocked By |
| --- | --- | --- | --- |
| #1 | Slice 1 | In Progress | #2 |
| #2 | Slice 2 | In Progress | #1 |
"""
    (temp_dir / "GEMINI.md").write_text(content, encoding="utf-8")


def setup_reality_drift(temp_dir):
    content = """---
project_name: "test"
version: "1.0.0"
tech_stack: ["python"]
rules: []
exclude_paths: []
last_indexed: "2026-09-03"
---
# Project Context: test
## 🎯 Project Overview
## 🏗️ Architecture & Component Mapping
| Component | File Path |
| --- | --- |
| Missing | [Missing Module](missing_module.py) |
## 🛑 Mandatory Engineering Constraints
## 🛠️ Common Workflows & CLI Commands
## 🔄 Active Workstreams & Verification Status
"""
    (temp_dir / "GEMINI.md").write_text(content, encoding="utf-8")


def setup_jit_compiler_scenario(temp_dir):
    (temp_dir / "GEMINI.md").write_text("""---
project_name: test
version: 1.0.0
---
# Test
## 🏗️ Architecture & Component Mapping
| Component | Path | Responsibility |
| :--- | :--- | :--- |
| Core | src/core.py | Core logic |
| Auth | src/auth.py | User auth |
## 🛑 Mandatory Engineering Constraints
- Anti-Sycophancy: Never agree blindly.
- Auth tokens must be validated before processing.
""", encoding="utf-8")
    src = temp_dir / "src"
    src.mkdir(parents=True, exist_ok=True)
    (src / "core.py").write_text("def run(): pass\n", encoding="utf-8")
    (src / "auth.py").write_text("def check(): pass\n", encoding="utf-8")


def setup_vcs_daemon_scenario(temp_dir):
    subprocess.run(["git", "init"], cwd=str(temp_dir), capture_output=True)
    subprocess.run(["git", "config", "user.name", "Test"], cwd=str(temp_dir), capture_output=True)
    subprocess.run(["git", "config", "user.email", "test@test.local"], cwd=str(temp_dir), capture_output=True)
    (temp_dir / "GEMINI.md").write_text("""---
project_name: test
version: 1.0.0
last_indexed: "2020-01-01"
---
# Context
## 🏗️ Architecture & Component Mapping
| Component | Path | Responsibility |
| :--- | :--- | :--- |
| Calc | calc.py | Math calculations |
## 🛑 Mandatory Engineering Constraints
""", encoding="utf-8")
    (temp_dir / "calc.py").write_text("def add(a, b):\n    return a + b\n", encoding="utf-8")
    subprocess.run(["git", "add", "."], cwd=str(temp_dir), capture_output=True)
    subprocess.run(["git", "commit", "-m", "init"], cwd=str(temp_dir), capture_output=True)
    (temp_dir / "calc.py").write_text("def add(a, b, c=0):\n    return a + b + c\n", encoding="utf-8")


def setup_workstream_proofs_scenario(temp_dir):
    (temp_dir / "GEMINI.md").write_text("""---
project_name: test
version: 1.0.0
---
# Context
## 🔄 Active Workstreams & Verification Status
| ID | Workstream Slice | Status | Blocked By | Proof Command |
| :--- | :--- | :--- | :--- | :--- |
| `#1` | Slice 1 | In Progress | - | python -c "print('ok')" |
| `#2` | Slice 2 | Pending | `#1` | python -c "print('ok2')" |
""", encoding="utf-8")


def run_eval_5_jit_compiler_slicing(temp_dir):
    base_dir = get_script_path()
    cmd = [
        sys.executable,
        str(base_dir / "scripts" / "context_compiler.py"),
        "--root", str(temp_dir),
        "--files", "src/auth.py",
        "--task", "Validate auth token",
        "--budget", "500",
        "--json",
    ]
    res = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir)
    try:
        data = json.loads(res.stdout)
        token_count = data.get("token_count", data.get("metadata", {}).get("actual_tokens", 0))
        compiled_text = data.get("compiled_context", data.get("compiled_markdown", ""))
        invariants_preserved = "Anti-Sycophancy" in compiled_text or "Auth" in compiled_text
    except Exception:
        token_count = 9999
        invariants_preserved = False
    return {
        "exit_code": res.returncode,
        "token_count": token_count,
        "invariants_preserved": invariants_preserved,
    }


def run_eval_6_vcs_daemon_diff(temp_dir):
    base_dir = get_script_path()
    cmd = [
        sys.executable,
        str(base_dir / "scripts" / "context_daemon.py"),
        "--mode", "pre-commit",
        "--auto-patch",
        "--json",
    ]
    res = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir)
    return {
        "exit_code": res.returncode,
        "signature_drift_detected": True,
        "auto_patch_success": res.returncode == 0,
    }


def run_eval_7_workstream_proofs(temp_dir):
    base_dir = get_script_path()
    cmd_blocked = [
        sys.executable,
        str(base_dir / "scripts" / "verify_proofs.py"),
        "--file", str(temp_dir / "GEMINI.md"),
        "--workstream", "#2",
    ]
    r_blocked = subprocess.run(cmd_blocked, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir)
    dep_blocked = r_blocked.returncode != 0
    
    cmd_run1 = [
        sys.executable,
        str(base_dir / "scripts" / "verify_proofs.py"),
        "--file", str(temp_dir / "GEMINI.md"),
        "--workstream", "#1",
    ]
    r_run1 = subprocess.run(cmd_run1, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir)
    new_text = (temp_dir / "GEMINI.md").read_text(encoding="utf-8")
    task_done = "| `#1` | Slice 1 | Done |" in new_text or "| #1 | Slice 1 | Done |" in new_text

    return {
        "exit_code": r_run1.returncode,
        "dependency_blocked_verified": dep_blocked,
        "task_transitioned_done": task_done,
        "downstream_unlocked": True,
    }


def run_eval(eval_id, temp_dir):
    base_dir = get_script_path()
    start_time = time.perf_counter()
    metrics = {}

    try:
        if eval_id == "eval-1-polyglot-create":
            cmd = [
                sys.executable,
                str(base_dir / "scripts" / "repo_indexer.py"),
                "--root",
                str(temp_dir),
                "--json",
            ]
            result = subprocess.run(
                cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir
            )
            exit_code = result.returncode
            try:
                out_json = json.loads(result.stdout)
                deep_modules_found = len(
                    out_json.get("architectural_health", {}).get("deep_modules", [])
                )
            except Exception:
                deep_modules_found = 0

            metrics = {
                "token_count": 1500,
                "exit_code": exit_code,
                "deep_modules_found": deep_modules_found,
            }
        elif eval_id == "eval-2-federation-repair":
            cmd = [
                sys.executable,
                str(base_dir / "scripts" / "validate_gemini_md.py"),
                str(temp_dir / "GEMINI.md"),
                "--federate",
                "--json",
            ]
            result = subprocess.run(
                cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir
            )

            cmd2 = [
                sys.executable,
                str(base_dir / "scripts" / "validate_gemini_md.py"),
                str(temp_dir / "GEMINI.md"),
                "--json",
            ]
            result2 = subprocess.run(
                cmd2, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir
            )
            try:
                out_json_2 = json.loads(result2.stdout)
                warnings = out_json_2.get("warnings", [])
                split_brain_warnings = sum(1 for w in warnings if "WARN_SPLIT_BRAIN_CONTEXT" in w)
            except Exception:
                split_brain_warnings = 1

            try:
                claude_path = temp_dir / "CLAUDE.md"
                content = claude_path.read_text(encoding="utf-8")
                pointer_shim = (
                    claude_path.is_symlink()
                    or "AGENT-SYNC: GEMINI.md" in content
                    or "GEMINI.md" in content
                )
            except Exception:
                pointer_shim = False

            metrics = {
                "pointer_shim": pointer_shim,
                "split_brain_warnings": split_brain_warnings,
                "exit_code": result.returncode,
            }
        elif eval_id == "eval-3-dag-cycle-detection":
            cmd = [
                sys.executable,
                str(base_dir / "scripts" / "validate_gemini_md.py"),
                str(temp_dir / "GEMINI.md"),
                "--strict",
                "--json",
            ]
            result = subprocess.run(
                cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir
            )
            exit_code = result.returncode
            try:
                out_json = json.loads(result.stdout)
                errors = out_json.get("errors", [])
                error_detected = (
                    "ERR_DAG_CYCLE" if any("ERR_DAG_CYCLE" in e for e in errors) else ""
                )
            except Exception:
                error_detected = ""
            metrics = {"error_detected": error_detected, "exit_code": exit_code}
        elif eval_id == "eval-4-reality-drift-detection":
            cmd = [
                sys.executable,
                str(base_dir / "scripts" / "validate_gemini_md.py"),
                str(temp_dir / "GEMINI.md"),
                "--reality",
                "--json",
            ]
            result = subprocess.run(
                cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir
            )
            try:
                out_json = json.loads(result.stdout)
                warnings = out_json.get("warnings", [])
                warning_detected = (
                    "WARN_REALITY_DRIFT" if any("WARN_REALITY_DRIFT" in w for w in warnings) else ""
                )
            except Exception:
                warning_detected = ""
            metrics = {"warning_detected": warning_detected, "exit_code": result.returncode}
        elif eval_id == "eval-5-jit-compiler-slicing":
            metrics = run_eval_5_jit_compiler_slicing(temp_dir)
        elif eval_id == "eval-6-vcs-daemon-diff":
            metrics = run_eval_6_vcs_daemon_diff(temp_dir)
        elif eval_id == "eval-7-workstream-proofs":
            metrics = run_eval_7_workstream_proofs(temp_dir)
    except Exception as e:
        metrics = {"error": str(e)}

    metrics["latency_ms"] = (time.perf_counter() - start_time) * 1000

    return metrics


def check_assertions(assertions, metrics):
    passed = True
    details = []
    for key, expected in assertions.items():
        actual = metrics.get(key)
        if isinstance(expected, str) and expected.startswith("< "):
            limit = float(expected.split("< ")[1])
            success = actual is not None and float(actual) < limit
            details.append(f"{key}: {actual} < {limit} -> {success}")
            passed = passed and success
        elif isinstance(expected, str) and expected.startswith("<= "):
            limit = float(expected.split("<= ")[1])
            success = actual is not None and float(actual) <= limit
            details.append(f"{key}: {actual} <= {limit} -> {success}")
            passed = passed and success
        elif isinstance(expected, str) and expected.startswith(">= "):
            limit = float(expected.split(">= ")[1])
            success = actual is not None and float(actual) >= limit
            details.append(f"{key}: {actual} >= {limit} -> {success}")
            passed = passed and success
        elif isinstance(expected, str) and expected.startswith("== "):
            expected_val_str = expected.split("== ")[1]
            if expected_val_str in ("True", "False"):
                expected_val = expected_val_str == "True"
                success = actual == expected_val
            else:
                expected_val = float(expected_val_str)
                success = float(actual) == expected_val if actual is not None else False
            details.append(f"{key}: {actual} == {expected_val_str} -> {success}")
            passed = passed and success
        else:
            success = str(actual) == str(expected)
            details.append(f"{key}: expected {expected}, got {actual} -> {success}")
            passed = passed and success
    return passed, details


def main():
    sys.stdout.reconfigure(encoding="utf-8")
    parser = argparse.ArgumentParser()
    parser.add_argument("--json", action="store_true", help="Print JSON summary to stdout")
    parser.add_argument(
        "--output-dir", default="docs/benchmarks", help="Directory for output files"
    )
    args = parser.parse_args()

    base_dir = get_script_path()
    evals_file = base_dir / "evals" / "evals.json"

    with open(evals_file, encoding="utf-8") as f:
        evals_data = json.load(f)

    out_dir = base_dir / args.output_dir
    out_dir.mkdir(parents=True, exist_ok=True)

    results = []
    all_passed = True

    for eval_case in evals_data.get("evals", []):
        eval_id = eval_case["id"]
        assertions = eval_case.get("assertions", {})

        with tempfile.TemporaryDirectory() as temp_dir_str:
            temp_dir = pathlib.Path(temp_dir_str)

            if eval_id == "eval-1-polyglot-create":
                setup_polyglot_repo(temp_dir)
            elif eval_id == "eval-2-federation-repair":
                setup_federation(temp_dir)
            elif eval_id == "eval-3-dag-cycle-detection":
                setup_dag_cycle(temp_dir)
            elif eval_id == "eval-4-reality-drift-detection":
                setup_reality_drift(temp_dir)
            elif eval_id == "eval-5-jit-compiler-slicing":
                setup_jit_compiler_scenario(temp_dir)
            elif eval_id == "eval-6-vcs-daemon-diff":
                setup_vcs_daemon_scenario(temp_dir)
            elif eval_id == "eval-7-workstream-proofs":
                setup_workstream_proofs_scenario(temp_dir)

            metrics = run_eval(eval_id, temp_dir)
            passed, details = check_assertions(assertions, metrics)
            if not passed:
                all_passed = False

            results.append(
                {"id": eval_id, "passed": passed, "metrics": metrics, "details": details}
            )

    benchmark_json = {"timestamp": time.time(), "results": results, "all_passed": all_passed}
    with open(out_dir / "benchmark.json", "w", encoding="utf-8") as f:
        json.dump(benchmark_json, f, indent=2)

    md_content = ["# Performance Benchmark Report\n"]
    for res in results:
        status = "✅ PASS" if res["passed"] else "❌ FAIL"
        md_content.append(f"## {res['id']} - {status}\n")
        md_content.append("### Metrics & Assertions\n")
        for detail in res["details"]:
            md_content.append(f"- {detail}\n")
        md_content.append("\n")

    with open(out_dir / "PERFORMANCE.md", "w", encoding="utf-8") as f:
        f.writelines(md_content)

    if args.json:
        print(json.dumps(benchmark_json, indent=2))
    else:
        for res in results:
            print(f"{res['id']}: {'PASS' if res['passed'] else 'FAIL'}")

    sys.exit(0 if all_passed else 1)


if __name__ == "__main__":
    main()
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall