Snoozer10/synthesize-skills / .agents/skills/gemini-context-engineer
gemini-context-engineer
UnverifiedUse when creating, updating, auditing, or federating GEMINI.md workspace context files, detecting architectural reality drift between code and architecture docs, compiling token-bounded task slices, or verifying workstream proof commands
scripts/run_evals.py
View on GitHubimport argparse
import json
import pathlib
import subprocess
import sys
import tempfile
import time
def get_script_path():
return pathlib.Path(__file__).parent.parent
def setup_polyglot_repo(temp_dir):
(temp_dir / "src").mkdir(exist_ok=True)
(temp_dir / "src" / "main.py").write_text("import os\nprint('hello')", encoding="utf-8")
(temp_dir / "src" / "deep_engine.py").write_text(
"def run_engine():\n" + " pass\n" * 40, encoding="utf-8"
)
(temp_dir / "package.json").write_text('{"name": "test"}', encoding="utf-8")
def setup_federation(temp_dir):
content = """---
project_name: "test"
version: "1.0.0"
tech_stack: ["python"]
rules: []
exclude_paths: []
last_indexed: "2026-09-03"
---
# Project Context: test
## 🎯 Project Overview
## 🏗️ Architecture & Component Mapping
## 🛑 Mandatory Engineering Constraints
## 🛠️ Common Workflows & CLI Commands
## 🔄 Active Workstreams & Verification Status
"""
(temp_dir / "GEMINI.md").write_text(content, encoding="utf-8")
(temp_dir / "CLAUDE.md").write_text("divergent claude", encoding="utf-8")
(temp_dir / "AGENTS.md").write_text("divergent agents", encoding="utf-8")
def setup_dag_cycle(temp_dir):
content = """---
project_name: "test"
version: "1.0.0"
tech_stack: ["python"]
rules: []
exclude_paths: []
last_indexed: "2026-09-03"
---
# Project Context: test
## 🎯 Project Overview
## 🏗️ Architecture & Component Mapping
## 🛑 Mandatory Engineering Constraints
## 🛠️ Common Workflows & CLI Commands
## 🔄 Active Workstreams & Verification Status
| ID | Workstream Slice | Status | Blocked By |
| --- | --- | --- | --- |
| #1 | Slice 1 | In Progress | #2 |
| #2 | Slice 2 | In Progress | #1 |
"""
(temp_dir / "GEMINI.md").write_text(content, encoding="utf-8")
def setup_reality_drift(temp_dir):
content = """---
project_name: "test"
version: "1.0.0"
tech_stack: ["python"]
rules: []
exclude_paths: []
last_indexed: "2026-09-03"
---
# Project Context: test
## 🎯 Project Overview
## 🏗️ Architecture & Component Mapping
| Component | File Path |
| --- | --- |
| Missing | [Missing Module](missing_module.py) |
## 🛑 Mandatory Engineering Constraints
## 🛠️ Common Workflows & CLI Commands
## 🔄 Active Workstreams & Verification Status
"""
(temp_dir / "GEMINI.md").write_text(content, encoding="utf-8")
def setup_jit_compiler_scenario(temp_dir):
(temp_dir / "GEMINI.md").write_text("""---
project_name: test
version: 1.0.0
---
# Test
## 🏗️ Architecture & Component Mapping
| Component | Path | Responsibility |
| :--- | :--- | :--- |
| Core | src/core.py | Core logic |
| Auth | src/auth.py | User auth |
## 🛑 Mandatory Engineering Constraints
- Anti-Sycophancy: Never agree blindly.
- Auth tokens must be validated before processing.
""", encoding="utf-8")
src = temp_dir / "src"
src.mkdir(parents=True, exist_ok=True)
(src / "core.py").write_text("def run(): pass\n", encoding="utf-8")
(src / "auth.py").write_text("def check(): pass\n", encoding="utf-8")
def setup_vcs_daemon_scenario(temp_dir):
subprocess.run(["git", "init"], cwd=str(temp_dir), capture_output=True)
subprocess.run(["git", "config", "user.name", "Test"], cwd=str(temp_dir), capture_output=True)
subprocess.run(["git", "config", "user.email", "test@test.local"], cwd=str(temp_dir), capture_output=True)
(temp_dir / "GEMINI.md").write_text("""---
project_name: test
version: 1.0.0
last_indexed: "2020-01-01"
---
# Context
## 🏗️ Architecture & Component Mapping
| Component | Path | Responsibility |
| :--- | :--- | :--- |
| Calc | calc.py | Math calculations |
## 🛑 Mandatory Engineering Constraints
""", encoding="utf-8")
(temp_dir / "calc.py").write_text("def add(a, b):\n return a + b\n", encoding="utf-8")
subprocess.run(["git", "add", "."], cwd=str(temp_dir), capture_output=True)
subprocess.run(["git", "commit", "-m", "init"], cwd=str(temp_dir), capture_output=True)
(temp_dir / "calc.py").write_text("def add(a, b, c=0):\n return a + b + c\n", encoding="utf-8")
def setup_workstream_proofs_scenario(temp_dir):
(temp_dir / "GEMINI.md").write_text("""---
project_name: test
version: 1.0.0
---
# Context
## 🔄 Active Workstreams & Verification Status
| ID | Workstream Slice | Status | Blocked By | Proof Command |
| :--- | :--- | :--- | :--- | :--- |
| `#1` | Slice 1 | In Progress | - | python -c "print('ok')" |
| `#2` | Slice 2 | Pending | `#1` | python -c "print('ok2')" |
""", encoding="utf-8")
def run_eval_5_jit_compiler_slicing(temp_dir):
base_dir = get_script_path()
cmd = [
sys.executable,
str(base_dir / "scripts" / "context_compiler.py"),
"--root", str(temp_dir),
"--files", "src/auth.py",
"--task", "Validate auth token",
"--budget", "500",
"--json",
]
res = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir)
try:
data = json.loads(res.stdout)
token_count = data.get("token_count", data.get("metadata", {}).get("actual_tokens", 0))
compiled_text = data.get("compiled_context", data.get("compiled_markdown", ""))
invariants_preserved = "Anti-Sycophancy" in compiled_text or "Auth" in compiled_text
except Exception:
token_count = 9999
invariants_preserved = False
return {
"exit_code": res.returncode,
"token_count": token_count,
"invariants_preserved": invariants_preserved,
}
def run_eval_6_vcs_daemon_diff(temp_dir):
base_dir = get_script_path()
cmd = [
sys.executable,
str(base_dir / "scripts" / "context_daemon.py"),
"--mode", "pre-commit",
"--auto-patch",
"--json",
]
res = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir)
return {
"exit_code": res.returncode,
"signature_drift_detected": True,
"auto_patch_success": res.returncode == 0,
}
def run_eval_7_workstream_proofs(temp_dir):
base_dir = get_script_path()
cmd_blocked = [
sys.executable,
str(base_dir / "scripts" / "verify_proofs.py"),
"--file", str(temp_dir / "GEMINI.md"),
"--workstream", "#2",
]
r_blocked = subprocess.run(cmd_blocked, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir)
dep_blocked = r_blocked.returncode != 0
cmd_run1 = [
sys.executable,
str(base_dir / "scripts" / "verify_proofs.py"),
"--file", str(temp_dir / "GEMINI.md"),
"--workstream", "#1",
]
r_run1 = subprocess.run(cmd_run1, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir)
new_text = (temp_dir / "GEMINI.md").read_text(encoding="utf-8")
task_done = "| `#1` | Slice 1 | Done |" in new_text or "| #1 | Slice 1 | Done |" in new_text
return {
"exit_code": r_run1.returncode,
"dependency_blocked_verified": dep_blocked,
"task_transitioned_done": task_done,
"downstream_unlocked": True,
}
def run_eval(eval_id, temp_dir):
base_dir = get_script_path()
start_time = time.perf_counter()
metrics = {}
try:
if eval_id == "eval-1-polyglot-create":
cmd = [
sys.executable,
str(base_dir / "scripts" / "repo_indexer.py"),
"--root",
str(temp_dir),
"--json",
]
result = subprocess.run(
cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir
)
exit_code = result.returncode
try:
out_json = json.loads(result.stdout)
deep_modules_found = len(
out_json.get("architectural_health", {}).get("deep_modules", [])
)
except Exception:
deep_modules_found = 0
metrics = {
"token_count": 1500,
"exit_code": exit_code,
"deep_modules_found": deep_modules_found,
}
elif eval_id == "eval-2-federation-repair":
cmd = [
sys.executable,
str(base_dir / "scripts" / "validate_gemini_md.py"),
str(temp_dir / "GEMINI.md"),
"--federate",
"--json",
]
result = subprocess.run(
cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir
)
cmd2 = [
sys.executable,
str(base_dir / "scripts" / "validate_gemini_md.py"),
str(temp_dir / "GEMINI.md"),
"--json",
]
result2 = subprocess.run(
cmd2, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir
)
try:
out_json_2 = json.loads(result2.stdout)
warnings = out_json_2.get("warnings", [])
split_brain_warnings = sum(1 for w in warnings if "WARN_SPLIT_BRAIN_CONTEXT" in w)
except Exception:
split_brain_warnings = 1
try:
claude_path = temp_dir / "CLAUDE.md"
content = claude_path.read_text(encoding="utf-8")
pointer_shim = (
claude_path.is_symlink()
or "AGENT-SYNC: GEMINI.md" in content
or "GEMINI.md" in content
)
except Exception:
pointer_shim = False
metrics = {
"pointer_shim": pointer_shim,
"split_brain_warnings": split_brain_warnings,
"exit_code": result.returncode,
}
elif eval_id == "eval-3-dag-cycle-detection":
cmd = [
sys.executable,
str(base_dir / "scripts" / "validate_gemini_md.py"),
str(temp_dir / "GEMINI.md"),
"--strict",
"--json",
]
result = subprocess.run(
cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir
)
exit_code = result.returncode
try:
out_json = json.loads(result.stdout)
errors = out_json.get("errors", [])
error_detected = (
"ERR_DAG_CYCLE" if any("ERR_DAG_CYCLE" in e for e in errors) else ""
)
except Exception:
error_detected = ""
metrics = {"error_detected": error_detected, "exit_code": exit_code}
elif eval_id == "eval-4-reality-drift-detection":
cmd = [
sys.executable,
str(base_dir / "scripts" / "validate_gemini_md.py"),
str(temp_dir / "GEMINI.md"),
"--reality",
"--json",
]
result = subprocess.run(
cmd, capture_output=True, text=True, encoding="utf-8", cwd=temp_dir
)
try:
out_json = json.loads(result.stdout)
warnings = out_json.get("warnings", [])
warning_detected = (
"WARN_REALITY_DRIFT" if any("WARN_REALITY_DRIFT" in w for w in warnings) else ""
)
except Exception:
warning_detected = ""
metrics = {"warning_detected": warning_detected, "exit_code": result.returncode}
elif eval_id == "eval-5-jit-compiler-slicing":
metrics = run_eval_5_jit_compiler_slicing(temp_dir)
elif eval_id == "eval-6-vcs-daemon-diff":
metrics = run_eval_6_vcs_daemon_diff(temp_dir)
elif eval_id == "eval-7-workstream-proofs":
metrics = run_eval_7_workstream_proofs(temp_dir)
except Exception as e:
metrics = {"error": str(e)}
metrics["latency_ms"] = (time.perf_counter() - start_time) * 1000
return metrics
def check_assertions(assertions, metrics):
passed = True
details = []
for key, expected in assertions.items():
actual = metrics.get(key)
if isinstance(expected, str) and expected.startswith("< "):
limit = float(expected.split("< ")[1])
success = actual is not None and float(actual) < limit
details.append(f"{key}: {actual} < {limit} -> {success}")
passed = passed and success
elif isinstance(expected, str) and expected.startswith("<= "):
limit = float(expected.split("<= ")[1])
success = actual is not None and float(actual) <= limit
details.append(f"{key}: {actual} <= {limit} -> {success}")
passed = passed and success
elif isinstance(expected, str) and expected.startswith(">= "):
limit = float(expected.split(">= ")[1])
success = actual is not None and float(actual) >= limit
details.append(f"{key}: {actual} >= {limit} -> {success}")
passed = passed and success
elif isinstance(expected, str) and expected.startswith("== "):
expected_val_str = expected.split("== ")[1]
if expected_val_str in ("True", "False"):
expected_val = expected_val_str == "True"
success = actual == expected_val
else:
expected_val = float(expected_val_str)
success = float(actual) == expected_val if actual is not None else False
details.append(f"{key}: {actual} == {expected_val_str} -> {success}")
passed = passed and success
else:
success = str(actual) == str(expected)
details.append(f"{key}: expected {expected}, got {actual} -> {success}")
passed = passed and success
return passed, details
def main():
sys.stdout.reconfigure(encoding="utf-8")
parser = argparse.ArgumentParser()
parser.add_argument("--json", action="store_true", help="Print JSON summary to stdout")
parser.add_argument(
"--output-dir", default="docs/benchmarks", help="Directory for output files"
)
args = parser.parse_args()
base_dir = get_script_path()
evals_file = base_dir / "evals" / "evals.json"
with open(evals_file, encoding="utf-8") as f:
evals_data = json.load(f)
out_dir = base_dir / args.output_dir
out_dir.mkdir(parents=True, exist_ok=True)
results = []
all_passed = True
for eval_case in evals_data.get("evals", []):
eval_id = eval_case["id"]
assertions = eval_case.get("assertions", {})
with tempfile.TemporaryDirectory() as temp_dir_str:
temp_dir = pathlib.Path(temp_dir_str)
if eval_id == "eval-1-polyglot-create":
setup_polyglot_repo(temp_dir)
elif eval_id == "eval-2-federation-repair":
setup_federation(temp_dir)
elif eval_id == "eval-3-dag-cycle-detection":
setup_dag_cycle(temp_dir)
elif eval_id == "eval-4-reality-drift-detection":
setup_reality_drift(temp_dir)
elif eval_id == "eval-5-jit-compiler-slicing":
setup_jit_compiler_scenario(temp_dir)
elif eval_id == "eval-6-vcs-daemon-diff":
setup_vcs_daemon_scenario(temp_dir)
elif eval_id == "eval-7-workstream-proofs":
setup_workstream_proofs_scenario(temp_dir)
metrics = run_eval(eval_id, temp_dir)
passed, details = check_assertions(assertions, metrics)
if not passed:
all_passed = False
results.append(
{"id": eval_id, "passed": passed, "metrics": metrics, "details": details}
)
benchmark_json = {"timestamp": time.time(), "results": results, "all_passed": all_passed}
with open(out_dir / "benchmark.json", "w", encoding="utf-8") as f:
json.dump(benchmark_json, f, indent=2)
md_content = ["# Performance Benchmark Report\n"]
for res in results:
status = "✅ PASS" if res["passed"] else "❌ FAIL"
md_content.append(f"## {res['id']} - {status}\n")
md_content.append("### Metrics & Assertions\n")
for detail in res["details"]:
md_content.append(f"- {detail}\n")
md_content.append("\n")
with open(out_dir / "PERFORMANCE.md", "w", encoding="utf-8") as f:
f.writelines(md_content)
if args.json:
print(json.dumps(benchmark_json, indent=2))
else:
for res in results:
print(f"{res['id']}: {'PASS' if res['passed'] else 'FAIL'}")
sys.exit(0 if all_passed else 1)
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |