MongLong0214/frontier-simplify / skills/frontier-simplify-review
frontier-simplify-review
UnverifiedReview a change for concrete defects, check them with a refuter and a reproducer, then check repairs and their regressions against the preserved review. Use for substantial PR reviews or repeated review rounds. Stops automatic review after at most three attempts per PR and names the calling agent's next action; does not guarantee safe merge convergence.
scripts/lib/evidence.py
View on GitHub"""Reviewer witnesses outlive the checkout they were written in and run again without a model.
A reviewer's reproduction used to live in the disposable checkout and die with it, so the next
reviewer rebuilt it, and an unfinished repair cost a whole review to discover. One consumer
round reported "the same defect remains on the queue path" -- a fact the previous round's own
witness would have shown in seconds. The host keeps review_evidence/ outside the checkout and
reruns its witnesses on the next head first. A witness is a claim the reviewer wrote down as a
program; its exit code is the only thing read, and a pass is not closure by itself.
"""
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import signal
import subprocess
import tempfile
import time
from protocol import digest, require, child_environment
import ledger
EVIDENCE = 'review_evidence'
SKIP_DIRECTORIES = {'node_modules', '__pycache__', '.pytest_cache', '.mypy_cache', '.ruff_cache',
'.venv', 'venv', '.git'}
MAX_FILE, MAX_TOTAL, MAX_FILES = 1 << 20, 16 << 20, 200
WITNESS_ID = re.compile(r'[A-Za-z0-9][A-Za-z0-9._-]{0,79}')
OBSOLETE = re.compile(r'^[ \t]*[-*][ \t]+witness_obsolete:[ \t]*`?([A-Za-z0-9][A-Za-z0-9._-]*)`?'
r'[ \t]*(?:--|—|–|:|\|)?[ \t]*(.*?)[ \t]*$', re.M)
def run(args, cwd=None):
return subprocess.run(list(map(str, args)), cwd=cwd, capture_output=True, env=child_environment())
def witness_environment():
"""A witness is reviewer-written code, and the host runs it outside the reviewer's sandbox.
So it gets what a shell needs and nothing the runner happens to hold: no tokens, keys or
host settings from the environment. File access is the wrapper's job (REVIEW_WITNESS_WRAPPER).
"""
keep = {'PATH', 'HOME', 'TMPDIR', 'LANG', 'LC_ALL', 'LC_CTYPE', 'USER', 'LOGNAME', 'SHELL', 'TERM'}
return dict({k: v for k, v in child_environment().items() if k in keep}, PYTHONDONTWRITEBYTECODE='1')
def evidence_path(rel):
path = PurePosixPath(rel)
require(not path.is_absolute() and '..' not in path.parts and len(path.parts) >= 2
and path.parts[0] == EVIDENCE and '\\' not in rel, 'witness-path',
f'preserved witness path escapes {EVIDENCE}/: {rel!r}')
return path
def remove(path):
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
else:
path.unlink()
def prepare(clone, overlay, replace=False):
"""Create review_evidence/ and restore the preserved files.
The reviewer's checkout keeps the tree's own files (replace=False): its tracked bytes are
observed afterwards. A disposable precheck clone replaces them (replace=True), because a
witness is only the reviewer's witness when every preserved file it reads is the preserved
one -- the tree under review must not supply a helper. Neither mode writes through a link.
"""
target = clone / EVIDENCE
if target.is_symlink() or (target.exists() and not target.is_dir()):
if not replace:
return None, [f'{EVIDENCE} is not a directory in this tree; witnesses are not preserved']
remove(target)
target.mkdir(exist_ok=True)
installed, notes = {}, []
for rel, source in sorted(overlay.items()):
parts = PurePosixPath(rel).parts
destination, blocked = clone, None
for index, part in enumerate(parts):
destination = destination / part
last = index == len(parts) - 1
if destination.is_symlink() or (destination.exists() and (last or not destination.is_dir())):
blocked = blocked or destination
if not replace:
break
remove(destination)
if not last:
destination.mkdir(exist_ok=True)
if blocked is not None and not replace:
notes.append(f'{rel}: the tree has {blocked.relative_to(clone).as_posix()} there; '
'the preserved copy was not installed')
continue
if blocked is not None:
notes.append(f'{rel}: the preserved copy replaced the tree\'s file for this run')
data = source.read_bytes()
destination.write_bytes(data)
destination.chmod(source.stat().st_mode & 0o777)
installed[rel] = digest(data)
return installed, notes
def intact(clone, overlay):
"""True when every preserved file sits in the clone byte for byte, with no link on its path."""
for rel, source in overlay.items():
path = clone
for part in PurePosixPath(rel).parts:
path = path / part
if path.is_symlink():
return False
if not path.is_file() or path.read_bytes() != source.read_bytes():
return False
return True
def collect(clone, directory, installed):
"""Preserve files the reviewer created or changed under review_evidence/. Returns True if any.
The finder and both validators of one attempt collect into the same bundle in turn. A path an
earlier reviewer of the attempt already preserved keeps its bytes; a different later copy is
listed as skipped, never written over it.
"""
base = clone / EVIDENCE
if installed is None or base.is_symlink() or not base.is_dir():
return (directory / 'WITNESSES.json').is_file()
listed = run(['git', '--literal-pathspecs', '-C', clone, 'ls-files', '-z', '--', EVIDENCE])
require(listed.returncode == 0, 'witness-collect', 'cannot list tracked evidence files')
tracked = {p.decode('utf-8', 'surrogateescape') for p in listed.stdout.split(b'\0') if p}
manifest = directory / 'WITNESSES.json'
earlier = json.loads(manifest.read_bytes()) if manifest.is_file() else {'files': [], 'skipped': []}
files, skipped = earlier['files'], earlier['skipped']
preserved = {item['path']: item['sha256'] for item in files}
total = sum(item['size'] for item in files)
for current, directories, names in os.walk(base, followlinks=False):
here = Path(current)
for name in sorted(directories):
if name in SKIP_DIRECTORIES or (here / name).is_symlink():
skipped.append({'path': (here / name).relative_to(clone).as_posix() + '/',
'reason': 'dependency, cache or linked directory'})
directories[:] = sorted(d for d in directories
if d not in SKIP_DIRECTORIES and not (here / d).is_symlink())
for name in sorted(names):
path = here / name
rel = path.relative_to(clone).as_posix()
if rel in tracked:
continue
if path.is_symlink() or not path.is_file() or '\n' in rel:
skipped.append({'path': rel, 'reason': 'not a regular file'})
continue
size = path.stat().st_size
if size > MAX_FILE or total + size > MAX_TOTAL or len(files) >= MAX_FILES:
skipped.append({'path': rel, 'reason': 'over the preservation size or count limit'})
continue
data = path.read_bytes()
if installed.get(rel) == digest(data) or preserved.get(rel) == digest(data):
continue
if rel in preserved:
skipped.append({'path': rel, 'reason': 'another reviewer of this attempt preserved this path'})
continue
target = directory / 'witnesses' / rel
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
target.chmod(0o755 if path.stat().st_mode & 0o111 else 0o644)
total += len(data)
files.append({'path': rel, 'sha256': digest(data), 'size': len(data)})
if not files and not skipped:
return False
skipped = [item for i, item in enumerate(skipped) if item not in skipped[:i]]
manifest.write_bytes(json.dumps({'schema_version': 1, 'files': files, 'skipped': skipped},
indent=2, sort_keys=True, ensure_ascii=False).encode() + b'\n')
return True
def view(root, starts, ends, since=1):
"""Witnesses of this PR's recorded reviews from attempt `since` on, including an attempt whose
finder review stayed usable after a required validator failed.
Later bytes replace earlier ones path by path. An ID is a top-level review_evidence/<ID>.sh.
`- witness_obsolete: <ID> -- <reason>` in a later recorded review retires it; a retirement
without a reason retires nothing.
"""
witnesses, overlay = {}, {}
for n in sorted(starts):
end = ends.get(n, {})
if n < since or not ledger.finder_recorded(starts[n], end):
continue
directory = root / f'round-{n:04}'
if 'WITNESSES.json' in end.get('outputs', {}):
manifest = json.loads((directory / 'WITNESSES.json').read_bytes())
require(manifest.get('schema_version') == 1 and isinstance(manifest.get('files'), list),
'witness-manifest', f'attempt {n} has an unknown witness manifest')
for item in manifest['files']:
rel = item['path']
path = evidence_path(rel)
source = directory / 'witnesses' / rel
require(source.is_file() and not source.is_symlink()
and digest(source.read_bytes()) == item['sha256'], 'witness-integrity',
f'attempt {n}: {rel} differs from its preserved digest')
overlay[rel] = source
if len(path.parts) == 2 and path.suffix == '.sh' and WITNESS_ID.fullmatch(path.stem):
witnesses[path.stem] = {'id': path.stem, 'path': rel, 'round': n,
'sha256': item['sha256'], 'status': 'active'}
text = (directory / 'ARTIFACT.md').read_text(encoding='utf-8', errors='replace')
for wid, reason in OBSOLETE.findall(text):
if wid in witnesses and reason.strip():
witnesses[wid].update(status='obsolete', retired_round=n, reason=reason.strip())
return witnesses, overlay
def execute(clone, rel, logs, timeout, wrapper=()):
"""Run one witness in its own process group. Exit 0 is PASS; anything else is not."""
logs.mkdir(parents=True)
started = time.monotonic()
with (logs / 'stdout.txt').open('wb') as out, (logs / 'stderr.txt').open('wb') as err:
child = None
try:
child = subprocess.Popen([*wrapper, 'bash', rel], cwd=clone, stdout=out, stderr=err,
stdin=subprocess.DEVNULL, env=witness_environment(),
start_new_session=True)
try:
rc = child.wait(timeout=timeout)
except subprocess.TimeoutExpired:
return 'TIMEOUT', None, time.monotonic() - started
except InterruptedError:
raise # a cancelled precheck stops; it is not one more failing witness
except OSError as error:
err.write(f'witness could not start: {type(error).__name__}\n'.encode())
return 'ERROR', None, time.monotonic() - started
finally:
if child is not None:
try:
os.killpg(child.pid, signal.SIGKILL)
except ProcessLookupError:
pass
child.wait()
return ('PASS' if rc == 0 else 'FAIL'), rc, time.monotonic() - started
def precheck(repo, head, witnesses, overlay, output, timeout, disputed, wrapper=()):
"""Rerun every active witness in a clean checkout of `head`; no model, no ledger write."""
rows = []
active = sorted((w for w in witnesses.values() if w['status'] == 'active'), key=lambda w: w['id'])
def cancelled(signum, frame):
raise InterruptedError(f'witness precheck interrupted by {signal.Signals(signum).name}')
previous = signal.signal(signal.SIGTERM, cancelled)
try:
if active:
output.mkdir(parents=True)
temp_base = '/private/tmp' if Path('/private/tmp').is_dir() else None
with tempfile.TemporaryDirectory(prefix='w', dir=temp_base) as temp:
clone = Path(temp).resolve() / 'repo'
cloned = run(['git', 'clone', '--quiet', '--no-hardlinks', '--no-checkout', repo, clone])
require(cloned.returncode == 0, 'witness-checkout', cloned.stderr.decode().strip())
for witness in active:
for command in (['checkout', '--quiet', '--force', '--detach', head],
['clean', '-ffdxq']):
result = run(['git', '-C', clone, *command])
require(result.returncode == 0, 'witness-checkout',
f'cannot reset the witness checkout to {head}')
_, notes = prepare(clone, overlay, replace=True)
if not intact(clone, overlay):
# Running anything else would let the repair stand in for the
# reviewer's witness or one of the files it reads.
rows.append(dict(witness, result='ERROR', exit=None, seconds=0.0,
notes=notes + ['the preserved files could not be restored intact'],
disputed=witness['id'] in disputed))
continue
result, rc, seconds = execute(clone, witness['path'], output / witness['id'],
timeout, wrapper)
rows.append(dict(witness, result=result, exit=rc, seconds=seconds, notes=notes,
disputed=witness['id'] in disputed))
finally:
signal.signal(signal.SIGTERM, previous)
return rows
def blocking(rows):
return [row['id'] for row in rows if row['result'] != 'PASS' and not row['disputed']]
def results_markdown(head, rows, witnesses, output=None):
lines = ['# Witness results', '', f'Head: {head}', '']
retired = [w for w in witnesses.values() if w['status'] == 'obsolete']
if not rows:
lines.append('No active preserved witness for this review sequence.')
else:
lines += ['Each active witness ran in a clean checkout of this head with review_evidence/ '
'restored. PASS means it exited 0. It supports closure only if the witness failed at '
'its origin head and still exercises its finding.', '',
'| witness | from attempt | result | exit | seconds | note |',
'|---|---|---|---|---|---|']
for row in rows:
note = '; '.join((['failing, disputed by the implementer'] if row['disputed']
and row['result'] != 'PASS' else []) + row['notes'])
exit_code = '' if row['exit'] is None else row['exit']
lines.append(f"| {row['id']} | {row['round']} | {row['result']} | {exit_code} | "
f"{row['seconds']:.1f} | {note.replace('|', '/')} |")
if retired:
lines += ['', 'Retired witnesses, not run:']
lines += [f"- {w['id']} (from attempt {w['round']}), retired in attempt {w['retired_round']}: "
f"{w['reason']}" for w in sorted(retired, key=lambda w: w['id'])]
if output is not None:
lines += ['', f'Logs: {output}']
return '\n'.join(lines) + '\n'
def ledger_rows(witnesses):
return [f"- {w['id']} from attempt {w['round']}: " +
('active' if w['status'] == 'active' else
f"retired in attempt {w['retired_round']}: {w['reason']}")
for w in sorted(witnesses.values(), key=lambda w: w['id'])]
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |