Parsa-29/loop-config / skills/last30days
last30days
FlaggedResearch what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web.
scripts/verify_v3.py
View on GitHub#!/usr/bin/env python3
"""Run the v3 verification bundle for last30days."""
from __future__ import annotations
import argparse
import json
import os
import statistics
import subprocess
import sys
import time
from pathlib import Path
SKILL_ROOT = Path(__file__).resolve().parents[1]
REPO_ROOT = Path(__file__).resolve().parents[3]
PYTHON = sys.executable
ENGINE = SKILL_ROOT / "scripts" / "last30days.py"
EVALUATOR = SKILL_ROOT / "scripts" / "evaluate_search_quality.py"
SMOKE_TOPIC = "openclaw skills"
SMOKE_CASES = [
("gemini", ["--quick", "--search=grounding,hackernews"]),
("openai", ["--quick", "--search=reddit,hackernews"]),
("xai", ["--quick", "--search=reddit,hackernews"]),
("auto", ["--quick", "--search=reddit,grounding,hackernews"]),
]
LATENCY_TOPICS = [
"openclaw skills",
"codex vs claude code",
"anthropic odds",
]
LATENCY_PROFILES = [
("quick", ["--quick", "--search=grounding,hackernews"]),
("default", ["--search=grounding,hackernews"]),
("deep", ["--deep", "--search=grounding,hackernews"]),
]
def run_command(cmd: list[str], *, env: dict[str, str] | None = None, timeout: int = 600) -> subprocess.CompletedProcess[str]:
return subprocess.run(
cmd,
cwd=REPO_ROOT,
env=env,
text=True,
capture_output=True,
timeout=timeout,
check=True,
)
def verify_unit() -> dict[str, str]:
run_command([PYTHON, "-m", "unittest", "discover", "-s", "tests", "-p", "test_*.py"], timeout=600)
run_command(
[
PYTHON,
"-m",
"py_compile",
*subprocess.run(
[
"rg",
"--files",
"skills/last30days/scripts",
"tests",
"-g",
"*.py",
"-g",
"!skills/last30days/scripts/lib/vendor/**",
],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
).stdout.split(),
],
timeout=600,
)
return {"status": "ok"}
def verify_diagnose() -> dict[str, object]:
result = run_command([PYTHON, str(ENGINE), "--diagnose"], timeout=120)
return json.loads(result.stdout)
def verify_smoke() -> list[dict[str, object]]:
rows: list[dict[str, object]] = []
for provider, extra in SMOKE_CASES:
env = os.environ.copy()
env["LAST30DAYS_REASONING_PROVIDER"] = provider
start = time.time()
result = run_command(
[PYTHON, str(ENGINE), SMOKE_TOPIC, "--emit=json", *extra],
env=env,
timeout=240,
)
duration = round(time.time() - start, 2)
report = json.loads(result.stdout)
rows.append(
{
"provider": provider,
"duration_seconds": duration,
"reasoning_provider": (report.get("provider_runtime") or {}).get("reasoning_provider"),
"cluster_count": len(report.get("clusters") or []),
"candidate_count": len(report.get("ranked_candidates") or []),
"error_sources": sorted((report.get("errors_by_source") or {}).keys()),
}
)
return rows
def verify_latency() -> dict[str, dict[str, object]]:
results: dict[str, dict[str, object]] = {}
for profile, extra in LATENCY_PROFILES:
timings = []
for topic in LATENCY_TOPICS:
start = time.time()
run_command(
[PYTHON, str(ENGINE), topic, "--emit=json", *extra],
timeout=300,
)
timings.append(time.time() - start)
results[profile] = {
"times": [round(value, 2) for value in timings],
"median_seconds": round(statistics.median(timings), 2),
"max_seconds": round(max(timings), 2),
}
return results
def verify_eval(
*,
baseline: str,
candidate: str,
output_dir: str,
quick: bool,
limit: int,
timeout: int,
) -> dict[str, object]:
cmd = [
PYTHON,
str(EVALUATOR),
f"--baseline={baseline}",
f"--candidate={candidate}",
f"--output-dir={output_dir}",
f"--limit={limit}",
f"--timeout={timeout}",
]
if quick:
cmd.append("--quick")
run_command(cmd, timeout=max(timeout * 8, 600))
output = Path(output_dir)
metrics = json.loads((output / "metrics.json").read_text())
summary = (output / "summary.md").read_text()
return {"metrics": metrics, "summary": summary}
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Run the v3 verification bundle")
parser.add_argument("--skip-eval", action="store_true", help="Skip the judged evaluator")
parser.add_argument("--skip-latency", action="store_true", help="Skip live latency sampling")
parser.add_argument("--baseline", default="HEAD~1")
parser.add_argument("--candidate", default="WORKTREE")
parser.add_argument("--output-dir", default="/tmp/last30days-v3-verify")
parser.add_argument("--quick-eval", action="store_true", help="Use evaluator quick mode")
parser.add_argument("--eval-limit", type=int, default=20)
parser.add_argument("--eval-timeout", type=int, default=240)
return parser
def main() -> int:
args = build_parser().parse_args()
summary: dict[str, object] = {}
summary["unit"] = verify_unit()
summary["diagnose"] = verify_diagnose()
summary["smoke"] = verify_smoke()
if not args.skip_latency:
summary["latency"] = verify_latency()
if not args.skip_eval:
summary["eval"] = verify_eval(
baseline=args.baseline,
candidate=args.candidate,
output_dir=args.output_dir,
quick=args.quick_eval,
limit=args.eval_limit,
timeout=args.eval_timeout,
)
print(json.dumps(summary, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| add-product-e2e-evalpaperclipai/paperclipAdd or extend a Paperclip full-stack runner E2E workflow, fixture, matcher, or report evidence path for local or Daytona execution. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| add-runner-evalpaperclipai/paperclipAdd or extend a Paperclip Runner protocol evaluation definition, roster, assertion, or report fixture with provenance and narrow validation. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| diagnose-why-work-stoppedpaperclipai/paperclipCompatibility entry point for failed, stalled, looping, or over-recovered Paperclip work. Use fix-failed-paperclip-run for the full diagnosis, regression, repair, real verification, and green-PR workflow; honor an explicitly requested diagnosis-only scope. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| fix-failed-paperclip-runpaperclipai/paperclipDiagnose and fix failed, stuck, slow, or incorrectly blocked Paperclip tasks and runs. Use with a task/run URL or failure report to trace the root cause, prove a regression red then green, check second-order effects, verify the real workflow, and deliver a PR with passing checks. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |