moonlight-lupin/agent-skills / plugins/skill-retrieval
skill-retrieval
VerifiedBM25-based skill retrieval plugin for Hermes Agent. Replaces the full skill list in the system prompt with a names-only compact view (~2K tokens) and injects top-K relevant skill descriptions per turn via BM25 retrieval (~300 tokens). Saves ~9K tokens/turn. Use when system prompt token overhead from skills is a concern, or when skill discovery quality matters.
scripts/rerank_ab_report.py
View on GitHub#!/usr/bin/env python3
"""Jev rerank A/B analysis — reads the per-turn JSONL log.
Usage:
python3 rerank_ab_report.py [LOG_PATH]
LOG_PATH defaults to $HERMES_HOME/data/jev-trial/rerank_ab_log.jsonl (else ~/.hermes/data/...)
(one JSON line per turn, written by jev_rerank.py).
Outputs A/B metrics for operator review:
- rank1 flip rate (Jev #1 vs BM25 #1)
- positional displacement (mean |rank change|, max)
- band distribution (decisive-high / band / decisive-low shares)
- fallback rate with reason breakdown
- latency p50/p95 and cost estimate
Controlled English output. No data leaves the machine.
"""
import json
import os
import sys
from collections import Counter
from pathlib import Path
_base = os.environ.get("HERMES_HOME") or (Path.home() / ".hermes")
DEFAULT_LOG = Path(_base) / "data" / "jev-trial" / "rerank_ab_log.jsonl"
BAND_LOW, BAND_HIGH = 0.35, 0.65
def load_records(path: Path) -> list[dict]:
records = []
if not path.exists():
return records
for line in path.read_text().splitlines():
line = line.strip()
if not line:
continue
try:
records.append(json.loads(line))
except json.JSONDecodeError:
print(f"WARN: unparseable line skipped: {line[:80]}")
return records
def percentile(values: list[float], pct: float) -> float:
if not values:
return 0.0
ordered = sorted(values)
idx = min(int(len(ordered) * pct / 100.0), len(ordered) - 1)
return ordered[idx]
def main() -> None:
path = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_LOG
records = load_records(path)
if not records:
print(f"No records in {path}")
print("The log fills as turns run with SKILL_RETRIEVAL_RERANK=jev.")
return
applied = [r for r in records if r.get("jev_order")]
skipped = [r for r in records if not r.get("jev_order")]
n_applied = len(applied)
print(f"Log: {path}")
print(f"Turns recorded: {len(records)} (applied {n_applied}, skipped {len(skipped)})")
if skipped:
reasons = Counter(r.get("fallback_reason") or "unknown" for r in skipped)
print("\nFallback rate:")
print(f" {len(skipped)}/{len(records)} = {len(skipped)/len(records):.0%}")
for reason, count in reasons.most_common():
print(f" {reason}: {count}")
if not n_applied:
print("\nNo applied reranks yet — insufficient evidence.")
return
flips = sum(1 for r in applied if r.get("rank1_flip"))
print("\nRank-1 flip rate (Jev #1 differs from BM25 #1):")
print(f" {sum(1 for r in applied if r.get('rank1_flip'))}/{n_applied}"
f" = {sum(1 for r in applied if r.get('rank1_flip'))/n_applied:.0%}")
displacements: list[float] = []
band_counts: Counter = Counter()
for r in applied:
probs = r.get("jev_probs") or {}
bm25 = r.get("bm25_order") or []
jev = r.get("jev_order") or []
bm25_rank = {sid: i for i, sid in enumerate(bm25)}
for sid in jev:
if sid in bm25_rank:
disp = abs(bm25_rank[sid] - jev.index(sid))
displacements.append(float(disp))
p = probs.get(sid)
if p is not None:
if p >= BAND_HIGH:
band_counts["decisive_high"] += 1
elif p > BAND_LOW:
band_counts["band_kept_bm25_position"] += 1
else:
band_counts["decisive_low"] += 1
print("\nPositional displacement (|BM25 rank - Jev rank|):")
if displacements:
mean_d = sum(displacements) / len(displacements)
print(f" mean {mean_d:.2f} max {max(displacements):.0f} n {len(displacements)}")
else:
print(" no data")
total_band = sum(band_counts.values()) or 1
print("\nBand distribution of Jev probabilities:")
for band in ("decisive_high", "band_kept_bm25_position", "decisive_low"):
count = band_counts.get(band, 0)
print(f" {band}: {count} ({count / total_band:.0%})")
latencies = [r["latency_ms"] for r in applied if isinstance(r.get("latency_ms"), (int, float))]
tokens = [r["input_tokens"] for r in applied if isinstance(r.get("input_tokens"), (int, float))]
if latencies:
print("\nJev overhead per turn:")
print(f" latency p50 {percentile(latencies, 50):.0f} ms"
f" p95 {percentile(latencies, 95):.0f} ms")
if tokens:
print(f" input tokens: mean {sum(tokens)/len(tokens):.0f}"
f" est cost/turn ${sum(tokens)/len(tokens)*0.042/1e6:.6f}")
if __name__ == "__main__":
main()Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |