moonlight-lupin/agent-skills / plugins/skill-retrieval
skill-retrieval
VerifiedBM25-based skill retrieval plugin for Hermes Agent. Replaces the full skill list in the system prompt with a names-only compact view (~2K tokens) and injects top-K relevant skill descriptions per turn via BM25 retrieval (~300 tokens). Saves ~9K tokens/turn. Use when system prompt token overhead from skills is a concern, or when skill discovery quality matters.
tests/test_issue8_round3.py
View on GitHub"""Regression tests for Issue #8 round 3 findings (audaimousa, close-out
comment, 2026-09-08).
Written by Hermes BEFORE the fix (honest RED contract). Each test encodes
the named-session snapshot-binding contract from the issue's final comment:
1. The build-time capability snapshot must bind to the ACTIVE Hermes
session identity via gateway.session_context.get_session_env(
"HERMES_SESSION_ID") — build_skills_system_prompt() has no
session_id parameter, so the ContextVar/env session is the only
identity source available at build time.
2. A NAMED session whose exact snapshot is missing (or evicted) must SKIP
BM25 injection for that turn (return no context) instead of
falling back to the fail-open zero-argument corpus. Missing
retrieval is safer than injecting a skill Hermes explicitly hid.
3. No cross-session fallback: a named session must never read another
named session's snapshot (isolation direction from 50daa54 kept).
The reporter's live reproduction at 50daa54: two named sessions with
disjoint capabilities, 100 interleaved A/B cycles across 5 fresh
processes — 200/200 hidden-skill leakage opportunities reproduced, because
build_skills_system_prompt() recorded the snapshot under the anonymous key
("") while pre_llm_call(session_id=A) looked up A, found nothing, and fell
back to the fail-open bare get_index().
"""
import importlib
import importlib.util
import pathlib
import sys
import types
import time
import pytest
import bm25_retriever as br
def _load_fresh():
importlib.reload(br)
return br
_PLUGIN_DIR = pathlib.Path(__file__).resolve().parent.parent
def _load_plugin_init(monkeypatch, tmp_path):
"""Load the plugin's __init__.py as a top-level module with scripts/ on
sys.path, so _on_pre_llm_call / get_index resolve. Returns the module
(holds _remember_capability_snapshot / _session_capability_snaps)."""
monkeypatch.syspath_prepend(str(_PLUGIN_DIR / "scripts"))
spec = importlib.util.spec_from_file_location(
"skill_retrieval_init_under_test", _PLUGIN_DIR / "__init__.py"
)
mod = importlib.util.module_from_spec(spec)
sys.modules["skill_retrieval_init_under_test"] = mod
spec.loader.exec_module(mod)
return mod
def _write_skill(root, rel, name, description, extra_frontmatter=""):
skill_dir = root / rel
skill_dir.mkdir(parents=True, exist_ok=True)
skill_md = skill_dir / "SKILL.md"
fm = f'name: {name}\ndescription: "{description}"'
if extra_frontmatter:
fm += "\n" + extra_frontmatter
skill_md.write_text(f"---\n{fm}\n---\n\n# {name}\n")
return skill_md
def _install_hermes_stubs(monkeypatch, *, conditions_map=None):
"""Stub agent.prompt_builder / agent.skill_utils with controllable gates.
Mirrors test_issue8_round2.py::_install_hermes_stubs (same four-family
gate evaluation as agent/prompt_builder.py). Kept separate so each
round's tests pin their own contract copy.
"""
pb = types.ModuleType("agent.prompt_builder")
pb._current_session_platform_hint = lambda: ""
pb.extract_skill_conditions = lambda fm: (conditions_map or {}).get(
fm.get("name", ""), {}
)
def _should_show(conditions, available_tools, available_toolsets, platform=None):
if available_tools is None and available_toolsets is None:
return True # fail-open, same as Hermes
at, ats = available_tools or set(), available_toolsets or set()
return not (
any(ts in ats for ts in conditions.get("fallback_for_toolsets", []))
or any(t in at for t in conditions.get("fallback_for_tools", []))
or any(ts not in ats for ts in conditions.get("requires_toolsets", []))
or any(t not in at for t in conditions.get("requires_tools", []))
)
pb._skill_should_show = _should_show
def _parse_like_hermes(skill_file):
name, desc, _body = br._parse_skill_md(skill_file)
return True, {"name": name}, desc
pb._parse_skill_file = _parse_like_hermes
def _build_snapshot_entry(skill_file, root, frontmatter, description):
rel = skill_file.relative_to(root)
return {
"category": "general",
"skill_name": rel.parent.name,
"frontmatter_name": frontmatter["name"],
"description": description,
}
pb._build_snapshot_entry = _build_snapshot_entry
su = types.ModuleType("agent.skill_utils")
su.get_disabled_skill_names = lambda platform=None: set()
su.get_project_skills_dirs = lambda: []
su.get_all_skills_dirs = lambda: []
su.iter_project_skill_files = lambda root: sorted(root.rglob("SKILL.md"))
su.iter_skill_index_files = lambda root, filename: sorted(root.rglob(filename))
agent_pkg = types.ModuleType("agent")
monkeypatch.setitem(sys.modules, "agent", agent_pkg)
monkeypatch.setitem(sys.modules, "agent.prompt_builder", pb)
monkeypatch.setitem(sys.modules, "agent.skill_utils", su)
return {"su": su, "pb": pb}
def _point_discovery_at(monkeypatch, stubs, *roots):
su = stubs["su"]
monkeypatch.setattr(su, "get_all_skills_dirs", lambda: list(roots))
class _FakeSessionContext:
"""Stub of gateway.session_context with a settable task-local session id.
Mirrors the real module's contract: get_session_env("HERMES_SESSION_ID")
returns the ContextVar-bound session id for the current task.
"""
session_id = ""
@classmethod
def get_session_env(cls, name, default=""):
if name == "HERMES_SESSION_ID":
return cls.session_id or default
return default
@classmethod
def install(cls, monkeypatch):
sc_mod = types.ModuleType("gateway.session_context")
sc_mod.get_session_env = cls.get_session_env
gw_pkg = types.ModuleType("gateway")
monkeypatch.setitem(sys.modules, "gateway", gw_pkg)
monkeypatch.setitem(sys.modules, "gateway.session_context", sc_mod)
return cls
# ─── Contract 1: build snapshot binds to the active session identity ────────
_GATED_VISIBLE = frozenset({"plain-skill", "extra-skill", "extra2-skill"})
def test_build_snapshot_binds_to_session_env_identity(monkeypatch, tmp_path):
"""Snapshot capture with an empty session_id must key to
get_session_env("HERMES_SESSION_ID") — the task-local identity Hermes
binds for the turn."""
mod = _load_plugin_init(monkeypatch, tmp_path)
SC = _FakeSessionContext.install(monkeypatch)
SC.session_id = "sess-A"
mod._remember_capability_snapshot("", frozenset({"alpha-skill"}))
snap = mod._session_capability_snaps.get("sess-A")
assert snap is not None, (
"snapshot must be keyed to the session-env identity, not the anonymous key"
)
assert snap[1] == frozenset({"alpha-skill"})
def test_build_snapshot_explicit_session_id_still_wins(monkeypatch, tmp_path):
"""An explicit session_id argument takes precedence over the session-env identity."""
mod = _load_plugin_init(monkeypatch, tmp_path)
SC = _FakeSessionContext.install(monkeypatch)
SC.session_id = "sess-ENV"
mod._remember_capability_snapshot("sess-EXPLICIT", frozenset({"t"}))
assert "sess-EXPLICIT" in mod._session_capability_snaps
assert "sess-ENV" not in mod._session_capability_snaps
# ─── Contract 2: named session with missing/stale snapshot must skip ────────
def _setup_plugin_module(br, monkeypatch, tmp_path, *, conditions_map):
"""Wire a skill corpus of capability-gated skills plus a filler doc and
patch __init__-level deps so _on_pre_llm_call runs against them."""
root = tmp_path / "skills"
_write_skill(root, "a", "alpha-skill", "alphaleak3f unique token")
_write_skill(root, "b", "beta-skill", "betaleak5g unique token")
_write_skill(root, "c", "gamma-skill", "always visible filler")
stubs = _install_hermes_stubs(
monkeypatch, conditions_map=conditions_map
)
_point_discovery_at(monkeypatch, stubs, root)
return root
def _write_gated_corpus(monkeypatch, tmp_path):
"""One capability-gated skill plus fillers. At least three documents so
the probe token and "filler" genuinely score (a vacuous empty result
cannot pass the skip/leak assertions below by accident)."""
root = tmp_path / "skills"
_write_skill(root, "plain", "plain-skill", "always visible filler")
_write_skill(root, "gated", "gated-skill", "quasarneedle9z")
_write_skill(root, "extra", "extra-skill", "unrelated filler vocabulary")
_write_skill(root, "extra2", "extra2-skill", "more unrelated words")
stubs = _install_hermes_stubs(
monkeypatch,
conditions_map={"gated-skill": {"requires_tools": ["special_tool"]}},
)
_point_discovery_at(monkeypatch, stubs, root)
return stubs
def _advance_clock(monkeypatch, seconds):
real = time.monotonic()
monkeypatch.setattr(time, "monotonic", lambda: real + seconds)
def test_named_session_missing_snapshot_skips_injection(monkeypatch, tmp_path):
"""A NAMED session with no exact snapshot must get NO injection rather
than the fail-open bare corpus (reporter contract #2: 'missing exact
named snapshot: FAIL — hidden Skill retrieved' must become a skip)."""
_load_fresh()
mod = _load_plugin_init(monkeypatch, tmp_path)
_write_gated_corpus(monkeypatch, tmp_path)
# Sanity: the fail-open corpus really would leak the gated skill.
leaked = mod._on_pre_llm_call(session_id="", user_message="quasarneedle9z")
assert leaked and "gated-skill" in leaked["context"]
# No snapshot recorded for "sess-A" at all.
result = mod._on_pre_llm_call(
session_id="sess-A", user_message="quasarneedle9z"
)
assert result is None, (
"named session with missing snapshot must skip injection, "
"not fall back to the fail-open corpus"
)
def test_named_session_snapshot_does_not_expire(monkeypatch, tmp_path):
"""Hermes builds the system prompt once per session and restores it from
the DB afterwards, so a named session's snapshot must keep applying on
later turns instead of expiring into the fail-open corpus."""
_load_fresh()
mod = _load_plugin_init(monkeypatch, tmp_path)
SC = _FakeSessionContext.install(monkeypatch)
_write_gated_corpus(monkeypatch, tmp_path)
SC.session_id = "sess-A"
mod._remember_capability_snapshot("sess-A", _GATED_VISIBLE)
_advance_clock(monkeypatch, 3600.0)
result = mod._on_pre_llm_call(
session_id="sess-A", user_message="quasarneedle9z filler"
)
assert result is not None and "plain-skill" in result["context"]
assert "gated-skill" not in result["context"]
def test_snapshot_cache_holds_more_than_eight_sessions(monkeypatch, tmp_path):
"""A busy gateway interleaves many sessions; a session's snapshot must
survive other sessions' prompt builds."""
_load_fresh()
mod = _load_plugin_init(monkeypatch, tmp_path)
SC = _FakeSessionContext.install(monkeypatch)
_write_gated_corpus(monkeypatch, tmp_path)
SC.session_id = "sess-A"
mod._remember_capability_snapshot("sess-A", _GATED_VISIBLE)
for i in range(20):
SC.session_id = f"other-{i}"
mod._remember_capability_snapshot(f"other-{i}", _GATED_VISIBLE)
result = mod._on_pre_llm_call(
session_id="sess-A", user_message="quasarneedle9z filler"
)
assert result is not None and "plain-skill" in result["context"]
assert "gated-skill" not in result["context"]
def test_named_session_evicted_snapshot_skips_injection(monkeypatch, tmp_path):
"""Once the bounded LRU evicts a named session's snapshot, that session
must skip injection, not fall back to the fail-open corpus."""
_load_fresh()
mod = _load_plugin_init(monkeypatch, tmp_path)
SC = _FakeSessionContext.install(monkeypatch)
_write_gated_corpus(monkeypatch, tmp_path)
SC.session_id = "sess-A"
mod._remember_capability_snapshot("sess-A", _GATED_VISIBLE)
for i in range(getattr(mod, "_MAX_SNAPSHOT_SESSIONS", 256) + 1):
SC.session_id = f"other-{i}"
mod._remember_capability_snapshot(f"other-{i}", _GATED_VISIBLE)
assert "sess-A" not in mod._session_capability_snaps
result = mod._on_pre_llm_call(
session_id="sess-A", user_message="quasarneedle9z"
)
assert result is None
def test_snapshot_captured_when_compaction_disabled(monkeypatch, tmp_path):
"""SKILL_RETRIEVAL_COMPACT=0 leaves the prompt untouched but must still
record the capability snapshot, or every named session would skip."""
_load_fresh()
mod = _load_plugin_init(monkeypatch, tmp_path)
_write_gated_corpus(monkeypatch, tmp_path)
full = (
"<available_skills>\n"
" cat: Desc\n"
" - plain-skill: always visible filler\n"
" - extra-skill: unrelated filler vocabulary\n"
" - extra2-skill: more unrelated words\n"
"</available_skills>"
)
monkeypatch.setattr(mod, "COMPACT_SYSTEM_PROMPT", False)
req = {"messages": [{"role": "system", "content": full}]}
out = mod._on_llm_request(request=req, session_id="sess-A")
assert out is None, "compaction disabled: request must be returned unchanged"
result = mod._on_pre_llm_call(
session_id="sess-A", user_message="quasarneedle9z filler"
)
assert result is not None and "plain-skill" in result["context"]
assert "gated-skill" not in result["context"]
def test_anonymous_session_keeps_fail_open(monkeypatch, tmp_path):
"""The anonymous path (session_id='') keeps Hermes' fail-open semantics:
bare get_index() fallback when no snapshot exists. Backward compat with
test_no_capability_args_fails_open."""
_load_fresh()
mod = _load_plugin_init(monkeypatch, tmp_path)
_write_gated_corpus(monkeypatch, tmp_path)
result = mod._on_pre_llm_call(session_id="", user_message="quasarneedle9z")
assert result is not None and result.get("context"), (
"anonymous session must keep the fail-open behavior"
)
assert "gated-skill" in result["context"]
# ─── Contract 3: cross-session isolation with binding ───────────────────────
def test_interleaved_named_sessions_no_leakage(monkeypatch, tmp_path):
"""Two named sessions with disjoint capabilities. Like Hermes, each
session builds its system prompt ONCE, then runs many interleaved turns
(restored prompt, no rebuild) spread over time. Neither session may ever
retrieve the other's hidden skill (reporter: 200/200 leakage at 50daa54
— must be 0), and each must keep retrieving its own skill."""
_load_fresh()
mod = _load_plugin_init(monkeypatch, tmp_path)
SC = _FakeSessionContext.install(monkeypatch)
root = tmp_path / "skills"
_write_skill(root, "a", "alpha-skill", "alphaleak3f unique token")
_write_skill(root, "b", "beta-skill", "betaleak5g unique token")
_write_skill(root, "x", "x-skill", "unrelated filler vocabulary")
_write_skill(root, "y", "y-skill", "more unrelated words here")
stubs = _install_hermes_stubs(
monkeypatch,
conditions_map={
"alpha-skill": {"requires_tools": ["tool_a"]},
"beta-skill": {"requires_tools": ["tool_b"]},
},
)
_point_discovery_at(monkeypatch, stubs, root)
SC.session_id = "sess-A"
mod._remember_capability_snapshot(
"sess-A", frozenset({"alpha-skill", "x-skill", "y-skill"})
)
SC.session_id = "sess-B"
mod._remember_capability_snapshot(
"sess-B", frozenset({"beta-skill", "x-skill", "y-skill"})
)
leak_count = 0
for cycle in range(20):
_advance_clock(monkeypatch, 31.0 * (cycle + 1))
result_a = mod._on_pre_llm_call(
session_id="sess-A", user_message="alphaleak3f betaleak5g"
)
result_b = mod._on_pre_llm_call(
session_id="sess-B", user_message="alphaleak3f betaleak5g"
)
assert result_a and "alpha-skill" in result_a["context"]
assert result_b and "beta-skill" in result_b["context"]
if "beta-skill" in result_a["context"]:
leak_count += 1
if "alpha-skill" in result_b["context"]:
leak_count += 1
assert leak_count == 0, (
f"cross-session hidden-skill leakage detected in {leak_count} probes"
)
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |