moonlight-lupin/agent-skills / plugins/skill-retrieval
skill-retrieval
VerifiedBM25-based skill retrieval plugin for Hermes Agent. Replaces the full skill list in the system prompt with a names-only compact view (~2K tokens) and injects top-K relevant skill descriptions per turn via BM25 retrieval (~300 tokens). Saves ~9K tokens/turn. Use when system prompt token overhead from skills is a concern, or when skill discovery quality matters.
tests/test_regressions.py
View on GitHub"""Regression tests for the four pre-release bugs (A1-A4) + coverage gaps (B).
These tests were written BEFORE the fixes. Each test encodes the correct
behaviour described in the fix brief. They fail against the buggy code and
pass after the fixes land.
"""
import importlib
import sys
import types
from pathlib import Path
import pytest
import bm25_retriever as br
SCRIPTS = Path(__file__).resolve().parent.parent / "scripts"
PLUGIN_DIR = Path(__file__).resolve().parent.parent
# ═══════════════════════════════════════════════════════════════════════════════
# A1 — frontmatter parser bleeds later YAML keys into the description
# ═══════════════════════════════════════════════════════════════════════════════
def test_parse_skill_md_does_not_bleed_later_keys(tmp_path):
"""Nested YAML keys after ``description:`` must NOT enter the description."""
skill_md = tmp_path / "SKILL.md"
skill_md.write_text(
"---\n"
"name: demo\n"
"description: Convert CSV files to JSON.\n"
"allowed-tools:\n"
" - Bash\n"
" - Read\n"
"metadata:\n"
" category: sausage-manufacturing\n"
" owner: bob\n"
"---\n"
)
name, desc, _body = br._parse_skill_md(skill_md)
assert name == "demo"
assert desc == "Convert CSV files to JSON."
assert "sausage" not in desc.lower()
assert "Bash" not in desc
assert "category" not in desc
assert "owner" not in desc
def test_parse_skill_md_own_plugin_skilmd_no_bleed():
"""The plugin's own SKILL.md must not bleed ``plugin_type`` / ``hooks``."""
skill_md = PLUGIN_DIR / "SKILL.md"
name, desc, _body = br._parse_skill_md(skill_md)
assert name == "skill-retrieval"
assert "plugin_type" not in desc
assert "hooks" not in desc
assert "[pre_llm_call]" not in desc
# ═══════════════════════════════════════════════════════════════════════════════
# A2 — YAML block-scalar indicators land in the corpus
# ═══════════════════════════════════════════════════════════════════════════════
@pytest.mark.parametrize("indicator", [">-", ">", "|-", "|"])
def test_parse_skill_md_block_scalar_indicator_stripped(tmp_path, indicator):
"""Block-scalar indicators (``>-``, ``>``, ``|-``, ``|``) must not appear."""
skill_md = tmp_path / "SKILL.md"
skill_md.write_text(
"---\n"
"name: demo\n"
f"description: {indicator}\n"
" A multi-line description.\n"
" Second line.\n"
"---\n"
)
_, desc, _body = br._parse_skill_md(skill_md)
assert not desc.startswith((">", "|"))
assert "multi-line description" in desc
assert "Second line" in desc
# ═══════════════════════════════════════════════════════════════════════════════
# A3 — _SKIP_DIRS substring-matches the absolute path, silently zeroing the index
# ═══════════════════════════════════════════════════════════════════════════════
def test_skip_dirs_ancestor_hub_workspace_still_indexes(tmp_path, monkeypatch):
"""An ancestor dir named ``.hub-workspace`` must NOT skip skills inside it."""
skills_root = tmp_path / ".hub-workspace" / "skills"
plugins_root = tmp_path / "plugins"
config_path = tmp_path / "config.yaml"
skill_dir = skills_root / "my-skill"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text(
"---\nname: my-skill\ndescription: Test skill\n---\n\n# Body\n"
)
plugins_root.mkdir()
config_path.write_text("skills:\n disabled: []\n")
monkeypatch.setattr(br, "SKILLS_ROOT", skills_root)
monkeypatch.setattr(br, "PLUGINS_ROOT", plugins_root)
monkeypatch.setattr(br, "CONFIG_PATH", config_path)
skills = br.load_active_skills()
ids = {s["skill_id"] for s in skills}
assert "my-skill" in ids
def test_skip_dirs_github_hub_not_skipped(tmp_path, monkeypatch):
"""``github.hub`` as a parent dir must not be skipped (only ``.hub`` as a path component)."""
skills_root = tmp_path / "skills"
plugins_root = tmp_path / "plugins"
config_path = tmp_path / "config.yaml"
skill_dir = skills_root / "github.hub" / "my-skill"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text(
"---\nname: gh-skill\ndescription: GH skill\n---\n\n# Body\n"
)
plugins_root.mkdir()
config_path.write_text("skills:\n disabled: []\n")
monkeypatch.setattr(br, "SKILLS_ROOT", skills_root)
monkeypatch.setattr(br, "PLUGINS_ROOT", plugins_root)
monkeypatch.setattr(br, "CONFIG_PATH", config_path)
skills = br.load_active_skills()
ids = {s["skill_id"] for s in skills}
assert "github.hub/my-skill" in ids
def test_skip_dirs_archive_live_not_skipped(tmp_path, monkeypatch):
"""``notes.archive-live`` must not be skipped."""
skills_root = tmp_path / "skills"
plugins_root = tmp_path / "plugins"
config_path = tmp_path / "config.yaml"
skill_dir = skills_root / "notes.archive-live" / "other-skill"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text(
"---\nname: other\ndescription: Other skill\n---\n\n# Body\n"
)
plugins_root.mkdir()
config_path.write_text("skills:\n disabled: []\n")
monkeypatch.setattr(br, "SKILLS_ROOT", skills_root)
monkeypatch.setattr(br, "PLUGINS_ROOT", plugins_root)
monkeypatch.setattr(br, "CONFIG_PATH", config_path)
skills = br.load_active_skills()
ids = {s["skill_id"] for s in skills}
assert "notes.archive-live/other-skill" in ids
def test_skip_dirs_genuine_archive_still_skipped(tmp_path, monkeypatch):
"""A genuine ``.archive/old`` component must still be skipped."""
skills_root = tmp_path / "skills"
plugins_root = tmp_path / "plugins"
config_path = tmp_path / "config.yaml"
live = skills_root / "live" / "ok"
live.mkdir(parents=True)
(live / "SKILL.md").write_text("---\nname: ok\ndescription: Live\n---\n\n# Body\n")
arch = skills_root / ".archive" / "old"
arch.mkdir(parents=True)
(arch / "SKILL.md").write_text("---\nname: old\ndescription: Old\n---\n\n# Body\n")
plugins_root.mkdir()
config_path.write_text("skills:\n disabled: []\n")
monkeypatch.setattr(br, "SKILLS_ROOT", skills_root)
monkeypatch.setattr(br, "PLUGINS_ROOT", plugins_root)
monkeypatch.setattr(br, "CONFIG_PATH", config_path)
skills = br.load_active_skills()
ids = {s["skill_id"] for s in skills}
assert "live/ok" in ids
assert not any(".archive" in i for i in ids)
# ═══════════════════════════════════════════════════════════════════════════════
# A4 — Phase 1 compaction mangles multi-line skill entries
# ═══════════════════════════════════════════════════════════════════════════════
def _load_plugin_module():
"""Load the plugin __init__.py as a module under a unique name."""
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
spec = importlib.util.spec_from_file_location(
"_sr_test_plugin", PLUGIN_DIR / "__init__.py"
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def _compact_block(plugin_mod, full_prompt):
"""Run names-only compaction through the public pure function + note."""
import re
match = re.search(r"<available_skills>(.*?)</available_skills>", full_prompt, re.DOTALL)
if not match:
return full_prompt, True
compact_inner = plugin_mod.compact_available_skills_block(match.group(1), True)
output = (
full_prompt[: match.start()]
+ "<available_skills>\n"
+ compact_inner
+ "\n</available_skills>"
+ full_prompt[match.end() :]
)
note = plugin_mod._compaction_note()
if "Skill descriptions are injected per-turn" not in output:
output += note
return output, True
def test_compact_wrapped_description_one_line():
"""A wrapped skill description must produce exactly one output line for the entry."""
mod = _load_plugin_module()
prompt = (
"<available_skills>\n"
" creative:\n"
" - deep-research: Think, search, extract, synthesize a cited report.\n"
' Use when the user asks for a literature review, market scan, or\n'
' "research X thoroughly". Produces citations.\n'
" - image-studio: fal.ai image generation\n"
" research:\n"
" - arxiv: Search arxiv papers\n"
"</available_skills>"
)
output, ok = _compact_block(mod, prompt)
assert ok is True
# Extract the skills block
import re
m = re.search(r"<available_skills>(.*?)</available_skills>", output, re.DOTALL)
assert m
block = m.group(1)
lines = [l for l in block.strip().split("\n") if l.strip()]
# deep-research must be a single line
dr_lines = [l for l in lines if "deep-research" in l]
assert len(dr_lines) == 1, f"Expected 1 line for deep-research, got {dr_lines}"
assert dr_lines[0].strip() == "- deep-research"
# No orphaned continuation lines about "literature review"
lit_lines = [l for l in lines if "literature" in l.lower()]
assert lit_lines == [], f"Orphaned continuation line survived: {lit_lines}"
def test_compact_wrapped_description_with_colon_in_continuation():
"""A wrapped continuation line containing ':' must be dropped, not
misclassified as a category header."""
mod = _load_plugin_module()
prompt = (
"<available_skills>\n"
" tools:\n"
" - demo: Performs several operations.\n"
" Use when: the user requests conversion.\n"
" - other: Another tool.\n"
"</available_skills>"
)
output, ok = _compact_block(mod, prompt)
assert ok is True
import re
m = re.search(r"<available_skills>(.*?)</available_skills>", output, re.DOTALL)
assert m
block = m.group(1)
lines = [l for l in block.strip().split("\n") if l.strip()]
# "Use when" must NOT appear as a category header
uw_lines = [l for l in lines if "Use when" in l]
assert uw_lines == [], f"Continuation with colon became category header: {uw_lines}"
# demo and other must both be present as skill entries
demo_lines = [l for l in lines if "demo" in l]
assert len(demo_lines) == 1
assert demo_lines[0].strip() == "- demo"
other_lines = [l for l in lines if "other" in l]
assert len(other_lines) == 1
assert other_lines[0].strip() == "- other"
def test_compact_keeps_names_only_category_line():
"""Hermes' demoted-category line (agent/prompt_builder.py
_render_skills_index) already lists names only; the skill names after
the colon must survive compaction."""
mod = _load_plugin_module()
names_only = " devops [names only]: docker-deploy, k8s-rollout"
prompt = (
"<available_skills>\n"
f"{names_only}\n"
" research: Research helpers\n"
" - arxiv: Search arXiv papers\n"
"</available_skills>"
)
output, ok = _compact_block(mod, prompt)
assert ok is True
assert names_only in output.split("\n")
assert " research:" in output.split("\n")
assert " - arxiv" in output.split("\n")
def test_compact_no_available_skills_block_returns_unchanged():
"""A prompt with no <available_skills> block must be left untouched."""
mod = _load_plugin_module()
prompt = "This is a system prompt with no skills block.\nJust text.\n"
result = mod._on_llm_request(
request={"messages": [{"role": "system", "content": prompt}]},
session_id="",
)
assert result is None
def test_compact_idempotence():
"""Rewriting an already-rewritten request must not double-append the trailing note."""
mod = _load_plugin_module()
prompt = (
"<available_skills>\n"
" test:\n"
" - foo: Bar\n"
"</available_skills>"
)
first = mod._on_llm_request(
request={"messages": [{"role": "system", "content": prompt}]},
session_id="",
)
assert first is not None
output1 = first["request"]["messages"][0]["content"]
note_text = "Skill descriptions are injected per-turn"
count1 = output1.count(note_text)
assert count1 == 1, f"First rewrite produced {count1} notes, expected 1"
second = mod._on_llm_request(request=first["request"], session_id="")
assert second is None
count2 = first["request"]["messages"][0]["content"].count(note_text)
assert count2 == 1, f"Second rewrite produced {count2} notes, expected 1 (idempotent)"
# ═══════════════════════════════════════════════════════════════════════════════
# B — coverage gaps: hook, compaction, singleton
# ═══════════════════════════════════════════════════════════════════════════════
# These hook tests use the anonymous session (""): a named session without a
# capability snapshot skips injection before reaching the index at all.
def test_on_pre_llm_call_empty_index(monkeypatch):
"""_on_pre_llm_call returns None when the index is empty / not built."""
mod = _load_plugin_module()
# Reset singleton state
br._index = None
br._skills_by_id = {}
# Point to empty roots
monkeypatch.setattr(br, "SKILLS_ROOT", Path("/nonexistent-sr-test"))
monkeypatch.setattr(br, "PLUGINS_ROOT", Path("/nonexistent-sr-test-plugins"))
monkeypatch.setattr(br, "CONFIG_PATH", Path("/nonexistent-sr-test-config.yaml"))
result = mod._on_pre_llm_call("", "search for images")
assert result is None
def test_on_pre_llm_call_no_results(monkeypatch):
"""_on_pre_llm_call returns None when BM25 finds no matches."""
mod = _load_plugin_module()
# Build a real index with unrelated skills
br._index = None
br._skills_by_id = {}
skills_root = Path("/tmp/_sr_test_noresults")
if skills_root.exists():
import shutil
shutil.rmtree(skills_root)
skills_root.mkdir(parents=True)
(skills_root / "a/SKILL.md").parent.mkdir(parents=True, exist_ok=True)
(skills_root / "a/SKILL.md").write_text("---\nname: alpha\ndescription: travel planning\n---\n\n# Body\n")
plugins_root = Path("/tmp/_sr_test_noresults_plugins")
if plugins_root.exists():
import shutil
shutil.rmtree(plugins_root)
plugins_root.mkdir(parents=True)
config_path = Path("/tmp/_sr_test_noresults_config.yaml")
config_path.write_text("skills:\n disabled: []\n")
monkeypatch.setattr(br, "SKILLS_ROOT", skills_root)
monkeypatch.setattr(br, "PLUGINS_ROOT", plugins_root)
monkeypatch.setattr(br, "CONFIG_PATH", config_path)
try:
result = mod._on_pre_llm_call("", "zzzzqqqqxxxx")
assert result is None
finally:
import shutil
shutil.rmtree(skills_root, ignore_errors=True)
shutil.rmtree(plugins_root, ignore_errors=True)
config_path.unlink(missing_ok=True)
br._index = None
br._skills_by_id = {}
def test_on_pre_llm_call_internal_exception(monkeypatch):
"""_on_pre_llm_call swallows internal exceptions and returns None, never raises."""
mod = _load_plugin_module()
# Force get_index to raise
def boom():
raise RuntimeError("simulated failure")
monkeypatch.setattr(br, "get_index", boom)
# Also patch the module-level reference in the plugin
monkeypatch.setattr(mod, "get_index", boom)
result = mod._on_pre_llm_call("", "search for images")
assert result is None
def test_on_pre_llm_call_hit(monkeypatch):
"""_on_pre_llm_call returns a {'context': ...} dict with expected shape on a hit."""
mod = _load_plugin_module()
# Build a real index
br._index = None
br._skills_by_id = {}
skills_root = Path("/tmp/_sr_test_hit")
if skills_root.exists():
import shutil
shutil.rmtree(skills_root)
skills_root.mkdir(parents=True)
# Need >=3 docs for positive Lucene IDF on a df=1 term
for rel, name, desc in [
("img/image-studio", "image-studio", "fal.ai image generation and photo editing"),
("travel/itinerary", "travel-itinerary", "plan trips flights hotels calendar"),
("research/arxiv", "arxiv", "search academic papers on arxiv"),
]:
d = skills_root / rel
d.mkdir(parents=True, exist_ok=True)
(d / "SKILL.md").write_text(
f"---\nname: {name}\ndescription: {desc}\n---\n\n# Body\n"
)
plugins_root = Path("/tmp/_sr_test_hit_plugins")
if plugins_root.exists():
import shutil
shutil.rmtree(plugins_root)
plugins_root.mkdir(parents=True)
config_path = Path("/tmp/_sr_test_hit_config.yaml")
config_path.write_text("skills:\n disabled: []\n")
monkeypatch.setattr(br, "SKILLS_ROOT", skills_root)
monkeypatch.setattr(br, "PLUGINS_ROOT", plugins_root)
monkeypatch.setattr(br, "CONFIG_PATH", config_path)
try:
result = mod._on_pre_llm_call("", "generate an image with fal.ai")
assert result is not None
assert isinstance(result, dict)
assert "context" in result
ctx = result["context"]
assert "Retrieved Skills" in ctx
assert "image-studio" in ctx
finally:
import shutil
shutil.rmtree(skills_root, ignore_errors=True)
shutil.rmtree(plugins_root, ignore_errors=True)
config_path.unlink(missing_ok=True)
br._index = None
br._skills_by_id = {}
def test_register_does_not_patch_prompt_builder(monkeypatch):
"""register() must not assign any prompt_builder attribute (catalog rule 9)."""
mod = _load_plugin_module()
agent_mod = types.ModuleType("agent")
agent_pb = types.ModuleType("agent.prompt_builder")
def original(*a, **kw):
return "<available_skills>\n test:\n - foo: Bar\n</available_skills>"
agent_pb.build_skills_system_prompt = original
agent_pb.clear_skills_system_prompt_cache = lambda **kw: None
agent_mod.prompt_builder = agent_pb
sys.modules["agent"] = agent_mod
sys.modules["agent.prompt_builder"] = agent_pb
class Ctx:
def register_hook(self, name, func):
pass
def register_middleware(self, name, func):
pass
try:
mod.register(Ctx())
assert agent_pb.build_skills_system_prompt is original
assert not getattr(agent_pb.build_skills_system_prompt, "_skill_retrieval_patched", False)
finally:
sys.modules.pop("agent", None)
sys.modules.pop("agent.prompt_builder", None)
def test_compact_available_skills_block_is_pure():
"""compact_available_skills_block is deterministic and sentinel-idempotent."""
mod = _load_plugin_module()
inner = " test:\n - foo: Bar description\n"
once = mod.compact_available_skills_block(inner, True)
twice = mod.compact_available_skills_block(once, True)
assert once == twice
assert mod._COMPACT_SENTINEL in once
assert "Bar description" not in once
assert mod.compact_available_skills_block(inner, False) == inner
# ─── Singleton: get_index / get_skill_info ──────────────────────────────────
@pytest.fixture
def reset_singleton():
"""Reset module-level singleton state before and after each test."""
br._index = None
br._skills_by_id = {}
yield
br._index = None
br._skills_by_id = {}
def test_get_index_builds_and_caches(tmp_path, monkeypatch, reset_singleton):
"""get_index builds on first call and returns the cached index on second."""
skills_root = tmp_path / "skills"
plugins_root = tmp_path / "plugins"
config_path = tmp_path / "config.yaml"
# Two docs so the query term discriminates between documents
(skills_root / "a/SKILL.md").parent.mkdir(parents=True, exist_ok=True)
(skills_root / "a/SKILL.md").write_text(
"---\nname: alpha\ndescription: travel planning flights\n---\n\n# Body\n"
)
(skills_root / "b/SKILL.md").parent.mkdir(parents=True, exist_ok=True)
(skills_root / "b/SKILL.md").write_text(
"---\nname: beta\ndescription: image generation editing\n---\n\n# Body\n"
)
plugins_root.mkdir()
config_path.write_text("skills:\n disabled: []\n")
monkeypatch.setattr(br, "SKILLS_ROOT", skills_root)
monkeypatch.setattr(br, "PLUGINS_ROOT", plugins_root)
monkeypatch.setattr(br, "CONFIG_PATH", config_path)
idx1 = br.get_index()
assert idx1 is not None
assert idx1._built
idx2 = br.get_index()
assert idx2 is idx1 # cached — same object
def test_get_index_returns_none_for_empty(monkeypatch, reset_singleton):
"""get_index returns None and logs a warning when no skills are found."""
monkeypatch.setattr(br, "SKILLS_ROOT", Path("/nonexistent-gi-test"))
monkeypatch.setattr(br, "PLUGINS_ROOT", Path("/nonexistent-gi-test-plugins"))
monkeypatch.setattr(br, "CONFIG_PATH", Path("/nonexistent-gi-test-config.yaml"))
assert br.get_index() is None
def test_get_skill_info_returns_none_for_unknown(reset_singleton):
"""get_skill_info returns None for an unknown skill id."""
assert br.get_skill_info("nonexistent-skill-id") is None
def test_get_skill_info_returns_info_after_build(tmp_path, monkeypatch, reset_singleton):
"""get_skill_info returns skill metadata after the index is built."""
skills_root = tmp_path / "skills"
plugins_root = tmp_path / "plugins"
config_path = tmp_path / "config.yaml"
# Need >=3 docs for positive Lucene IDF
(skills_root / "a/my-skill").mkdir(parents=True)
(skills_root / "a/my-skill/SKILL.md").write_text(
"---\nname: my-skill\ndescription: A test skill for things\n---\n\n# Body\n"
)
(skills_root / "b/other").mkdir(parents=True)
(skills_root / "b/other/SKILL.md").write_text(
"---\nname: other\ndescription: Another skill for testing\n---\n\n# Body\n"
)
(skills_root / "c/third").mkdir(parents=True)
(skills_root / "c/third/SKILL.md").write_text(
"---\nname: third\ndescription: Third skill for coverage\n---\n\n# Body\n"
)
plugins_root.mkdir()
config_path.write_text("skills:\n disabled: []\n")
monkeypatch.setattr(br, "SKILLS_ROOT", skills_root)
monkeypatch.setattr(br, "PLUGINS_ROOT", plugins_root)
monkeypatch.setattr(br, "CONFIG_PATH", config_path)
br.get_index()
info = br.get_skill_info("a/my-skill")
assert info is not None
assert info["name"] == "my-skill"
assert "test skill" in info["description"].lower()Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Unverified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |