moonlight-lupin/agent-skills / plugins/skill-retrieval
skill-retrieval
VerifiedBM25-based skill retrieval plugin for Hermes Agent. Replaces the full skill list in the system prompt with a names-only compact view (~2K tokens) and injects top-K relevant skill descriptions per turn via BM25 retrieval (~300 tokens). Saves ~9K tokens/turn. Use when system prompt token overhead from skills is a concern, or when skill discovery quality matters.
scripts/bm25_retriever.py
View on GitHub"""BM25 skill retriever — indexes active skill descriptions and ranks them per query.
Adapted from SR-Agents (oneal2000/SR-Agents, MIT license).
Source: https://github.com/oneal2000/SR-Agents/blob/main/src/sragents/retrieve/bm25.py
Indexed lazily and cached (invalidated on skill/config changes); retrieval is
sub-millisecond for 128 skills.
"""
import inspect
import os
import re
import sys
import time
import math
import logging
import threading
from pathlib import Path
logger = logging.getLogger(__name__)
_YAML_MODULE = None
def _yaml_module():
"""Return a YAML module. PyYAML has no CPython 3.14 wheel, so prefer
the host's hermes_yaml (ruamel) facade when available.
Resolved once and cached in a module global: re-running the import
machinery per SKILL.md parse is wasted work when the facade is absent,
and if BOTH modules are missing the ModuleNotFoundError must surface
here (inside a controlled call site), never from a bare top-level
``import yaml`` — _parse_skill_md's contract is never-raises.
"""
global _YAML_MODULE
if _YAML_MODULE is not None:
return _YAML_MODULE
try:
import hermes_yaml
_YAML_MODULE = hermes_yaml
except Exception:
import yaml
_YAML_MODULE = yaml
return _YAML_MODULE
# ─── Config ───────────────────────────────────────────────────────────────────
try:
from hermes_constants import get_config_path, get_hermes_home, get_skills_dir
except ImportError: # pragma: no cover - standalone test fallback
def get_hermes_home() -> Path:
val = os.environ.get("HERMES_HOME", "").strip()
return Path(val).expanduser() if val else Path.home() / ".hermes"
def get_config_path() -> Path:
return get_hermes_home() / "config.yaml"
def get_skills_dir() -> Path:
return get_hermes_home() / "skills"
def get_plugins_dir() -> Path:
"""Return the profile-scoped plugins directory."""
return get_hermes_home() / "plugins"
# Explicit test override points for the standalone loader; None at runtime.
# Runtime code resolves paths through Hermes' official home helpers on every
# load so multiplexed/named profiles (and Hermes' context-local home override)
# never read the profile that happened to be active at import time.
SKILLS_ROOT: "Path | None" = None
PLUGINS_ROOT: "Path | None" = None
CONFIG_PATH: "Path | None" = None
# Directories under any plugin's skills/ tree to skip (mirrors the
# .archive / .curator_backups / .hub exclusions used for standalone skills).
_SKIP_DIRS = (".archive", ".curator_backups", ".hub")
# Copied from agent/skill_utils.py. Do not import those private sets.
_EXCLUDED_SKILL_DIRS = frozenset((
".git", ".github", ".hub", ".archive", ".curator_backups", ".locks",
".venv", "venv", "node_modules", "site-packages", "__pycache__",
".tox", ".nox", ".pytest_cache", ".mypy_cache", ".ruff_cache",
))
_SKILL_SUPPORT_DIRS = frozenset(("references", "templates", "assets", "scripts"))
# Org mirrors live at ``<root>/_org/<org_id>/``. Core gates them on
# ``<root>/_org/.active_org`` (agent/skill_utils.py read_active_org_id).
_ORG_MIRROR_DIR = "_org"
_ORG_ACTIVE_MARKER = ".active_org"
_ABSENT_ORG_MARKER = ("absent",)
K1 = 1.5
B = 0.75
# ─── Tokenizer ────────────────────────────────────────────────────────────────
# CJK-like scripts the bigram tokenizer covers: kana (fullwidth + halfwidth),
# ideographs (Unified + Extension A/B) and hangul syllables. Everything below
# U+3040 with a word boundary in the corpus behaves like the whitespace
# languages.
CJK_RANGES = (
(0x3040, 0x30FF), # Hiragana + Katakana
(0x3400, 0x4DBF), # CJK Extension A
(0x4E00, 0x9FFF), # CJK Unified Ideographs
(0xAC00, 0xD7AF), # Hangul syllables
(0xFF66, 0xFF9F), # Halfwidth Katakana
(0x20000, 0x2A6DF), # CJK Extension B (supplementary plane)
)
def _has_cjk(text: str) -> bool:
"""True when any character sits in one of the bigram-covered CJK ranges."""
return any(
lo <= ord(ch) <= hi for ch in text for lo, hi in CJK_RANGES
)
# Matches maximal runs of the covered CJK scripts inside a preview.
_CJK_RUN_RE = re.compile(
"[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uff66-\uff9f"
"\U00020000-\U0002a6df]+"
)
# Splits a lowercased token into ASCII runs, covered-CJK runs, and single
# leftover non-ASCII characters (so a mixed-script token keeps its Latin
# terms instead of welding them into a bigram with a CJK character).
_SCRIPT_SPLIT_RE = re.compile(
"[a-z0-9]+|[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uff66-\uff9f"
"\U00020000-\U0002a6df]+|[^a-z0-9\\s]"
)
#: Capped budget for the joined CJK runs of one document: enough to hit
#: content-bearing text at the top of a SKILL.md for the short CJK queries
#: the tokenizer serves, small enough that BM25's length normalization
#: (b=0.75) does not push the skill's English-description matches far down
#: when it ranks against CJK-free competitors (a CJK-body skill is exactly
#: the motivating case for English queries too).
CJK_RUN_BUDGET = 200
def _cjk_runs(body: str) -> str:
"""Extract the covered CJK runs from a body preview, space-joined.
Only the runs join the document — never the 3000-char English body
around them — and the joined text is capped at CJK_RUN_BUDGET, so a
single CJK mention cannot put ~2500 bigrams into an English document
and a Chinese body cannot dominate the length normalization.
"""
runs = " ".join(m.group() for m in _CJK_RUN_RE.finditer(body))
return runs[:CJK_RUN_BUDGET]
def tokenize(text: str) -> list[str]:
"""Simple whitespace + punctuation tokenizer, lowercased.
Defends against non-str queries: Telegram can deliver the user message
as a list of content parts (str or {"text": ...} dicts). Flatten to a
plain string before tokenizing; anything else is stringified.
"""
if not isinstance(text, str):
if isinstance(text, (list, tuple)):
parts: list[str] = []
for part in text:
if isinstance(part, str):
parts.append(part)
elif isinstance(part, dict):
value = part.get("text") or part.get("caption")
if isinstance(value, str):
parts.append(value)
text = " ".join(parts)
else:
text = str(text)
text = text.lower()
text = re.sub(r"[^\w\s]", " ", text)
tokens: list[str] = []
for token in text.split():
# CJK runs have no whitespace boundaries: a whole Chinese/ Japanese/
# Korean sentence would otherwise become ONE token and match nothing
# in the index. Emit bigrams so such a query can hit corpus text.
# ASCII tokens pass through unchanged.
if len(token) == 1 or not _has_cjk(token):
tokens.append(token)
continue
# Mixed-script tokens: split into ASCII runs, CJK runs, and per-char
# leftover (e.g. 'é'), so '用python写' keeps the 'python' term,
# 'Python으로' keeps the 'python' term, and 'café中文' does not weld
# 'é' and '中' into one token.
for part in _SCRIPT_SPLIT_RE.findall(token):
if part[0].isascii():
tokens.append(part)
continue
if len(part) == 1:
# Single CJK character: emitted as itself (covered by the
# len==1 rule for whole tokens).
tokens.append(part)
continue
# Bigrams only. For a 2-character run the single bigram IS the
# run; appending it and its bigram would emit the same string
# twice and double-count its TF on documents and queries.
for i in range(len(part) - 1):
tokens.append(part[i : i + 2])
return tokens
# ─── Skill corpus loader ──────────────────────────────────────────────────────
def _parse_skill_md(skill_md: Path) -> tuple[str, str, str]:
"""Return (name, description, body_preview) parsed from a SKILL.md frontmatter.
Returns ("", "", "") for valid frontmatter with missing name/description.
Returns ("", "", "") for invalid frontmatter (no fences, malformed YAML,
non-mapping) and for a file that cannot be read. Never raises, so one
unreadable skill cannot abort the whole index build.
"""
try:
yaml = _yaml_module()
except Exception:
logger.warning("No YAML module available (neither hermes_yaml nor PyYAML)")
return "", "", ""
try:
text = skill_md.read_text(errors="replace")
except OSError as exc:
logger.warning("Cannot read %s: %s", skill_md, exc)
return "", "", ""
lines = text.splitlines()
# Frontmatter must start at the first line (allowing optional BOM).
first_line = lines[0].lstrip("\ufeff").strip() if lines else ""
if first_line != "---":
logger.warning("No YAML frontmatter in %s (first line is not '---')", skill_md)
return "", "", ""
# Find the closing fence.
close_idx = None
for i in range(1, len(lines)):
if lines[i].strip() == "---":
close_idx = i
break
if close_idx is None:
logger.warning("No closing YAML frontmatter fence in %s", skill_md)
return "", "", ""
fm_text = "\n".join(lines[1:close_idx])
try:
data = yaml.safe_load(fm_text)
except yaml.YAMLError:
logger.warning("Malformed YAML frontmatter in %s", skill_md)
return "", "", ""
if not isinstance(data, dict):
logger.warning("Non-mapping YAML frontmatter in %s", skill_md)
return "", "", ""
name = data.get("name")
desc = data.get("description")
if name is None or desc is None:
logger.warning("Missing name or description in frontmatter of %s", skill_md)
name_str = "" if name is None else str(name)
desc_str = "" if desc is None else str(desc)
# Body preview (first ~3000 chars after the frontmatter) joins the BM25
# corpus: Chinese queries match skill CONTENT, not just the (usually
# English) description. Size-capped so one huge SKILL.md cannot bloat
# the index build.
body_lines = lines[close_idx + 1 :]
body_preview = " ".join(line.strip() for line in body_lines if line.strip())[:3000]
return name_str, desc_str, body_preview
def _iter_skill_files(root: Path, prefix: str = "") -> list[tuple[Path, str, str]]:
"""Walk `root` for SKILL.md files. Returns [(path, skill_id, leaf_name), ...].
`prefix` is prepended to the relative id (e.g. "chief-of-staff:") so
plugin-bundled skills can't collide with standalone ones.
"""
out: list[tuple[Path, str, str]] = []
if not root.exists():
return out
for skill_md in sorted(root.rglob("SKILL.md")):
rel_parts = skill_md.relative_to(root).parts
if any(part in _SKIP_DIRS for part in rel_parts):
continue
skill_dir = skill_md.parent
# Normalize to forward slashes: on Windows str(Path) yields
# backslashes, which break skill_id lookups and cross-platform tests.
rel_parts = skill_dir.relative_to(root).parts
rel_name = prefix + "/".join(rel_parts)
leaf_name = skill_dir.name
out.append((skill_md, rel_name, leaf_name))
return out
def _runtime_paths_are_overridden() -> bool:
"""Return True when tests explicitly set the legacy path constants.
Only an explicit (non-None) ``SKILLS_ROOT`` / ``CONFIG_PATH`` counts; a
Hermes home change is a profile switch, not a test override.
``PLUGINS_ROOT`` is *not* a signal: Hermes-discovery tests (and Issue #8
registry tests) patch ``get_plugins_dir`` while still wanting the
registry-aware loader.
"""
return SKILLS_ROOT is not None or CONFIG_PATH is not None
def _skill_id_from_entry(entry: dict, prefix: str = "") -> str:
category = str(entry.get("category") or "general")
skill_name = str(entry.get("skill_name") or entry.get("frontmatter_name") or "")
if category and category != "general":
return f"{prefix}{category}/{skill_name}"
return f"{prefix}{skill_name}"
def _frontmatter_keywords(frontmatter: dict) -> tuple[list[str], list[str]]:
"""Extract (triggers, tags) lists from parsed SKILL.md frontmatter.
Only str-list shapes are accepted; anything else yields ([], []).
Never raises — one malformed skill must not abort the corpus build.
"""
try:
trig = frontmatter.get("triggers")
triggers = [str(t) for t in trig if isinstance(t, str)] if isinstance(trig, list) else []
meta = frontmatter.get("metadata")
hermes = meta.get("hermes") if isinstance(meta, dict) else None
tags_fm = hermes.get("tags") if isinstance(hermes, dict) else None
tags = [str(t) for t in tags_fm if isinstance(t, str)] if isinstance(tags_fm, list) else []
except Exception: # defensive: frontmatter shapes vary across sources
return [], []
return triggers, tags
def _index_text_join(base: str, body: object) -> str:
"""Extract the CJK runs of a SKILL.md body preview into its document.
The runs join ONLY when the body is a str containing CJK characters,
space-joined and capped at CJK_RUN_BUDGET: the bigram tokenizer gets
target text for CJK queries, while the English body text around the
runs contributes nothing (no recall or length shift in the English
corpus — the query-relevant skill keeps its description score).
Anything else (missing key, wrong type, CJK-free body) degrades to the
base text unchanged — the same contract the description-only corpus had.
"""
if isinstance(body, str) and _has_cjk(body):
runs = _cjk_runs(body)
if runs:
return base + "\n" + runs
return base
def _record_skill(skills: list[dict], seen_names: set[str], entry: dict, *, prefix: str = "") -> None:
"""Append a parsed Hermes skill entry while preserving first-seen precedence."""
name = str(entry.get("frontmatter_name") or entry.get("skill_name") or "").strip()
if not name or name in seen_names:
return
seen_names.add(name)
desc = str(entry.get("description") or "")
# Trigger phrases, tags and the body preview broaden BM25 recall beyond
# the description: the indexed document is the retrieval surface
# (name: desc + lists + body preview), while only the description is
# injected. Trigger phrases cover queries the short description cannot;
# the CJK bigram tokenizer needs Chinese body text to match Chinese
# queries against.
triggers = entry.pop("_triggers", None)
tags = entry.pop("_tags", None)
body = entry.pop("_body_preview", None)
text = f"{name}: {desc}"
if triggers:
text += " triggers: " + " ".join(triggers)
if tags:
text += " tags: " + " ".join(tags)
# _index_text_join degrades a missing/wrong-type preview to the base text
# (same contract _parse_skill_md's str-only previews already guaranteed)
# and joins a preview only when it contains CJK, so English documents
# stay byte-identical to the description-only corpus.
text = _index_text_join(text, body)
skills.append({
"skill_id": _skill_id_from_entry(entry, prefix=prefix),
"leaf_name": str(entry.get("skill_name") or name),
"name": name,
"frontmatter_name": str(entry.get("frontmatter_name") or name),
"description": desc,
"text": text,
"plugin_origin": entry.get("plugin_origin") is True,
})
def _try_list_plugin_skill_metadata():
"""Return ``(metadata, plugin_manager)`` from Hermes' live registry, or None.
None means the registry is unreachable and the caller should fall back to
a raw directory scan so the corpus never silently empties.
``discover_plugins`` may import installed plugin packages, which insert
their ``scripts/`` dirs onto ``sys.path``. Restore the path afterwards so
a later ``importlib.reload(bm25_retriever)`` still finds *this* file
rather than an older copy under ``~/.hermes/plugins``.
"""
saved_path = list(sys.path)
try:
from hermes_cli.plugins import discover_plugins, get_plugin_manager
discover_plugins()
pm = get_plugin_manager()
if pm is None:
return None
metadata = pm.list_plugin_skill_metadata() if pm else []
return list(metadata or []), pm
except Exception as exc:
logger.debug("Plugin skill registry unavailable: %s", exc)
return None
finally:
sys.path[:] = saved_path
def _skill_doc_is_visible(skill: dict, visible_names: "set[str] | frozenset[str] | None") -> bool:
"""True when ``visible_names`` is None (fail-open) or a rendered identity is listed.
Core's ``<available_skills>`` block lists the frontmatter name, not a
``category/skill`` path and not a ``:``-suffix of a qualified name.
Match those rendered strings exactly. A colon in the name is not
provenance. Only entries tagged ``plugin_origin`` during the plugin
directory scan are exempt. They never appear in the block.
"""
if visible_names is None:
return True
if skill.get("plugin_origin") is True:
return True
candidates = {
str(skill.get("leaf_name") or ""),
str(skill.get("name") or ""),
str(skill.get("frontmatter_name") or ""),
str(skill.get("skill_name") or ""),
str(skill.get("skill_id") or ""),
}
candidates.discard("")
return any(c in visible_names for c in candidates)
def _index_cache_key(
home_key: str,
available_tools: "set[str] | None",
available_toolsets: "set[str] | None",
platform_hint: "str | None" = None,
disabled: "frozenset[str] | None" = None,
visible_names: "frozenset[str] | None" = None,
):
"""Cache key for a BM25 index.
The corpus also depends on the session platform (``platform_disabled``
and ``session_platforms`` gates), so the platform hint and the resolved
disabled set are part of the key, mirroring Hermes' own skills prompt
cache key. Fail-open with no platform/disabled info stays a plain home
string.
"""
if (
available_tools is None
and available_toolsets is None
and not platform_hint
and not disabled
and visible_names is None
):
return home_key
return (
home_key,
frozenset(available_tools or ()),
frozenset(available_toolsets or ()),
platform_hint or None,
disabled or frozenset(),
None if visible_names is None else frozenset(visible_names),
)
def _session_platform_key_parts() -> "tuple[str | None, frozenset[str] | None]":
"""Return ``(platform_hint, frozenset(disabled))`` for the index cache key.
Same inputs ``load_active_skills`` gates on. Never raises: any failure
degrades that part to None (the key then matches the loader's own
fail-open behavior as closely as the host allows).
"""
try:
from agent.prompt_builder import _current_session_platform_hint
from agent.skill_utils import get_disabled_skill_names
except Exception:
return None, None
try:
platform_hint = _current_session_platform_hint() or None
except Exception:
platform_hint = None
try:
disabled = frozenset(get_disabled_skill_names(platform_hint) or ())
except Exception:
disabled = None
return platform_hint, disabled
def _runtime_cache_key(
available_tools: "set[str] | None",
available_toolsets: "set[str] | None",
visible_names: "frozenset[str] | None" = None,
platform_hint: "str | None" = None,
):
"""Runtime cache key shared by ``get_index`` and ``get_skill_info``."""
home_key = str(get_hermes_home().expanduser().resolve(strict=False))
live_platform, disabled = _session_platform_key_parts()
if platform_hint is None:
platform_hint = live_platform
return home_key, _index_cache_key(
home_key, available_tools, available_toolsets, platform_hint, disabled,
visible_names=None if visible_names is None else frozenset(visible_names),
)
def _load_active_skills_legacy(
visible_names: "set[str] | frozenset[str] | None" = None,
) -> list[dict]:
"""Standalone loader: explicit test overrides, or no Hermes helpers.
Unset path constants resolve from the current Hermes home at call time.
"""
try:
yaml = _yaml_module()
except Exception:
logger.warning("No YAML module available — legacy loader returns no skills")
return []
skills_root = SKILLS_ROOT if SKILLS_ROOT is not None else get_skills_dir()
plugins_root = PLUGINS_ROOT if PLUGINS_ROOT is not None else get_plugins_dir()
config_path = CONFIG_PATH if CONFIG_PATH is not None else get_config_path()
# Load disabled list and profile-configured external skill directories.
disabled = set()
if config_path.exists():
# A malformed or unreadable config must not abort the index build —
# that would leave the agent with a compacted prompt and no retrieval.
try:
config = yaml.safe_load(config_path.read_text()) or {}
except (OSError, yaml.YAMLError) as exc:
logger.warning(
"Cannot read %s: %s — treating no skills as disabled",
config_path, exc,
)
config = {}
if not isinstance(config, dict):
config = {}
skills_cfg = config.get("skills")
if isinstance(skills_cfg, dict):
disabled = set(skills_cfg.get("disabled") or [])
# Collect candidate skill files from both roots.
candidates: list[tuple[Path, str, str]] = []
candidates += _iter_skill_files(skills_root)
if plugins_root.exists():
for plugin_dir in sorted(plugins_root.iterdir()):
if not plugin_dir.is_dir() or plugin_dir.name.startswith("."):
continue
plugin_skills = plugin_dir / "skills"
if plugin_skills.is_dir():
candidates += _iter_skill_files(plugin_skills, prefix=f"{plugin_dir.name}:")
skills: list[dict] = []
for skill_md, rel_name, leaf_name in candidates:
# Disabled list uses bare leaf names and rel_names; match either.
if leaf_name in disabled or rel_name in disabled:
continue
name, desc, body_preview = _parse_skill_md(skill_md)
if not name:
name = leaf_name
skills.append({
"skill_id": rel_name,
"leaf_name": leaf_name,
"name": name,
"description": desc,
# Body preview joins the indexed text so CJK queries (bigrams) can
# hit skill content beyond the frontmatter — most writing-skill
# descriptions are English while queries are Chinese. The shared
# helper applies the same CJK gate as the primary paths.
"text": _index_text_join(f"{name}: {desc}", body_preview),
# Legacy/fallback entries are not the plugin directory scan.
"plugin_origin": False,
})
if visible_names is not None:
skills = [s for s in skills if _skill_doc_is_visible(s, visible_names)]
return skills
def load_active_skills(
available_tools: "set[str] | None" = None,
available_toolsets: "set[str] | None" = None,
visible_names: "set[str] | frozenset[str] | None" = None,
) -> list[dict]:
"""Load active skills using Hermes' own profile-aware discovery helpers.
Runtime path/config resolution is delegated to Hermes core helpers so named
profiles, context-local home overrides, ``skills.external_dirs``, trusted
project-local skills, disabled lists, platform gates, and condition gates
match the agent's normal skill index as closely as a plugin can. Plugin-
bundled skills come from Hermes' plugin registry when available, with a
directory-scan fallback so the corpus never silently empties.
``available_tools`` / ``available_toolsets`` are forwarded to
``_skill_should_show``. Both default to None, which is Hermes' fail-open
(index everything) when the session's capability snapshot is unknown.
``visible_names`` is an allowlist of skill leaf/names currently shown in
the core ``<available_skills>`` block. A skill whose leaf/name is not in
the set is excluded. None is fail-open (no allowlist filter). Entries
from the plugin directory scan are exempt. A colon in a local name is not.
"""
if _runtime_paths_are_overridden():
return _load_active_skills_legacy(visible_names=visible_names)
# A mid-scan failure in a read-only helper (e.g. a signature change in
# _parse_skill_file/_build_snapshot_entry on a Hermes upgrade) must not
# leave a silently empty corpus — an empty corpus plus the llm_request
# rewrite would strip every skill description from the prompt with
# nothing injected back. Fall back to the standalone loader instead.
try:
return _load_active_skills_hermes(
available_tools, available_toolsets, visible_names,
)
except Exception as exc:
logger.warning(
"Hermes discovery helpers raised mid-scan: %s — falling back "
"to the standalone loader so the corpus is not silently empty",
exc,
)
return _load_active_skills_legacy(visible_names=visible_names)
def _load_active_skills_hermes(
available_tools: "set[str] | None",
available_toolsets: "set[str] | None",
visible_names: "set[str] | frozenset[str] | None",
) -> list[dict]:
"""Hermes-helper skill scan (raises; caller falls back to legacy)."""
try:
from agent.prompt_builder import (
_build_snapshot_entry,
_current_session_platform_hint,
_parse_skill_file,
_skill_should_show,
extract_skill_conditions,
)
from agent.skill_utils import (
get_all_skills_dirs,
get_disabled_skill_names,
get_project_skills_dirs,
iter_project_skill_files,
iter_skill_index_files,
)
except ImportError as exc:
logger.warning("Cannot import Hermes skill discovery helpers: %s", exc)
return _load_active_skills_legacy(visible_names=visible_names)
# Compat shim for Hermes ≤0.20.x, where _skill_should_show takes 3
# positional arguments (no session_platform). plugin.yaml now requires
# >=0.21.1 (4-arg), but the shim is kept for hosts that skip the
# requires_hermes gate. Wrap the
# 3-arg host function in a 4-arg adapter once, at load time, by
# inspecting its declared parameters. Fall back to assuming the
# 4-arg form when inspection fails, because the shim must not mask
# a real TypeError from a future signature change.
# If any read-only helper changed signature (TypeError), fall back to the
# standalone loader rather than returning a silently empty corpus — an
# empty corpus plus the llm_request rewrite would strip every skill
# description from the prompt with nothing injected back.
_show_params = None
try:
_show_params = inspect.signature(_skill_should_show).parameters
except (TypeError, ValueError):
_show_params = None
if _show_params is not None and len(_show_params) < 4:
_raw_should_show = _skill_should_show
def _skill_should_show( # noqa: F811 - deliberate host-function wrapper
conditions, available_tools, available_toolsets, session_platform=None,
):
return _raw_should_show(conditions, available_tools, available_toolsets)
platform_hint = _current_session_platform_hint() or None
disabled = get_disabled_skill_names(platform_hint)
skills: list[dict] = []
seen_names: set[str] = set()
skill_matches_platform = getattr(
sys.modules.get("agent.skill_utils"), "skill_matches_platform", lambda _fm: True,
)
def add_skill_file(
skill_file: Path, root: Path, *, prefix: str = "", qualify_name: bool = False,
plugin_origin: bool = False,
) -> None:
# Parse errors are per-file (a malformed SKILL.md must not abort the
# scan). But helper-signature failures (TypeError/AttributeError) are
# systemic: swallowing them per file yields an empty corpus while the
# llm_request rewrite still strips all descriptions. Re-raise those so
# load_active_skills' outer guard can fall back to the legacy loader.
try:
is_compatible, frontmatter, desc = _parse_skill_file(skill_file)
if not is_compatible:
return
entry = dict(_build_snapshot_entry(skill_file, root, frontmatter, desc))
bare_frontmatter = entry["frontmatter_name"]
bare_skill = entry["skill_name"]
# Qualified identity before the disabled check, so ``foo:hidden``
# in the disabled set matches the name the scan will publish.
if qualify_name and prefix:
qname = f"{prefix}{bare_frontmatter}"
entry["frontmatter_name"] = qname
entry["skill_name"] = qname
prefix = ""
entry["plugin_origin"] = plugin_origin
entry["_triggers"], entry["_tags"] = _frontmatter_keywords(frontmatter)
if (
entry["frontmatter_name"] in disabled
or entry["skill_name"] in disabled
or bare_frontmatter in disabled
or bare_skill in disabled
):
return
if not _skill_should_show(
extract_skill_conditions(frontmatter),
available_tools,
available_toolsets,
platform_hint,
):
return
if not _skill_doc_is_visible(entry, visible_names):
return
# Body preview joins the indexed document (see _record_skill): the
# CJK bigram tokenizer needs Chinese body text to match Chinese
# queries against — most skill descriptions are English.
# _parse_skill_md is a tested in-repo helper that always returns a
# 3-tuple of str and never raises; the tuple-guard degenerates a
# future stub's wrong shape to no preview instead of an IndexError
# escaping the (TypeError, AttributeError) systemic handler.
parsed3 = _parse_skill_md(skill_file)
entry["_body_preview"] = (
parsed3[2] if isinstance(parsed3, tuple) and len(parsed3) >= 3 else ""
)
_record_skill(skills, seen_names, entry, prefix=prefix)
except (TypeError, AttributeError) as exc:
# Signature change in a core helper is systemic, not per-file:
# re-raise so the outer guard switches to the standalone loader.
logger.warning(
"Core helper signature issue (%s) while reading %s — "
"aborting Hermes-helper scan for standalone fallback",
exc, skill_file,
)
raise
except Exception as exc:
logger.debug("Error reading skill %s: %s", skill_file, exc)
def scan_plugin_directories(*, qualify_name: bool) -> None:
plugins_root = get_plugins_dir()
if not plugins_root.exists():
return
for plugin_dir in sorted(plugins_root.iterdir()):
if not plugin_dir.is_dir() or plugin_dir.name.startswith("."):
continue
plugin_skills = plugin_dir / "skills"
if not plugin_skills.is_dir():
continue
for skill_file in iter_skill_index_files(plugin_skills, "SKILL.md"):
add_skill_file(
skill_file, plugin_skills,
prefix=f"{plugin_dir.name}:",
qualify_name=qualify_name,
plugin_origin=True,
)
def add_registry_plugin_skill(plugin_skill: dict, pm) -> None:
plugin_skill = dict(plugin_skill)
frontmatter = dict(plugin_skill.pop("frontmatter", {}) or {})
qualified = str(plugin_skill.get("name") or "").strip()
if not qualified:
return
desc = str(plugin_skill.get("description") or "")
# Parse on-disk SKILL.md before platform/disabled/capability gates so
# gating uses file frontmatter. Registry metadata is the fallback when
# the file cannot be resolved or read.
skill_path = plugin_skill.get("path")
finder = getattr(pm, "find_plugin_skill", None) if pm is not None else None
if skill_path is None and callable(finder):
try:
skill_path = finder(qualified)
except Exception as exc:
logger.debug("find_plugin_skill(%s) failed: %s", qualified, exc)
skill_path = None
if skill_path:
try:
path_obj = Path(skill_path)
if path_obj.is_file():
_ok, file_fm, file_desc = _parse_skill_file(path_obj)
if file_desc:
desc = file_desc
if file_fm:
frontmatter = file_fm
except Exception as exc:
logger.debug("Error reading registry skill %s: %s", skill_path, exc)
try:
if not skill_matches_platform(frontmatter):
return
except Exception as exc:
logger.debug("skill_matches_platform failed for %s: %s", qualified, exc)
bare = str(
frontmatter.get("name")
or plugin_skill.get("bare_name")
or qualified.split(":")[-1]
)
if qualified in disabled or bare in disabled:
return
if not _skill_should_show(
extract_skill_conditions(frontmatter),
available_tools,
available_toolsets,
platform_hint,
):
return
entry = {
"category": "general",
"skill_name": qualified,
"frontmatter_name": qualified,
"description": desc,
}
entry["_triggers"], entry["_tags"] = _frontmatter_keywords(frontmatter)
# Same body-preview join as the directory scan (see _record_skill).
# skill_path should be the SKILL.md file; resolve one level deeper if
# a directory is passed, so registry skills are not silently left
# with no body preview.
body_src = Path(str(skill_path)) if skill_path else None
if body_src is not None and body_src.is_dir():
body_src = body_src / "SKILL.md"
if body_src is not None:
parsed3 = _parse_skill_md(body_src)
entry["_body_preview"] = (
parsed3[2] if isinstance(parsed3, tuple) and len(parsed3) >= 3 else ""
)
else:
entry["_body_preview"] = ""
# Registry ids are ``plugin:skill`` and are absent from <available_skills>.
_record_skill(skills, seen_names, entry)
# Precedence mirrors Hermes: trusted project-local → profile-local →
# configured external dirs. First seen name wins.
for project_dir in get_project_skills_dirs():
for skill_file in iter_project_skill_files(project_dir):
add_skill_file(skill_file, project_dir)
all_skill_dirs = list(get_all_skills_dirs() or [])
for skill_root in all_skill_dirs:
for skill_file in iter_skill_index_files(skill_root, "SKILL.md"):
add_skill_file(skill_file, skill_root)
registry = _try_list_plugin_skill_metadata()
if registry is not None:
metadata, pm = registry
for plugin_skill in metadata:
if isinstance(plugin_skill, dict):
add_registry_plugin_skill(plugin_skill, pm)
else:
scan_plugin_directories(qualify_name=True)
return skills
# ─── BM25 Index ───────────────────────────────────────────────────────────────
class BM25Index:
"""BM25 Okapi retriever using a pure-stdlib inverted index.
At build time, the full BM25 weight for every (term, document) pair is
precomputed and stored in an inverted index: ``dict[str, list[tuple[int, float]]]``
mapping each term to a posting list of (doc_index, weight). A query then
sums weights over the posting lists of the query terms only — no scan over
the full corpus.
"""
def __init__(self, k1: float = K1, b: float = B):
self.k1 = k1
self.b = b
self._built = False
def build(self, corpus_ids: list[str], corpus_texts: list[str]) -> None:
self._corpus_ids = corpus_ids
t0 = time.time()
# Tokenize, build vocabulary, record document lengths.
vocab: dict[str, int] = {}
tokenized: list[list[str]] = []
doc_lens: list[int] = []
for text in corpus_texts:
tokens = tokenize(text)
tokenized.append(tokens)
doc_lens.append(len(tokens))
for t in tokens:
if t not in vocab:
vocab[t] = len(vocab)
n_docs = len(corpus_texts)
avgdl = sum(doc_lens) / n_docs if n_docs else 1.0
n_terms = len(vocab)
self._vocab = vocab
# Document frequency per term.
df: dict[int, int] = {}
for tokens in tokenized:
for tid in set(vocab[t] for t in tokens):
df[tid] = df.get(tid, 0) + 1
# Lucene BM25 IDF: log(1 + (N - df + 0.5) / (df + 0.5)). Always > 0, so
# terms in >= half the docs (and 1-2 doc corpora) still score.
idf: dict[int, float] = {}
for tid, df_count in df.items():
idf[tid] = math.log(1.0 + (n_docs - df_count + 0.5) / (df_count + 0.5))
# Build inverted index with precomputed BM25 weights.
# term → list of (doc_index, weight)
k1, b = self.k1, self.b
nnz = 0
postings: dict[int, list[tuple[int, float]]] = {}
for i, tokens in enumerate(tokenized):
if not tokens:
continue
counts: dict[int, int] = {}
for t in tokens:
tid = vocab[t]
counts[tid] = counts.get(tid, 0) + 1
dl = doc_lens[i]
denom = k1 * (1.0 - b + b * dl / avgdl)
for tid, tf in counts.items():
# BM25 Okapi TF saturation.
sat = (tf * (k1 + 1.0)) / (tf + denom)
weight = sat * idf[tid]
postings.setdefault(tid, []).append((i, weight))
nnz += 1
self._postings = postings
self._built = True
logger.info("BM25 index built: %d docs, %d terms, %d nnz in %.3fs",
n_docs, n_terms, nnz, time.time() - t0)
def retrieve(self, query: str, top_k: int = 5) -> list[tuple[str, float]]:
"""Return [(skill_id, score), ...] sorted by descending score."""
if not self._built:
return []
tokens = tokenize(query)
seen: set[int] = set()
scores: dict[int, float] = {}
for t in tokens:
tid = self._vocab.get(t)
if tid is None or tid in seen:
continue
seen.add(tid)
for doc_idx, weight in self._postings.get(tid, ()):
scores[doc_idx] = scores.get(doc_idx, 0.0) + weight
if not scores:
return []
# Sort by descending score, break ties by ascending doc index.
ranked = sorted(scores.items(), key=lambda x: (-x[1], x[0]))
results = [
(self._corpus_ids[idx], score)
for idx, score in ranked[:top_k]
if score > 0
]
return results
# ─── Singleton index ─────────────────────────────────────────────────────────
# Bounded caches: distinct (home × capability-snapshot) combinations could grow
# without limit in long-lived processes; evict the oldest snapshot per home
# beyond the cap so stale snapshots cannot accumulate.
_MAX_CACHED_CAP_KEYS_PER_HOME = 8
_MAX_CACHED_HOMES = 8
_indexes_by_home: dict = {}
_skills_by_home_and_id: dict = {}
_cap_key_order_by_home: dict = {}
_home_order: list = []
# Gateway threads share these lists. The lock covers mutations and removals
# only. Dict reads stay unlocked. A torn get is a miss and rebuilds.
_cache_lock = threading.Lock()
_index: BM25Index | None = None
_skills_by_id: dict[str, dict] = {}
# Runtime-override (legacy-test) caches keyed by capability snapshot. The
# zero-arg path keeps using ``_index`` / ``_skills_by_id`` exactly as before.
_override_indexes_by_cap: dict = {}
_override_skills_by_cap: dict = {}
# Runtime entries are validated against a cheap on-disk manifest (see
# _corpus_manifest) so skills added/removed or config edits made outside this
# process are picked up without a restart. An empty corpus is cached too
# (value None) so zero-skill installs don't rescan every root each turn.
_manifest_by_key: dict = {}
_warned_empty_keys: set = set()
def clear_index_cache() -> None:
"""Drop every cached BM25 index and skill-info map.
Called from the plugin's ``on_skill_lifecycle`` observer (skill_manage
create/patch, hub install, ``/skills`` toggles) and whenever the recursive
corpus manifest detects an on-disk change.
"""
global _index, _skills_by_id
with _cache_lock:
_indexes_by_home.clear()
_skills_by_home_and_id.clear()
_cap_key_order_by_home.clear()
_home_order.clear()
_manifest_by_key.clear()
_warned_empty_keys.clear()
_override_indexes_by_cap.clear()
_override_skills_by_cap.clear()
_index = None
_skills_by_id = {}
def _stat_signature(path: str) -> "tuple[int, int] | None":
try:
st = os.stat(path)
except OSError:
return None
return st.st_mtime_ns, st.st_size
def _read_active_org_id(marker_path: str) -> "str | None":
"""Org id from ``_org/.active_org``, or None when the marker is missing.
Mirrors core ``read_active_org_id``: utf-8-sig, strip, empty means no
org skills. The manifest does not import that helper.
"""
try:
with open(marker_path, "r", encoding="utf-8-sig") as handle:
text = handle.read()
except (OSError, UnicodeDecodeError):
return None
return text.strip() or None
def _corpus_manifest() -> "tuple | None":
"""Cheap signature of the on-disk inputs to the runtime corpus.
Walks each skills root and plugin ``skills/`` dir, recording a
(mtime_ns, size) signature for ``SKILL.md`` and ``DESCRIPTION.md`` only,
plus config.yaml. Skill-package support dirs and VCS/cache dirs are not
walked. A symlink cycle is cut by a visited-realpath set. Adding or
removing a skill changes the manifest because its ``SKILL.md`` entry
appears or disappears. Each root also records ``_org/.active_org``.
With no marker the ``_org`` tree is not walked. With a marker only
``_org/<active_id>/`` is walked. Returns None when the manifest cannot
be computed (the cache is then trusted as before); never raises.
"""
try:
roots: list[str] = []
try:
from agent.skill_utils import get_all_skills_dirs, get_project_skills_dirs
roots += [str(d) for d in (get_project_skills_dirs() or [])]
roots += [str(d) for d in (get_all_skills_dirs() or [])]
except Exception:
roots.append(str(get_skills_dir()))
plugins_root = str(get_plugins_dir())
entries: list = [("config", _stat_signature(str(get_config_path())))]
entries.append((plugins_root, _stat_signature(plugins_root)))
try:
with os.scandir(plugins_root) as it:
roots += sorted(
os.path.join(e.path, "skills") for e in it if e.is_dir()
)
except OSError:
pass
seen_real: set[str] = set()
def _record_org_marker(root: str) -> "str | None":
marker_path = os.path.join(root, _ORG_MIRROR_DIR, _ORG_ACTIVE_MARKER)
sig = _stat_signature(marker_path)
if sig is None:
# Creating the marker must change the manifest even when the
# new org mirror has no SKILL.md yet.
entries.append((marker_path, _ABSENT_ORG_MARKER))
return None
# Core's snapshot keys the marker on stat. Resolution keys on the
# stripped text. This stat pair has no inode or ctime, so a
# same-size rewrite that restores mtime would not invalidate.
# The text is the org id, so it is part of the signature.
active = _read_active_org_id(marker_path)
entries.append((marker_path, (sig, active)))
return active
def _walk_root(root: str, active_org: "str | None") -> None:
entries.append((root, _stat_signature(root)))
org_root = os.path.normpath(os.path.join(root, _ORG_MIRROR_DIR))
root_key = os.path.normpath(root)
try:
walker = os.walk(root, followlinks=True)
except OSError:
return
try:
for dirpath, dirnames, filenames in walker:
try:
real = os.path.realpath(dirpath)
except OSError:
dirnames[:] = []
continue
if real in seen_real:
dirnames[:] = []
continue
seen_real.add(real)
has_skill_md = "SKILL.md" in filenames
here = os.path.normpath(dirpath)
# Same org gate as core iter_skill_index_files: no marker
# skips ``_org`` entirely; a marker descends only into
# ``_org/<active_id>/``.
if here == root_key and active_org is None:
if _ORG_MIRROR_DIR in dirnames:
dirnames.remove(_ORG_MIRROR_DIR)
elif here == org_root:
dirnames[:] = [d for d in dirnames if d == active_org]
# Same prune as core's iter_skill_index_files: never walk
# VCS/cache dirs, and never walk support dirs of a skill
# package (scripts, assets, references, templates).
dirnames[:] = sorted(
d for d in dirnames
if d not in _EXCLUDED_SKILL_DIRS
and not (has_skill_md and d in _SKILL_SUPPORT_DIRS)
)
for filename in ("DESCRIPTION.md", "SKILL.md"):
if filename in filenames:
path = os.path.join(dirpath, filename)
entries.append((path, _stat_signature(path)))
except OSError:
return
for root in roots:
_walk_root(root, _record_org_marker(root))
return tuple(entries)
except Exception as exc:
logger.debug("Skill corpus manifest unavailable: %s", exc)
return None
def _load_active_skills_for_index(
available_tools: "set[str] | None",
available_toolsets: "set[str] | None",
visible_names: "set[str] | frozenset[str] | None" = None,
) -> list[dict]:
"""Call ``load_active_skills`` without kwargs when every snapshot is unknown.
Existing tests monkeypatch ``load_active_skills`` with a zero-arg fake;
fail-open ``get_index()`` must keep calling it that way.
"""
if available_tools is None and available_toolsets is None and visible_names is None:
return load_active_skills()
kwargs = {}
if available_tools is not None or available_toolsets is not None:
kwargs["available_tools"] = available_tools
kwargs["available_toolsets"] = available_toolsets
if visible_names is not None:
kwargs["visible_names"] = visible_names
return load_active_skills(**kwargs)
def _drop_cached_key(key) -> None:
_indexes_by_home.pop(key, None)
_skills_by_home_and_id.pop(key, None)
_manifest_by_key.pop(key, None)
_warned_empty_keys.discard(key)
def _touch_cap_key(home_key: str, cache_key) -> None:
"""LRU of capability keys for one home. A hit moves the key to the end
so an idle key is the one rebuilt when the cap is exceeded."""
with _cache_lock:
order = _cap_key_order_by_home.setdefault(home_key, [])
try:
order.remove(cache_key)
except ValueError:
pass
order.append(cache_key)
if len(order) > _MAX_CACHED_CAP_KEYS_PER_HOME:
stale = order[:-_MAX_CACHED_CAP_KEYS_PER_HOME]
del order[:-_MAX_CACHED_CAP_KEYS_PER_HOME]
for key in stale:
_drop_cached_key(key)
def _touch_cached_home(home_key: str) -> None:
"""LRU of Hermes homes. Evict the oldest home past the bound, and every
capability key stored for it."""
with _cache_lock:
if home_key in _home_order:
_home_order.remove(home_key)
_home_order.append(home_key)
while len(_home_order) > _MAX_CACHED_HOMES:
stale_home = _home_order.pop(0)
for key in _cap_key_order_by_home.pop(stale_home, ()):
_drop_cached_key(key)
def get_index(
available_tools: "set[str] | None" = None,
available_toolsets: "set[str] | None" = None,
visible_names: "set[str] | frozenset[str] | None" = None,
platform_hint: "str | None" = None,
) -> BM25Index | None:
global _index, _skills_by_id
if _runtime_paths_are_overridden():
if available_tools is None and available_toolsets is None and visible_names is None:
if _index is not None:
return _index
skills = _load_active_skills_for_index(available_tools, available_toolsets, visible_names)
if not skills:
logger.warning("No active skills found for BM25 index")
return None
_index = BM25Index()
_index.build(
[s["skill_id"] for s in skills],
[s["text"] for s in skills],
)
_skills_by_id = {s["skill_id"]: s for s in skills}
return _index
cap_key = _index_cache_key(
"", available_tools, available_toolsets, visible_names=None if visible_names is None else frozenset(visible_names),
)
cached = _override_indexes_by_cap.get(cap_key)
if cached is not None:
return cached
skills = _load_active_skills_for_index(available_tools, available_toolsets, visible_names)
if not skills:
logger.warning("No active skills found for BM25 index")
return None
index = BM25Index()
index.build(
[s["skill_id"] for s in skills],
[s["text"] for s in skills],
)
_override_indexes_by_cap[cap_key] = index
_override_skills_by_cap[cap_key] = {s["skill_id"]: s for s in skills}
return index
home_key, cache_key = _runtime_cache_key(
available_tools, available_toolsets, visible_names=visible_names, platform_hint=platform_hint,
)
manifest = _corpus_manifest()
if cache_key in _indexes_by_home and _manifest_by_key.get(cache_key) == manifest:
cached = _indexes_by_home[cache_key]
_touch_cap_key(home_key, cache_key)
_touch_cached_home(home_key)
return cached
skills = _load_active_skills_for_index(available_tools, available_toolsets, visible_names)
index = None
if skills:
_warned_empty_keys.discard(cache_key)
index = BM25Index()
index.build(
[s["skill_id"] for s in skills],
[s["text"] for s in skills],
)
elif cache_key not in _warned_empty_keys:
_warned_empty_keys.add(cache_key)
logger.warning("No active skills found for BM25 index")
_indexes_by_home[cache_key] = index
_skills_by_home_and_id[cache_key] = {s["skill_id"]: s for s in skills}
_manifest_by_key[cache_key] = manifest
_touch_cap_key(home_key, cache_key)
_touch_cached_home(home_key)
return index
def get_skill_info(
skill_id: str,
available_tools: "set[str] | None" = None,
available_toolsets: "set[str] | None" = None,
visible_names: "set[str] | frozenset[str] | None" = None,
platform_hint: "str | None" = None,
) -> dict | None:
if _runtime_paths_are_overridden():
if available_tools is None and available_toolsets is None and visible_names is None:
return _skills_by_id.get(skill_id)
cap_key = _index_cache_key(
"", available_tools, available_toolsets,
visible_names=None if visible_names is None else frozenset(visible_names),
)
return _override_skills_by_cap.get(cap_key, {}).get(skill_id)
_home_key, cache_key = _runtime_cache_key(
available_tools, available_toolsets, visible_names=visible_names, platform_hint=platform_hint,
)
return _skills_by_home_and_id.get(cache_key, {}).get(skill_id)Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |