Skip to content

Dianel555/DSkills / skills/agent-wiki

agent-wiki

Unverified

Incremental LLM-friendly wiki generator for Obsidian note vaults. Use when: (1) Building wiki from notes, (2) Ingesting notes to wiki, (3) Obsidian LLM wiki, (4) Incremental knowledge base management. Triggers: 'build wiki from notes', 'ingest notes to wiki', 'Obsidian LLM wiki', 'incremental knowledge base'.

FileKindSize
.env.exampleother408 B
README.mdmarkdown15.3 KB
SKILL.mdskill md23.3 KB
pyproject.tomlmanifest2.1 KB
references/cli-matrix.mdmarkdown5.9 KB
references/homepage.mdmarkdown6.1 KB
references/index-schema.mdmarkdown6.9 KB
references/site-export.mdmarkdown3.1 KB
references/topic-authoring.mdmarkdown5.1 KB
scripts/agent_wiki/__init__.pyscript103 B
scripts/agent_wiki/authors.pyscript3.9 KB
scripts/agent_wiki/bases.pyscript4.0 KB
scripts/agent_wiki/batch.pyscript3.7 KB
scripts/agent_wiki/boundary.pyscript55.8 KB
scripts/agent_wiki/cache.pyscript5.5 KB
scripts/agent_wiki/canvas.pyscript8.0 KB
scripts/agent_wiki/cleanup.pyscript2.4 KB
scripts/agent_wiki/cli.pyscript97 B
scripts/agent_wiki/commands.pyscript28.3 KB
scripts/agent_wiki/config.pyscript4.2 KB
scripts/agent_wiki/coverage.pyscript2.1 KB
scripts/agent_wiki/frontmatter.pyscript1.6 KB
scripts/agent_wiki/home.pyscript7.9 KB
scripts/agent_wiki/keywords.pyscript2.0 KB
scripts/agent_wiki/links.pyscript14.7 KB
scripts/agent_wiki/plugins.pyscript2.1 KB
scripts/agent_wiki/quality.pyscript7.6 KB
scripts/agent_wiki/scanner.pyscript6.7 KB
scripts/agent_wiki/site.pyscript39.2 KB
scripts/agent_wiki/source_type.pyscript3.2 KB
scripts/agent_wiki/wiki_index.pyscript17.4 KB
scripts/agent_wiki/worklist.pyscript5.7 KB
scripts/agent_wiki_cli.pyscript5.5 KB
templates/home/academic.mdmarkdown2.9 KB
templates/home/dashboard.mdmarkdown2.7 KB
templates/home/magazine.mdmarkdown2.5 KB
templates/query/research.mdmarkdown787 B
tests/conftest.pyscript177 B
tests/fixtures/sample_vault/attachments/image.pngbinary26 B
tests/fixtures/sample_vault/课程/双缝实验.mdmarkdown47 B
tests/fixtures/sample_vault/课程/量子力学.mdmarkdown75 B
tests/test_alias_resolution.pyscript7.5 KB
tests/test_authors.pyscript2.2 KB
tests/test_backlinks.pyscript7.0 KB
tests/test_bases.pyscript4.7 KB
tests/test_batch.pyscript3.0 KB
tests/test_benchmark_index.pyscript2.1 KB
tests/test_boundary_platform.pyscript695 B
tests/test_cache.pyscript2.1 KB
tests/test_cache_strict.pyscript2.1 KB
tests/test_canvas.pyscript8.3 KB
tests/test_capture.pyscript5.5 KB
tests/test_cleanup.pyscript1.2 KB
tests/test_cleanup_init_strict.pyscript2.1 KB
tests/test_cli.pyscript6.2 KB
tests/test_config.pyscript2.0 KB
tests/test_coverage.pyscript7.2 KB
tests/test_e2e.pyscript2.5 KB
tests/test_evaluate_enhancements.pyscript14.7 KB
tests/test_featured_base.pyscript4.3 KB
tests/test_final_security_review.pyscript6.6 KB
tests/test_frontmatter.pyscript1.1 KB
tests/test_gen_home.pyscript4.9 KB
tests/test_gen_site_cli.pyscript3.9 KB
tests/test_home.pyscript7.7 KB
tests/test_immutability.pyscript1.4 KB
tests/test_immutability_capture.pyscript2.4 KB
tests/test_index_capture.pyscript4.5 KB
tests/test_index_cli.pyscript5.4 KB
tests/test_index_extension.pyscript17.0 KB
tests/test_index_source_count.pyscript3.0 KB
tests/test_package_structure.pyscript3.4 KB
tests/test_plugins.pyscript3.1 KB
tests/test_quality.pyscript21.3 KB
tests/test_quality_cli.pyscript5.5 KB
tests/test_quality_prose_weight.pyscript3.8 KB
tests/test_quality_script_fairness.pyscript4.7 KB
tests/test_quality_tier_refactor.pyscript4.0 KB
tests/test_scanner.pyscript2.6 KB
tests/test_scanner_fast_path.pyscript3.5 KB
tests/test_scanner_strict.pyscript2.1 KB
tests/test_secure_temporary_writes.pyscript978 B
tests/test_site.pyscript20.2 KB
tests/test_site_import_isolation.pyscript3.9 KB
tests/test_site_slug_refactor.pyscript3.6 KB
tests/test_source_type.pyscript3.5 KB
tests/test_status_metrics.pyscript4.1 KB
tests/test_status_quality.pyscript5.3 KB
tests/test_status_worklist.pyscript3.0 KB
tests/test_structure.pyscript501 B
tests/test_topic_category.pyscript4.2 KB
tests/test_topic_path_boundary.pyscript2.2 KB
tests/test_vault_boundary.pyscript24.3 KB
tests/test_wiki_index.pyscript13.3 KB
tests/test_worklist.pyscript12.3 KB
tests/test_worklist_cli.pyscript1.9 KB

scripts/agent_wiki/wiki_index.py

View on GitHub
"""Derived retrieval index (``wiki/.wiki-index.json``).

The index normalizes common research metadata from ``wiki/topics/*.md``
frontmatter into a deterministic JSON cache for fast Agent routing. Topic
markdown stays the single source of truth; the index is never written back
into topic files.
"""

from __future__ import annotations

import json
import re
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

from . import boundary, config, frontmatter, links, quality, source_type
from .config import nfc as _nfc

INDEX_VERSION = 2
EPOCH = "1970-01-01T00:00:00Z"
_SUMMARY_LIMIT = 1000
_YEAR_RE = re.compile(r"\d{4}")
_COMMON_ENTRY_FIELDS = frozenset({
    "path", "title", "sources", "last_updated", "year_start", "year_end", "authors",
    "source_type", "institutions", "methods", "technical_routes", "research_trends", "summary",
    "keywords", "kind", "links", "link_records", "mtime_ns", "citekey", "doi", "library_id",
    "review_status", "reviewed_at",
})
_TOPIC_ENTRY_FIELDS = frozenset({
    "type", "topic_category", "aliases", "quality_tier", "featured", "backlinks",
})
_STRING_ENTRY_FIELDS = frozenset({
    "path", "title", "last_updated", "source_type", "summary", "kind", "citekey", "doi", "library_id",
    "review_status", "reviewed_at",
})
_STRING_LIST_ENTRY_FIELDS = frozenset({
    "sources", "authors", "institutions", "methods", "technical_routes", "research_trends", "keywords", "links",
})


def _cache_entry_is_current(entry: object, kind: str) -> bool:
    if not isinstance(entry, dict) or entry.get("kind") != kind:
        return False
    required = _COMMON_ENTRY_FIELDS | (_TOPIC_ENTRY_FIELDS if kind == "topic" else frozenset())
    if not required.issubset(entry):
        return False
    if not all(isinstance(entry[field], str) for field in _STRING_ENTRY_FIELDS):
        return False
    if not all(
        isinstance(entry[field], list) and all(isinstance(item, str) for item in entry[field])
        for field in _STRING_LIST_ENTRY_FIELDS
    ):
        return False
    if not isinstance(entry["link_records"], list) or not all(isinstance(item, dict) for item in entry["link_records"]):
        return False
    if type(entry["mtime_ns"]) is not int:
        return False
    if any(entry[field] is not None and type(entry[field]) is not int for field in ("year_start", "year_end")):
        return False
    if kind != "topic":
        return True
    return (
        isinstance(entry["type"], str)
        and isinstance(entry["topic_category"], str)
        and isinstance(entry["aliases"], list)
        and all(isinstance(item, str) for item in entry["aliases"])
        and isinstance(entry["quality_tier"], str)
        and type(entry["featured"]) is bool
        and type(entry["backlinks"]) is int
    )


class NormalizedPathCollisionError(Exception):
    def __init__(self, path: str) -> None:
        self.path = path
        super().__init__(path)


class IndexWriteError(OSError):
    pass


def empty_schema() -> dict[str, Any]:
    return {"version": INDEX_VERSION, "generated_at": EPOCH, "topics": {}, "queries": {}, "alias_index": {}}


def _str_list(value: Any) -> list[str]:
    if value is None:
        return []
    if isinstance(value, list):
        return [_nfc(item) for item in value]
    return [_nfc(value)]


def _str_field(value: Any) -> str:
    if value is None:
        return ""
    if isinstance(value, list):
        return _nfc(value[0]) if value else ""
    return _nfc(value)


def _title(value: Any, stem: str) -> str:
    if value is None:
        return _nfc(stem)
    if isinstance(value, list):
        return _nfc(" ".join(str(item) for item in value))
    return _nfc(value)


def _sources(value: Any) -> list[str]:
    if value is None:
        return []
    items = value if isinstance(value, list) else [value]
    return [config.normalize_relpath(str(item)) for item in items]


def _year(value: Any) -> int | None:
    if isinstance(value, bool):
        return None
    if isinstance(value, int):
        return value
    if isinstance(value, list):
        return _year(value[0]) if value else None
    if not isinstance(value, str):
        return None
    match = _YEAR_RE.search(value)
    return int(match.group()) if match else None


def _summary(value: Any) -> str:
    if value is None:
        text = ""
    elif isinstance(value, list):
        text = "; ".join(str(item) for item in value)
    else:
        text = str(value)
    return _nfc(text)[:_SUMMARY_LIMIT]


def _parse_links(body: str) -> list[str]:
    """Return compatibility targets from the shared Obsidian/Markdown parser."""
    return links.unique_targets(links.parse(body))


def _entry(
    rel: str,
    meta: dict[str, Any],
    stem: str,
    kind: str,
    links: list[str],
    body: str = "",
    mtime_ns: int = 0,
    link_records: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
    """Build an index entry. Topic entries include extended fields; query entries preserve their current schema."""
    sources = _sources(meta.get("sources"))
    entry: dict[str, Any] = {
        "path": rel,
        "title": _title(meta.get("title"), stem),
        "sources": sources,
        "last_updated": _str_field(meta.get("last_updated")),
        "year_start": _year(meta.get("year_start")),
        "year_end": _year(meta.get("year_end")),
        "authors": _str_list(meta.get("authors")),
        "source_type": source_type.classify_sources(sources),
        "institutions": _str_list(meta.get("institutions")),
        "methods": _str_list(meta.get("methods")),
        "technical_routes": _str_list(meta.get("technical_routes")),
        "research_trends": _str_list(meta.get("research_trends")),
        "summary": _summary(meta.get("summary")),
        "keywords": _str_list(meta.get("keywords")),
        "kind": kind,
        "links": links,
        "link_records": link_records or [],
        "mtime_ns": mtime_ns,
    }

    # Add topic-only fields
    if kind == "topic":
        # Optional page kind (orthogonal to derived source_type)
        type_value = meta.get("type")
        if isinstance(type_value, str):
            entry["type"] = _nfc(type_value)
        else:
            entry["type"] = ""

        # Agent-assigned subject category (orthogonal to the `type` genre);
        # drives static-site grouping, falling back to `type` when absent.
        entry["topic_category"] = _str_field(meta.get("topic_category"))

        # aliases: order-preserved list (not deduplicated)
        entry["aliases"] = _str_list(meta.get("aliases"))

        # quality_tier: computed from body with source grounding
        # Use deduplicated source count (per D3.3)
        unique_sources = len(set(sources))
        entry["quality_tier"] = quality.compute_tier(body, source_count=unique_sources)

        # featured: strict boolean coercion
        featured_value = meta.get("featured")
        entry["featured"] = featured_value is True

        # backlinks: initialized to 0, computed later in rebuild
        entry["backlinks"] = 0

    # Academic metadata is optional; empty strings keep the JSON shape stable.
    for field in ("citekey", "doi", "library_id", "review_status", "reviewed_at"):
        entry[field] = _str_field(meta.get(field))

    return entry


def _iso_utc(mtime_ns: int) -> str:
    seconds = mtime_ns // 1_000_000_000
    return datetime.fromtimestamp(seconds, tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ")


def _parse_doc(job: tuple[str, Path, str]) -> tuple[str, int, dict[str, Any] | None, dict[str, Any] | None]:
    """Parse one topic/query file into ``(rel, mtime_ns, entry, error)``. Runs in worker threads.

    Stats the file here (tight against ``read_text``) so the entry's
    ``mtime_ns`` tracks the content actually read, not the earlier stat done
    in ``_index_dir`` for the reuse decision — closes the bulk of the TOCTOU
    window; a residual stat→read gap self-heals on the next rebuild.
    """
    rel, path, kind = job
    try:
        st = boundary.stat(path)
        text = boundary.read_text(path, encoding="utf-8-sig")
    except boundary.BoundaryError:
        # A link now sits on the path to this topic: its content is outside the vault.
        return rel, 0, None, {"path": rel, "error": "outside_vault"}
    except (UnicodeDecodeError, OSError):
        return rel, 0, None, {"path": rel, "error": "topic_decode_failed"}
    if st is None:
        return rel, 0, None, {"path": rel, "error": "topic_decode_failed"}
    mtime_ns = st.st_mtime_ns
    try:
        meta, body = frontmatter.parse(text)
    except frontmatter.FrontmatterError:
        return rel, mtime_ns, None, {"path": rel, "error": "frontmatter_parse_failed"}
    refs = links.parse(body)
    return rel, mtime_ns, _entry(
        rel, meta, path.stem, kind, links.unique_targets(refs), body, mtime_ns, links.serialize(refs)
    ), None


def _index_dir(
    directory: Path,
    key_root: Path,
    kind: str,
    entries: dict[str, Any],
    errors: list[dict[str, Any]],
    mtimes: list[int],
    *,
    existing_index: dict[str, Any] | None = None,
    workers: int = 8,
) -> None:
    """Index every ``*.md`` under ``directory`` into ``entries`` keyed by its NFC
    POSIX path relative to ``key_root``; per-directory collision detection.

    When ``existing_index`` is provided, entries whose stored ``mtime_ns``
    matches the file's current mtime are reused without re-parsing. Changed and
    new files are parsed in parallel with a bounded thread pool.

    Enumeration and every read go through :mod:`boundary`, so a link planted on the
    directory (or on any of its ancestors) yields no entries instead of publishing
    content from outside the vault."""
    try:
        files = [entry.path for entry in boundary.entries(directory, ".md")]
    except boundary.BoundaryError:
        errors.append({"path": directory.name, "error": "outside_vault"})
        return
    files.sort(key=lambda p: config.normalize_relpath(p.relative_to(key_root).as_posix()))
    seen: set[str] = set()
    jobs: list[tuple[str, Path, str]] = []
    for path in files:
        rel = config.normalize_relpath(path.relative_to(key_root).as_posix())
        if rel in seen:
            raise NormalizedPathCollisionError(rel)
        seen.add(rel)
        try:
            st = boundary.stat(path)
            if st is None:
                raise FileNotFoundError(2, "missing topic", str(path))
            mtime_ns = st.st_mtime_ns
        except (boundary.BoundaryError, OSError):
            # Same code the parse path would have reported
            errors.append({"path": rel, "error": "topic_decode_failed"})
            continue
        cached = existing_index.get(rel) if existing_index else None
        if (
            isinstance(cached, dict)
            and _cache_entry_is_current(cached, kind)
            and cached.get("path") == rel
            and cached.get("mtime_ns") == mtime_ns
        ):
            entries[rel] = cached
            mtimes.append(mtime_ns)
        else:
            jobs.append((rel, path, kind))

    if len(jobs) < 2:
        parsed = [_parse_doc(job) for job in jobs]
    else:
        with ThreadPoolExecutor(max_workers=min(workers, len(jobs))) as pool:
            parsed = list(pool.map(_parse_doc, jobs))
    for rel, mtime_ns, entry, error in parsed:
        if entry is not None:
            entries[rel] = entry
            mtimes.append(mtime_ns)
        if error is not None:
            errors.append(error)


def rebuild(vault: str | Path, *, incremental: bool = False) -> tuple[dict[str, Any], list[dict[str, Any]]]:
    """Build the index from topic and query frontmatter.

    When ``incremental=True``, reuses existing index entries whose stored
    mtime matches the file on disk; only changed and new files are re-parsed.
    Topic keys are ``wiki/topics/``-relative (bare ``<name>.md``); query
    keys are ``wiki/``-relative (``queries/<name>.md``). Returns ``(data,
    errors)``. Decode/parse failures are skipped and reported; a normalized-key
    collision within a directory is fatal and raises ``NormalizedPathCollisionError``.
    """
    boundary.bind(vault)
    wiki = config.wiki_root(vault)
    topics_root = config.topics_dir(vault)

    # Load existing index if incremental mode. Validate top-level shape and the
    # topics/queries maps so a corrupted/hand-edited index (null, array, or a
    # non-dict entry) falls back to a full rebuild instead of crashing.
    existing_topics: dict[str, Any] | None = None
    existing_queries: dict[str, Any] | None = None
    if incremental:
        index_path = config.index_path(vault)
        if boundary.exists(index_path):
            try:
                existing_data = json.loads(boundary.read_text(index_path, encoding="utf-8"))
            except (json.JSONDecodeError, OSError, UnicodeDecodeError, boundary.BoundaryError):
                existing_data = None
            if isinstance(existing_data, dict) and existing_data.get("version") == INDEX_VERSION:
                topics_field = existing_data.get("topics", {})
                queries_field = existing_data.get("queries", {})
                if isinstance(topics_field, dict):
                    existing_topics = topics_field
                if isinstance(queries_field, dict):
                    existing_queries = queries_field

    topics: dict[str, Any] = {}
    queries: dict[str, Any] = {}
    errors: list[dict[str, Any]] = []
    mtimes: list[int] = []

    _index_dir(topics_root, topics_root, "topic", topics, errors, mtimes, existing_index=existing_topics)
    _index_dir(config.queries_dir(vault), wiki, "query", queries, errors, mtimes, existing_index=existing_queries)

    # Build alias_index from frontmatter aliases + optional .wiki-aliases.json
    alias_index: dict[str, str] = {}
    alias_sources: dict[str, list[str]] = {}  # Track sources for conflict detection

    # Collect frontmatter aliases
    for topic_key, topic_entry in topics.items():
        for alias in topic_entry.get("aliases", []):
            alias_nfc = _nfc(alias)
            if alias_nfc not in alias_sources:
                alias_sources[alias_nfc] = []
            alias_sources[alias_nfc].append(topic_key)

    # Merge optional .wiki-aliases.json
    aliases_file = wiki / ".wiki-aliases.json"
    if boundary.exists(aliases_file):
        try:
            aliases_map = json.loads(boundary.read_text(aliases_file, encoding="utf-8"))
            if not isinstance(aliases_map, dict):
                errors.append({"error": "alias_map_invalid"})
            else:
                for alias, target in aliases_map.items():
                    if not isinstance(alias, str) or not isinstance(target, str):
                        errors.append({"error": "alias_map_invalid"})
                        continue
                    alias_nfc = _nfc(alias)
                    target_nfc = _nfc(target)
                    if alias_nfc not in alias_sources:
                        alias_sources[alias_nfc] = []
                    alias_sources[alias_nfc].append(target_nfc)
        except (json.JSONDecodeError, UnicodeDecodeError, boundary.BoundaryError):
            errors.append({"error": "alias_map_invalid"})

    # Resolve aliases: check for conflicts and missing targets
    topic_keys = set(topics.keys())
    for alias_nfc, targets in alias_sources.items():
        # Deduplicate targets
        unique_targets = sorted(set(targets))

        # Check if alias conflicts with a real topic key
        if alias_nfc in topic_keys:
            unique_targets.append(alias_nfc)
            unique_targets = sorted(set(unique_targets))

        # Check for missing targets
        valid_targets = [t for t in unique_targets if t in topic_keys]

        # Report missing targets
        for target in unique_targets:
            if target not in topic_keys:
                errors.append({"alias": alias_nfc, "error": "alias_target_missing", "target": target})

        # Check for conflicts
        if len(valid_targets) > 1:
            errors.append({"alias": alias_nfc, "error": "alias_conflict", "candidates": valid_targets})
        elif len(valid_targets) == 1:
            alias_index[alias_nfc] = valid_targets[0]

    # Compute backlinks (inbound link count per topic) using the same resolver
    # as worklist, Canvas, and the HTML export.
    backlinks: dict[str, set[str]] = {key: set() for key in topic_keys}
    all_entries = list(topics.items()) + list(queries.items())
    query_keys = set(queries)

    for source_key, source_entry in all_entries:
        for ref in links.from_entry(source_entry):
            resolution = links.resolve(ref.target, topic_keys, query_keys, alias_index)
            target_key = resolution.key
            if resolution.status == "resolved" and target_key in topic_keys and target_key != source_key:
                backlinks[target_key].add(source_key)

    # Update topic entries with backlink counts
    for topic_key in topic_keys:
        topics[topic_key]["backlinks"] = len(backlinks[topic_key])

    data = {
        "version": INDEX_VERSION,
        "generated_at": _iso_utc(max(mtimes)) if mtimes else EPOCH,
        "topics": topics,
        "queries": queries,
        "alias_index": dict(sorted(alias_index.items())),
    }
    return data, errors


def serialize(data: dict[str, Any]) -> str:
    return json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True) + "\n"


def save_index(vault: str | Path, data: dict[str, Any]) -> None:
    boundary.bind(vault)
    path = config.index_path(vault)
    try:
        boundary.ensure_directory(path.parent)
        config.atomic_write_text(path, serialize(data))
    except OSError as exc:
        raise IndexWriteError(str(exc)) from exc
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall