Skip to content

yya007/SkillFinder

skill-finder

Unverified

Discover and recommend agent skills from SkillsMP, ClawHub, SkillHub, and the Anthropic marketplace. Search a local FAISS index with a natural language description of your task; get back ranked matches with install commands.

FileKindSize
SKILL.mdskill md8.5 KB
scripts/__init__.pyscript0 B
scripts/fetch_skill.pyscript10.4 KB
scripts/npm-release.shscript2.5 KB
scripts/requirements.txtmanifest54 B
scripts/search.pyscript19.6 KB
scripts/trial_crawl.shexecutablescript2.1 KB
scripts/update_index.pyscript11.5 KB

scripts/update_index.py

View on GitHub
"""
scripts/update_index.py — Pull the latest SkillFinder index release.

Downloads the pre-built FAISS index from the latest GitHub Release and
extracts it into the local data/ directory atomically.

Usage:
    python scripts/update_index.py              # update if newer available
    python scripts/update_index.py --check      # report status, no download
    python scripts/update_index.py --force      # download even if up to date
"""

from __future__ import annotations

import argparse
import hashlib
import re
import shutil
import sys
import tarfile
import tempfile
from datetime import date
from pathlib import Path
from typing import Optional

import requests

# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------

REPO = "yya007/SkillFinder"
GITHUB_API = "https://api.github.com"
RELEASE_URL = f"{GITHUB_API}/repos/{REPO}/releases/latest"
_DEFAULT_DATA_DIR = str(Path(__file__).parent / ".." / "data")


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def read_local_version(data_dir: str) -> Optional[str]:
    """Read the local index date from data_dir/version.txt.

    Returns the date string "YYYY-MM-DD", or None if version.txt is absent.
    Raises ValueError if version.txt exists but lacks a valid ``date:`` field.
    """
    version_path = Path(data_dir) / "version.txt"
    if not version_path.exists():
        return None

    content = version_path.read_text(encoding="utf-8")
    for line in content.splitlines():
        if line.startswith("date:"):
            raw = line.split(":", 1)[1].strip()
            # Validate format YYYY-MM-DD
            try:
                date.fromisoformat(raw)
            except ValueError:
                raise ValueError(
                    f"version.txt contains an invalid date: {raw!r}. Expected YYYY-MM-DD."
                )
            return raw

    raise ValueError(
        "version.txt exists but has no 'date:' field. "
        "Run a full re-install or delete data/ and retry."
    )


def get_latest_release() -> dict:
    """Fetch the latest GitHub Release metadata for this repo.

    Returns the full release dict (tag_name, assets, body, ...).
    Raises RuntimeError on non-200 responses.
    """
    try:
        resp = requests.get(
            RELEASE_URL,
            headers={"Accept": "application/vnd.github+json"},
            timeout=30,
        )
    except requests.RequestException as exc:
        raise RuntimeError(f"Failed to reach GitHub API: {exc}") from exc

    if resp.status_code != 200:
        raise RuntimeError(
            f"GitHub API returned {resp.status_code}: {resp.text[:200]}"
        )
    return resp.json()


def needs_update(local_version: Optional[str], tag_name: str) -> bool:
    """Return True if the release tag is newer than the local index.

    Args:
        local_version: Local date string "YYYY-MM-DD", or None (no index).
        tag_name:      GitHub tag, e.g. "index-20260310" or "20260310".

    Returns:
        True if an update should be downloaded.
    """
    if local_version is None:
        return True

    # Extract YYYYMMDD from tag (strip "index-" prefix if present)
    raw = tag_name.removeprefix("index-").replace("-", "")
    if len(raw) != 8 or not raw.isdigit():
        # Unknown tag format — assume update needed
        return True

    release_date_str = f"{raw[:4]}-{raw[4:6]}-{raw[6:]}"
    try:
        release_date = date.fromisoformat(release_date_str)
        local_date = date.fromisoformat(local_version)
    except ValueError:
        return True

    return release_date > local_date


def parse_release_sha256(body: str) -> Optional[str]:
    """Extract the SHA256 checksum from a GitHub Release body.

    Looks for patterns like:
        **SHA256:** `abc123def456`
        SHA256: abc123def456

    Returns the hex string, or None if not found.
    """
    if not body:
        return None
    # Try bold/backtick style first: **SHA256:** `<hash>`
    m = re.search(r"\*\*SHA256:\*\*\s*`([0-9a-fA-F]+)`", body)
    if m:
        return m.group(1)
    # Plain style: SHA256: <hash>
    m = re.search(r"SHA256:\s*([0-9a-fA-F]{8,})", body)
    if m:
        return m.group(1)
    return None


def verify_sha256(path: str, expected: str) -> bool:
    """Verify the SHA256 checksum of a file.

    Args:
        path:     Path to the file to verify.
        expected: Expected hex digest (case-insensitive).

    Returns:
        True if the checksum matches, False otherwise.

    Raises:
        FileNotFoundError: If the file does not exist.
    """
    sha = hashlib.sha256()
    with Path(path).open("rb") as fh:
        for chunk in iter(lambda: fh.read(65536), b""):
            sha.update(chunk)
    return sha.hexdigest().lower() == expected.lower()


def download_file(url: str, dest: str) -> None:
    """Download a file from url and write it to dest.

    Raises:
        RuntimeError: If the HTTP response is not 200.
        requests.RequestException: On network errors.
    """
    resp = requests.get(url, stream=True, timeout=120)
    if resp.status_code != 200:
        raise RuntimeError(
            f"Download failed (HTTP {resp.status_code}) from {url}"
        )
    with Path(dest).open("wb") as fh:
        for chunk in resp.iter_content(chunk_size=65536):
            fh.write(chunk)


def extract_artifact(tar_path: str, data_dir: str) -> None:
    """Extract a tar.gz index artifact into data_dir atomically.

    Extracts to a temporary ``data_dir/.new/`` staging directory first,
    then moves each file into place.  No partial writes are visible to
    readers, and ``.new/`` is always removed on completion.

    Args:
        tar_path: Path to the downloaded .tar.gz file.
        data_dir: Destination directory (created if absent).
    """
    data_path = Path(data_dir)
    data_path.mkdir(parents=True, exist_ok=True)
    staging = data_path / ".new"

    # Clean up any leftover staging dir from a previous failed run
    if staging.exists():
        shutil.rmtree(staging)
    staging.mkdir()

    try:
        with tarfile.open(tar_path, "r:gz") as tar:
            # Extract only safe members (no absolute paths, no ../  traversal)
            for member in tar.getmembers():
                member_path = Path(member.name)
                if member_path.is_absolute() or ".." in member_path.parts:
                    continue
                # Extract flat: strip any directory prefix, put files at staging root
                member.name = member_path.name
                tar.extract(member, path=staging, filter="data")

        # Verify required index files were extracted
        required = {"index.faiss", "metadata.jsonl", "version.txt"}
        extracted = {f.name for f in staging.iterdir()}
        missing = required - extracted
        if missing:
            raise RuntimeError(
                f"Extraction incomplete — missing: {', '.join(sorted(missing))}. "
                "The release artifact may be corrupt."
            )

        # Move staged files into data_dir
        for staged_file in staging.iterdir():
            staged_file.replace(data_path / staged_file.name)
    finally:
        if staging.exists():
            shutil.rmtree(staging)


def run_update(
    data_dir: str = _DEFAULT_DATA_DIR,
    check_only: bool = False,
    force: bool = False,
) -> dict:
    """Orchestrate an index update end to end.

    Args:
        data_dir:   Directory containing (or to receive) index files.
        check_only: If True, report status without downloading anything.
        force:      If True, download even if the local index is current.

    Returns:
        Dict with "status" ("up_to_date" | "update_available" | "updated" | "error")
        and "message".
    """
    try:
        local_version = read_local_version(data_dir)
    except ValueError as exc:
        return {"status": "error", "message": str(exc)}

    try:
        release = get_latest_release()
    except RuntimeError as exc:
        return {"status": "error", "message": str(exc)}

    tag = release.get("tag_name", "")

    if not force and not needs_update(local_version, tag):
        return {
            "status": "up_to_date",
            "message": f"Local index ({local_version}) matches latest release ({tag}).",
        }

    if check_only:
        return {
            "status": "update_available",
            "message": f"New release available: {tag} (local: {local_version}).",
        }

    # Find the tar.gz asset
    assets = release.get("assets", [])
    tar_asset = next(
        (a for a in assets if a.get("name", "").endswith(".tar.gz")), None
    )
    if not tar_asset:
        return {
            "status": "error",
            "message": f"No .tar.gz asset found in release {tag}.",
        }

    url = tar_asset["browser_download_url"]
    expected_sha = parse_release_sha256(release.get("body", ""))
    if not expected_sha and not force:
        return {
            "status": "error",
            "message": (
                "SHA256 checksum not found in release body — cannot verify download. "
                "Use --force to bypass (not recommended)."
            ),
        }
    if not expected_sha and force:
        print(
            "Warning: SHA256 checksum not found in release body — "
            "skipping integrity verification (--force). "
            "Only use this if you trust the release source.",
            file=sys.stderr,
            flush=True,
        )

    # Download to a temp file
    with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp:
        tmp_path = tmp.name

    try:
        download_file(url, tmp_path)

        if expected_sha and not verify_sha256(tmp_path, expected_sha):
            return {
                "status": "error",
                "message": "SHA256 checksum mismatch — download may be corrupt.",
            }

        extract_artifact(tmp_path, data_dir)
    except (OSError, RuntimeError, requests.RequestException) as exc:
        return {"status": "error", "message": str(exc)}
    finally:
        Path(tmp_path).unlink(missing_ok=True)

    return {
        "status": "updated",
        "message": f"Index updated to {tag}.",
    }


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------

def _parse_args(argv=None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Download the latest SkillFinder index from GitHub Releases.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument(
        "--check",
        action="store_true",
        help="Check for updates without downloading.",
    )
    parser.add_argument(
        "--force",
        action="store_true",
        help="Force download even if local index is current.",
    )
    parser.add_argument(
        "--data",
        default=_DEFAULT_DATA_DIR,
        metavar="DIR",
        help="Path to the data directory (default: data/ next to this script).",
    )
    return parser.parse_args(argv)


def main(argv=None) -> int:
    args = _parse_args(argv)
    result = run_update(
        data_dir=args.data,
        check_only=args.check,
        force=args.force,
    )
    status = result.get("status", "error")
    msg = result.get("message", "")
    print(f"[{status}] {msg}")
    return 0 if status in ("up_to_date", "updated", "update_available") else 1


if __name__ == "__main__":
    sys.exit(main())
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall