Skip to content

anbeime/skill

xiaoyue-companion

Verified

小跃虚拟伴侣 - 使用智谱 AI 提供温暖的对话陪伴和静态图片分享

scripts/sync_skills.py

View on GitHub
#!/usr/bin/env python3
"""
多源技能仓库同步脚本
从多个上游源提取技能仓库链接并汇总
"""

import json
import re
import os
import sys
from datetime import datetime
import urllib.request
import ssl

# 上游源配置
UPSTREAM_SOURCES = {
    "openai_skills": {
        "name": "OpenAI Skills",
        "url": "https://raw.githubusercontent.com/openai/skills/main/README.md",
        "type": "directory_structure"
    },
    "voltagent_awesome": {
        "name": "VoltAgent Awesome Agent Skills",
        "url": "https://raw.githubusercontent.com/VoltAgent/awesome-agent-skills/main/README.md",
        "type": "awesome_list"
    },
    "awesome_dsh_plugin": {
        "name": "Awesome DSH Plugin",
        "url": "https://raw.githubusercontent.com/awesome-dsh-plugin/awesome-dsh-plugin/main/README.md",
        "type": "awesome_list"
    },
    "mattpocock_skills": {
        "name": "Matt Pocock Skills",
        # mono-repo:README 列的是本地 skill 路径(./skills/<cat>/<name>/SKILL.md),
        # 不是 GitHub 链接列表,所以不能用 awesome_list(会落空)。
        # 用专属解析按 skill 颗粒度收录,repo_id 形如 mattpocock/skills/engineering/tdd
        "url": "https://raw.githubusercontent.com/mattpocock/skills/main/README.md",
        "type": "mattpocock_skills"
    },
    "voltagent_design_md": {
        "name": "VoltAgent Awesome DESIGN.md",
        # 73 个真实网站 DESIGN.md 提取(Claude / Cursor / Vercel / Stripe / Apple ...)
        # 格式特殊:列表项是 getdesign.md/<slug>/design-md 而不是 github.com/owner/repo
        # 所以 sync 脚本只记录元数据(仓库总数 + 上游元链接),
        # 实际 DESIGN.md 文件请用户从 https://github.com/VoltAgent/awesome-design-md 手动浏览
        "url": "https://raw.githubusercontent.com/VoltAgent/awesome-design-md/main/README.md",
        "type": "design_md_catalog",
        "self_repo": ("VoltAgent", "awesome-design-md")
    },
    "inkos_story_ai": {
        "name": "Narcooo/inkos - Story Creation AI Agent",
        # v2.9.0 · 16 个内置 SKILL.md(顶层 inkos + 15 个 inkos-* 子 skill)·
        # 长篇写作 / 商业短篇 / 剧本 / 分镜 / 互动影游 / Play 开放世界 / 翻译 / 拆稿 / 市场研究 / 导入 / 封面 / 去 AI 味
        # AGPL-3.0 · 安装:npm i -g @actalk/inkos
        # 收编策略(D-25 方案 D):只贴 README 到 docs/skill-collection/inkos-master.md,不双层嵌套 13.5 MB
        "url": "https://raw.githubusercontent.com/Narcooo/inkos/main/README.md",
        "type": "awesome_list",
        "self_repo": ("Narcooo", "inkos")
    }
}

def create_ssl_context():
    context = ssl.create_default_context()
    context.check_hostname = False
    context.verify_mode = ssl.CERT_NONE
    return context

def fetch_content(url):
    try:
        context = create_ssl_context()
        with urllib.request.urlopen(url, context=context, timeout=30) as response:
            return response.read().decode('utf-8')
    except Exception as e:
        print(f"❌ 获取内容失败 {url}: {e}")
        return None

def extract_github_repos(text):
    pattern = r'github\.com/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)'
    matches = re.findall(pattern, text)
    repos = set()
    skip = {'issues', 'pulls', 'discussions', 'actions', 'projects',
            'wiki', 'pulse', 'graphs', 'settings', 'sponsors',
            'codespaces', 'security', 'network', 'stargazers', 'watchers',
            'forks', 'branches', 'tags', 'releases', 'packages',
            'milestones', 'labels', 'compare', 'tree', 'blob', 'raw',
            'commits', 'blame', 'edit', 'delete'}
    for owner, repo in matches:
        if repo.lower() in skip:
            continue
        if owner.lower() in {'www', 'apps', 'api', 'docs', 'support',
                              'enterprise', 'features', 'pricing',
                              'customer-stories', 'readme'}:
            continue
        repos.add((owner, repo))
    return repos

def extract_openai_skills(text):
    repos = set()
    skill_patterns = [
        r'skills/\.system/([a-zA-Z0-9_-]+)',
        r'skills/\.curated/([a-zA-Z0-9_-]+)',
        r'skills/\.experimental/([a-zA-Z0-9_-]+)'
    ]
    for pattern in skill_patterns:
        matches = re.findall(pattern, text)
        for skill_name in matches:
            repos.add(('openai', f'skills/{skill_name}'))
    return repos

def parse_awesome_list(text, self_repo=None):
    repos = extract_github_repos(text)
    if self_repo:
        owner, name = self_repo
        repos.discard((owner, name))
    return repos

def extract_mattpocock_skills(text):
    """Matt Pocock 是 mono-repo,README 用相对路径列出各 skill:
    ./skills/<category>/<name>/SKILL.md
    按 skill 颗粒度收录,repo_id 形如 mattpocock/skills/engineering/tdd。
    """
    repos = set()
    matches = re.findall(r'skills/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)/SKILL\.md', text)
    for category, name in matches:
        repos.add(('mattpocock', f'skills/{category}/{name}'))
    return repos

def load_existing_sources():
    if os.path.exists('SKILL_SOURCES.json'):
        try:
            with open('SKILL_SOURCES.json', 'r', encoding='utf-8') as f:
                return json.load(f)
        except Exception as e:
            print(f"⚠️ 读取现有文件失败: {e}")
    return {"sources": {}, "last_updated": None, "total_count": 0}

def save_sources(sources_data):
    with open('SKILL_SOURCES.json', 'w', encoding='utf-8') as f:
        json.dump(sources_data, f, ensure_ascii=False, indent=2)
    print(f"✅ 已保存 SKILL_SOURCES.json,共 {sources_data['total_count']} 个技能源")

def github_home(info: dict) -> str:
    u = info.get("url", "")
    m = re.search(r"github\.com/([^/]+)/([^/]+)", u)
    if m:
        return f"https://github.com/{m.group(1)}/{m.group(2)}"
    return u


def _apply_readme(path: str, content: str) -> None:
    if not os.path.exists(path):
        open(path, "w", encoding="utf-8").close()
    existing = open(path, encoding="utf-8").read()
    MARKER_START = "<!-- AUTO-SYNC-SKILLS-START -->"
    MARKER_END = "<!-- AUTO-SYNC-SKILLS-END -->"
    if MARKER_START in existing and MARKER_END in existing:
        new = re.sub(f"{MARKER_START}.*?{MARKER_END}", content, existing, flags=re.DOTALL)
    else:
        new = existing.rstrip() + "\n\n" + content
    open(path, "w", encoding="utf-8").write(new)


def neutral_group_label(idx: int, lang: str = "zh") -> str:
    """README 分组只用中性序号,不点名上游仓库。

    上游名称只保留在 UPSTREAM_SOURCES 配置与 SKILL_SOURCES.json 里(溯源用),
    避免 README 每天被机器人重写时反复强调「几个上游 / 分别叫什么」。
    """
    return f"索引分组 {idx}" if lang == "zh" else f"Index group {idx}"


# 头部(自动段之前)不允许出现的「数量 + 点名」宣称,用于防回潮
CLAIM_NAME_PATTERNS = (
    "OpenAI Skills", "VoltAgent Awesome", "Awesome DSH Plugin",
    "Matt Pocock Skills", "Narcooo/inkos", "Narcooo-inkos",
    "VoltAgent DESIGN.md", "VoltAgent Awesome DESIGN.md",
)
CLAIM_COUNT_PATTERNS = (
    re.compile(r"\d+\s*个上游"),
    re.compile(r"\d+\s*upstream\s+(?:repos|repositories|sources)", re.I),
)
# 头部同样不允许出现「实现细节 / 整改自证」表述:README 面向用户,只讲有什么、怎么用,
# 实现机制属于代码注释与 CONTRIBUTING,不写入正文。
CLAIM_META_PATTERNS = (
    "非手工填写", "无需手工维护", "不在正文写死", "由脚本从数据文件实时计算",
    "计数由脚本", "由自动化脚本", "统一写入 `public/", "前端搜索数据源",
    "真·动态", "防回潮", "scripts/sync_skills.py", "UPSTREAM_SOURCES",
    "never hard-coded", "no manual edits", "computed from data files",
    "by the sync script", "generated by the sync script", "Source configuration lives",
)


def guard_readme_claims(path: str = "README.md") -> int:
    """防回潮守卫:清掉 README 头部(自动段之前)的上游数量/名称宣称与实现细节自证。

    bot 每次同步后调用一次;即便有人手工改回营销文案或整改说明,次日也会被自动清掉。
    只作用于头部,不碰 AUTO-SYNC 自动段里的仓库链接列表。
    """
    if not os.path.exists(path):
        return 0
    text = open(path, encoding="utf-8").read()
    marker = "<!-- AUTO-SYNC-SKILLS-START -->"
    head, sep, tail = text.partition(marker)
    if not sep:
        head, tail = text, ""
    kept, removed = [], 0
    for line in head.split("\n"):
        if (any(p in line for p in CLAIM_NAME_PATTERNS)
                or any(p in line for p in CLAIM_META_PATTERNS)
                or any(p.search(line) for p in CLAIM_COUNT_PATTERNS)):
            removed += 1
            continue
        kept.append(line)
    if removed:
        open(path, "w", encoding="utf-8").write("\n".join(kept) + sep + tail)
        print(f"⚠️ {os.path.basename(path)} 防回潮:清除 {removed} 行上游数量/名称宣称或实现细节自证")
    return removed


def update_readme(sources_data):
    MARKER_START = "<!-- AUTO-SYNC-SKILLS-START -->"
    MARKER_END = "<!-- AUTO-SYNC-SKILLS-END -->"

    skills_content = f"""{MARKER_START}

## 📦 社区技能仓库聚合

> 社区技能仓库索引,按分组列出,便于检索。

**最后更新**: {sources_data['last_updated']}

### 技能仓库列表

"""

    for group_idx, (source_key, source_info) in enumerate(UPSTREAM_SOURCES.items(), 1):
        skills_content += f"#### {neutral_group_label(group_idx, 'zh')}\n\n"

        count = 0
        for repo_id, repo_data in sources_data['sources'].items():
            if repo_data.get('upstream_source') == source_key:
                parts = repo_id.split('/')
                if len(parts) == 2:
                    owner, repo = parts
                else:
                    owner = parts[0]
                    repo = '/'.join(parts[1:])

                repo_url = f"https://github.com/{owner}/{repo}"
                skills_content += f"- [{repo}]({repo_url})\n"
                count += 1

        if count == 0:
            skills_content += "- 暂无数据\n"

        skills_content += "\n"

    skills_content += f"{MARKER_END}"

    existing_content = ""
    if os.path.exists('README.md'):
        with open('README.md', 'r', encoding='utf-8') as f:
            existing_content = f.read()

    if MARKER_START in existing_content and MARKER_END in existing_content:
        new_content = re.sub(
            f"{MARKER_START}.*?{MARKER_END}",
            skills_content.replace('\\', '\\\\'),
            existing_content,
            flags=re.DOTALL
        )
    else:
        new_content = existing_content.rstrip() + "\n\n" + skills_content

    _apply_readme("README.md", skills_content)
    print("✅ 已在 README.md 写入技能列表(自动段)")

def update_readme_en(sources_data):
    """English AUTO-SYNC section for README_EN.md."""
    MARKER_START = "<!-- AUTO-SYNC-SKILLS-START -->"
    MARKER_END = "<!-- AUTO-SYNC-SKILLS-END -->"
    content = f"""{MARKER_START}

## 📦 Community Skill Repositories Aggregation

> Community skill repositories, grouped for easy browsing.

**Last updated**: {sources_data['last_updated']}

### Skill Repositories

"""
    for group_idx, (key, info) in enumerate(UPSTREAM_SOURCES.items(), 1):
        content += f"#### {neutral_group_label(group_idx, 'en')}\n\n"
        cnt = 0
        for rid, rd in sources_data['sources'].items():
            if rd.get('upstream_source') == key:
                parts = rid.split('/')
                owner = parts[0]
                repo = '/'.join(parts[1:]) if len(parts) > 2 else parts[1]
                content += f"- [{repo}](https://github.com/{owner}/{repo})\n"
                cnt += 1
        if cnt == 0:
            content += "- No data yet\n"
        content += "\n"
    content += MARKER_END
    _apply_readme("README_EN.md", content)
    print("✅ 已在 README_EN.md 写入技能列表(自动段)")


# 各上游对应的自身仓库(用于 awesome list 排除自身)
SELF_REPOS = {
    "voltagent_awesome": ("VoltAgent", "awesome-agent-skills"),
    "awesome_dsh_plugin": ("awesome-dsh-plugin", "awesome-dsh-plugin"),
    "inkos_story_ai": ("Narcooo", "inkos"),
}

# ===== README 统计数据自动注入 =====
# 与仓库列表的 AUTO-SYNC-SKILLS 标记区分,单独用 STATS 标记圈住「统计数据」段,
# 由 sync_skills.py(及 sync_skills_to_search.py)每日用真实计数覆盖。
STATS_START = "<!-- AUTO-SYNC-STATS-START -->"
STATS_END = "<!-- AUTO-SYNC-STATS-END -->"


def count_local_skills(path: str = "data/local_skills.json") -> int:
    """本地精选策展技能数(data/local_skills.json)。"""
    try:
        d = json.load(open(path, encoding="utf-8"))
    except Exception:
        return 0
    if isinstance(d.get("skills"), list):
        return len(d["skills"])
    if "categories" in d:
        return sum(len(v.get("skills", [])) for v in d["categories"].values())
    if "total_skills" in d:
        return d["total_skills"]
    return 0


def count_search_skills(path: str = "public/data/skills.json") -> int:
    """前端搜索库技能数(public/data/skills.json),以数组实际长度为准。"""
    try:
        d = json.load(open(path, encoding="utf-8"))
    except Exception:
        return 0
    skills = d.get("skills") or []
    return len(skills) if skills else d.get("total", 0)


def count_trending_skills(path: str = "public/data/skills.json") -> int:
    """热门技能榜单条数(skills.json 中 trending:true 的条目,每日随 GitHub 抓取变化)。"""
    try:
        d = json.load(open(path, encoding="utf-8"))
    except Exception:
        return 0
    return sum(1 for s in (d.get("skills") or []) if s.get("trending") is True)


def inject_readme_stats(upstream: int, local: int, search: int, trending: int = 0,
                        path: str = "README.md", lang: str = "zh") -> None:
    """把真实计数注入 README 的 AUTO-SYNC-STATS 标记段(bot 每日自保持准确)。

    口径说明:upstream 是自动抓取的「上游索引条目」(含重复/待筛选链接),
    不标为「官方技能」,避免夸大;trending 来自 GitHub Search API 实时抓取。
    带时间戳,让「每日刷新」可见。
    """
    ts = datetime.now().strftime("%Y-%m-%d %H:%M")
    if lang == "en":
        block = (
            f"{STATS_START}\n"
            f"- **Index entries** (pending curation): {upstream:,}\n"
            f"- **Local curated skills**: {local:,}\n"
            f"- **Search index** (skills.json): {search:,}\n"
            f"- **Trending board** (by stars): {trending:,}\n"
            f"- **Last synced**: {ts} (auto-refreshed daily)\n"
            f"{STATS_END}"
        )
    else:
        block = (
            f"{STATS_START}\n"
            f"- **索引条目**(含待筛选): {upstream:,} 条\n"
            f"- **本地技能(精选策展)**: {local:,} 个\n"
            f"- **搜索索引(skills.json)**: {search:,} 个\n"
            f"- **热门技能榜单**(按 Star 排序): {trending:,} 个\n"
            f"- **最后同步**: {ts}(每日自动刷新)\n"
            f"{STATS_END}"
        )
    if not os.path.exists(path):
        return
    content = open(path, encoding="utf-8").read()
    if STATS_START in content and STATS_END in content:
        new = re.sub(f"{STATS_START}.*?{STATS_END}", block, content, flags=re.DOTALL)
    else:
        new = content.rstrip() + "\n\n" + block + "\n"
    open(path, "w", encoding="utf-8").write(new)
    print(f"✅ {os.path.basename(path)} 统计段已注入:索引 {search:,} / 热门 {trending:,} / 本地 {local:,} / 上游 {upstream:,}")


def main():
    print("🚀 开始同步技能仓库...")
    print(f"⏰ 当前时间: {datetime.now().isoformat()}")

    sources_data = load_existing_sources()
    all_repos = {}

    for source_key, source_info in UPSTREAM_SOURCES.items():
        print(f"\n📥 正在处理: {source_info['name']}")

        content = fetch_content(source_info['url'])
        if not content:
            continue

        if source_info['type'] == 'directory_structure':
            repos = extract_openai_skills(content)
        elif source_info['type'] == 'mattpocock_skills':
            repos = extract_mattpocock_skills(content)
        else:
            self_repo = SELF_REPOS.get(source_key)
            repos = parse_awesome_list(content, self_repo=self_repo)

        print(f"   发现 {len(repos)} 个仓库")

        for owner, repo in repos:
            repo_id = f"{owner}/{repo}"
            all_repos[repo_id] = {
                "upstream_source": source_key,
                "discovered_at": datetime.now().isoformat(),
                "description": ""
            }

    existing_sources = sources_data.get('sources', {})
    for repo_id, repo_data in all_repos.items():
        if repo_id in existing_sources:
            repo_data['discovered_at'] = existing_sources[repo_id].get('discovered_at', repo_data['discovered_at'])

    # 保留手动收录的技能(不在任何上游自动发现列表中的条目),避免被自动同步清空
    merged = dict(all_repos)
    for repo_id, repo_data in existing_sources.items():
        if repo_id not in merged:
            merged[repo_id] = repo_data
    sources_data['sources'] = merged
    sources_data['last_updated'] = datetime.now().isoformat()
    sources_data['total_count'] = len(merged)

    save_sources(sources_data)
    refresh_all_readme(sources_data)

    print(f"\n✨ 同步完成!共发现 {len(all_repos)} 个技能仓库(索引条目 {sources_data['total_count']:,} 条)")


def refresh_all_readme(sources_data) -> None:
    """重建中英 README 的自动段 / 统计段,并执行防回潮守卫(不出网)。"""
    update_readme(sources_data)
    update_readme_en(sources_data)
    stats = dict(
        upstream=sources_data.get('total_count', len(sources_data.get('sources', {}))),
        local=count_local_skills(),
        search=count_search_skills(),
        trending=count_trending_skills(),
    )
    inject_readme_stats(**stats, path="README.md", lang="zh")
    inject_readme_stats(**stats, path="README_EN.md", lang="en")
    guard_readme_claims("README.md")
    guard_readme_claims("README_EN.md")


def rebuild_readme_only() -> None:
    """--readme-only:用本地 SKILL_SOURCES.json 重建 README,不抓取上游(用于改模板后立即生效)。"""
    d = load_existing_sources()
    refresh_all_readme(d)
    print("✅ README / README_EN 已按新模板重建(未出网)")


if __name__ == '__main__':
    if '--readme-only' in sys.argv:
        rebuild_readme_only()
    else:
        main()
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall