#!/usr/bin/env python3
"""
多源技能仓库同步脚本
从多个上游源提取技能仓库链接并汇总
"""
import json
import re
import os
import sys
from datetime import datetime
import urllib.request
import ssl
# 上游源配置
UPSTREAM_SOURCES = {
"openai_skills": {
"name": "OpenAI Skills",
"url": "https://raw.githubusercontent.com/openai/skills/main/README.md",
"type": "directory_structure"
},
"voltagent_awesome": {
"name": "VoltAgent Awesome Agent Skills",
"url": "https://raw.githubusercontent.com/VoltAgent/awesome-agent-skills/main/README.md",
"type": "awesome_list"
},
"awesome_dsh_plugin": {
"name": "Awesome DSH Plugin",
"url": "https://raw.githubusercontent.com/awesome-dsh-plugin/awesome-dsh-plugin/main/README.md",
"type": "awesome_list"
},
"mattpocock_skills": {
"name": "Matt Pocock Skills",
# mono-repo:README 列的是本地 skill 路径(./skills/<cat>/<name>/SKILL.md),
# 不是 GitHub 链接列表,所以不能用 awesome_list(会落空)。
# 用专属解析按 skill 颗粒度收录,repo_id 形如 mattpocock/skills/engineering/tdd
"url": "https://raw.githubusercontent.com/mattpocock/skills/main/README.md",
"type": "mattpocock_skills"
},
"voltagent_design_md": {
"name": "VoltAgent Awesome DESIGN.md",
# 73 个真实网站 DESIGN.md 提取(Claude / Cursor / Vercel / Stripe / Apple ...)
# 格式特殊:列表项是 getdesign.md/<slug>/design-md 而不是 github.com/owner/repo
# 所以 sync 脚本只记录元数据(仓库总数 + 上游元链接),
# 实际 DESIGN.md 文件请用户从 https://github.com/VoltAgent/awesome-design-md 手动浏览
"url": "https://raw.githubusercontent.com/VoltAgent/awesome-design-md/main/README.md",
"type": "design_md_catalog",
"self_repo": ("VoltAgent", "awesome-design-md")
},
"inkos_story_ai": {
"name": "Narcooo/inkos - Story Creation AI Agent",
# v2.9.0 · 16 个内置 SKILL.md(顶层 inkos + 15 个 inkos-* 子 skill)·
# 长篇写作 / 商业短篇 / 剧本 / 分镜 / 互动影游 / Play 开放世界 / 翻译 / 拆稿 / 市场研究 / 导入 / 封面 / 去 AI 味
# AGPL-3.0 · 安装:npm i -g @actalk/inkos
# 收编策略(D-25 方案 D):只贴 README 到 docs/skill-collection/inkos-master.md,不双层嵌套 13.5 MB
"url": "https://raw.githubusercontent.com/Narcooo/inkos/main/README.md",
"type": "awesome_list",
"self_repo": ("Narcooo", "inkos")
}
}
def create_ssl_context():
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
return context
def fetch_content(url):
try:
context = create_ssl_context()
with urllib.request.urlopen(url, context=context, timeout=30) as response:
return response.read().decode('utf-8')
except Exception as e:
print(f"❌ 获取内容失败 {url}: {e}")
return None
def extract_github_repos(text):
pattern = r'github\.com/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)'
matches = re.findall(pattern, text)
repos = set()
skip = {'issues', 'pulls', 'discussions', 'actions', 'projects',
'wiki', 'pulse', 'graphs', 'settings', 'sponsors',
'codespaces', 'security', 'network', 'stargazers', 'watchers',
'forks', 'branches', 'tags', 'releases', 'packages',
'milestones', 'labels', 'compare', 'tree', 'blob', 'raw',
'commits', 'blame', 'edit', 'delete'}
for owner, repo in matches:
if repo.lower() in skip:
continue
if owner.lower() in {'www', 'apps', 'api', 'docs', 'support',
'enterprise', 'features', 'pricing',
'customer-stories', 'readme'}:
continue
repos.add((owner, repo))
return repos
def extract_openai_skills(text):
repos = set()
skill_patterns = [
r'skills/\.system/([a-zA-Z0-9_-]+)',
r'skills/\.curated/([a-zA-Z0-9_-]+)',
r'skills/\.experimental/([a-zA-Z0-9_-]+)'
]
for pattern in skill_patterns:
matches = re.findall(pattern, text)
for skill_name in matches:
repos.add(('openai', f'skills/{skill_name}'))
return repos
def parse_awesome_list(text, self_repo=None):
repos = extract_github_repos(text)
if self_repo:
owner, name = self_repo
repos.discard((owner, name))
return repos
def extract_mattpocock_skills(text):
"""Matt Pocock 是 mono-repo,README 用相对路径列出各 skill:
./skills/<category>/<name>/SKILL.md
按 skill 颗粒度收录,repo_id 形如 mattpocock/skills/engineering/tdd。
"""
repos = set()
matches = re.findall(r'skills/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)/SKILL\.md', text)
for category, name in matches:
repos.add(('mattpocock', f'skills/{category}/{name}'))
return repos
def load_existing_sources():
if os.path.exists('SKILL_SOURCES.json'):
try:
with open('SKILL_SOURCES.json', 'r', encoding='utf-8') as f:
return json.load(f)
except Exception as e:
print(f"⚠️ 读取现有文件失败: {e}")
return {"sources": {}, "last_updated": None, "total_count": 0}
def save_sources(sources_data):
with open('SKILL_SOURCES.json', 'w', encoding='utf-8') as f:
json.dump(sources_data, f, ensure_ascii=False, indent=2)
print(f"✅ 已保存 SKILL_SOURCES.json,共 {sources_data['total_count']} 个技能源")
def github_home(info: dict) -> str:
u = info.get("url", "")
m = re.search(r"github\.com/([^/]+)/([^/]+)", u)
if m:
return f"https://github.com/{m.group(1)}/{m.group(2)}"
return u
def _apply_readme(path: str, content: str) -> None:
if not os.path.exists(path):
open(path, "w", encoding="utf-8").close()
existing = open(path, encoding="utf-8").read()
MARKER_START = "<!-- AUTO-SYNC-SKILLS-START -->"
MARKER_END = "<!-- AUTO-SYNC-SKILLS-END -->"
if MARKER_START in existing and MARKER_END in existing:
new = re.sub(f"{MARKER_START}.*?{MARKER_END}", content, existing, flags=re.DOTALL)
else:
new = existing.rstrip() + "\n\n" + content
open(path, "w", encoding="utf-8").write(new)
def neutral_group_label(idx: int, lang: str = "zh") -> str:
"""README 分组只用中性序号,不点名上游仓库。
上游名称只保留在 UPSTREAM_SOURCES 配置与 SKILL_SOURCES.json 里(溯源用),
避免 README 每天被机器人重写时反复强调「几个上游 / 分别叫什么」。
"""
return f"索引分组 {idx}" if lang == "zh" else f"Index group {idx}"
# 头部(自动段之前)不允许出现的「数量 + 点名」宣称,用于防回潮
CLAIM_NAME_PATTERNS = (
"OpenAI Skills", "VoltAgent Awesome", "Awesome DSH Plugin",
"Matt Pocock Skills", "Narcooo/inkos", "Narcooo-inkos",
"VoltAgent DESIGN.md", "VoltAgent Awesome DESIGN.md",
)
CLAIM_COUNT_PATTERNS = (
re.compile(r"\d+\s*个上游"),
re.compile(r"\d+\s*upstream\s+(?:repos|repositories|sources)", re.I),
)
# 头部同样不允许出现「实现细节 / 整改自证」表述:README 面向用户,只讲有什么、怎么用,
# 实现机制属于代码注释与 CONTRIBUTING,不写入正文。
CLAIM_META_PATTERNS = (
"非手工填写", "无需手工维护", "不在正文写死", "由脚本从数据文件实时计算",
"计数由脚本", "由自动化脚本", "统一写入 `public/", "前端搜索数据源",
"真·动态", "防回潮", "scripts/sync_skills.py", "UPSTREAM_SOURCES",
"never hard-coded", "no manual edits", "computed from data files",
"by the sync script", "generated by the sync script", "Source configuration lives",
)
def guard_readme_claims(path: str = "README.md") -> int:
"""防回潮守卫:清掉 README 头部(自动段之前)的上游数量/名称宣称与实现细节自证。
bot 每次同步后调用一次;即便有人手工改回营销文案或整改说明,次日也会被自动清掉。
只作用于头部,不碰 AUTO-SYNC 自动段里的仓库链接列表。
"""
if not os.path.exists(path):
return 0
text = open(path, encoding="utf-8").read()
marker = "<!-- AUTO-SYNC-SKILLS-START -->"
head, sep, tail = text.partition(marker)
if not sep:
head, tail = text, ""
kept, removed = [], 0
for line in head.split("\n"):
if (any(p in line for p in CLAIM_NAME_PATTERNS)
or any(p in line for p in CLAIM_META_PATTERNS)
or any(p.search(line) for p in CLAIM_COUNT_PATTERNS)):
removed += 1
continue
kept.append(line)
if removed:
open(path, "w", encoding="utf-8").write("\n".join(kept) + sep + tail)
print(f"⚠️ {os.path.basename(path)} 防回潮:清除 {removed} 行上游数量/名称宣称或实现细节自证")
return removed
def update_readme(sources_data):
MARKER_START = "<!-- AUTO-SYNC-SKILLS-START -->"
MARKER_END = "<!-- AUTO-SYNC-SKILLS-END -->"
skills_content = f"""{MARKER_START}
## 📦 社区技能仓库聚合
> 社区技能仓库索引,按分组列出,便于检索。
**最后更新**: {sources_data['last_updated']}
### 技能仓库列表
"""
for group_idx, (source_key, source_info) in enumerate(UPSTREAM_SOURCES.items(), 1):
skills_content += f"#### {neutral_group_label(group_idx, 'zh')}\n\n"
count = 0
for repo_id, repo_data in sources_data['sources'].items():
if repo_data.get('upstream_source') == source_key:
parts = repo_id.split('/')
if len(parts) == 2:
owner, repo = parts
else:
owner = parts[0]
repo = '/'.join(parts[1:])
repo_url = f"https://github.com/{owner}/{repo}"
skills_content += f"- [{repo}]({repo_url})\n"
count += 1
if count == 0:
skills_content += "- 暂无数据\n"
skills_content += "\n"
skills_content += f"{MARKER_END}"
existing_content = ""
if os.path.exists('README.md'):
with open('README.md', 'r', encoding='utf-8') as f:
existing_content = f.read()
if MARKER_START in existing_content and MARKER_END in existing_content:
new_content = re.sub(
f"{MARKER_START}.*?{MARKER_END}",
skills_content.replace('\\', '\\\\'),
existing_content,
flags=re.DOTALL
)
else:
new_content = existing_content.rstrip() + "\n\n" + skills_content
_apply_readme("README.md", skills_content)
print("✅ 已在 README.md 写入技能列表(自动段)")
def update_readme_en(sources_data):
"""English AUTO-SYNC section for README_EN.md."""
MARKER_START = "<!-- AUTO-SYNC-SKILLS-START -->"
MARKER_END = "<!-- AUTO-SYNC-SKILLS-END -->"
content = f"""{MARKER_START}
## 📦 Community Skill Repositories Aggregation
> Community skill repositories, grouped for easy browsing.
**Last updated**: {sources_data['last_updated']}
### Skill Repositories
"""
for group_idx, (key, info) in enumerate(UPSTREAM_SOURCES.items(), 1):
content += f"#### {neutral_group_label(group_idx, 'en')}\n\n"
cnt = 0
for rid, rd in sources_data['sources'].items():
if rd.get('upstream_source') == key:
parts = rid.split('/')
owner = parts[0]
repo = '/'.join(parts[1:]) if len(parts) > 2 else parts[1]
content += f"- [{repo}](https://github.com/{owner}/{repo})\n"
cnt += 1
if cnt == 0:
content += "- No data yet\n"
content += "\n"
content += MARKER_END
_apply_readme("README_EN.md", content)
print("✅ 已在 README_EN.md 写入技能列表(自动段)")
# 各上游对应的自身仓库(用于 awesome list 排除自身)
SELF_REPOS = {
"voltagent_awesome": ("VoltAgent", "awesome-agent-skills"),
"awesome_dsh_plugin": ("awesome-dsh-plugin", "awesome-dsh-plugin"),
"inkos_story_ai": ("Narcooo", "inkos"),
}
# ===== README 统计数据自动注入 =====
# 与仓库列表的 AUTO-SYNC-SKILLS 标记区分,单独用 STATS 标记圈住「统计数据」段,
# 由 sync_skills.py(及 sync_skills_to_search.py)每日用真实计数覆盖。
STATS_START = "<!-- AUTO-SYNC-STATS-START -->"
STATS_END = "<!-- AUTO-SYNC-STATS-END -->"
def count_local_skills(path: str = "data/local_skills.json") -> int:
"""本地精选策展技能数(data/local_skills.json)。"""
try:
d = json.load(open(path, encoding="utf-8"))
except Exception:
return 0
if isinstance(d.get("skills"), list):
return len(d["skills"])
if "categories" in d:
return sum(len(v.get("skills", [])) for v in d["categories"].values())
if "total_skills" in d:
return d["total_skills"]
return 0
def count_search_skills(path: str = "public/data/skills.json") -> int:
"""前端搜索库技能数(public/data/skills.json),以数组实际长度为准。"""
try:
d = json.load(open(path, encoding="utf-8"))
except Exception:
return 0
skills = d.get("skills") or []
return len(skills) if skills else d.get("total", 0)
def count_trending_skills(path: str = "public/data/skills.json") -> int:
"""热门技能榜单条数(skills.json 中 trending:true 的条目,每日随 GitHub 抓取变化)。"""
try:
d = json.load(open(path, encoding="utf-8"))
except Exception:
return 0
return sum(1 for s in (d.get("skills") or []) if s.get("trending") is True)
def inject_readme_stats(upstream: int, local: int, search: int, trending: int = 0,
path: str = "README.md", lang: str = "zh") -> None:
"""把真实计数注入 README 的 AUTO-SYNC-STATS 标记段(bot 每日自保持准确)。
口径说明:upstream 是自动抓取的「上游索引条目」(含重复/待筛选链接),
不标为「官方技能」,避免夸大;trending 来自 GitHub Search API 实时抓取。
带时间戳,让「每日刷新」可见。
"""
ts = datetime.now().strftime("%Y-%m-%d %H:%M")
if lang == "en":
block = (
f"{STATS_START}\n"
f"- **Index entries** (pending curation): {upstream:,}\n"
f"- **Local curated skills**: {local:,}\n"
f"- **Search index** (skills.json): {search:,}\n"
f"- **Trending board** (by stars): {trending:,}\n"
f"- **Last synced**: {ts} (auto-refreshed daily)\n"
f"{STATS_END}"
)
else:
block = (
f"{STATS_START}\n"
f"- **索引条目**(含待筛选): {upstream:,} 条\n"
f"- **本地技能(精选策展)**: {local:,} 个\n"
f"- **搜索索引(skills.json)**: {search:,} 个\n"
f"- **热门技能榜单**(按 Star 排序): {trending:,} 个\n"
f"- **最后同步**: {ts}(每日自动刷新)\n"
f"{STATS_END}"
)
if not os.path.exists(path):
return
content = open(path, encoding="utf-8").read()
if STATS_START in content and STATS_END in content:
new = re.sub(f"{STATS_START}.*?{STATS_END}", block, content, flags=re.DOTALL)
else:
new = content.rstrip() + "\n\n" + block + "\n"
open(path, "w", encoding="utf-8").write(new)
print(f"✅ {os.path.basename(path)} 统计段已注入:索引 {search:,} / 热门 {trending:,} / 本地 {local:,} / 上游 {upstream:,}")
def main():
print("🚀 开始同步技能仓库...")
print(f"⏰ 当前时间: {datetime.now().isoformat()}")
sources_data = load_existing_sources()
all_repos = {}
for source_key, source_info in UPSTREAM_SOURCES.items():
print(f"\n📥 正在处理: {source_info['name']}")
content = fetch_content(source_info['url'])
if not content:
continue
if source_info['type'] == 'directory_structure':
repos = extract_openai_skills(content)
elif source_info['type'] == 'mattpocock_skills':
repos = extract_mattpocock_skills(content)
else:
self_repo = SELF_REPOS.get(source_key)
repos = parse_awesome_list(content, self_repo=self_repo)
print(f" 发现 {len(repos)} 个仓库")
for owner, repo in repos:
repo_id = f"{owner}/{repo}"
all_repos[repo_id] = {
"upstream_source": source_key,
"discovered_at": datetime.now().isoformat(),
"description": ""
}
existing_sources = sources_data.get('sources', {})
for repo_id, repo_data in all_repos.items():
if repo_id in existing_sources:
repo_data['discovered_at'] = existing_sources[repo_id].get('discovered_at', repo_data['discovered_at'])
# 保留手动收录的技能(不在任何上游自动发现列表中的条目),避免被自动同步清空
merged = dict(all_repos)
for repo_id, repo_data in existing_sources.items():
if repo_id not in merged:
merged[repo_id] = repo_data
sources_data['sources'] = merged
sources_data['last_updated'] = datetime.now().isoformat()
sources_data['total_count'] = len(merged)
save_sources(sources_data)
refresh_all_readme(sources_data)
print(f"\n✨ 同步完成!共发现 {len(all_repos)} 个技能仓库(索引条目 {sources_data['total_count']:,} 条)")
def refresh_all_readme(sources_data) -> None:
"""重建中英 README 的自动段 / 统计段,并执行防回潮守卫(不出网)。"""
update_readme(sources_data)
update_readme_en(sources_data)
stats = dict(
upstream=sources_data.get('total_count', len(sources_data.get('sources', {}))),
local=count_local_skills(),
search=count_search_skills(),
trending=count_trending_skills(),
)
inject_readme_stats(**stats, path="README.md", lang="zh")
inject_readme_stats(**stats, path="README_EN.md", lang="en")
guard_readme_claims("README.md")
guard_readme_claims("README_EN.md")
def rebuild_readme_only() -> None:
"""--readme-only:用本地 SKILL_SOURCES.json 重建 README,不抓取上游(用于改模板后立即生效)。"""
d = load_existing_sources()
refresh_all_readme(d)
print("✅ README / README_EN 已按新模板重建(未出网)")
if __name__ == '__main__':
if '--readme-only' in sys.argv:
rebuild_readme_only()
else:
main()