Skip to content

luffysolution-svg/omnischolar / skills/obsidian-literature-base

obsidian-literature-base

Verified

Build and refresh an Obsidian Base and optional topic Wiki for OmniScholar MinerU publications and Zotero reading records, including bibliographic views, prefixed or AI-assisted topics, cover images, foldable image galleries, and preserved topic synthesis pages. Use when organizing multiple OmniScholar literature outputs in an Obsidian vault; do not use for reading-status workflows or modifying Zotero.

FileKindSize
SKILL.mdskill md5.0 KB
references/schema.mdmarkdown4.0 KB
scripts/build_literature_base.pyscript31.8 KB

scripts/build_literature_base.py

View on GitHub
#!/usr/bin/env python3
"""Build an Obsidian literature Base from OmniScholar publication outputs."""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import re
import sys
import unicodedata
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import unquote

import yaml

GENERATED_BY = "obsidian-literature-base"
BASE_MARKER = f"# Generated by {GENERATED_BY}; manual edits may be overwritten."
MARKDOWN_MARKER = f"generatedBy: {GENERATED_BY}"
GENERATED_COMMENT = f"<!-- Generated by {GENERATED_BY}; manual edits may be overwritten. -->"
MANUAL_START = "<!-- MANUAL:START -->"
MANUAL_END = "<!-- MANUAL:END -->"
AUTO_START = "<!-- AUTO:START -->"
AUTO_END = "<!-- AUTO:END -->"
DEFAULT_TOPIC_PREFIXES = ("topic/", "主题/")
IMAGE_SUFFIXES = {".avif", ".gif", ".jpeg", ".jpg", ".png", ".svg", ".webp"}
MAX_MARKDOWN_BYTES = 64 * 1024 * 1024
STANDARD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\(([^)]+)\)")
WIKI_IMAGE_RE = re.compile(r"!\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")


@dataclass(slots=True)
class PaperPlan:
    directory: Path
    sidecar: dict[str, Any]
    markdown_path: Path
    source_properties: dict[str, Any]
    zotero_record: Path | None
    images: list[str]
    topics: list[str]
    topic_source: str
    title: str
    index_path: Path


def _confined(root: Path, relative: str) -> Path:
    candidate = (root / relative).resolve()
    try:
        candidate.relative_to(root)
    except ValueError as exc:
        raise ValueError(f"Path escapes the vault root: {relative}") from exc
    return candidate


def _read_json(path: Path) -> dict[str, Any]:
    value = json.loads(path.read_text(encoding="utf-8"))
    if not isinstance(value, dict):
        raise ValueError(f"Expected a JSON object: {path}")
    return value


def _frontmatter(path: Path) -> dict[str, Any]:
    with path.open("r", encoding="utf-8", errors="replace") as handle:
        if handle.readline().strip() != "---":
            return {}
        lines: list[str] = []
        size = 0
        for line in handle:
            size += len(line.encode("utf-8"))
            if size > 256 * 1024:
                return {}
            if line.strip() == "---":
                value = yaml.safe_load("".join(lines)) or {}
                return value if isinstance(value, dict) else {}
            lines.append(line)
    return {}


def _publication_sidecars(vault_root: Path) -> list[tuple[Path, dict[str, Any]]]:
    records: list[tuple[Path, dict[str, Any]]] = []
    for path in sorted(vault_root.rglob("metadata.json"), key=lambda item: item.as_posix().casefold()):
        relative_parts = path.relative_to(vault_root).parts
        if ".omnischolar" in relative_parts:
            continue
        try:
            document = _read_json(path)
        except (OSError, UnicodeError, json.JSONDecodeError, ValueError):
            continue
        if (
            document.get("schemaVersion") == 1
            and isinstance(document.get("publication"), dict)
            and isinstance(document.get("zotero"), dict)
        ):
            records.append((path.parent, document))
    return records


def _creator_names(value: Any) -> list[str]:
    if not isinstance(value, list):
        return []
    result: list[str] = []
    for creator in value:
        if not isinstance(creator, dict):
            continue
        name = str(creator.get("name") or "").strip()
        if not name:
            name = " ".join(
                part
                for key in ("firstName", "lastName")
                if (part := str(creator.get(key) or "").strip())
            )
        if name and name not in result:
            result.append(name)
    return result


def _source_tags(value: Any) -> list[str]:
    if not isinstance(value, list):
        return []
    result: list[str] = []
    for entry in value:
        tag = entry.get("tag") if isinstance(entry, dict) else entry
        if isinstance(tag, str) and tag.strip() and tag.strip() not in result:
            result.append(tag.strip())
    return result


def _obsidian_tag(value: str) -> str | None:
    tag = unicodedata.normalize("NFKC", value).strip().lstrip("#").strip()
    tag = re.sub(r"\s+", "-", tag)
    tag = re.sub(r"[^\w/-]+", "-", tag, flags=re.UNICODE)
    tag = re.sub(r"-{2,}", "-", tag).strip("-_/ ")
    if not tag:
        return None
    return f"tag-{tag}" if tag.isdecimal() else tag


def _dedupe(values: list[str]) -> list[str]:
    result: list[str] = []
    seen: set[str] = set()
    for value in values:
        text = unicodedata.normalize("NFKC", str(value)).strip()
        key = text.casefold()
        if text and key not in seen:
            seen.add(key)
            result.append(text)
    return result


def _prefixed_topics(tags: list[str], prefixes: tuple[str, ...]) -> list[str]:
    topics: list[str] = []
    for tag in tags:
        folded = tag.casefold()
        for prefix in prefixes:
            if folded.startswith(prefix.casefold()):
                topic = tag[len(prefix) :].strip(" /:-")
                if topic:
                    topics.append(topic)
                break
    return _dedupe(topics)


def _year(paper: dict[str, Any]) -> int | None:
    for value in (paper.get("year"), paper.get("date")):
        match = re.search(r"(?:19|20)\d{2}", str(value or ""))
        if match:
            return int(match.group(0))
    return None


def _mineru_markdown(directory: Path, index_name: str) -> tuple[Path, dict[str, Any]]:
    fallback: tuple[Path, dict[str, Any]] | None = None
    for path in sorted(directory.glob("*.md"), key=lambda item: item.name.casefold()):
        if path.name == index_name:
            continue
        properties = _frontmatter(path)
        if properties.get("recordType") == "mineru-publication":
            return path, properties
        fallback = fallback or (path, properties)
    if fallback is None:
        raise ValueError(f"No top-level MinerU Markdown found in {directory}")
    return fallback


def _zotero_record(directory: Path, index_name: str, mineru_path: Path) -> Path | None:
    for path in sorted(directory.rglob("*.md"), key=lambda item: item.as_posix().casefold()):
        if path == mineru_path or path.name == index_name:
            continue
        if _frontmatter(path).get("recordType") == "zotero-reading-record":
            return path
    return None


def _image_target(raw: str) -> str:
    target = raw.strip().strip("<>")
    if " \"" in target or " '" in target:
        target = target.split(" ", 1)[0]
    return unquote(target.split("#", 1)[0].split("?", 1)[0]).replace("\\", "/")


def _images(directory: Path, markdown_path: Path) -> list[str]:
    if markdown_path.stat().st_size > MAX_MARKDOWN_BYTES:
        raise ValueError(f"MinerU Markdown exceeds {MAX_MARKDOWN_BYTES} bytes: {markdown_path}")
    text = markdown_path.read_text(encoding="utf-8", errors="replace")
    candidates = [*STANDARD_IMAGE_RE.findall(text), *WIKI_IMAGE_RE.findall(text)]
    result: list[str] = []
    for raw in candidates:
        target = _image_target(raw)
        if not target or "://" in target:
            continue
        try:
            resolved = _confined(directory, target)
        except ValueError:
            continue
        if resolved.is_file() and resolved.suffix.casefold() in IMAGE_SUFFIXES:
            relative = resolved.relative_to(directory).as_posix()
            if relative not in result:
                result.append(relative)
    assets = directory / "assets"
    if assets.is_dir():
        for path in sorted(assets.rglob("*"), key=lambda item: item.as_posix().casefold()):
            if path.is_file() and path.suffix.casefold() in IMAGE_SUFFIXES:
                relative = path.relative_to(directory).as_posix()
                if relative not in result:
                    result.append(relative)
    return result


def _load_ai_topics(path: Path | None) -> dict[str, list[str]]:
    if path is None:
        return {}
    document = _read_json(path)
    result: dict[str, list[str]] = {}
    for key, value in document.items():
        if not isinstance(key, str) or not isinstance(value, list):
            raise ValueError("AI topic mappings must map string identifiers to string lists")
        topics = _dedupe([item for item in value if isinstance(item, str)])
        if len(topics) > 3:
            raise ValueError(f"AI topics for {key!r} exceed the maximum of three")
        result[key] = topics
    return result


def _ai_topics_for(
    mapping: dict[str, list[str]], paper: dict[str, Any], publication: dict[str, Any]
) -> list[str]:
    keys = (
        publication.get("id"),
        paper.get("zoteroKey"),
        paper.get("doi"),
        paper.get("DOI"),
        paper.get("title"),
    )
    for key in keys:
        if key is not None and str(key) in mapping:
            return mapping[str(key)]
    return []


def _callout(kind: str, title: str, content: str) -> list[str]:
    lines = [f"> [!{kind}]- {title}"]
    if not content.strip():
        return [*lines, "> _无内容。_"]
    for line in content.splitlines():
        lines.append(f"> {line}" if line else ">")
    return lines


def _safe_wiki_stem(value: str) -> str:
    text = unicodedata.normalize("NFC", re.sub(r"<[^>]*>", "", value))
    text = re.sub(r'[<>:"/\\|?*\x00-\x1f]+', " ", text)
    text = re.sub(r"\s+", " ", text).strip(" .") or "Untitled"
    if re.match(r"^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)", text, re.IGNORECASE):
        text = f"_{text}"
    if len(text.encode("utf-8")) > 120:
        suffix = f"-{hashlib.sha256(text.encode('utf-8')).hexdigest()[:8]}"
        while text and len(f"{text}{suffix}".encode("utf-8")) > 120:
            text = text[:-1]
        text = f"{text.rstrip(' .-')}{suffix}"
    return text


def _topic_paths(topics_root: Path, topics: list[str]) -> dict[str, Path]:
    result: dict[str, Path] = {}
    occupied: dict[str, str] = {}
    for topic in sorted(topics, key=str.casefold):
        stem = _safe_wiki_stem(topic)
        key = stem.casefold()
        if key in occupied and occupied[key] != topic:
            suffix = hashlib.sha256(topic.encode("utf-8")).hexdigest()[:8]
            stem = f"{stem}-{suffix}"
            key = stem.casefold()
        occupied[key] = topic
        result[topic] = topics_root / f"{stem}.md"
    return result


def _default_manual_section() -> str:
    return "\n".join(
        [
            "## 定义与边界",
            "",
            "_在这里记录主题定义、范围、同义词与排除项。_",
            "",
            "## 核心研究问题",
            "",
            "_在这里记录该主题需要回答的关键问题。_",
            "",
            "## 当前共识",
            "",
            "_每条判断应链接支持它的文献。_",
            "",
            "## 尚存争议",
            "",
            "_并列记录不同解释及其证据来源。_",
            "",
            "## 研究空白",
            "",
            "_记录证据缺口、条件冲突和待检索关键词。_",
            "",
            "## 相关主题",
            "",
            "_使用 Wikilink 连接相关主题。_",
        ]
    )


def _manual_section(path: Path) -> str:
    if not path.is_file():
        return _default_manual_section()
    text = path.read_text(encoding="utf-8", errors="replace")
    start = text.find(MANUAL_START)
    end = text.find(MANUAL_END, start + len(MANUAL_START))
    if start < 0 or end < 0:
        return _default_manual_section()
    return text[start + len(MANUAL_START) : end].strip("\n")


def _paper_wikilink(vault_root: Path, plan: PaperPlan) -> str:
    relative = plan.index_path.relative_to(vault_root).as_posix()
    return f"[[{relative}|{plan.title}]]"


def _topic_document(vault_root: Path, topic: str, members: list[PaperPlan], path: Path) -> str:
    years = sorted(
        {year for plan in members if (year := _year(plan.sidecar["zotero"])) is not None}
    )
    year_range = None
    if years:
        year_range = str(years[0]) if len(years) == 1 else f"{years[0]}–{years[-1]}"
    existing_properties = _frontmatter(path) if path.is_file() else {}
    aliases = existing_properties.get("aliases")
    properties = {
        "recordType": "literature-topic",
        "generatedBy": GENERATED_BY,
        "topic": topic,
        "aliases": aliases if isinstance(aliases, list) else [],
        "paperCount": len(members),
        "yearRange": year_range,
        "reviewStatus": existing_properties.get("reviewStatus") or "draft",
    }
    frontmatter = yaml.safe_dump(
        properties,
        allow_unicode=True,
        default_flow_style=False,
        sort_keys=False,
        width=100_000,
    ).rstrip()
    related: list[str] = []
    ordered = sorted(
        members,
        key=lambda plan: (
            -(_year(plan.sidecar["zotero"]) or 0),
            plan.title.casefold(),
        ),
    )
    for plan in ordered:
        paper = plan.sidecar["zotero"]
        details = [str(value) for value in (_year(paper), paper.get("publicationTitle")) if value]
        suffix = f" — {', '.join(details)}" if details else ""
        related.append(f"- {_paper_wikilink(vault_root, plan)}{suffix}")
    return "\n".join(
        [
            "---",
            frontmatter,
            "---",
            GENERATED_COMMENT,
            "",
            f"# {topic}",
            "",
            MANUAL_START,
            _manual_section(path),
            MANUAL_END,
            "",
            AUTO_START,
            "## 相关文献",
            "",
            *related,
            AUTO_END,
            "",
        ]
    )


def _wiki_index_document(
    vault_root: Path,
    base_path: Path,
    topic_members: dict[str, list[PaperPlan]],
    topic_paths: dict[str, Path],
    paper_count: int,
) -> str:
    properties = {
        "recordType": "literature-wiki",
        "generatedBy": GENERATED_BY,
        "title": "文献知识库",
        "paperCount": paper_count,
        "topicCount": len(topic_paths),
    }
    frontmatter = yaml.safe_dump(
        properties,
        allow_unicode=True,
        default_flow_style=False,
        sort_keys=False,
        width=100_000,
    ).rstrip()
    base_relative = base_path.relative_to(vault_root).as_posix()
    topic_lines = [
        f"- [[{topic_paths[topic].relative_to(vault_root).as_posix()}|{topic}]]"
        f"({len(topic_members[topic])} 篇)"
        for topic in sorted(topic_paths, key=str.casefold)
    ]
    return "\n".join(
        [
            "---",
            frontmatter,
            "---",
            GENERATED_COMMENT,
            "",
            "# 文献知识库",
            "",
            f"> [!info] 当前收录 {paper_count} 篇文献、{len(topic_paths)} 个共享主题。",
            "",
            "## 按主题浏览",
            "",
            f"![[{base_relative}#按主题]]",
            "",
            "## 文献总表",
            "",
            f"![[{base_relative}#文献总表]]",
            "",
            "## 主题词条",
            "",
            *(topic_lines or ["_暂时没有达到建页阈值的共享主题。_"]),
            "",
        ]
    )


def _index_document(
    directory: Path,
    sidecar: dict[str, Any],
    markdown_path: Path,
    source_properties: dict[str, Any],
    zotero_record: Path | None,
    images: list[str],
    topics: list[str],
    topic_source: str,
    topic_links: list[str],
) -> str:
    paper = sidecar["zotero"]
    publication = sidecar["publication"]
    title = str(paper.get("title") or source_properties.get("title") or "Untitled")
    authors = _creator_names(paper.get("creators"))
    raw_tags = _source_tags(paper.get("tags"))
    rendered_tags = source_properties.get("tags")
    tags = (
        _dedupe([str(item) for item in rendered_tags if isinstance(item, str)])
        if isinstance(rendered_tags, list)
        else _dedupe([tag for value in raw_tags if (tag := _obsidian_tag(value))])
    )
    zotero_key = str(paper.get("zoteroKey") or "")
    zotero_record_relative = (
        zotero_record.relative_to(directory).as_posix() if zotero_record is not None else None
    )
    properties: dict[str, Any] = {
        "recordType": "literature-index",
        "generatedBy": GENERATED_BY,
        "title": title,
        "authors": authors,
        "firstAuthor": authors[0] if authors else None,
        "year": _year(paper),
        "publicationTitle": paper.get("publicationTitle"),
        "topics": topics,
        "topicLinks": topic_links,
        "topicSource": topic_source,
        "tags": tags,
        "abstract": paper.get("abstract"),
        "cover": f"[[{images[0]}]]" if images else None,
        "imageCount": len(images),
        "mineruDocument": f"[[{markdown_path.name}]]",
        "zoteroRecord": f"[[{zotero_record_relative}]]" if zotero_record_relative else None,
        "zoteroLink": f"zotero://select/library/items/{zotero_key}" if zotero_key else None,
        "publicationId": publication.get("id"),
        "zoteroKey": zotero_key or None,
        "DOI": paper.get("doi") or paper.get("DOI"),
        "URL": paper.get("url") or paper.get("URL"),
    }
    frontmatter = yaml.safe_dump(
        properties,
        allow_unicode=True,
        default_flow_style=False,
        sort_keys=False,
        width=100_000,
    ).rstrip()
    source_lines = [f"- MinerU:[[{markdown_path.name}]]"]
    if zotero_record_relative:
        source_lines.append(f"- Zotero 笔记与批注:[[{zotero_record_relative}]]")
    if properties["zoteroLink"]:
        source_lines.append(f"- Zotero 条目:[打开]({properties['zoteroLink']})")
    gallery = "_MinerU 未输出可预览图片。_"
    if images:
        rows = [
            " ".join(f"![[{image}|240]]" for image in images[index : index + 3])
            for index in range(0, len(images), 3)
        ]
        gallery = "\n".join(rows)
    body = [
        "---",
        frontmatter,
        "---",
        GENERATED_COMMENT,
        "",
        f"# {title}",
        "",
        *(
            ["主题 Wiki:" + " · ".join(topic_links), ""]
            if topic_links
            else []
        ),
        *_callout("info", "来源", "\n".join(source_lines)),
        "",
        *_callout("abstract", "摘要", str(properties["abstract"] or "")),
        "",
        "## 图片画廊",
        "",
        *_callout("example", f"全部图片({len(images)})", gallery),
        "",
    ]
    return "\n".join(body)


def _record_filter() -> dict[str, Any]:
    return {"and": ['recordType == "literature-index"']}


def _base_document() -> str:
    columns = [
        "formula.paper_link",
        "authors",
        "year",
        "publicationTitle",
        "topics",
        "tags",
        "imageCount",
        "DOI",
        "zoteroLink",
    ]
    views: list[dict[str, Any]] = [
        {
            "type": "table",
            "name": "文献总表",
            "filters": _record_filter(),
            "order": columns,
            "sort": [{"property": "title", "direction": "ASC"}],
        },
        {
            "type": "table",
            "name": "原始产物",
            "filters": {
                "or": [
                    'recordType == "mineru-publication"',
                    'recordType == "zotero-reading-record"',
                ]
            },
            "groupBy": {"property": "recordType", "direction": "ASC"},
            "order": [
                "formula.paper_link",
                "recordType",
                "zoteroKey",
                "file.path",
                "file.mtime",
            ],
            "sort": [{"property": "file.mtime", "direction": "DESC"}],
        },
        {
            "type": "table",
            "name": "按年份",
            "filters": _record_filter(),
            "groupBy": {"property": "year", "direction": "DESC"},
            "order": columns,
            "sort": [
                {"property": "year", "direction": "DESC"},
                {"property": "title", "direction": "ASC"},
            ],
        },
        {
            "type": "table",
            "name": "按作者",
            "filters": _record_filter(),
            "groupBy": {"property": "firstAuthor", "direction": "ASC"},
            "order": columns,
            "sort": [{"property": "firstAuthor", "direction": "ASC"}],
        },
        {
            "type": "table",
            "name": "按期刊",
            "filters": _record_filter(),
            "groupBy": {"property": "publicationTitle", "direction": "ASC"},
            "order": columns,
            "sort": [{"property": "publicationTitle", "direction": "ASC"}],
        },
        {
            "type": "cards",
            "name": "按主题",
            "filters": _record_filter(),
            "groupBy": {"property": "topics", "direction": "ASC"},
            "order": [
                "formula.paper_link",
                "authors",
                "year",
                "publicationTitle",
                "topics",
                "abstract",
                "imageCount",
            ],
            "sort": [{"property": "title", "direction": "ASC"}],
            "image": "note.cover",
            "imageAspectRatio": 0.62,
            "cardSize": 260,
        },
        {
            "type": "table",
            "name": "按标签",
            "filters": _record_filter(),
            "groupBy": {"property": "tags", "direction": "ASC"},
            "order": columns,
            "sort": [{"property": "title", "direction": "ASC"}],
        },
    ]
    document: dict[str, Any] = {
        "filters": {
            "or": [
                'recordType == "literature-index"',
                'recordType == "mineru-publication"',
                'recordType == "zotero-reading-record"',
            ]
        },
        "formulas": {"paper_link": "link(file.path, title)"},
        "properties": {
            "formula.paper_link": {"displayName": "文献"},
            "recordType": {"displayName": "产物类型"},
            "authors": {"displayName": "作者"},
            "firstAuthor": {"displayName": "第一作者"},
            "year": {"displayName": "年份"},
            "publicationTitle": {"displayName": "期刊"},
            "topics": {"displayName": "主题"},
            "topicSource": {"displayName": "主题来源"},
            "tags": {"displayName": "标签"},
            "abstract": {"displayName": "摘要"},
            "imageCount": {"displayName": "图片数"},
            "DOI": {"displayName": "DOI"},
            "zoteroLink": {"displayName": "Zotero"},
            "file.path": {"displayName": "路径"},
            "file.mtime": {"displayName": "更新时间"},
        },
        "views": views,
    }
    return BASE_MARKER + "\n" + yaml.safe_dump(
        document,
        allow_unicode=True,
        default_flow_style=False,
        sort_keys=False,
        width=100_000,
    )


def _write_generated(path: Path, content: str, marker: str, *, force: bool = False) -> None:
    if path.exists():
        existing = path.read_text(encoding="utf-8", errors="replace")[:4096]
        if marker not in existing and not force:
            raise FileExistsError(f"Refusing to overwrite an unowned file: {path}")
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
    temporary.write_text(content, encoding="utf-8", newline="\n")
    os.replace(temporary, path)


def _check_generated_target(path: Path, marker: str, *, force: bool = False) -> None:
    if not path.exists() or force:
        return
    existing = path.read_text(encoding="utf-8", errors="replace")[:4096]
    if marker not in existing:
        raise FileExistsError(f"Refusing to overwrite an unowned file: {path}")


def build(
    vault_root: Path,
    *,
    base_relative: str,
    index_name: str,
    topic_prefixes: tuple[str, ...],
    ai_topics_path: Path | None,
    wiki: bool,
    wiki_index_relative: str,
    wiki_topics_relative: str,
    wiki_min_papers: int,
    force_base: bool,
    dry_run: bool,
) -> dict[str, Any]:
    root = vault_root.resolve()
    if not root.is_dir():
        raise ValueError(f"Vault root is not a directory: {root}")
    if Path(index_name).name != index_name or not index_name.lower().endswith(".md"):
        raise ValueError("--index-name must be a Markdown filename without directories")
    base_path = _confined(root, base_relative)
    if base_path.suffix.casefold() != ".base":
        raise ValueError("--base-path must end with .base")
    if wiki_min_papers < 1:
        raise ValueError("--wiki-min-papers must be at least 1")
    wiki_index_path = _confined(root, wiki_index_relative)
    if wiki and wiki_index_path.suffix.casefold() != ".md":
        raise ValueError("--wiki-index must end with .md")
    wiki_topics_root = _confined(root, wiki_topics_relative)
    if wiki and wiki_topics_root.exists() and not wiki_topics_root.is_dir():
        raise ValueError("--wiki-topics-dir must identify a directory")
    mapping = _load_ai_topics(ai_topics_path)
    records = _publication_sidecars(root)
    if not records:
        raise ValueError(f"No OmniScholar publication metadata found under {root}")

    plans: list[PaperPlan] = []
    topic_sources = {"zotero-tag": 0, "ai": 0, "missing": 0}
    without_images: list[str] = []
    without_topics: list[str] = []
    for directory, sidecar in records:
        paper = sidecar["zotero"]
        publication = sidecar["publication"]
        markdown_path, source_properties = _mineru_markdown(directory, index_name)
        zotero_record = _zotero_record(directory, index_name, markdown_path)
        images = _images(directory, markdown_path)
        raw_tags = _source_tags(paper.get("tags"))
        topics = _prefixed_topics(raw_tags, topic_prefixes)
        source = "zotero-tag"
        if not topics:
            topics = _ai_topics_for(mapping, paper, publication)
            source = "ai" if topics else "missing"
        topic_sources[source] += 1
        title = str(paper.get("title") or markdown_path.stem)
        if not images:
            without_images.append(title)
        if not topics:
            without_topics.append(title)
        plans.append(
            PaperPlan(
                directory=directory,
                sidecar=sidecar,
                markdown_path=markdown_path,
                source_properties=source_properties,
                zotero_record=zotero_record,
                images=images,
                topics=topics,
                topic_source=source,
                title=title,
                index_path=directory / index_name,
            )
        )

    topic_members: dict[str, list[PaperPlan]] = {}
    for plan in plans:
        for topic in plan.topics:
            topic_members.setdefault(topic, []).append(plan)
    shared_topics = {
        topic: members
        for topic, members in topic_members.items()
        if len(members) >= wiki_min_papers
    }
    topic_paths = _topic_paths(wiki_topics_root, list(shared_topics)) if wiki else {}

    planned: list[tuple[Path, str]] = []
    for plan in plans:
        topic_links = [
            f"[[{topic_paths[topic].relative_to(root).as_posix()}|{topic}]]"
            for topic in plan.topics
            if topic in topic_paths
        ]
        planned.append(
            (
                plan.index_path,
                _index_document(
                    plan.directory,
                    plan.sidecar,
                    plan.markdown_path,
                    plan.source_properties,
                    plan.zotero_record,
                    plan.images,
                    plan.topics,
                    plan.topic_source,
                    topic_links,
                ),
            )
        )

    wiki_documents: list[tuple[Path, str]] = []
    if wiki:
        wiki_documents.append(
            (
                wiki_index_path,
                _wiki_index_document(root, base_path, shared_topics, topic_paths, len(plans)),
            )
        )
        for topic, path in topic_paths.items():
            wiki_documents.append(
                (path, _topic_document(root, topic, shared_topics[topic], path))
            )

    all_generated_paths = [path for path, _ in (*planned, *wiki_documents)]
    if len(set(all_generated_paths)) != len(all_generated_paths):
        raise ValueError("Generated index and Wiki paths must be distinct")
    stale_wiki_pages: list[str] = []
    if wiki and wiki_topics_root.is_dir():
        expected = set(topic_paths.values())
        for path in sorted(wiki_topics_root.rglob("*.md"), key=lambda item: item.as_posix().casefold()):
            if path in expected:
                continue
            if MARKDOWN_MARKER in path.read_text(encoding="utf-8", errors="replace")[:4096]:
                stale_wiki_pages.append(path.relative_to(root).as_posix())

    if not dry_run:
        for path, _ in (*planned, *wiki_documents):
            _check_generated_target(path, MARKDOWN_MARKER)
        _check_generated_target(base_path, BASE_MARKER, force=force_base)
        for path, content in (*planned, *wiki_documents):
            _write_generated(path, content, MARKDOWN_MARKER)
        _write_generated(base_path, _base_document(), BASE_MARKER, force=force_base)
    return {
        "vaultRoot": str(root),
        "basePath": str(base_path),
        "papers": len(plans),
        "topicSources": topic_sources,
        "withoutTopics": without_topics,
        "withoutImages": without_images,
        "wiki": {
            "enabled": wiki,
            "indexPath": str(wiki_index_path) if wiki else None,
            "topicPages": len(topic_paths),
            "minimumPapers": wiki_min_papers,
            "staleTopicPages": stale_wiki_pages,
        },
        "dryRun": dry_run,
    }


def _parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("vault_root", type=Path, help="Obsidian vault containing OmniScholar outputs")
    parser.add_argument("--base-path", default="Literature.base", help="Base path relative to vault")
    parser.add_argument("--index-name", default="literature-index.md", help="Generated index filename")
    parser.add_argument(
        "--topic-prefix",
        action="append",
        dest="topic_prefixes",
        help="Zotero topic tag prefix; repeat to supply several prefixes",
    )
    parser.add_argument("--ai-topics", type=Path, help="Optional JSON topic mapping")
    parser.add_argument(
        "--wiki",
        action="store_true",
        help="Generate a Wiki hub and shared-topic pages with preserved manual sections",
    )
    parser.add_argument(
        "--wiki-index",
        default="文献知识库.md",
        help="Wiki hub path relative to the vault",
    )
    parser.add_argument(
        "--wiki-topics-dir",
        default="Literature Wiki/Topics",
        help="Generated topic-page directory relative to the vault",
    )
    parser.add_argument(
        "--wiki-min-papers",
        type=int,
        default=2,
        help="Minimum papers sharing a topic before a Wiki page is generated",
    )
    parser.add_argument(
        "--force-base",
        action="store_true",
        help="Replace an existing unowned Base; requires explicit user authorization",
    )
    parser.add_argument("--dry-run", action="store_true", help="Inspect and report without writing")
    return parser


def main() -> int:
    args = _parser().parse_args()
    prefixes = tuple(args.topic_prefixes or DEFAULT_TOPIC_PREFIXES)
    try:
        summary = build(
            args.vault_root,
            base_relative=args.base_path,
            index_name=args.index_name,
            topic_prefixes=prefixes,
            ai_topics_path=args.ai_topics,
            wiki=args.wiki,
            wiki_index_relative=args.wiki_index,
            wiki_topics_relative=args.wiki_topics_dir,
            wiki_min_papers=args.wiki_min_papers,
            force_base=args.force_base,
            dry_run=args.dry_run,
        )
    except (OSError, UnicodeError, json.JSONDecodeError, ValueError, FileExistsError) as exc:
        print(f"error: {exc}", file=sys.stderr)
        return 2
    print(json.dumps(summary, ensure_ascii=False, indent=2))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
add-product-e2e-evalpaperclipai/paperclipAdd or extend a Paperclip full-stack runner E2E workflow, fixture, matcher, or report evidence path for local or Daytona execution.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
add-runner-evalpaperclipai/paperclipAdd or extend a Paperclip Runner protocol evaluation definition, roster, assertion, or report fixture with provenance and narrow validation.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
diagnose-why-work-stoppedpaperclipai/paperclipCompatibility entry point for failed, stalled, looping, or over-recovered Paperclip work. Use fix-failed-paperclip-run for the full diagnosis, regression, repair, real verification, and green-PR workflow; honor an explicitly requested diagnosis-only scope.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
fix-failed-paperclip-runpaperclipai/paperclipDiagnose and fix failed, stuck, slow, or incorrectly blocked Paperclip tasks and runs. Use with a task/run URL or failure report to trace the root cause, prove a regression red then green, check second-order effects, verify the real workflow, and deliver a PR with passing checks.Stars: 115kUpdated Claude CodeCodexVerifiedInstall