luffysolution-svg/omnischolar / skills/obsidian-literature-base
obsidian-literature-base
VerifiedBuild and refresh an Obsidian Base and optional topic Wiki for OmniScholar MinerU publications and Zotero reading records, including bibliographic views, prefixed or AI-assisted topics, cover images, foldable image galleries, and preserved topic synthesis pages. Use when organizing multiple OmniScholar literature outputs in an Obsidian vault; do not use for reading-status workflows or modifying Zotero.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 5.0 KB |
| references/schema.md | markdown | 4.0 KB |
| scripts/build_literature_base.py | script | 31.8 KB |
scripts/build_literature_base.py
View on GitHub#!/usr/bin/env python3
"""Build an Obsidian literature Base from OmniScholar publication outputs."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import sys
import unicodedata
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import unquote
import yaml
GENERATED_BY = "obsidian-literature-base"
BASE_MARKER = f"# Generated by {GENERATED_BY}; manual edits may be overwritten."
MARKDOWN_MARKER = f"generatedBy: {GENERATED_BY}"
GENERATED_COMMENT = f"<!-- Generated by {GENERATED_BY}; manual edits may be overwritten. -->"
MANUAL_START = "<!-- MANUAL:START -->"
MANUAL_END = "<!-- MANUAL:END -->"
AUTO_START = "<!-- AUTO:START -->"
AUTO_END = "<!-- AUTO:END -->"
DEFAULT_TOPIC_PREFIXES = ("topic/", "主题/")
IMAGE_SUFFIXES = {".avif", ".gif", ".jpeg", ".jpg", ".png", ".svg", ".webp"}
MAX_MARKDOWN_BYTES = 64 * 1024 * 1024
STANDARD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\(([^)]+)\)")
WIKI_IMAGE_RE = re.compile(r"!\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")
@dataclass(slots=True)
class PaperPlan:
directory: Path
sidecar: dict[str, Any]
markdown_path: Path
source_properties: dict[str, Any]
zotero_record: Path | None
images: list[str]
topics: list[str]
topic_source: str
title: str
index_path: Path
def _confined(root: Path, relative: str) -> Path:
candidate = (root / relative).resolve()
try:
candidate.relative_to(root)
except ValueError as exc:
raise ValueError(f"Path escapes the vault root: {relative}") from exc
return candidate
def _read_json(path: Path) -> dict[str, Any]:
value = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(value, dict):
raise ValueError(f"Expected a JSON object: {path}")
return value
def _frontmatter(path: Path) -> dict[str, Any]:
with path.open("r", encoding="utf-8", errors="replace") as handle:
if handle.readline().strip() != "---":
return {}
lines: list[str] = []
size = 0
for line in handle:
size += len(line.encode("utf-8"))
if size > 256 * 1024:
return {}
if line.strip() == "---":
value = yaml.safe_load("".join(lines)) or {}
return value if isinstance(value, dict) else {}
lines.append(line)
return {}
def _publication_sidecars(vault_root: Path) -> list[tuple[Path, dict[str, Any]]]:
records: list[tuple[Path, dict[str, Any]]] = []
for path in sorted(vault_root.rglob("metadata.json"), key=lambda item: item.as_posix().casefold()):
relative_parts = path.relative_to(vault_root).parts
if ".omnischolar" in relative_parts:
continue
try:
document = _read_json(path)
except (OSError, UnicodeError, json.JSONDecodeError, ValueError):
continue
if (
document.get("schemaVersion") == 1
and isinstance(document.get("publication"), dict)
and isinstance(document.get("zotero"), dict)
):
records.append((path.parent, document))
return records
def _creator_names(value: Any) -> list[str]:
if not isinstance(value, list):
return []
result: list[str] = []
for creator in value:
if not isinstance(creator, dict):
continue
name = str(creator.get("name") or "").strip()
if not name:
name = " ".join(
part
for key in ("firstName", "lastName")
if (part := str(creator.get(key) or "").strip())
)
if name and name not in result:
result.append(name)
return result
def _source_tags(value: Any) -> list[str]:
if not isinstance(value, list):
return []
result: list[str] = []
for entry in value:
tag = entry.get("tag") if isinstance(entry, dict) else entry
if isinstance(tag, str) and tag.strip() and tag.strip() not in result:
result.append(tag.strip())
return result
def _obsidian_tag(value: str) -> str | None:
tag = unicodedata.normalize("NFKC", value).strip().lstrip("#").strip()
tag = re.sub(r"\s+", "-", tag)
tag = re.sub(r"[^\w/-]+", "-", tag, flags=re.UNICODE)
tag = re.sub(r"-{2,}", "-", tag).strip("-_/ ")
if not tag:
return None
return f"tag-{tag}" if tag.isdecimal() else tag
def _dedupe(values: list[str]) -> list[str]:
result: list[str] = []
seen: set[str] = set()
for value in values:
text = unicodedata.normalize("NFKC", str(value)).strip()
key = text.casefold()
if text and key not in seen:
seen.add(key)
result.append(text)
return result
def _prefixed_topics(tags: list[str], prefixes: tuple[str, ...]) -> list[str]:
topics: list[str] = []
for tag in tags:
folded = tag.casefold()
for prefix in prefixes:
if folded.startswith(prefix.casefold()):
topic = tag[len(prefix) :].strip(" /:-")
if topic:
topics.append(topic)
break
return _dedupe(topics)
def _year(paper: dict[str, Any]) -> int | None:
for value in (paper.get("year"), paper.get("date")):
match = re.search(r"(?:19|20)\d{2}", str(value or ""))
if match:
return int(match.group(0))
return None
def _mineru_markdown(directory: Path, index_name: str) -> tuple[Path, dict[str, Any]]:
fallback: tuple[Path, dict[str, Any]] | None = None
for path in sorted(directory.glob("*.md"), key=lambda item: item.name.casefold()):
if path.name == index_name:
continue
properties = _frontmatter(path)
if properties.get("recordType") == "mineru-publication":
return path, properties
fallback = fallback or (path, properties)
if fallback is None:
raise ValueError(f"No top-level MinerU Markdown found in {directory}")
return fallback
def _zotero_record(directory: Path, index_name: str, mineru_path: Path) -> Path | None:
for path in sorted(directory.rglob("*.md"), key=lambda item: item.as_posix().casefold()):
if path == mineru_path or path.name == index_name:
continue
if _frontmatter(path).get("recordType") == "zotero-reading-record":
return path
return None
def _image_target(raw: str) -> str:
target = raw.strip().strip("<>")
if " \"" in target or " '" in target:
target = target.split(" ", 1)[0]
return unquote(target.split("#", 1)[0].split("?", 1)[0]).replace("\\", "/")
def _images(directory: Path, markdown_path: Path) -> list[str]:
if markdown_path.stat().st_size > MAX_MARKDOWN_BYTES:
raise ValueError(f"MinerU Markdown exceeds {MAX_MARKDOWN_BYTES} bytes: {markdown_path}")
text = markdown_path.read_text(encoding="utf-8", errors="replace")
candidates = [*STANDARD_IMAGE_RE.findall(text), *WIKI_IMAGE_RE.findall(text)]
result: list[str] = []
for raw in candidates:
target = _image_target(raw)
if not target or "://" in target:
continue
try:
resolved = _confined(directory, target)
except ValueError:
continue
if resolved.is_file() and resolved.suffix.casefold() in IMAGE_SUFFIXES:
relative = resolved.relative_to(directory).as_posix()
if relative not in result:
result.append(relative)
assets = directory / "assets"
if assets.is_dir():
for path in sorted(assets.rglob("*"), key=lambda item: item.as_posix().casefold()):
if path.is_file() and path.suffix.casefold() in IMAGE_SUFFIXES:
relative = path.relative_to(directory).as_posix()
if relative not in result:
result.append(relative)
return result
def _load_ai_topics(path: Path | None) -> dict[str, list[str]]:
if path is None:
return {}
document = _read_json(path)
result: dict[str, list[str]] = {}
for key, value in document.items():
if not isinstance(key, str) or not isinstance(value, list):
raise ValueError("AI topic mappings must map string identifiers to string lists")
topics = _dedupe([item for item in value if isinstance(item, str)])
if len(topics) > 3:
raise ValueError(f"AI topics for {key!r} exceed the maximum of three")
result[key] = topics
return result
def _ai_topics_for(
mapping: dict[str, list[str]], paper: dict[str, Any], publication: dict[str, Any]
) -> list[str]:
keys = (
publication.get("id"),
paper.get("zoteroKey"),
paper.get("doi"),
paper.get("DOI"),
paper.get("title"),
)
for key in keys:
if key is not None and str(key) in mapping:
return mapping[str(key)]
return []
def _callout(kind: str, title: str, content: str) -> list[str]:
lines = [f"> [!{kind}]- {title}"]
if not content.strip():
return [*lines, "> _无内容。_"]
for line in content.splitlines():
lines.append(f"> {line}" if line else ">")
return lines
def _safe_wiki_stem(value: str) -> str:
text = unicodedata.normalize("NFC", re.sub(r"<[^>]*>", "", value))
text = re.sub(r'[<>:"/\\|?*\x00-\x1f]+', " ", text)
text = re.sub(r"\s+", " ", text).strip(" .") or "Untitled"
if re.match(r"^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)", text, re.IGNORECASE):
text = f"_{text}"
if len(text.encode("utf-8")) > 120:
suffix = f"-{hashlib.sha256(text.encode('utf-8')).hexdigest()[:8]}"
while text and len(f"{text}{suffix}".encode("utf-8")) > 120:
text = text[:-1]
text = f"{text.rstrip(' .-')}{suffix}"
return text
def _topic_paths(topics_root: Path, topics: list[str]) -> dict[str, Path]:
result: dict[str, Path] = {}
occupied: dict[str, str] = {}
for topic in sorted(topics, key=str.casefold):
stem = _safe_wiki_stem(topic)
key = stem.casefold()
if key in occupied and occupied[key] != topic:
suffix = hashlib.sha256(topic.encode("utf-8")).hexdigest()[:8]
stem = f"{stem}-{suffix}"
key = stem.casefold()
occupied[key] = topic
result[topic] = topics_root / f"{stem}.md"
return result
def _default_manual_section() -> str:
return "\n".join(
[
"## 定义与边界",
"",
"_在这里记录主题定义、范围、同义词与排除项。_",
"",
"## 核心研究问题",
"",
"_在这里记录该主题需要回答的关键问题。_",
"",
"## 当前共识",
"",
"_每条判断应链接支持它的文献。_",
"",
"## 尚存争议",
"",
"_并列记录不同解释及其证据来源。_",
"",
"## 研究空白",
"",
"_记录证据缺口、条件冲突和待检索关键词。_",
"",
"## 相关主题",
"",
"_使用 Wikilink 连接相关主题。_",
]
)
def _manual_section(path: Path) -> str:
if not path.is_file():
return _default_manual_section()
text = path.read_text(encoding="utf-8", errors="replace")
start = text.find(MANUAL_START)
end = text.find(MANUAL_END, start + len(MANUAL_START))
if start < 0 or end < 0:
return _default_manual_section()
return text[start + len(MANUAL_START) : end].strip("\n")
def _paper_wikilink(vault_root: Path, plan: PaperPlan) -> str:
relative = plan.index_path.relative_to(vault_root).as_posix()
return f"[[{relative}|{plan.title}]]"
def _topic_document(vault_root: Path, topic: str, members: list[PaperPlan], path: Path) -> str:
years = sorted(
{year for plan in members if (year := _year(plan.sidecar["zotero"])) is not None}
)
year_range = None
if years:
year_range = str(years[0]) if len(years) == 1 else f"{years[0]}–{years[-1]}"
existing_properties = _frontmatter(path) if path.is_file() else {}
aliases = existing_properties.get("aliases")
properties = {
"recordType": "literature-topic",
"generatedBy": GENERATED_BY,
"topic": topic,
"aliases": aliases if isinstance(aliases, list) else [],
"paperCount": len(members),
"yearRange": year_range,
"reviewStatus": existing_properties.get("reviewStatus") or "draft",
}
frontmatter = yaml.safe_dump(
properties,
allow_unicode=True,
default_flow_style=False,
sort_keys=False,
width=100_000,
).rstrip()
related: list[str] = []
ordered = sorted(
members,
key=lambda plan: (
-(_year(plan.sidecar["zotero"]) or 0),
plan.title.casefold(),
),
)
for plan in ordered:
paper = plan.sidecar["zotero"]
details = [str(value) for value in (_year(paper), paper.get("publicationTitle")) if value]
suffix = f" — {', '.join(details)}" if details else ""
related.append(f"- {_paper_wikilink(vault_root, plan)}{suffix}")
return "\n".join(
[
"---",
frontmatter,
"---",
GENERATED_COMMENT,
"",
f"# {topic}",
"",
MANUAL_START,
_manual_section(path),
MANUAL_END,
"",
AUTO_START,
"## 相关文献",
"",
*related,
AUTO_END,
"",
]
)
def _wiki_index_document(
vault_root: Path,
base_path: Path,
topic_members: dict[str, list[PaperPlan]],
topic_paths: dict[str, Path],
paper_count: int,
) -> str:
properties = {
"recordType": "literature-wiki",
"generatedBy": GENERATED_BY,
"title": "文献知识库",
"paperCount": paper_count,
"topicCount": len(topic_paths),
}
frontmatter = yaml.safe_dump(
properties,
allow_unicode=True,
default_flow_style=False,
sort_keys=False,
width=100_000,
).rstrip()
base_relative = base_path.relative_to(vault_root).as_posix()
topic_lines = [
f"- [[{topic_paths[topic].relative_to(vault_root).as_posix()}|{topic}]]"
f"({len(topic_members[topic])} 篇)"
for topic in sorted(topic_paths, key=str.casefold)
]
return "\n".join(
[
"---",
frontmatter,
"---",
GENERATED_COMMENT,
"",
"# 文献知识库",
"",
f"> [!info] 当前收录 {paper_count} 篇文献、{len(topic_paths)} 个共享主题。",
"",
"## 按主题浏览",
"",
f"![[{base_relative}#按主题]]",
"",
"## 文献总表",
"",
f"![[{base_relative}#文献总表]]",
"",
"## 主题词条",
"",
*(topic_lines or ["_暂时没有达到建页阈值的共享主题。_"]),
"",
]
)
def _index_document(
directory: Path,
sidecar: dict[str, Any],
markdown_path: Path,
source_properties: dict[str, Any],
zotero_record: Path | None,
images: list[str],
topics: list[str],
topic_source: str,
topic_links: list[str],
) -> str:
paper = sidecar["zotero"]
publication = sidecar["publication"]
title = str(paper.get("title") or source_properties.get("title") or "Untitled")
authors = _creator_names(paper.get("creators"))
raw_tags = _source_tags(paper.get("tags"))
rendered_tags = source_properties.get("tags")
tags = (
_dedupe([str(item) for item in rendered_tags if isinstance(item, str)])
if isinstance(rendered_tags, list)
else _dedupe([tag for value in raw_tags if (tag := _obsidian_tag(value))])
)
zotero_key = str(paper.get("zoteroKey") or "")
zotero_record_relative = (
zotero_record.relative_to(directory).as_posix() if zotero_record is not None else None
)
properties: dict[str, Any] = {
"recordType": "literature-index",
"generatedBy": GENERATED_BY,
"title": title,
"authors": authors,
"firstAuthor": authors[0] if authors else None,
"year": _year(paper),
"publicationTitle": paper.get("publicationTitle"),
"topics": topics,
"topicLinks": topic_links,
"topicSource": topic_source,
"tags": tags,
"abstract": paper.get("abstract"),
"cover": f"[[{images[0]}]]" if images else None,
"imageCount": len(images),
"mineruDocument": f"[[{markdown_path.name}]]",
"zoteroRecord": f"[[{zotero_record_relative}]]" if zotero_record_relative else None,
"zoteroLink": f"zotero://select/library/items/{zotero_key}" if zotero_key else None,
"publicationId": publication.get("id"),
"zoteroKey": zotero_key or None,
"DOI": paper.get("doi") or paper.get("DOI"),
"URL": paper.get("url") or paper.get("URL"),
}
frontmatter = yaml.safe_dump(
properties,
allow_unicode=True,
default_flow_style=False,
sort_keys=False,
width=100_000,
).rstrip()
source_lines = [f"- MinerU:[[{markdown_path.name}]]"]
if zotero_record_relative:
source_lines.append(f"- Zotero 笔记与批注:[[{zotero_record_relative}]]")
if properties["zoteroLink"]:
source_lines.append(f"- Zotero 条目:[打开]({properties['zoteroLink']})")
gallery = "_MinerU 未输出可预览图片。_"
if images:
rows = [
" ".join(f"![[{image}|240]]" for image in images[index : index + 3])
for index in range(0, len(images), 3)
]
gallery = "\n".join(rows)
body = [
"---",
frontmatter,
"---",
GENERATED_COMMENT,
"",
f"# {title}",
"",
*(
["主题 Wiki:" + " · ".join(topic_links), ""]
if topic_links
else []
),
*_callout("info", "来源", "\n".join(source_lines)),
"",
*_callout("abstract", "摘要", str(properties["abstract"] or "")),
"",
"## 图片画廊",
"",
*_callout("example", f"全部图片({len(images)})", gallery),
"",
]
return "\n".join(body)
def _record_filter() -> dict[str, Any]:
return {"and": ['recordType == "literature-index"']}
def _base_document() -> str:
columns = [
"formula.paper_link",
"authors",
"year",
"publicationTitle",
"topics",
"tags",
"imageCount",
"DOI",
"zoteroLink",
]
views: list[dict[str, Any]] = [
{
"type": "table",
"name": "文献总表",
"filters": _record_filter(),
"order": columns,
"sort": [{"property": "title", "direction": "ASC"}],
},
{
"type": "table",
"name": "原始产物",
"filters": {
"or": [
'recordType == "mineru-publication"',
'recordType == "zotero-reading-record"',
]
},
"groupBy": {"property": "recordType", "direction": "ASC"},
"order": [
"formula.paper_link",
"recordType",
"zoteroKey",
"file.path",
"file.mtime",
],
"sort": [{"property": "file.mtime", "direction": "DESC"}],
},
{
"type": "table",
"name": "按年份",
"filters": _record_filter(),
"groupBy": {"property": "year", "direction": "DESC"},
"order": columns,
"sort": [
{"property": "year", "direction": "DESC"},
{"property": "title", "direction": "ASC"},
],
},
{
"type": "table",
"name": "按作者",
"filters": _record_filter(),
"groupBy": {"property": "firstAuthor", "direction": "ASC"},
"order": columns,
"sort": [{"property": "firstAuthor", "direction": "ASC"}],
},
{
"type": "table",
"name": "按期刊",
"filters": _record_filter(),
"groupBy": {"property": "publicationTitle", "direction": "ASC"},
"order": columns,
"sort": [{"property": "publicationTitle", "direction": "ASC"}],
},
{
"type": "cards",
"name": "按主题",
"filters": _record_filter(),
"groupBy": {"property": "topics", "direction": "ASC"},
"order": [
"formula.paper_link",
"authors",
"year",
"publicationTitle",
"topics",
"abstract",
"imageCount",
],
"sort": [{"property": "title", "direction": "ASC"}],
"image": "note.cover",
"imageAspectRatio": 0.62,
"cardSize": 260,
},
{
"type": "table",
"name": "按标签",
"filters": _record_filter(),
"groupBy": {"property": "tags", "direction": "ASC"},
"order": columns,
"sort": [{"property": "title", "direction": "ASC"}],
},
]
document: dict[str, Any] = {
"filters": {
"or": [
'recordType == "literature-index"',
'recordType == "mineru-publication"',
'recordType == "zotero-reading-record"',
]
},
"formulas": {"paper_link": "link(file.path, title)"},
"properties": {
"formula.paper_link": {"displayName": "文献"},
"recordType": {"displayName": "产物类型"},
"authors": {"displayName": "作者"},
"firstAuthor": {"displayName": "第一作者"},
"year": {"displayName": "年份"},
"publicationTitle": {"displayName": "期刊"},
"topics": {"displayName": "主题"},
"topicSource": {"displayName": "主题来源"},
"tags": {"displayName": "标签"},
"abstract": {"displayName": "摘要"},
"imageCount": {"displayName": "图片数"},
"DOI": {"displayName": "DOI"},
"zoteroLink": {"displayName": "Zotero"},
"file.path": {"displayName": "路径"},
"file.mtime": {"displayName": "更新时间"},
},
"views": views,
}
return BASE_MARKER + "\n" + yaml.safe_dump(
document,
allow_unicode=True,
default_flow_style=False,
sort_keys=False,
width=100_000,
)
def _write_generated(path: Path, content: str, marker: str, *, force: bool = False) -> None:
if path.exists():
existing = path.read_text(encoding="utf-8", errors="replace")[:4096]
if marker not in existing and not force:
raise FileExistsError(f"Refusing to overwrite an unowned file: {path}")
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
temporary.write_text(content, encoding="utf-8", newline="\n")
os.replace(temporary, path)
def _check_generated_target(path: Path, marker: str, *, force: bool = False) -> None:
if not path.exists() or force:
return
existing = path.read_text(encoding="utf-8", errors="replace")[:4096]
if marker not in existing:
raise FileExistsError(f"Refusing to overwrite an unowned file: {path}")
def build(
vault_root: Path,
*,
base_relative: str,
index_name: str,
topic_prefixes: tuple[str, ...],
ai_topics_path: Path | None,
wiki: bool,
wiki_index_relative: str,
wiki_topics_relative: str,
wiki_min_papers: int,
force_base: bool,
dry_run: bool,
) -> dict[str, Any]:
root = vault_root.resolve()
if not root.is_dir():
raise ValueError(f"Vault root is not a directory: {root}")
if Path(index_name).name != index_name or not index_name.lower().endswith(".md"):
raise ValueError("--index-name must be a Markdown filename without directories")
base_path = _confined(root, base_relative)
if base_path.suffix.casefold() != ".base":
raise ValueError("--base-path must end with .base")
if wiki_min_papers < 1:
raise ValueError("--wiki-min-papers must be at least 1")
wiki_index_path = _confined(root, wiki_index_relative)
if wiki and wiki_index_path.suffix.casefold() != ".md":
raise ValueError("--wiki-index must end with .md")
wiki_topics_root = _confined(root, wiki_topics_relative)
if wiki and wiki_topics_root.exists() and not wiki_topics_root.is_dir():
raise ValueError("--wiki-topics-dir must identify a directory")
mapping = _load_ai_topics(ai_topics_path)
records = _publication_sidecars(root)
if not records:
raise ValueError(f"No OmniScholar publication metadata found under {root}")
plans: list[PaperPlan] = []
topic_sources = {"zotero-tag": 0, "ai": 0, "missing": 0}
without_images: list[str] = []
without_topics: list[str] = []
for directory, sidecar in records:
paper = sidecar["zotero"]
publication = sidecar["publication"]
markdown_path, source_properties = _mineru_markdown(directory, index_name)
zotero_record = _zotero_record(directory, index_name, markdown_path)
images = _images(directory, markdown_path)
raw_tags = _source_tags(paper.get("tags"))
topics = _prefixed_topics(raw_tags, topic_prefixes)
source = "zotero-tag"
if not topics:
topics = _ai_topics_for(mapping, paper, publication)
source = "ai" if topics else "missing"
topic_sources[source] += 1
title = str(paper.get("title") or markdown_path.stem)
if not images:
without_images.append(title)
if not topics:
without_topics.append(title)
plans.append(
PaperPlan(
directory=directory,
sidecar=sidecar,
markdown_path=markdown_path,
source_properties=source_properties,
zotero_record=zotero_record,
images=images,
topics=topics,
topic_source=source,
title=title,
index_path=directory / index_name,
)
)
topic_members: dict[str, list[PaperPlan]] = {}
for plan in plans:
for topic in plan.topics:
topic_members.setdefault(topic, []).append(plan)
shared_topics = {
topic: members
for topic, members in topic_members.items()
if len(members) >= wiki_min_papers
}
topic_paths = _topic_paths(wiki_topics_root, list(shared_topics)) if wiki else {}
planned: list[tuple[Path, str]] = []
for plan in plans:
topic_links = [
f"[[{topic_paths[topic].relative_to(root).as_posix()}|{topic}]]"
for topic in plan.topics
if topic in topic_paths
]
planned.append(
(
plan.index_path,
_index_document(
plan.directory,
plan.sidecar,
plan.markdown_path,
plan.source_properties,
plan.zotero_record,
plan.images,
plan.topics,
plan.topic_source,
topic_links,
),
)
)
wiki_documents: list[tuple[Path, str]] = []
if wiki:
wiki_documents.append(
(
wiki_index_path,
_wiki_index_document(root, base_path, shared_topics, topic_paths, len(plans)),
)
)
for topic, path in topic_paths.items():
wiki_documents.append(
(path, _topic_document(root, topic, shared_topics[topic], path))
)
all_generated_paths = [path for path, _ in (*planned, *wiki_documents)]
if len(set(all_generated_paths)) != len(all_generated_paths):
raise ValueError("Generated index and Wiki paths must be distinct")
stale_wiki_pages: list[str] = []
if wiki and wiki_topics_root.is_dir():
expected = set(topic_paths.values())
for path in sorted(wiki_topics_root.rglob("*.md"), key=lambda item: item.as_posix().casefold()):
if path in expected:
continue
if MARKDOWN_MARKER in path.read_text(encoding="utf-8", errors="replace")[:4096]:
stale_wiki_pages.append(path.relative_to(root).as_posix())
if not dry_run:
for path, _ in (*planned, *wiki_documents):
_check_generated_target(path, MARKDOWN_MARKER)
_check_generated_target(base_path, BASE_MARKER, force=force_base)
for path, content in (*planned, *wiki_documents):
_write_generated(path, content, MARKDOWN_MARKER)
_write_generated(base_path, _base_document(), BASE_MARKER, force=force_base)
return {
"vaultRoot": str(root),
"basePath": str(base_path),
"papers": len(plans),
"topicSources": topic_sources,
"withoutTopics": without_topics,
"withoutImages": without_images,
"wiki": {
"enabled": wiki,
"indexPath": str(wiki_index_path) if wiki else None,
"topicPages": len(topic_paths),
"minimumPapers": wiki_min_papers,
"staleTopicPages": stale_wiki_pages,
},
"dryRun": dry_run,
}
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("vault_root", type=Path, help="Obsidian vault containing OmniScholar outputs")
parser.add_argument("--base-path", default="Literature.base", help="Base path relative to vault")
parser.add_argument("--index-name", default="literature-index.md", help="Generated index filename")
parser.add_argument(
"--topic-prefix",
action="append",
dest="topic_prefixes",
help="Zotero topic tag prefix; repeat to supply several prefixes",
)
parser.add_argument("--ai-topics", type=Path, help="Optional JSON topic mapping")
parser.add_argument(
"--wiki",
action="store_true",
help="Generate a Wiki hub and shared-topic pages with preserved manual sections",
)
parser.add_argument(
"--wiki-index",
default="文献知识库.md",
help="Wiki hub path relative to the vault",
)
parser.add_argument(
"--wiki-topics-dir",
default="Literature Wiki/Topics",
help="Generated topic-page directory relative to the vault",
)
parser.add_argument(
"--wiki-min-papers",
type=int,
default=2,
help="Minimum papers sharing a topic before a Wiki page is generated",
)
parser.add_argument(
"--force-base",
action="store_true",
help="Replace an existing unowned Base; requires explicit user authorization",
)
parser.add_argument("--dry-run", action="store_true", help="Inspect and report without writing")
return parser
def main() -> int:
args = _parser().parse_args()
prefixes = tuple(args.topic_prefixes or DEFAULT_TOPIC_PREFIXES)
try:
summary = build(
args.vault_root,
base_relative=args.base_path,
index_name=args.index_name,
topic_prefixes=prefixes,
ai_topics_path=args.ai_topics,
wiki=args.wiki,
wiki_index_relative=args.wiki_index,
wiki_topics_relative=args.wiki_topics_dir,
wiki_min_papers=args.wiki_min_papers,
force_base=args.force_base,
dry_run=args.dry_run,
)
except (OSError, UnicodeError, json.JSONDecodeError, ValueError, FileExistsError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
print(json.dumps(summary, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| add-product-e2e-evalpaperclipai/paperclipAdd or extend a Paperclip full-stack runner E2E workflow, fixture, matcher, or report evidence path for local or Daytona execution. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| add-runner-evalpaperclipai/paperclipAdd or extend a Paperclip Runner protocol evaluation definition, roster, assertion, or report fixture with provenance and narrow validation. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| diagnose-why-work-stoppedpaperclipai/paperclipCompatibility entry point for failed, stalled, looping, or over-recovered Paperclip work. Use fix-failed-paperclip-run for the full diagnosis, regression, repair, real verification, and green-PR workflow; honor an explicitly requested diagnosis-only scope. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| fix-failed-paperclip-runpaperclipai/paperclipDiagnose and fix failed, stuck, slow, or incorrectly blocked Paperclip tasks and runs. Use with a task/run URL or failure report to trace the root cause, prove a regression red then green, check second-order effects, verify the real workflow, and deliver a PR with passing checks. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |