Dianel555/DSkills / skills/literature-searcher
literature-searcher
VerifiedSearch CrossRef, OpenAlex, PubMed, Semantic Scholar, and optional Scopus; deduplicate results, download open-access PDFs by DOI, classify papers, monitor new results, and analyze coverage. Use when asked to search literature, monitor a topic, download an open-access paper, classify papers, or analyze literature gaps.
| File | Kind | Size |
|---|---|---|
| .env.example | other | 743 B |
| README.md | markdown | 3.7 KB |
| SKILL.md | skill md | 2.9 KB |
| literature_searcher/__init__.py | script | 3.1 KB |
| literature_searcher/classifier.py | script | 11.2 KB |
| literature_searcher/crossref_searcher.py | script | 2.8 KB |
| literature_searcher/downloader.py | script | 5.3 KB |
| literature_searcher/insight.py | script | 6.0 KB |
| literature_searcher/monitor.py | script | 14.3 KB |
| literature_searcher/openalex_searcher.py | script | 3.0 KB |
| literature_searcher/pubmed_searcher.py | script | 6.1 KB |
| literature_searcher/scopus_searcher.py | script | 2.1 KB |
| literature_searcher/searcher.py | script | 3.4 KB |
| literature_searcher/semantic_scholar_searcher.py | script | 3.7 KB |
| monitor_config.json.template | other | 443 B |
| tests/test_literature_searcher.py | script | 9.7 KB |
| tests/test_retry_budget.py | script | 898 B |
literature_searcher/downloader.py
View on GitHub"""
PDF下载器: Unpaywall + scihub-cli
"""
import json
import shutil
import subprocess
import tempfile
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Optional, Tuple
from . import (
UNPAYWALL_URL, USER_AGENT, LITERATURE_EMAIL, UVX_FALLBACK_CMD,
generate_random_email, unique_path, safe_filename,
)
class Downloader:
"""PDF 下载器(Unpaywall + scihub-cli)。"""
@staticmethod
def _download_with_url(url: str, outdir: Path, filename_base: str,
timeout: int = 60, email: str = None) -> Tuple[bool, Optional[str], Optional[str]]:
"""直接从URL下载PDF"""
try:
headers = {"User-Agent": USER_AGENT}
if email:
headers["From"] = email
req = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(req, timeout=timeout) as resp:
content_type = resp.headers.get("Content-Type", "")
if "pdf" not in content_type.lower() and "octet-stream" not in content_type.lower():
return False, url, f"非PDF: {content_type}"
data = resp.read()
if len(data) < 1000 or data[:4] != b"%PDF":
return False, url, "内容不是有效PDF"
outdir.mkdir(parents=True, exist_ok=True)
target = unique_path(outdir / f"{filename_base}.pdf")
with open(target, "wb") as f:
f.write(data)
return True, str(target), None
except Exception as e:
return False, url, str(e)
@staticmethod
def _unpaywall_doi(doi: str, outdir: Path, filename_base: str,
email: str, timeout: int = 60) -> Tuple[bool, Optional[str], Optional[str]]:
"""通过Unpaywall获取免费PDF"""
try:
url = f"{UNPAYWALL_URL}/{doi}?email={email}"
req = urllib.request.Request(url)
req.add_header("User-Agent", USER_AGENT)
with urllib.request.urlopen(req, timeout=20) as resp:
data = json.loads(resp.read().decode("utf-8"))
oa = data.get("best_oa_location")
if not oa:
return False, None, "无开放获取版本"
pdf_url = oa.get("url_for_pdf") or oa.get("url")
if not pdf_url:
return False, None, "无PDF链接"
return Downloader._download_with_url(pdf_url, outdir, filename_base, timeout, email)
except Exception as e:
return False, None, f"Unpaywall错误: {e}"
@staticmethod
def _scihub_fallback(doi: str, outdir: Path, filename_base: str,
timeout: int = 180) -> Tuple[bool, Optional[str], Optional[str]]:
"""通过scihub-cli下载"""
cmd = None
if shutil.which("scihub-cli"):
cmd = ["scihub-cli"]
elif shutil.which("uvx"):
cmd = UVX_FALLBACK_CMD.copy()
else:
return False, None, "scihub-cli未安装"
with tempfile.TemporaryDirectory() as td:
tmp = Path(td)
input_file = tmp / "input.txt"
input_file.write_text(f"{doi}\n")
tmp_out = tmp / "out"
tmp_out.mkdir()
full_cmd = cmd + [str(input_file), "-o", str(tmp_out), "-t", "30", "-r", "2", "-p", "1"]
try:
proc = subprocess.run(full_cmd, capture_output=True, text=True,
timeout=timeout, check=False)
except subprocess.TimeoutExpired:
return False, None, "scihub超时"
except Exception as e:
return False, None, f"scihub执行错误: {e}"
pdfs = list(tmp_out.rglob("*.pdf"))
if not pdfs:
return False, None, "scihub未返回PDF"
best = max(pdfs, key=lambda p: p.stat().st_size)
if best.read_bytes()[:4] != b"%PDF":
return False, None, "无效PDF"
outdir.mkdir(parents=True, exist_ok=True)
target = unique_path(outdir / f"{filename_base}.pdf")
shutil.copy2(best, target)
return True, str(target), None
@staticmethod
def download_by_doi(doi: str, outdir: str = "./downloads", email: str = None,
scihub_fallback: str = "auto",
timeout: int = 120) -> dict:
"""通过DOI下载PDF (Unpaywall → scihub)"""
outdir = Path(outdir)
email = email or LITERATURE_EMAIL or generate_random_email()
filename_base = safe_filename(doi, "paper")
# 1) Unpaywall
success, path_or_url, err = Downloader._unpaywall_doi(
doi, outdir, filename_base, email, timeout
)
if success:
return {"doi": doi, "status": "downloaded", "source": "unpaywall", "path": path_or_url}
# 2) SciHub fallback
if scihub_fallback != "off":
success, path, err2 = Downloader._scihub_fallback(doi, outdir, filename_base, timeout)
if success:
return {"doi": doi, "status": "downloaded", "source": "scihub", "path": path}
return {"doi": doi, "status": "failed", "unpaywall_error": err, "scihub_error": err2}
return {"doi": doi, "status": "failed", "error": err}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |