Skip to content

Yuan1z0825/nature-skills / skills/nature-academic-search

nature-academic-search

Flagged

Search literature across sources, verify or manage citations, and build MeSH strategies or citation-impact audits. Use for 文献检索、引文核对、参考文献管理、严格他引 and evidence-backed citer profiles; not for translating a paper or drafting manuscript prose.

FileKindSize
README.mdmarkdown2.6 KB
README_EN.mdmarkdown3.0 KB
SKILL.mdexecutableskill md3.5 KB
agents/openai.yamlconfig265 B
config/mcp-snippet.jsonconfig502 B
config/settings-snippet.jsonconfig59 B
config/triggers-academic-search.tomlconfig978 B
install.shexecutablescript4.3 KB
manifest.yamlconfig3.1 KB
mcp-server/README.mdmarkdown1.9 KB
mcp-server/academic_search_server.pyscript20.9 KB
mcp-server/config.tomlconfig249 B
mcp-server/requirements.txtmanifest134 B
mcp-server/sources/__init__.pyscript352 B
mcp-server/sources/arxiv.pyscript11.4 KB
mcp-server/sources/crossref.pyscript8.0 KB
mcp-server/sources/elsevier_common.pyscript2.4 KB
mcp-server/sources/pubmed.pyscript13.7 KB
mcp-server/sources/sciencedirect.pyscript6.9 KB
mcp-server/sources/scopus.pyscript19.0 KB
mcp-server/tests/__init__.pyscript40 B
mcp-server/tests/test_arxiv_error_entries.pyscript1.4 KB
mcp-server/tests/test_citation_errors.pyscript1.6 KB
mcp-server/tests/test_crossref_version_relations.pyscript3.3 KB
mcp-server/tests/test_elsevier_live.pyscript2.6 KB
mcp-server/tests/test_format_converter_hardening.pyscript4.2 KB
mcp-server/tests/test_mcp_tools.pyscript2.6 KB
mcp-server/tests/test_pubmed_parsing.pyscript4.4 KB
mcp-server/tests/test_scopus_empty_result.pyscript1006 B
mcp-server/tests/test_script_converters.pyscript2.9 KB
mcp-server/tests/test_sources.pyscript30.5 KB
mcp-server/utils/__init__.pyscript359 B
mcp-server/utils/config.pyscript1.4 KB
mcp-server/utils/errors.pyscript620 B
mcp-server/utils/logging.pyscript1.1 KB
references/citation-parser.mdmarkdown2.7 KB
references/dedup-engine.mdmarkdown1.9 KB
references/pubmed-28344011.bibexecutablescript3.3 KB
references/pubmed-28344011.nbibexecutablescript11.9 KB
references/pubmed-28344011.risexecutablescript3.4 KB
references/ris-bibtex-format.mdexecutablescript4.5 KB
references/search-strategy.mdexecutablescript2.0 KB
references/source-tiers.mdmarkdown2.7 KB
references/workflows/wf1-multi-source-search.mdmarkdown1.9 KB
references/workflows/wf2-citation-verification.mdmarkdown1.6 KB
references/workflows/wf3-mesh-strategy.mdmarkdown538 B
references/workflows/wf4-citation-file-mgmt.mdmarkdown1.9 KB
references/workflows/wf5-reference-mgmt.mdmarkdown1.5 KB
references/workflows/wf6-strict-other-citation-impact-audit.mdmarkdown11.2 KB
scripts/academic_search.pyscript23.2 KB
scripts/converters.pyscript18.5 KB
scripts/format-converter.pyscript19.5 KB
scripts/preflight.pyscript3.7 KB
static/core/routing-and-ops.mdmarkdown6.1 KB
static/core/tools.mdmarkdown3.0 KB

scripts/format-converter.py

View on GitHub
# -*- coding: utf-8 -*-
"""
Multi-source citation downloader with format conversion.
Sources: PubMed (NCBI E-utilities), CrossRef (REST API), arXiv (Atom API).
Outputs: .nbib (PubMed only), .ris, .bib, .enw.

Usage:
  python format-converter.py --pmid 28344011
  python format-converter.py --pmid 28344011,10645439 --format ris
  python format-converter.py --doi 10.1038/nature14539 --format bib
  python format-converter.py --arxiv 1706.03762 --format ris
  python format-converter.py --query "TB-Profiler AND Bioinformatics[Journal]"
  python format-converter.py --input refs.txt
  python format-converter.py --input refs.txt --format ris
  python format-converter.py --interactive

refs.txt format:
  PMID:28344011
  DOI:10.1038/nature14539
  ARXIV:1706.03762
  QUERY:TB-Profiler AND Bioinformatics[Journal]
  AUTHOR:Dheda TITLE:drug-resistant tuberculosis
  # Lines starting with # are comments
"""

import os
import sys
import time
import json
import argparse
import defusedxml.ElementTree as ET
import requests
from urllib.parse import urlencode, urlparse
from urllib.request import urlopen

from converters import (
    convert_from_medline,
    convert_from_crossref,
    convert_from_arxiv,
    get_extension,
)

EUTILS_BASE = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
CROSSREF_BASE = "https://api.crossref.org/works"
ARXIV_BASE = "https://export.arxiv.org/api/query"
DELAY = 0.5
ALLOWED_HOSTS = {"eutils.ncbi.nlm.nih.gov", "api.crossref.org", "export.arxiv.org"}


def _is_allowed_url(url):
    """Only allow requests to the known, hardcoded API hosts (mitigates SSRF)."""
    return urlparse(url).hostname in ALLOWED_HOSTS


# ── PubMed ──────────────────────────────────────────────────────

def esearch(query, max_results=5):
    params = {"db": "pubmed", "term": query, "retmax": max_results, "retmode": "xml"}
    url = f"{EUTILS_BASE}/esearch.fcgi?{urlencode(params)}"
    try:
        if not _is_allowed_url(url):
            print("  ESearch error: URL host not in allowlist")
            return []
        resp = requests.get(url, timeout=30)
        xml_data = resp.text
        root = ET.fromstring(xml_data)
        id_list = root.find("IdList")
        if id_list is not None:
            return [e.text for e in id_list.findall("Id")]
        return []
    except Exception as e:
        print(f"  ESearch error: {e}")
        return []


def efetch_medline(pmid, retries=1):
    params = {"db": "pubmed", "id": pmid, "rettype": "medline", "retmode": "text"}
    url = f"{EUTILS_BASE}/efetch.fcgi?{urlencode(params)}"
    if not _is_allowed_url(url):
        print(f"  EFetch error for PMID {pmid}: URL host not in allowlist")
        return None
    attempts = max(1, retries)  # --retry 0 must still try once
    for attempt in range(attempts):
        try:
            with urlopen(url, timeout=30) as resp:
                return resp.read().decode("utf-8")
        except Exception as e:
            if attempt == attempts - 1:
                print(f"  EFetch error for PMID {pmid}: {e}")
                return None
            time.sleep(DELAY * (attempt + 1))


def download_pubmed(pmid, output_dir, fmt, retries=1):
    """Download citation by PMID. Returns (success, filename_or_error)."""
    pmid = pmid.strip()
    if not pmid:
        return False, "Empty PMID"

    print(f"  Downloading PMID: {pmid}")
    time.sleep(DELAY)
    nbib_text = efetch_medline(pmid, retries=retries)

    if not nbib_text or not nbib_text.strip():
        return False, f"PMID {pmid} not found or empty response"

    content = convert_from_medline(nbib_text, fmt)
    ext = get_extension(fmt)
    safe_pmid = pmid.replace("/", "_")[:60]
    filename = f"pubmed-{safe_pmid}{ext}"
    filepath = os.path.join(output_dir, filename)
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(content)

    # Extract title for display
    for line in nbib_text.split("\n"):
        if line.startswith("TI  -"):
            print(f"  Title: {line[5:].strip()[:100]}")
            break

    print(f"  Saved: {filename} (format: {fmt})")
    return True, filename


def search_pubmed(query, output_dir, fmt, retries=1):
    print(f"  Searching: {query[:80]}...")
    time.sleep(DELAY)
    pmids = esearch(query)
    if not pmids:
        return False, f"No results for query: {query[:60]}"
    pmid = pmids[0]
    print(f"  Found PMID: {pmid}")
    return download_pubmed(pmid, output_dir, fmt, retries=retries)


# ── CrossRef ────────────────────────────────────────────────────

def download_crossref(doi, output_dir, fmt, retries=1):
    """Download citation by DOI from CrossRef. Returns (success, filename_or_error)."""
    doi = doi.strip()
    if not doi:
        return False, "Empty DOI"

    print(f"  Downloading DOI: {doi}")
    time.sleep(DELAY)
    url = f"{CROSSREF_BASE}/{doi}"
    if not _is_allowed_url(url):
        return False, f"CrossRef URL host not in allowlist: {url}"
    attempts = max(1, retries)  # --retry 0 must still try once
    for attempt in range(attempts):
        try:
            with urlopen(url, timeout=30) as resp:
                data = json.loads(resp.read().decode("utf-8"))
            break
        except Exception as e:
            if attempt == attempts - 1:
                return False, f"CrossRef API error for DOI {doi}: {e}"
            time.sleep(DELAY * (attempt + 1))

    content = convert_from_crossref(data, fmt)
    ext = get_extension(fmt)
    safe_doi = doi.replace("/", "_").replace(".", "_")[:60]
    filename = f"crossref-{safe_doi}{ext}"
    filepath = os.path.join(output_dir, filename)
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(content)

    msg = data.get("message", data)
    title = msg.get("title", [])
    if title:
        print(f"  Title: {title[0][:100]}")

    print(f"  Saved: {filename} (format: {fmt})")
    return True, filename


# ── arXiv ───────────────────────────────────────────────────────

def download_arxiv(arxiv_id, output_dir, fmt, retries=1):
    """Download citation by arXiv ID. Returns (success, filename_or_error)."""
    arxiv_id = arxiv_id.strip()
    if not arxiv_id:
        return False, "Empty arXiv ID"

    print(f"  Downloading arXiv: {arxiv_id}")
    time.sleep(DELAY)
    params = {"id_list": arxiv_id, "max_results": 1}
    url = f"{ARXIV_BASE}?{urlencode(params)}"
    if not _is_allowed_url(url):
        return False, f"arXiv URL host not in allowlist: {url}"
    attempts = max(1, retries)  # --retry 0 must still try once
    for attempt in range(attempts):
        try:
            with urlopen(url, timeout=30) as resp:
                xml_data = resp.read().decode("utf-8")
            break
        except Exception as e:
            if attempt == attempts - 1:
                return False, f"arXiv API error for ID {arxiv_id}: {e}"
            time.sleep(DELAY * (attempt + 1))

    root = ET.fromstring(xml_data)
    content = convert_from_arxiv(root, fmt)
    if not content:
        return False, f"arXiv ID {arxiv_id}: no entry found in response"

    ext = get_extension(fmt)
    safe_id = arxiv_id.replace("/", "_")[:60]
    filename = f"arxiv-{safe_id}{ext}"
    filepath = os.path.join(output_dir, filename)
    with open(filepath, "w", encoding="utf-8") as f:
        f.write(content)

    print(f"  Saved: {filename} (format: {fmt})")
    return True, filename


# ── Input parsing ───────────────────────────────────────────────

def parse_input_line(line):
    """Parse a single input line into (type, value)."""
    line = line.strip()
    if not line or line.startswith("#"):
        return None, None

    upper = line.upper()
    if upper.startswith("PMID:"):
        return "pmid", line[5:].strip()
    if upper.startswith("DOI:"):
        return "doi", line[4:].strip()
    if upper.startswith("ARXIV:"):
        return "arxiv", line[6:].strip()
    if upper.startswith("QUERY:"):
        return "query", line[6:].strip()
    if upper.startswith("AUTHOR:"):
        parts = line.split("TITLE:", 1)
        author = parts[0][7:].strip()
        title = parts[1].strip() if len(parts) > 1 else ""
        return "author_title", (author, title)

    # Default: free-text search query
    return "query", line


def process_entry(entry_type, value, output_dir, fmt, retries=1):
    if entry_type == "pmid":
        return download_pubmed(value, output_dir, fmt, retries=retries)
    elif entry_type == "doi":
        return download_crossref(value, output_dir, fmt, retries=retries)
    elif entry_type == "arxiv":
        return download_arxiv(value, output_dir, fmt, retries=retries)
    elif entry_type == "author_title":
        author, title = value
        query_parts = []
        if author:
            query_parts.append(f"{author}[Author]")
        if title:
            query_parts.append(f"{title}[Title]")
        query = " AND ".join(query_parts) if query_parts else ""
        if not query:
            return False, "Empty author and title"
        return search_pubmed(query, output_dir, fmt, retries=retries)
    elif entry_type == "query":
        return search_pubmed(value, output_dir, fmt, retries=retries)
    return False, f"Unknown entry type: {entry_type}"


def process_file(input_file, output_dir, fmt, retries=1):
    if not os.path.exists(input_file):
        print(f"Error: Input file not found: {input_file}")
        return 0, 0, [f"File not found: {input_file}"]

    with open(input_file, "r", encoding="utf-8") as f:
        lines = f.readlines()

    success, failed, errors = 0, 0, []
    for i, line in enumerate(lines, 1):
        entry_type, value = parse_input_line(line)
        if entry_type is None:
            continue
        print(f"\n[Line {i}] Processing: {line.strip()[:60]}")
        ok, result = process_entry(entry_type, value, output_dir, fmt, retries=retries)
        if ok:
            success += 1
        else:
            failed += 1
            errors.append(f"Line {i}: {result}")
    return success, failed, errors


def interactive_mode(output_dir, fmt, retries=1):
    print(f"Interactive mode (format: {fmt}) - enter references (one per line, empty line to finish):")
    print("Formats: PMID:12345 | DOI:10.xxx | ARXIV:2301.xxx | AUTHOR:Name TITLE:keywords | QUERY:...")
    print("-" * 60)

    success, failed, errors = 0, 0, []
    while True:
        try:
            line = input("> ").strip()
        except (EOFError, KeyboardInterrupt):
            break
        if not line:
            break
        entry_type, value = parse_input_line(line)
        if entry_type is None:
            continue
        ok, result = process_entry(entry_type, value, output_dir, fmt, retries=retries)
        if ok:
            success += 1
        else:
            failed += 1
            errors.append(result)
    return success, failed, errors


# ── Main ────────────────────────────────────────────────────────

def self_test():
    """Run self-check on format converter pipeline."""
    print("FORMAT CONVERTER SELF-TEST")
    print("-" * 40)

    # 1. Module import check
    try:
        from converters import convert_from_medline, convert_from_crossref, convert_from_arxiv
        print("  [OK] Module imports")
    except Exception as e:
        print(f"  [FAIL] Module imports: {e}")
        return

    # 2. PubMed endpoint (known PMID: 28344011)
    pmid = "28344011"
    print(f"  Testing PubMed (PMID {pmid})...")
    try:
        import time
        time.sleep(0.5)
        nbib_text = efetch_medline(pmid)
        if nbib_text and nbib_text.strip():
            ris_content = convert_from_medline(nbib_text, "ris")
            bib_content = convert_from_medline(nbib_text, "bib")
            enw_content = convert_from_medline(nbib_text, "enw")
            if ris_content.strip() and bib_content.strip() and enw_content.strip():
                print(f"  [OK] PubMed endpoint (RIS: {len(ris_content)}B, BibTeX: {len(bib_content)}B, ENW: {len(enw_content)}B)")
            else:
                print(f"  [FAIL] PubMed conversion produced empty output")
        else:
            print(f"  [FAIL] PubMed returned empty response for PMID {pmid}")
    except Exception as e:
        print(f"  [FAIL] PubMed endpoint: {e}")

    # 3. CrossRef endpoint (known DOI)
    doi = "10.1038/nature14539"
    print(f"  Testing CrossRef (DOI {doi})...")
    try:
        from urllib.request import urlopen
        import json
        time.sleep(0.5)
        url = f"https://api.crossref.org/works/{doi}"
        with urlopen(url, timeout=30) as resp:
            data = json.loads(resp.read().decode("utf-8"))
        ris_content = convert_from_crossref(data, "ris")
        bib_content = convert_from_crossref(data, "bib")
        enw_content = convert_from_crossref(data, "enw")
        if ris_content.strip() and bib_content.strip() and enw_content.strip():
            print(f"  [OK] CrossRef endpoint (RIS: {len(ris_content)}B, BibTeX: {len(bib_content)}B, ENW: {len(enw_content)}B)")
        else:
            print(f"  [FAIL] CrossRef conversion produced empty output")
    except Exception as e:
        print(f"  [FAIL] CrossRef endpoint: {e}")

    print("-" * 40)
    print("Self-test complete.")


def main():
    parser = argparse.ArgumentParser(
        description="Multi-source citation downloader with format conversion (.nbib/.ris/.bib)",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  %(prog)s --pmid 28344011
  %(prog)s --pmid 28344011,10645439 --format ris
  %(prog)s --doi 10.1038/nature14539 --format bib
  %(prog)s --doi 10.1038/nature14539,10.1038/s41586-020-2649-2 --format ris
  %(prog)s --arxiv 1706.03762 --format bib
  %(prog)s --arxiv 1706.03762,2302.13971 --format ris
  %(prog)s --query "TB-Profiler AND Bioinformatics[Journal]"
  %(prog)s --input refs.txt
  %(prog)s --input refs.txt --format ris
  %(prog)s --interactive

refs.txt format:
  PMID:28344011
  DOI:10.1038/nature14539
  ARXIV:1706.03762
  QUERY:TB-Profiler AND Bioinformatics[Journal]
  AUTHOR:Dheda TITLE:drug-resistant tuberculosis
  # Lines starting with # are comments
        """,
    )
    parser.add_argument("--pmid", help="PMID(s), comma-separated")
    parser.add_argument("--doi", help="DOI(s), comma-separated")
    parser.add_argument("--arxiv", help="arXiv ID(s), comma-separated")
    parser.add_argument("--author", help="Author name for PubMed search")
    parser.add_argument("--title", help="Title keywords for PubMed search")
    parser.add_argument("--query", help="PubMed search query")
    parser.add_argument("--input", help="Input file with references")
    parser.add_argument("--interactive", action="store_true", help="Interactive mode")
    parser.add_argument(
        "--format", choices=["nbib", "ris", "bib", "enw"], default="nbib",
        help="Output format: nbib (default, MEDLINE), ris (EndNote/Zotero), bib (BibTeX/LaTeX), enw (EndNote tagged)",
    )
    parser.add_argument(
        "--output", default="./references/",
        help="Output directory (default: ./references/)",
    )
    parser.add_argument("--version", action="version", version="format-converter 1.0.0")
    parser.add_argument("--test", action="store_true", help="Run self-test on format converter pipeline")
    parser.add_argument("--retry", type=int, default=1, help="Retry count for HTTP calls")
    parser.add_argument("--preflight", action="store_true", help="Run connectivity check on API endpoints")

    args = parser.parse_args()

    if args.test:
        self_test()
        return

    if args.preflight:
        import sys as _sys
        _sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
        from preflight import check_endpoints
        results = check_endpoints()
        # Print report
        print("PRE-FLIGHT REPORT")
        all_ok = True
        for name, info in results.items():
            status = "OK" if info["ok"] else "FAIL"
            detail = f"({info['time']:.1f}s)" if info["ok"] else f"({info['error']})"
            print(f"  {name:25s}: {status} {detail}")
            if not info["ok"]:
                all_ok = False
        reachable = sum(1 for v in results.values() if v["ok"])
        total = len(results)
        print(f"  {reachable}/{total} endpoints reachable.")
        if not all_ok:
            print("  Affected: format-converter downloads for unreachable endpoints (MCP tools unaffected).")
            _sys.exit(1)
        return

    if not any([args.pmid, args.doi, args.arxiv, args.author, args.query, args.input, args.interactive]):
        parser.error("Specify at least one input method")

    fmt = args.format
    output_dir = os.path.abspath(args.output)
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
        print(f"Created output directory: {output_dir}")

    print(f"Output directory: {output_dir}")
    print(f"Format: {fmt}")
    print("=" * 60)

    total_success, total_failed, all_errors = 0, 0, []

    def handle_list(ids_str, handler, retries=1):
        nonlocal total_success, total_failed
        ids = [x.strip() for x in ids_str.split(",") if x.strip()]
        for item_id in ids:
            print(f"\nProcessing: {item_id}")
            ok, result = handler(item_id, output_dir, fmt, retries=retries)
            if ok:
                total_success += 1
            else:
                total_failed += 1
                all_errors.append(result)

    # --pmid
    if args.pmid:
        handle_list(args.pmid, download_pubmed, retries=args.retry)

    # --doi
    if args.doi:
        if fmt == "nbib":
            print("Warning: CrossRef does not provide .nbib (MEDLINE) format. Falling back to .ris")
            fmt = "ris"
        handle_list(args.doi, download_crossref, retries=args.retry)

    # --arxiv
    if args.arxiv:
        if fmt == "nbib":
            print("Warning: arXiv does not provide .nbib (MEDLINE) format. Falling back to .ris")
            fmt = "ris"
        handle_list(args.arxiv, download_arxiv, retries=args.retry)

    # --author / --title / --query
    if args.author or args.title or args.query:
        if args.query:
            query = args.query
        else:
            query_parts = []
            if args.author:
                query_parts.append(f"{args.author}[Author]")
            if args.title:
                query_parts.append(f"{args.title}[Title]")
            query = " AND ".join(query_parts)
        print(f"\nProcessing search: {query}")
        ok, result = search_pubmed(query, output_dir, fmt, retries=args.retry)
        if ok:
            total_success += 1
        else:
            total_failed += 1
            all_errors.append(result)

    # --input
    if args.input:
        print(f"\nProcessing file: {args.input}")
        s, f, e = process_file(args.input, output_dir, fmt, retries=args.retry)
        total_success += s
        total_failed += f
        all_errors.extend(e)

    # --interactive
    if args.interactive:
        s, f, e = interactive_mode(output_dir, fmt, retries=args.retry)
        total_success += s
        total_failed += f
        all_errors.extend(e)

    # Summary
    print("\n" + "=" * 60)
    print("SUMMARY")
    print(f"  Success: {total_success}")
    print(f"  Failed:  {total_failed}")
    if all_errors:
        print("  Errors:")
        for err in all_errors:
            print(f"    - {err}")
    print(f"  Output:  {output_dir}")
    print("=" * 60)


if __name__ == "__main__":
    main()
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 196kUpdated Claude CodeCodexVerifiedInstall