Skip to content

Yuan1z0825/nature-skills / skills/nature-academic-search

nature-academic-search

Flagged

Search literature across sources, verify or manage citations, and build MeSH strategies or citation-impact audits. Use for 文献检索、引文核对、参考文献管理、严格他引 and evidence-backed citer profiles; not for translating a paper or drafting manuscript prose.

FileKindSize
README.mdmarkdown2.6 KB
README_EN.mdmarkdown3.0 KB
SKILL.mdexecutableskill md3.5 KB
agents/openai.yamlconfig265 B
config/mcp-snippet.jsonconfig502 B
config/settings-snippet.jsonconfig59 B
config/triggers-academic-search.tomlconfig978 B
install.shexecutablescript4.3 KB
manifest.yamlconfig3.1 KB
mcp-server/README.mdmarkdown1.9 KB
mcp-server/academic_search_server.pyscript20.9 KB
mcp-server/config.tomlconfig249 B
mcp-server/requirements.txtmanifest134 B
mcp-server/sources/__init__.pyscript352 B
mcp-server/sources/arxiv.pyscript11.4 KB
mcp-server/sources/crossref.pyscript8.0 KB
mcp-server/sources/elsevier_common.pyscript2.4 KB
mcp-server/sources/pubmed.pyscript13.7 KB
mcp-server/sources/sciencedirect.pyscript6.9 KB
mcp-server/sources/scopus.pyscript19.0 KB
mcp-server/tests/__init__.pyscript40 B
mcp-server/tests/test_arxiv_error_entries.pyscript1.4 KB
mcp-server/tests/test_citation_errors.pyscript1.6 KB
mcp-server/tests/test_crossref_version_relations.pyscript3.3 KB
mcp-server/tests/test_elsevier_live.pyscript2.6 KB
mcp-server/tests/test_format_converter_hardening.pyscript4.2 KB
mcp-server/tests/test_mcp_tools.pyscript2.6 KB
mcp-server/tests/test_pubmed_parsing.pyscript4.4 KB
mcp-server/tests/test_scopus_empty_result.pyscript1006 B
mcp-server/tests/test_script_converters.pyscript2.9 KB
mcp-server/tests/test_sources.pyscript30.5 KB
mcp-server/utils/__init__.pyscript359 B
mcp-server/utils/config.pyscript1.4 KB
mcp-server/utils/errors.pyscript620 B
mcp-server/utils/logging.pyscript1.1 KB
references/citation-parser.mdmarkdown2.7 KB
references/dedup-engine.mdmarkdown1.9 KB
references/pubmed-28344011.bibexecutablescript3.3 KB
references/pubmed-28344011.nbibexecutablescript11.9 KB
references/pubmed-28344011.risexecutablescript3.4 KB
references/ris-bibtex-format.mdexecutablescript4.5 KB
references/search-strategy.mdexecutablescript2.0 KB
references/source-tiers.mdmarkdown2.7 KB
references/workflows/wf1-multi-source-search.mdmarkdown1.9 KB
references/workflows/wf2-citation-verification.mdmarkdown1.6 KB
references/workflows/wf3-mesh-strategy.mdmarkdown538 B
references/workflows/wf4-citation-file-mgmt.mdmarkdown1.9 KB
references/workflows/wf5-reference-mgmt.mdmarkdown1.5 KB
references/workflows/wf6-strict-other-citation-impact-audit.mdmarkdown11.2 KB
scripts/academic_search.pyscript23.2 KB
scripts/converters.pyscript18.5 KB
scripts/format-converter.pyscript19.5 KB
scripts/preflight.pyscript3.7 KB
static/core/routing-and-ops.mdmarkdown6.1 KB
static/core/tools.mdmarkdown3.0 KB

mcp-server/sources/arxiv.py

View on GitHub
"""arXiv data source via REST API (Atom XML feed)."""

from __future__ import annotations

import re
import time
import urllib.parse
from datetime import datetime

import defusedxml.ElementTree as ET
from xml.etree.ElementTree import Element
import requests

from utils.config import get_config
from utils.errors import DataSourceError
from utils.logging import setup_logging

ARXIV_API_URL = "https://export.arxiv.org/api/query"

ARXIV_NS = {
    "atom": "http://www.w3.org/2005/Atom",
    "arxiv": "http://arxiv.org/schemas/atom",
}

_MIN_REQUEST_INTERVAL = 3.0  # seconds between requests
_SOURCE_NAME = "arxiv"

logger = setup_logging("INFO")


class ArxivSource:
    """arXiv search and retrieval via the Atom API."""

    def __init__(self):
        self._last_request_time: float = 0.0
        self._timeout: int | None = None

    # ------------------------------------------------------------------
    # Public interface
    # ------------------------------------------------------------------

    def search(
        self,
        query: str,
        rows: int = 5,
        categories: list[str] | None = None,
        date_from: str | None = None,
        date_to: str | None = None,
    ) -> dict:
        """Search arXiv.

        Parameters
        ----------
        query : str
            Free-text search query.
        rows : int
            Max number of results to return.
        categories : list[str] | None
            arXiv categories to restrict to (e.g. ["cs.AI", "cs.LG"]).
        date_from : str | None
            Start date in YYYY-MM-DD format.
        date_to : str | None
            End date in YYYY-MM-DD format.

        Returns
        -------
        dict
            {"results": [...], "total": int, "source": "arxiv"}
        """
        search_query = self._build_query(query, categories)
        date_filter = self._build_date_filter(date_from, date_to)

        params = {
            "search_query": search_query,
            "start": 0,
            "max_results": rows,
            "sortBy": "relevance",
            "sortOrder": "descending",
        }
        if date_filter:
            params["search_query"] = f"{search_query}+AND+{date_filter}"

        raw = self._request(params)
        results = self._parse_feed(raw)
        return {
            "results": results,
            "total": len(results),
            "source": _SOURCE_NAME,
        }

    def get_by_id(self, arxiv_id: str) -> dict:
        """Retrieve a single paper by arXiv ID.

        Parameters
        ----------
        arxiv_id : str
            arXiv identifier, e.g. "2401.12345" or "2401.12345v1".

        Returns
        -------
        dict
            Paper record in unified format.
        """
        clean_id = self._normalize_id(arxiv_id)
        params = {
            "id_list": clean_id,
            "max_results": 1,
        }
        raw = self._request(params)
        results = self._parse_feed(raw)
        if not results:
            raise DataSourceError(
                _SOURCE_NAME,
                f"Paper not found: {arxiv_id}",
            )
        return results[0]

    # ------------------------------------------------------------------
    # Query construction
    # ------------------------------------------------------------------

    @staticmethod
    def _build_query(user_query: str, categories: list[str] | None = None) -> str:
        """Build the search_query parameter.

        Combines user query with optional category restrictions.
        Returns a string with +AND+/+OR+ connectors (URL-ready).
        """
        parts: list[str] = []
        parts.append(f"({user_query})")
        if categories:
            cat_expr = " OR ".join(f"cat:{cat}" for cat in categories)
            parts.append(f"({cat_expr})")
        combined = " AND ".join(parts)
        # Replace connectors and spaces for URL embedding
        combined = (
            combined.replace(" AND ", "+AND+")
            .replace(" OR ", "+OR+")
            .replace(" ", "+")
        )
        return combined

    @staticmethod
    def _build_date_filter(
        date_from: str | None, date_to: str | None
    ) -> str:
        """Build submittedDate filter expression.

        The arXiv API requires the literal +TO+ syntax and 14-digit
        timestamps in the format YYYYMMDDHHMM.

        Returns empty string when no date bounds are given.
        """
        if not date_from and not date_to:
            return ""

        # Default boundaries
        start_ts = "000000000000"
        end_ts = "999912312359"

        if date_from:
            start_ts = _date_to_ts(date_from)
        if date_to:
            end_ts = _date_to_ts(date_to, end_of_day=True)

        return f"submittedDate:[{start_ts}+TO+{end_ts}]"

    # ------------------------------------------------------------------
    # HTTP
    # ------------------------------------------------------------------

    def _request(self, params: dict) -> str:
        """Execute an HTTP GET to the arXiv API with rate limiting."""
        self._enforce_rate_limit()
        url = f"{ARXIV_API_URL}?{urllib.parse.urlencode(params)}"
        timeout = self._get_timeout()
        logger.debug("arXiv request: %s", url)

        try:
            resp = requests.get(
                url,
                headers={"User-Agent": "academic-search/1.0"},
                timeout=timeout,
            )
            resp.raise_for_status()
            return resp.text
        except requests.exceptions.HTTPError as exc:
            code = exc.response.status_code if exc.response is not None else 0
            if code in (429, 503):
                raise DataSourceError(
                    _SOURCE_NAME,
                    f"Rate limited or unavailable (HTTP {code})",
                    original_error=exc,
                ) from exc
            raise DataSourceError(
                _SOURCE_NAME,
                f"HTTP error {code}: {exc}",
                original_error=exc,
            ) from exc
        except requests.exceptions.Timeout as exc:
            raise DataSourceError(
                _SOURCE_NAME,
                f"Request timed out after {timeout}s",
                original_error=exc,
            ) from exc
        except requests.exceptions.RequestException as exc:
            raise DataSourceError(
                _SOURCE_NAME,
                f"Network error: {exc}",
                original_error=exc,
            ) from exc

    def _enforce_rate_limit(self) -> None:
        elapsed = time.monotonic() - self._last_request_time
        if elapsed < _MIN_REQUEST_INTERVAL:
            time.sleep(_MIN_REQUEST_INTERVAL - elapsed)
        self._last_request_time = time.monotonic()

    def _get_timeout(self) -> int:
        if self._timeout is None:
            self._timeout = get_config().arxiv_timeout
        return self._timeout

    # ------------------------------------------------------------------
    # XML parsing
    # ------------------------------------------------------------------

    def _parse_feed(self, xml_text: str) -> list[dict]:
        """Parse arXiv Atom XML into a list of unified result dicts."""
        try:
            root = ET.fromstring(xml_text)
        except ET.ParseError as exc:
            raise DataSourceError(
                _SOURCE_NAME,
                f"Malformed XML response: {exc}",
                original_error=exc,
            ) from exc

        entries: list[dict] = []
        for entry in root.findall("atom:entry", ARXIV_NS):
            parsed = self._parse_entry(entry)
            if parsed:
                entries.append(parsed)
        return entries

    def _parse_entry(self, entry: Element) -> dict | None:
        """Extract a single paper record from an Atom <entry>."""
        arxiv_id_raw = _text(entry, "atom:id", ARXIV_NS)
        if not arxiv_id_raw:
            return None
        # arXiv reports a bad request as an <entry> whose id is http://arxiv.org/api/errors#...
        if "/api/errors" in arxiv_id_raw:
            return None

        title = _text(entry, "atom:title", ARXIV_NS) or ""
        title = re.sub(r"\s+", " ", title).strip()

        summary = _text(entry, "atom:summary", ARXIV_NS) or ""
        summary = re.sub(r"\s+", " ", summary).strip()

        authors = [
            name
            for name in (
                _text(author, "atom:name", ARXIV_NS)
                for author in entry.findall("atom:author", ARXIV_NS)
            )
            if name
        ]

        primary_cat = entry.find("arxiv:primary_category", ARXIV_NS)
        categories = []
        if primary_cat is not None:
            term = primary_cat.get("term")
            if term:
                categories.append(term)
        # Also collect secondary categories from atom:category elements
        for cat in entry.findall("atom:category", ARXIV_NS):
            scheme = cat.get("scheme", "")
            term = cat.get("term", "")
            if term and "arxiv" in scheme.lower() and term not in categories:
                categories.append(term)

        published = _text(entry, "atom:published", ARXIV_NS) or ""
        year = _extract_year(published)

        pdf_url = ""
        for link in entry.findall("atom:link", ARXIV_NS):
            if link.get("title") == "pdf":
                pdf_url = link.get("href", "")
                break

        arxiv_id = self._normalize_id(arxiv_id_raw)

        return {
            "title": title,
            "authors": authors,
            "year": year,
            "arxiv_id": arxiv_id,
            "categories": categories,
            "abstract": summary,
            "pdf_url": pdf_url,
            "source": _SOURCE_NAME,
        }

    # ------------------------------------------------------------------
    # Helpers
    # ------------------------------------------------------------------

    @staticmethod
    def _normalize_id(raw: str) -> str:
        """Strip URL prefix and version suffix from an arXiv ID.

        http://arxiv.org/abs/2401.12345v1 -> 2401.12345
        """
        arxiv_id = raw.strip()
        # Remove URL prefix
        match = re.search(r"(\d{4}\.\d{4,5})(v\d+)?$", arxiv_id)
        if match:
            return match.group(1)
        # Fallback: strip known prefixes
        for prefix in ("https://arxiv.org/abs/", "http://arxiv.org/abs/"):
            if arxiv_id.startswith(prefix):
                arxiv_id = arxiv_id[len(prefix):]
                break
        # Strip version
        arxiv_id = re.sub(r"v\d+$", "", arxiv_id)
        return arxiv_id


# ------------------------------------------------------------------
# Module-level helpers
# ------------------------------------------------------------------


def _text(parent: Element, xpath: str, ns: dict) -> str | None:
    """Return stripped text of a sub-element, or None."""
    el = parent.find(xpath, ns)
    if el is not None and el.text:
        return el.text.strip()
    return None


def _date_to_ts(date_str: str, end_of_day: bool = False) -> str:
    """Convert YYYY-MM-DD to YYYYMMDDHHMM (14-digit timestamp).

    Parameters
    ----------
    date_str : str
        Date in YYYY-MM-DD format.
    end_of_day : bool
        If True, use 2359 as HHMM; otherwise 0000.
    """
    dt = datetime.strptime(date_str, "%Y-%m-%d")
    suffix = "2359" if end_of_day else "0000"
    return dt.strftime("%Y%m%d") + suffix


def _extract_year(published: str) -> int | None:
    """Extract year from ISO datetime string (e.g. 2024-01-15T...)."""
    try:
        return int(published[:4])
    except (ValueError, IndexError):
        return None
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 196kUpdated Claude CodeCodexVerifiedInstall