Skip to content

Dianel555/DSkills / skills/literature-searcher

literature-searcher

Verified

Search CrossRef, OpenAlex, PubMed, Semantic Scholar, and optional Scopus; deduplicate results, download open-access PDFs by DOI, classify papers, monitor new results, and analyze coverage. Use when asked to search literature, monitor a topic, download an open-access paper, classify papers, or analyze literature gaps.

literature_searcher/semantic_scholar_searcher.py

View on GitHub
"""
Semantic Scholar API 搜索
"""

import json
import math
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
import urllib.error
import urllib.parse
import urllib.request
import time
from typing import List, Optional

from . import SEMANTIC_SCHOLAR_API, USER_AGENT, SEMANTIC_SCHOLAR_API_KEY, Paper


MAX_RETRY_WAIT = 30.0


def _retry_after_seconds(value: str | None, default: float) -> float:
    try:
        try:
            delay = float(value) if value else default
        except ValueError:
            date = parsedate_to_datetime(value)
            if date.tzinfo is None:
                date = date.replace(tzinfo=timezone.utc)
            delay = (date - datetime.now(timezone.utc)).total_seconds()
        if not math.isfinite(delay):
            delay = default
        return max(0.0, min(MAX_RETRY_WAIT, delay))
    except (TypeError, ValueError, OverflowError):
        return min(MAX_RETRY_WAIT, default)


class SemanticScholarSearcher:
    """Semantic Scholar API 搜索"""

    @staticmethod
    def search(query: str, limit: int = 20, year_from: Optional[int] = None,
               year_to: Optional[int] = None) -> List[Paper]:
        papers = []
        try:
            params = {
                "query": query,
                "limit": min(limit, 100),
                "fields": "title,abstract,authors,year,citationCount,journal,externalIds,publicationTypes",
            }
            if year_from or year_to:
                year_range = f"{year_from or ''}-{year_to or ''}"
                params["year"] = year_range

            url = f"{SEMANTIC_SCHOLAR_API}/paper/search?{urllib.parse.urlencode(params)}"
            req = urllib.request.Request(url)
            req.add_header("User-Agent", USER_AGENT)
            if SEMANTIC_SCHOLAR_API_KEY:
                req.add_header("x-api-key", SEMANTIC_SCHOLAR_API_KEY)

            data = SemanticScholarSearcher._request_with_retry(req)
            if data is None:
                return papers

            for item in data.get("data", []):
                authors = [a.get("name", "") for a in item.get("authors", []) if a.get("name")]
                ext_ids = item.get("externalIds", {}) or {}
                doi = ext_ids.get("DOI", "")

                journal_info = item.get("journal", {}) or {}
                journal_name = journal_info.get("name", "")

                papers.append(Paper(
                    title=item.get("title", ""),
                    abstract=item.get("abstract", ""),
                    authors=authors,
                    doi=doi,
                    year=item.get("year"),
                    citations=item.get("citationCount", 0),
                    journal=journal_name,
                    source_platform="semantic_scholar",
                ))
        except Exception as e:
            print(f"[SemanticScholar] 搜索错误: {e}")

        return papers[:limit]

    @staticmethod
    def _request_with_retry(req: urllib.request.Request, max_attempts: int = 3) -> Optional[dict]:
        for attempt in range(max_attempts):
            try:
                with urllib.request.urlopen(req, timeout=30) as resp:
                    return json.loads(resp.read().decode("utf-8"))
            except urllib.error.HTTPError as e:
                if e.code != 429 or attempt == max_attempts - 1:
                    print(f"[SemanticScholar] HTTP错误 {e.code}: {e.reason}")
                    return None

                retry_after = e.headers.get("Retry-After") if e.headers else None
                delay = _retry_after_seconds(retry_after, min(MAX_RETRY_WAIT, 2 ** min(attempt, 5)))
                print(f"[SemanticScholar] 配额已用尽,{delay:.0f} 秒后重试...")
                time.sleep(delay)

        return None
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 323kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 323kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 323kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall