OpenSenseNova/SenseNova-Skills / skills/sn-search-academic
sn-search-academic
Verified用于学术调研、论文精读、相关工作梳理、百科知识查询和引用链追溯。
scripts/arxiv_crawler_search.py
View on GitHub#!/usr/bin/env python3
"""arXiv search-result crawler using Camoufox browser disguise.
Usage:
python3 arxiv_crawler_search.py --query "qwen"
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
import re
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.parse import quote_plus
BASE_URL = "https://arxiv.org"
DEFAULT_QUERY = "qwen"
def parse_limit(value: str | None) -> int | None:
if value is None or value == "":
return 10
if value.lower() == "all":
return None
try:
limit = int(value)
except ValueError as exc:
raise argparse.ArgumentTypeError("--limit must be a positive integer or 'all'") from exc
if limit <= 0:
raise argparse.ArgumentTypeError("--limit must be a positive integer or 'all'")
return limit
def ancestors_from(path: Path) -> list[Path]:
resolved = path.resolve()
start = resolved.parent if resolved.is_file() or resolved.suffix else resolved
return [start, *start.parents]
def has_camoufox_js(directory: Path) -> bool:
return (directory / "node_modules" / "camoufox-js").exists()
def find_upwards(start: Path) -> Path | None:
for directory in ancestors_from(start):
if has_camoufox_js(directory):
return directory
return None
def find_camoufox_project_root(
script_path: Path | None = None,
env: dict[str, str] | None = None,
cwd: Path | None = None,
) -> Path:
env = env or os.environ
script_path = script_path or Path(__file__)
cwd = cwd or Path.cwd()
npm_package_json = env.get("npm_package_json")
candidates: list[Path] = []
if npm_package_json:
candidates.append(Path(npm_package_json))
candidates.extend([script_path, cwd])
for candidate in candidates:
root = find_upwards(candidate)
if root:
return root
raise SystemExit(
"Could not find node_modules/camoufox-js. Search order:\n"
" 1. npm_package_json directory and parents\n"
" 2. crawler script directory and parents\n"
" 3. current working directory and parents\n"
"Install camoufox-js in the current or an ancestor npm project, for example: npm install camoufox-js"
)
def host_os_name() -> str:
if sys.platform == "darwin":
return "macos"
if sys.platform.startswith("linux"):
return "linux"
if sys.platform.startswith("win"):
return "windows"
return "macos"
def normalize_space(value: str | None) -> str:
return re.sub(r"\s+", " ", value or "").strip()
def camoufox_launch_options_script(headless: bool, os_name: str | None = None) -> str:
headless_js = "true" if headless else "false"
os_js = json.dumps(os_name or host_os_name())
return (
"import { launchOptions } from 'camoufox-js';\n"
f"const options = await launchOptions({{headless:{headless_js}, os:{os_js}, humanize:true, enable_cache:true}});\n"
"console.log(JSON.stringify(options));\n"
)
def camoufox_launch_options(headless: bool) -> dict[str, Any]:
project_root = find_camoufox_project_root()
try:
raw = subprocess.check_output(
["node", "--input-type=module", "-e", camoufox_launch_options_script(headless=headless)],
cwd=project_root,
text=True,
stderr=subprocess.PIPE,
)
except FileNotFoundError as exc:
raise SystemExit("缺少 Node.js。此 crawler 需要 Node.js 和 camoufox-js 来生成浏览器启动参数。") from exc
except subprocess.CalledProcessError as exc:
raise SystemExit(
"Failed to generate Camoufox launch options via camoufox-js.\n"
f"stderr:\n{exc.stderr}"
) from exc
return json.loads(raw)
async def create_page(headless: bool):
try:
from playwright.async_api import async_playwright
except ModuleNotFoundError as exc:
skill_dir = Path(__file__).resolve().parents[1]
requirements = skill_dir / "requirements.txt"
raise SystemExit(
"缺少依赖 playwright。请运行:\n"
f" python3 -m pip install -r {requirements}\n"
" python3 -m playwright install firefox"
) from exc
options = camoufox_launch_options(headless=headless)
playwright = await async_playwright().start()
browser = await playwright.firefox.launch(
executable_path=options["executablePath"],
headless=options.get("headless", headless),
args=options.get("args") or [],
env={**os.environ, **(options.get("env") or {})},
firefox_user_prefs=options.get("firefoxUserPrefs") or {},
proxy=options.get("proxy") or None,
)
context = await browser.new_context(
viewport={"width": 1280, "height": 720},
permissions=["geolocation"],
locale="en-US",
timezone_id="America/Los_Angeles",
geolocation={"latitude": 37.7749, "longitude": -122.4194},
)
page = await context.new_page()
return playwright, browser, context, page
async def dismiss_popups(page) -> None:
for label in ["Accept", "I agree", "Got it", "Close"]:
try:
button = page.get_by_role("button", name=re.compile(rf"^{re.escape(label)}$", re.I))
if await button.count():
await button.first.click(timeout=1000)
except Exception:
pass
def format_stdout_result(result: dict[str, Any]) -> str:
return json.dumps(result, ensure_ascii=False, indent=2)
def format_cli_result(result: dict[str, Any], output: Path) -> str:
cli_result = {**result, "output_path": str(output.resolve())}
return format_stdout_result(cli_result)
def extract_year(*values: str | None) -> int | None:
for value in values:
match = re.search(r"\b(19|20)\d{2}\b", value or "")
if match:
return int(match.group(0))
return None
def extract_primary_category(*values: str | None) -> str | None:
for value in values:
match = re.search(r"\[([^\]]+)\]", value or "")
if match:
category = match.group(1)
if re.search(r"\b(pdf|ps|other)\b", category, flags=re.I):
continue
return category
return None
def clean_arxiv_id(arxiv_id: str | None) -> str | None:
match = re.search(r"(?:arXiv:)?([0-9]{4}\.[0-9]{4,5}(?:v\d+)?|[a-z.-]+/[0-9]{7}(?:v\d+)?)", arxiv_id or "", re.I)
if match:
return match.group(1)
return normalize_space(arxiv_id) or None
def normalize_subject_category(subject: str | None) -> str | None:
subject = normalize_space(subject)
paren_match = re.search(r"\(([a-z.-]+\.[A-Z]{2}|[a-z.-]+)\)", subject)
if paren_match:
return paren_match.group(1)
code_match = re.search(r"\b([a-z.-]+\.[A-Z]{2}|[a-z.-]+)\b", subject)
if code_match:
return code_match.group(1)
return subject or None
def format_search_row(row: dict[str, Any]) -> dict[str, Any]:
title = normalize_space(row.get("title"))
raw_arxiv_id = normalize_space(row.get("arxiv_id"))
announced = normalize_space(row.get("announced"))
abstract = normalize_space(row.get("abstract"))
comments = normalize_space(row.get("comments"))
raw_text = normalize_space(row.get("raw_text"))
subjects = row.get("subjects") or []
return {
"title": title,
"arxiv_id": clean_arxiv_id(raw_arxiv_id),
"primary_category": normalize_subject_category(subjects[0]) if subjects else extract_primary_category(raw_arxiv_id, raw_text),
"url": row.get("url"),
"pdf_url": row.get("pdf_url"),
"authors": row.get("authors") or [],
"subjects": subjects,
"year": extract_year(announced, raw_text),
"announced": announced or None,
"abstract": abstract or None,
"comments": comments or None,
}
def search_page_size(limit: int | None) -> int:
if limit is None:
return 200
if limit <= 25:
return 25
if limit <= 50:
return 50
if limit <= 100:
return 100
return 200
def build_search_url(query: str, searchtype: str, order: str, limit: int | None) -> str:
return (
f"{BASE_URL}/search/?query={quote_plus(query)}"
f"&searchtype={quote_plus(searchtype)}"
"&abstracts=show"
f"&order={quote_plus(order)}"
f"&size={search_page_size(limit)}"
)
async def wait_for_search_results(page) -> None:
await page.wait_for_selector("li.arxiv-result", timeout=30000)
async def extract_search_results(page, limit: int | None) -> list[dict[str, Any]]:
rows = await page.evaluate(
"""() => {
const normalize = (s) => (s || '').replace(/\\s+/g, ' ').trim();
const absolute = (href) => new URL(href, location.origin).href;
const textOf = (root, selector) => normalize(root.querySelector(selector)?.innerText || root.querySelector(selector)?.textContent);
const linkHref = (root, selector) => {
const href = root.querySelector(selector)?.getAttribute('href');
return href ? absolute(href) : null;
};
const sectionText = (root, label) => {
const paragraph = [...root.querySelectorAll('p')].find((p) => {
const text = normalize(p.innerText || p.textContent);
return text.toLowerCase().startsWith(label.toLowerCase());
});
if (!paragraph) return '';
return normalize((paragraph.innerText || paragraph.textContent).replace(new RegExp(`^${label}:?`, 'i'), ''));
};
return [...document.querySelectorAll('li.arxiv-result')].map((item) => {
const absLink = item.querySelector('p.list-title a[href*="/abs/"]');
const pdfLink = [...item.querySelectorAll('a[href*="/pdf/"]')][0];
const fullAbstract = item.querySelector('span.abstract-full');
const abstract = fullAbstract
? normalize((fullAbstract.innerText || fullAbstract.textContent).replace(/\\s*△ Less\\s*$/i, ''))
: sectionText(item, 'Abstract');
return {
title: textOf(item, 'p.title'),
arxiv_id: normalize(item.querySelector('p.list-title')?.innerText || item.querySelector('p.list-title')?.textContent).replace(/^arXiv:\\s*/i, ''),
url: absLink ? absolute(absLink.getAttribute('href')) : null,
pdf_url: pdfLink ? absolute(pdfLink.getAttribute('href')) : null,
authors: [...item.querySelectorAll('p.authors a')].map((a) => normalize(a.innerText || a.textContent)).filter(Boolean),
subjects: [...item.querySelectorAll('.tags .tag, span.tag')].map((tag) => normalize(tag.innerText || tag.textContent)).filter(Boolean),
announced: textOf(item, 'p.is-size-7'),
abstract,
comments: sectionText(item, 'Comments'),
raw_text: normalize(item.innerText || item.textContent),
};
});
}"""
)
results: list[dict[str, Any]] = []
seen_ids: set[str] = set()
for row in rows:
formatted = format_search_row(row)
arxiv_id = formatted.get("arxiv_id")
if not formatted.get("title") or not arxiv_id or arxiv_id in seen_ids:
continue
seen_ids.add(arxiv_id)
results.append(formatted)
if limit and len(results) >= limit:
break
return results
async def crawl_search(
query: str,
output: Path,
headless: bool,
limit: int | None,
searchtype: str,
order: str,
) -> dict[str, Any]:
playwright, browser, context, page = await create_page(headless=headless)
try:
search_url = build_search_url(query, searchtype, order, limit)
await page.goto(search_url, wait_until="domcontentloaded")
await page.wait_for_load_state("networkidle", timeout=30000)
await dismiss_popups(page)
await wait_for_search_results(page)
papers = await extract_search_results(page, limit)
result = {
"query": query,
"search_url": search_url,
"results_count": len(papers),
"papers": papers,
"created_at": datetime.now(timezone.utc).isoformat(),
}
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(format_stdout_result(result), encoding="utf-8")
return result
finally:
await browser.close()
await playwright.stop()
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Crawl arXiv search results.")
parser.add_argument("--query", default=DEFAULT_QUERY, help="Search query.")
parser.add_argument("--output", default="arxiv_search_output.json", help="JSON output path.")
parser.add_argument("--limit", default="10", help="Maximum papers to collect, or 'all'. Defaults to 10.")
parser.add_argument(
"--searchtype",
default="all",
choices=["all", "title", "author", "abstract", "comments", "journal_ref", "acm_class", "msc_class", "report_num"],
help="arXiv search type. Defaults to all.",
)
parser.add_argument("--order", default="-announced_date_first", help="arXiv order parameter.")
parser.add_argument("--headless", choices=["true", "false"], default="true", help="Run Camoufox headless.")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv or sys.argv[1:])
output = Path(args.output)
result = asyncio.run(
crawl_search(
query=args.query,
output=output,
headless=args.headless == "true",
limit=parse_limit(args.limit),
searchtype=args.searchtype,
order=args.order,
)
)
print(format_cli_result(result, output))
return 0
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |