openJiuwen-ai/sciencediscovery / skills/literature-searcher
literature-searcher
VerifiedUse this skill when a research workflow needs verified academic source retrieval through literature-search MCP interfaces available in the current session before evidence extraction. Discovers and selects relevant interfaces adaptively, then produces deduplicated literature JSON and coverage notes. Not for full-text reading, evidence extraction, final report writing, or workflow coordination.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 22.5 KB |
| scripts/arxiv_search.py | script | 13.5 KB |
| scripts/crossref_search.py | script | 13.6 KB |
| scripts/deduplicate.py | script | 10.0 KB |
| scripts/external_urls.py | script | 3.1 KB |
| scripts/output_utils.py | script | 5.9 KB |
| scripts/pubmed_search.py | script | 14.3 KB |
| scripts/query_preprocessor.py | script | 9.3 KB |
| scripts/requirements.txt | manifest | 633 B |
scripts/deduplicate.py
View on GitHub# Copyright (C) 2026-2026 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Literature source deduplication script.
Deduplicates sources across multiple database search results by:
1. DOI exact match
2. URL exact match
3. Title similarity (threshold configurable, default 0.85)
Produces a deduplicated JSON output file and optional deduplication report file.
Stdout emits a compact metadata JSON for the calling agent.
Output mode (file intermediary pattern):
Full JSON result → written to --output file (compact JSON, no indent)
Dedup report → written to --report file (if specified)
Stdout → metadata JSON: {output_file, report_file, file_size_bytes, total_before_dedup, duplicates_removed, final_unique}
Usage:
python deduplicate.py results_arxiv.json results_s2.json results_pubmed.json \
--output deduplicated_results.json --report dedup_report.json
# With auto-detection of team-workspace output directory:
python deduplicate.py artifacts/literature_search/results_*.json \
--output artifacts/literature_search/deduplicated_results.json \
--report artifacts/literature_search/dedup_report.json
# Or let deduplicate.py auto-detect paths (no --output/--report):
python deduplicate.py artifacts/literature_search/results_*.json
"""
import argparse
import glob as glob_mod
import json
import os
import sys
from output_utils import get_output_dir, LITERATURE_SEARCH_SUBDIR
from external_urls import external_url_list
def resolve_input_files(raw_patterns: list) -> list:
resolved = []
for pattern in raw_patterns:
expanded = glob_mod.glob(pattern)
if expanded:
resolved.extend(sorted(expanded))
elif os.path.isfile(pattern):
resolved.append(pattern)
else:
print(f"Warning: input pattern '{pattern}' matched no files", file=sys.stderr)
return resolved
def normalize_doi(doi: str) -> str:
if not doi:
return ""
doi = doi.strip().lower()
for prefix in external_url_list("data_sources.doi.accepted_prefixes"):
if doi.startswith(prefix):
doi = doi[len(prefix):]
break
return doi
def normalize_url(url: str) -> str:
if not url:
return ""
url = url.strip().lower()
if url.endswith("/"):
url = url[:-1]
return url
def normalize_title(title: str) -> str:
return title.strip().lower().replace("\n", " ").replace(" ", " ")
def title_similarity(t1: str, t2: str) -> float:
t1 = normalize_title(t1)
t2 = normalize_title(t2)
if not t1 or not t2:
return 0.0
if t1 == t2:
return 1.0
words1 = set(t1.split())
words2 = set(t2.split())
intersection = words1 & words2
union = words1 | words2
if not union:
return 0.0
return len(intersection) / len(union)
def metadata_richness(source: dict) -> int:
score = 0
if source.get("title"):
score += 1
if source.get("abstract"):
score += 2
if source.get("doi"):
score += 1
if source.get("authors") and len(source.get("authors", [])) > 0:
score += 1
if source.get("year"):
score += 1
if source.get("venue"):
score += 1
return score
def deduplicate(input_files: list, title_threshold: float = 0.85) -> tuple:
all_sources = []
for filepath in input_files:
with open(filepath, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
sources = data
else:
sources = data.get("sources", [])
all_sources.extend(sources)
seen_dois = {}
seen_urls = {}
seen_titles = {}
unique_sources = []
duplicates_removed = []
duplicate_details = []
for source in all_sources:
doi = normalize_doi(source.get("doi", ""))
url = normalize_url(source.get("url", ""))
title = normalize_title(source.get("title", ""))
db = source.get("source_database", "unknown")
is_duplicate = False
matched_existing_idx = -1
match_reason = ""
if doi and doi in seen_dois:
is_duplicate = True
matched_existing_idx = seen_dois[doi]
match_reason = "DOI match"
if not is_duplicate and url and url in seen_urls:
is_duplicate = True
matched_existing_idx = seen_urls[url]
match_reason = "URL match"
if not is_duplicate and title:
for idx, existing_title in seen_titles.items():
sim = title_similarity(title, existing_title)
if sim > title_threshold:
is_duplicate = True
matched_existing_idx = idx
match_reason = f"Title similarity ({sim:.2f})"
break
if is_duplicate and matched_existing_idx >= 0:
existing = unique_sources[matched_existing_idx]
existing_db = existing.get("source_database", "unknown")
duplicate_details.append({
"title": source.get("title", ""),
"databases": [existing_db, db],
"match_reason": match_reason,
"kept_source": existing_db if metadata_richness(existing) >= metadata_richness(source) else db,
})
if metadata_richness(source) > metadata_richness(existing):
existing_url = normalize_url(existing.get("url", ""))
if existing_url and existing_url != url and existing_url in seen_urls:
del seen_urls[existing_url]
unique_sources[matched_existing_idx] = source
if doi:
seen_dois[doi] = matched_existing_idx
if url:
seen_urls[url] = matched_existing_idx
seen_titles[matched_existing_idx] = title
duplicates_removed.append(source)
else:
idx = len(unique_sources)
unique_sources.append(source)
if doi:
seen_dois[doi] = idx
if url:
seen_urls[url] = idx
seen_titles[idx] = title
report = {
"total_before_dedup": len(all_sources),
"duplicates_removed": len(duplicates_removed),
"final_unique": len(unique_sources),
"dedup_methods": ["DOI exact match", "URL exact match", f"Title similarity > {title_threshold}"],
"duplicate_details": duplicate_details,
"input_files": input_files,
}
return unique_sources, report
def main():
base_dir = get_output_dir()
default_output_dir = os.path.join(base_dir, LITERATURE_SEARCH_SUBDIR)
parser = argparse.ArgumentParser(description="Deduplicate literature search results across databases")
parser.add_argument("input_files", nargs="+",
help="JSON files from search scripts to deduplicate. Supports glob patterns (e.g., 'artifacts/literature_search/results_*.json')")
parser.add_argument("--output", default=None,
help=f"Deduplicated output JSON file path. Defaults to {default_output_dir}/deduplicated_results.json")
parser.add_argument("--report", default=None,
help=f"Deduplication report JSON file path. Defaults to {default_output_dir}/dedup_report.json if --output is auto-detected")
parser.add_argument("--title-similarity-threshold", type=float, default=0.85,
help="Title similarity threshold for near-duplicate detection (default: 0.85)")
args = parser.parse_args()
resolved_inputs = resolve_input_files(args.input_files)
if not resolved_inputs:
print("Error: no input files found after resolving glob patterns", file=sys.stderr)
sys.exit(1)
for filepath in resolved_inputs:
try:
with open(filepath, "r", encoding="utf-8") as f:
json.load(f)
except Exception as e:
print(f"Error reading {filepath}: {e}", file=sys.stderr)
sys.exit(1)
unique_sources, report = deduplicate(resolved_inputs, args.title_similarity_threshold)
result = {
"sources": unique_sources,
"total_results": len(unique_sources),
"dedup_report": report,
}
if args.output:
output_path = args.output
else:
os.makedirs(default_output_dir, exist_ok=True)
output_path = os.path.join(default_output_dir, "deduplicated_results.json")
os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(result, f, ensure_ascii=False, separators=(",", ":"))
output_size = os.path.getsize(output_path)
report_path = None
if args.report:
report_path = args.report
elif not args.output:
report_path = os.path.join(default_output_dir, "dedup_report.json")
if report_path:
os.makedirs(os.path.dirname(report_path) or ".", exist_ok=True)
with open(report_path, "w", encoding="utf-8") as f:
json.dump(report, f, ensure_ascii=False, separators=(",", ":"))
metadata = {
"output_file": output_path,
"output_size_bytes": output_size,
"output_size_human": f"{output_size / 1024:.1f}KB",
"report_file": report_path,
"total_before_dedup": report["total_before_dedup"],
"duplicates_removed": report["duplicates_removed"],
"final_unique": report["final_unique"],
"input_files": resolved_inputs,
}
print(json.dumps(metadata, indent=2, ensure_ascii=False))
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |