moonlight-lupin/agent-skills / research/deep-research
deep-research
VerifiedUse when the user asks for "deep research", a "research report on", a "comprehensive analysis of", to "look into X in depth", to "write a report on X", or any question needing multi-source synthesis beyond a single search — an autonomous multi-step deep research engine implementing an iterative Think → Search → Extract → Synthesize → Stop loop. The LLM drives every decision: what to search, what's relevant, what's missing, and when to stop. Produces a cited, magazine-quality report with inline citations, category-specific formatting, and research stats. For entity vetting/dossiers use entity-research; for news digests use news-monitoring; for source-grounded Q&A use notebooklm-mode.
tests/test_source_db.py
View on GitHub#!/usr/bin/env python3
"""Tests for the citation-database scripts (absorbed source-tracker).
Run: python -m pytest research/deep-research/tests/test_source_db.py -v
"""
from __future__ import annotations
import csv
import json
import sqlite3
import sys
import tempfile
from pathlib import Path
from unittest.mock import patch
SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "scripts"
sys.path.insert(0, str(SCRIPTS_DIR))
import source_db # noqa: E402
import url_health # noqa: E402
class FakeHeadResponse:
"""Context-manager response object for mocked HEAD requests."""
def __init__(self, status: int) -> None:
"""Store an HTTP status code."""
self.status = status
def __enter__(self) -> "FakeHeadResponse":
"""Return the fake response."""
return self
def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
"""Close the fake response context."""
return None
def getcode(self) -> int:
"""Return the HTTP status code."""
return self.status
def temp_db_path() -> str:
"""Create a temporary SQLite file path for a test."""
handle = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
handle.close()
return handle.name
def connect_temp() -> tuple[str, sqlite3.Connection]:
"""Create and connect to a temporary source database."""
db_path = temp_db_path()
return db_path, source_db.connect(db_path)
def test_add_and_search() -> None:
"""Add three sources, search by topic, and verify only matching rows return."""
_db_path, conn = connect_temp()
with conn:
source_db.add_source(conn, "https://example.com/alpha", "topic-a", "Alpha", "first", "web")
source_db.add_source(conn, "https://example.com/beta", "topic-a", "Beta", "second", "pdf")
source_db.add_source(conn, "https://example.com/gamma", "topic-b", "Gamma", "third", "news")
results = source_db.search_sources(conn, "topic-a")
assert len(results) == 2
assert {row["title"] for row in results} == {"Alpha", "Beta"}
assert all(row["topic"] == "topic-a" for row in results)
pdf_results = source_db.search_sources(conn, "topic-a", source_type="pdf", verified=True)
assert len(pdf_results) == 1
assert pdf_results[0]["title"] == "Beta"
def test_dedup() -> None:
"""Add http/www/trailing-slash variants and verify dedup merges them."""
_db_path, conn = connect_temp()
with conn:
source_db.add_source(conn, "http://example.com", "dedup-topic", "HTTP", "note one", "web")
source_db.add_source(conn, "https://www.example.com/", "dedup-topic", "HTTPS", "note two", "web")
before = source_db.search_sources(conn, "dedup-topic")
assert len(before) == 2
summary = source_db.deduplicate_sources(conn)
after = source_db.search_sources(conn, "dedup-topic")
assert summary["merged_groups"] == 1
assert summary["removed_rows"] == 1
assert len(after) == 1
assert "note one" in after[0]["notes"]
assert "note two" in after[0]["notes"]
def test_export_markdown() -> None:
"""Export sources as Markdown and verify bibliography formatting."""
_db_path, conn = connect_temp()
with conn:
source_db.add_source(conn, "https://example.com/report", "exports", "Report Title", "useful note", "report")
output = source_db.export_sources(conn, "exports", "markdown")
assert "## exports" in output
assert "- [Report Title](https://example.com/report) — useful note (" in output
def test_export_bibtex() -> None:
"""Export sources as BibTeX and verify required fields."""
_db_path, conn = connect_temp()
with conn:
source_db.add_source(conn, "https://example.com/report", "exports", "Report Title", "useful note", "report")
output = source_db.export_sources(conn, "exports", "bibtex")
assert "@misc{" in output
assert "title={Report Title}" in output
assert "url={https://example.com/report}" in output
assert "note={useful note}" in output
assert "urldate={" in output
def test_stats() -> None:
"""Add sources across topics/types and verify stats counts."""
_db_path, conn = connect_temp()
with conn:
source_db.add_source(conn, "https://example.com/a", "topic-a", "A", "", "web")
source_db.add_source(conn, "https://example.com/b", "topic-a", "B", "", "pdf")
source_db.add_source(conn, "https://example.com/c", "topic-b", "C", "", "pdf")
conn.execute("UPDATE sources SET verified = 0 WHERE url = ?", ("https://example.com/c",))
conn.commit()
data = source_db.stats(conn)
assert data["total"] == 3
assert data["by_topic"] == {"topic-a": 2, "topic-b": 1}
assert data["by_type"] == {"pdf": 2, "web": 1}
assert data["by_verification"] == {"verified": 2, "unverified": 1}
def test_url_normalization() -> None:
"""Verify fragment stripping, lowercase host/scheme, and trailing slash rules."""
assert source_db.canonicalize_url("HTTPS://WWW.Example.COM/path/#frag") == "https://www.example.com/path"
assert source_db.canonicalize_url("https://Example.com/path/") == "https://example.com/path"
assert source_db.canonicalize_url("https://Example.com/") == "https://example.com/"
assert source_db.dedup_key("HTTPS://WWW.Example.COM/path/#frag") == "//example.com/path"
assert source_db.dedup_key("http://example.com/") == source_db.dedup_key("https://www.example.com/")
def test_rejects_non_web_schemes() -> None:
"""file://, ftp:// and other non-web schemes must be rejected — never stored or fetched."""
for bad in ("file://localhost/etc/passwd", "ftp://example.com/download", "javascript:alert(1)"):
try:
source_db.canonicalize_url(bad)
except ValueError:
pass
else:
raise AssertionError(f"expected ValueError for {bad!r}")
def test_health_check() -> None:
"""Mock URL requests and verify url_health updates verified flags."""
db_path, conn = connect_temp()
with conn:
source_db.add_source(conn, "https://alive.example.com", "health", "Alive", "", "web")
source_db.add_source(conn, "https://dead.example.com", "health", "Dead", "", "web")
def fake_urlopen(request: object, timeout: int = 10) -> FakeHeadResponse:
url = getattr(request, "full_url", "")
if "alive" in url:
return FakeHeadResponse(204)
return FakeHeadResponse(404)
with patch("url_health.urllib.request.urlopen", side_effect=fake_urlopen):
with url_health.connect(db_path) as health_conn:
summary = url_health.check_sources(health_conn, stale_days=0, timeout=1, batch_size=10)
with source_db.connect(db_path) as check_conn:
rows = {row["url"]: row for row in source_db.search_sources(check_conn, "health")}
assert summary == {"checked": 2, "alive": 1, "dead": 1}
assert rows["https://alive.example.com/"]["verified"] is True
assert rows["https://dead.example.com/"]["verified"] is False
assert rows["https://alive.example.com/"]["last_checked"] is not None
def test_cli_json_search_roundtrip() -> None:
"""Exercise CLI-style JSON output generation for search results."""
_db_path, conn = connect_temp()
with conn:
source_db.add_source(conn, "https://example.com/json", "json-topic", "JSON", "note", "api")
rendered = json.dumps(source_db.search_sources(conn, "json-topic"), sort_keys=True)
assert '"source_type": "api"' in rendered
def test_csv_export_header() -> None:
"""Verify CSV export uses the documented compact field order."""
_db_path, conn = connect_temp()
with conn:
source_db.add_source(conn, "https://example.com/csv", "csv-topic", "CSV", "note", "dataset")
rendered = source_db.export_sources(conn, "csv-topic", "csv")
rows = list(csv.reader(rendered.splitlines()))
assert rows[0] == ["id", "url", "title", "topic", "source_type", "accessed_at", "notes", "verified", "last_checked"]
assert rows[1][4] == "dataset"
if __name__ == "__main__":
import pytest
raise SystemExit(pytest.main([__file__, "-v"]))
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| deprecation-and-migrationaddyosmani/agent-skillsManages deprecation and migration. Use when removing old systems, APIs, or features. Use when migrating users from one implementation to another. Use when migrating a database schema in production, such as renaming or dropping a column without downtime (expand/contract). Use when deciding whether to maintain or sunset existing code. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| performance-optimizationaddyosmani/agent-skillsOptimizes application performance across frontend, backend, queries, and databases. Use when performance requirements exist, when you suspect performance regressions, when Core Web Vitals or load times need improvement, when N+1 query patterns need fixing, or when profiling reveals bottlenecks. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |
| agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |