moonlight-lupin/agent-skills / research/deep-research
deep-research
VerifiedUse when the user asks for "deep research", a "research report on", a "comprehensive analysis of", to "look into X in depth", to "write a report on X", or any question needing multi-source synthesis beyond a single search — an autonomous multi-step deep research engine implementing an iterative Think → Search → Extract → Synthesize → Stop loop. The LLM drives every decision: what to search, what's relevant, what's missing, and when to stop. Produces a cited, magazine-quality report with inline citations, category-specific formatting, and research stats. For entity vetting/dossiers use entity-research; for news digests use news-monitoring; for source-grounded Q&A use notebooklm-mode.
scripts/url_health.py
View on GitHub#!/usr/bin/env python3
"""Cron-compatible link health checker for citation-database databases."""
from __future__ import annotations
import argparse
import datetime as _dt
import os
import sqlite3
import sys
import urllib.error
import urllib.parse
import urllib.request
from typing import Optional, Sequence
DEFAULT_DB_PATH = "./sources.db"
SCHEMA = """
CREATE TABLE IF NOT EXISTS sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url TEXT NOT NULL UNIQUE,
url_normalized TEXT NOT NULL,
title TEXT,
topic TEXT NOT NULL,
source_type TEXT DEFAULT 'web',
accessed_at TEXT NOT NULL,
notes TEXT,
verified INTEGER DEFAULT 1,
last_checked TEXT,
session_id TEXT
);
CREATE INDEX IF NOT EXISTS idx_topic ON sources(topic);
CREATE INDEX IF NOT EXISTS idx_normalized ON sources(url_normalized);
"""
def resolve_db_path(db_path: Optional[str]) -> str:
"""Resolve DB path from CLI option, environment, or default."""
return db_path or os.environ.get("SOURCE_TRACKER_DB") or DEFAULT_DB_PATH
def connect(db_path: str) -> sqlite3.Connection:
"""Open a SQLite connection and ensure the source schema exists."""
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
conn.executescript(SCHEMA)
conn.commit()
return conn
def is_alive(url: str, timeout: int) -> bool:
"""Return True when a HEAD request receives HTTP 200-399.
Servers that reject HEAD (405/501) get one GET retry so they aren't
misreported as dead. Non-web schemes are never probed.
"""
if urllib.parse.urlsplit(url).scheme.lower() not in ("http", "https"):
return False
for method in ("HEAD", "GET"):
try:
request = urllib.request.Request(url, method=method, headers={"User-Agent": "deep-research-citation-db/1.0"})
with urllib.request.urlopen(request, timeout=timeout) as response:
code = getattr(response, "status", None)
if code is None:
code = response.getcode()
return 200 <= int(code) <= 399
except urllib.error.HTTPError as exc:
if method == "HEAD" and int(exc.code) in (405, 501):
continue
return 200 <= int(exc.code) <= 399
except Exception:
return False
return False
def stale_cutoff(stale_days: int) -> str:
"""Return the ISO date before which link checks are considered stale."""
return (_dt.date.today() - _dt.timedelta(days=stale_days)).isoformat()
def select_stale_sources(conn: sqlite3.Connection, stale_days: int, batch_size: int) -> list[sqlite3.Row]:
"""Select sources that have never been checked or are older than the cutoff."""
cutoff = stale_cutoff(stale_days)
return conn.execute(
"""
SELECT id, url
FROM sources
WHERE last_checked IS NULL OR last_checked < ?
ORDER BY COALESCE(last_checked, ''), id
LIMIT ?
""",
(cutoff, batch_size),
).fetchall()
def check_sources(conn: sqlite3.Connection, stale_days: int = 30, timeout: int = 10, batch_size: int = 50) -> dict[str, int]:
"""Check a batch of stale sources and update verified/last_checked fields."""
rows = select_stale_sources(conn, stale_days, batch_size)
checked = alive = dead = 0
today = _dt.date.today().isoformat()
for row in rows:
ok = is_alive(row["url"], timeout)
conn.execute(
"UPDATE sources SET verified = ?, last_checked = ? WHERE id = ?",
(1 if ok else 0, today, int(row["id"])),
)
checked += 1
if ok:
alive += 1
else:
dead += 1
conn.commit()
return {"checked": checked, "alive": alive, "dead": dead}
def build_parser() -> argparse.ArgumentParser:
"""Build the command-line argument parser."""
parser = argparse.ArgumentParser(description="Check stale URLs in a citation-database SQLite database.")
parser.add_argument("--db-path", help="SQLite database path (default: SOURCE_TRACKER_DB or ./sources.db)")
parser.add_argument("--stale-days", type=int, default=30, help="Only check URLs not checked in this many days (default: 30).")
parser.add_argument("--timeout", type=int, default=10, help="HTTP HEAD timeout per URL in seconds (default: 10).")
parser.add_argument("--batch-size", type=int, default=50, help="Maximum URLs to check per run (default: 50).")
return parser
def main(argv: Optional[Sequence[str]] = None) -> int:
"""Run the URL health checker and always exit successfully."""
parser = build_parser()
args = parser.parse_args(argv)
db_path = resolve_db_path(args.db_path)
try:
with connect(db_path) as conn:
result = check_sources(conn, args.stale_days, args.timeout, args.batch_size)
print(f"Checked {result['checked']} URLs: {result['alive']} alive, {result['dead']} dead")
except Exception as exc:
print(f"Checked 0 URLs: 0 alive, 0 dead", file=sys.stdout)
print(f"warning: health check failed: {exc}", file=sys.stderr)
return 0
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| deprecation-and-migrationaddyosmani/agent-skillsManages deprecation and migration. Use when removing old systems, APIs, or features. Use when migrating users from one implementation to another. Use when migrating a database schema in production, such as renaming or dropping a column without downtime (expand/contract). Use when deciding whether to maintain or sunset existing code. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| performance-optimizationaddyosmani/agent-skillsOptimizes application performance across frontend, backend, queries, and databases. Use when performance requirements exist, when you suspect performance regressions, when Core Web Vitals or load times need improvement, when N+1 query patterns need fixing, or when profiling reveals bottlenecks. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |
| agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |