moonlight-lupin/agent-skills / research/library-rag
library-rag
UnverifiedSemantic search over a personal library using Nemotron-3-Embed-1B embeddings + sqlite-vec. Index books, documents, any text corpus; query by meaning. Includes EPUB→Markdown conversion and MCP server for auto-available search tools.
| File | Kind | Size |
|---|---|---|
| .gitignore | other | 305 B |
| LICENSE | other | 1.3 KB |
| README.md | markdown | 11.8 KB |
| SKILL.md | skill md | 18.9 KB |
| references/chunking-strategies.md | markdown | 5.8 KB |
| references/epub-conversion.md | markdown | 3.8 KB |
| references/openrouter-embeddings.md | markdown | 4.1 KB |
| references/portable-rag-per-skill.md | markdown | 12.1 KB |
| references/rag-pipeline-review.md | markdown | 9.7 KB |
| requirements-dev.txt | markdown | 167 B |
| requirements.txt | manifest | 286 B |
| scripts/convert_epub_library.py | script | 13.0 KB |
| scripts/convert_pdf_library.py | script | 6.0 KB |
| scripts/mcp_server.py | script | 10.9 KB |
| scripts/rag_common.py | script | 8.7 KB |
| scripts/rag_index.py | script | 23.4 KB |
| scripts/rag_query.py | script | 6.8 KB |
| tests/conftest.py | script | 188 B |
| tests/test_chunking.py | script | 5.0 KB |
| tests/test_convert.py | script | 4.6 KB |
| tests/test_convert_pdf.py | script | 4.1 KB |
| tests/test_index_search.py | script | 4.8 KB |
| tests/test_indexing_atomicity.py | script | 4.8 KB |
| tests/test_prune_vacuum.py | script | 5.5 KB |
| tests/test_rag_common.py | script | 4.9 KB |
scripts/rag_query.py
View on GitHub#!/usr/bin/env python3
"""
RAG Query — semantic search over a personal library.
Usage:
python3 rag_query.py "your search query"
python3 rag_query.py "search terms" --top-k 5
python3 rag_query.py "query" --source my-source-type --verbose
python3 rag_query.py --stats
Can also be imported:
from rag_query import search
results = search("query text", top_k=10)
"""
import os, sys, json, argparse
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from rag_common import load_api_key, get_embedding, float_to_blob, connect_db
# ─── Config ───────────────────────────────────────────────────────────────────
LIBRARY_ROOT = Path(os.environ.get('LIBRARY_ROOT', os.path.expanduser('~/.hermes/library')))
DB_PATH = LIBRARY_ROOT / 'rag_index.db'
# ─── Search ───────────────────────────────────────────────────────────────────
def search(query, top_k=10, source_type=None, api_key=None):
"""
Semantic search over the library.
Args:
query: Natural language search query
top_k: Number of results to return
source_type: Optional filter by source type (top-level directory name)
api_key: Embedding API key (loaded from env if not provided)
Returns:
List of dicts with: rank, similarity, source_type, source_book,
book, chapter, section_title, chunk_text, source_file
"""
if api_key is None:
api_key = load_api_key()
if not DB_PATH.exists():
raise FileNotFoundError(f"Index DB not found at {DB_PATH}. Run rag_index.py first.")
query_emb = get_embedding(query, api_key)
query_blob = float_to_blob(query_emb)
conn = connect_db(DB_PATH)
total_vecs = conn.execute('SELECT COUNT(*) FROM vec_chunks').fetchone()[0]
if total_vecs == 0:
conn.close()
return []
# sqlite-vec MATCH/k can't combine with WHERE clauses, so we over-fetch and
# post-filter by source_type. When a source type is sparse, the first over-fetch
# may not yield top_k matches — iteratively grow k (capped at the corpus size)
# until we have enough or have scanned everything.
fetch_k = min(top_k * 5 if source_type else top_k, total_vecs)
results = []
while True:
raw_results = conn.execute('''
SELECT rowid, distance FROM vec_chunks
WHERE embedding MATCH ? AND k = ?
ORDER BY distance
''', (query_blob, fetch_k)).fetchall()
results = []
for rowid, distance in raw_results:
row = conn.execute('''
SELECT source_type, source_book, source_file, book, chapter,
section_title, chunk_text
FROM chunks WHERE id = ?
''', (rowid,)).fetchone()
if not row:
continue
if source_type and row[0] != source_type:
continue
sim = max(0, 1 - distance ** 2 / 2)
results.append({
'rowid': rowid,
'distance': distance,
'similarity': sim,
'source_type': row[0],
'source_book': row[1],
'source_file': row[2],
'book': row[3],
'chapter': row[4],
'section_title': row[5],
'chunk_text': row[6],
})
if len(results) >= top_k:
break
if len(results) >= top_k or fetch_k >= total_vecs or not source_type:
break
fetch_k = min(fetch_k * 2, total_vecs)
conn.close()
results.sort(key=lambda x: x['similarity'], reverse=True)
for i, r in enumerate(results):
r['rank'] = i + 1
return results
def stats():
"""Print index statistics."""
conn = connect_db(DB_PATH)
total = conn.execute('SELECT COUNT(*) FROM chunks').fetchone()[0]
by_type = conn.execute('''
SELECT source_type, COUNT(*) as cnt FROM chunks
GROUP BY source_type ORDER BY cnt DESC
''').fetchall()
files = conn.execute('SELECT COUNT(*) FROM indexed_files').fetchone()[0]
print(f"📚 RAG Index Statistics")
print(f" DB: {DB_PATH}")
print(f" Total chunks: {total:,}")
print(f" Files indexed: {files}")
print(f"\n By source type:")
for stype, cnt in by_type:
print(f" {stype:25s} {cnt:>7,}")
db_size = DB_PATH.stat().st_size
print(f"\n DB size: {db_size / 1024 / 1024:.1f} MB")
conn.close()
# ─── CLI ──────────────────────────────────────────────────────────────────────
def format_result(r, verbose=False):
"""Format a single result for display."""
citation_parts = []
if r['book']:
citation_parts.append(r['book'])
if r['chapter']:
citation_parts.append(r['chapter'])
if r['section_title']:
citation_parts.append(r['section_title'])
citation = ' — '.join(citation_parts) if citation_parts else 'Unknown section'
source_label = f"[{r['source_type']}]"
book_label = f"({r['source_book']})" if r['source_book'] else ""
sim_pct = f"{r['similarity']:.1%}"
text = r['chunk_text']
if not verbose and len(text) > 400:
text = text[:400] + '...'
return f"{r['rank']}. {source_label} {book_label} {citation} ({sim_pct})\n {text}"
def main():
parser = argparse.ArgumentParser(description='RAG Query')
parser.add_argument('query', nargs='?', help='Search query')
parser.add_argument('--top-k', type=int, default=10, help='Number of results')
parser.add_argument('--source', type=str, help='Filter by source type')
parser.add_argument('--stats', action='store_true', help='Show index statistics')
parser.add_argument('--verbose', action='store_true', help='Show full chunk text')
parser.add_argument('--json', action='store_true', help='Output as JSON')
args = parser.parse_args()
if args.stats:
stats()
return
if not args.query:
parser.print_help()
return
results = search(
args.query,
top_k=args.top_k,
source_type=args.source,
)
if args.json:
print(json.dumps(results, ensure_ascii=False, indent=2))
else:
if not results:
print("No results found.")
return
print(f"🔍 Query: \"{args.query}\"")
if args.source:
print(f" Filter: source={args.source}")
print(f" Results: {len(results)}\n")
for r in results:
print(format_result(r, verbose=args.verbose))
print()
if __name__ == '__main__':
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| deprecation-and-migrationaddyosmani/agent-skillsManages deprecation and migration. Use when removing old systems, APIs, or features. Use when migrating users from one implementation to another. Use when migrating a database schema in production, such as renaming or dropping a column without downtime (expand/contract). Use when deciding whether to maintain or sunset existing code. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| performance-optimizationaddyosmani/agent-skillsOptimizes application performance across frontend, backend, queries, and databases. Use when performance requirements exist, when you suspect performance regressions, when Core Web Vitals or load times need improvement, when N+1 query patterns need fixing, or when profiling reveals bottlenecks. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |
| agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |