moonlight-lupin/agent-skills / research/library-rag
library-rag
UnverifiedSemantic search over a personal library using Nemotron-3-Embed-1B embeddings + sqlite-vec. Index books, documents, any text corpus; query by meaning. Includes EPUB→Markdown conversion and MCP server for auto-available search tools.
| File | Kind | Size |
|---|---|---|
| .gitignore | other | 305 B |
| LICENSE | other | 1.3 KB |
| README.md | markdown | 11.8 KB |
| SKILL.md | skill md | 18.9 KB |
| references/chunking-strategies.md | markdown | 5.8 KB |
| references/epub-conversion.md | markdown | 3.8 KB |
| references/openrouter-embeddings.md | markdown | 4.1 KB |
| references/portable-rag-per-skill.md | markdown | 12.1 KB |
| references/rag-pipeline-review.md | markdown | 9.7 KB |
| requirements-dev.txt | markdown | 167 B |
| requirements.txt | manifest | 286 B |
| scripts/convert_epub_library.py | script | 13.0 KB |
| scripts/convert_pdf_library.py | script | 6.0 KB |
| scripts/mcp_server.py | script | 10.9 KB |
| scripts/rag_common.py | script | 8.7 KB |
| scripts/rag_index.py | script | 23.4 KB |
| scripts/rag_query.py | script | 6.8 KB |
| tests/conftest.py | script | 188 B |
| tests/test_chunking.py | script | 5.0 KB |
| tests/test_convert.py | script | 4.6 KB |
| tests/test_convert_pdf.py | script | 4.1 KB |
| tests/test_index_search.py | script | 4.8 KB |
| tests/test_indexing_atomicity.py | script | 4.8 KB |
| tests/test_prune_vacuum.py | script | 5.5 KB |
| tests/test_rag_common.py | script | 4.9 KB |
tests/test_indexing_atomicity.py
View on GitHub"""Tests for atomic per-file indexing (index_file): partial-failure rollback
and stale-chunk replacement. No API calls — embeddings are injected."""
import sqlite3
import pytest
import rag_index as ri
def fake_embeddings_ok(dim):
return lambda texts, key, retries=3: ([[0.1] * dim for _ in texts],
{"prompt_tokens": 1, "cost": 0.0})
def make_doc(path, paras=8):
body = "\n\n".join(f"Paragraph {i} " + "word " * 40 for i in range(paras))
path.write_text(f"---\nbook: B\n---\n\n## Heading\n\n{body}")
def test_index_file_rolls_back_on_failure(tmp_path, monkeypatch):
db = tmp_path / "db.sqlite"
conn = sqlite3.connect(str(db))
ri.init_db(conn, rebuild=True)
p = tmp_path / "doc.md"
make_doc(p)
monkeypatch.setattr(ri, "BATCH_SIZE", 1) # force multiple batches
dim = ri.EMBEDDING_DIMS
calls = {"n": 0}
def flaky(texts, key, retries=3):
calls["n"] += 1
if calls["n"] >= 2:
raise RuntimeError("simulated API failure")
return [[0.1] * dim for _ in texts], {"prompt_tokens": 1, "cost": 0.0}
monkeypatch.setattr(ri, "get_embeddings", flaky)
with pytest.raises(RuntimeError):
ri.index_file(conn, str(p), "books", ri.chunk_markdown, "key")
# Partial chunks rolled back; file NOT recorded as indexed -> retried next run.
assert conn.execute("SELECT COUNT(*) FROM chunks").fetchone()[0] == 0
assert conn.execute("SELECT COUNT(*) FROM vec_chunks").fetchone()[0] == 0
assert conn.execute(
"SELECT COUNT(*) FROM indexed_files WHERE file_path = ?", (str(p),)
).fetchone()[0] == 0
def test_index_file_replaces_stale_chunks(tmp_path, monkeypatch):
db = tmp_path / "db.sqlite"
conn = sqlite3.connect(str(db))
ri.init_db(conn, rebuild=True)
p = tmp_path / "doc.md"
make_doc(p)
dim = ri.EMBEDDING_DIMS
monkeypatch.setattr(ri, "get_embeddings", fake_embeddings_ok(dim))
stored1, _ = ri.index_file(conn, str(p), "books", ri.chunk_markdown, "key")
assert stored1 > 0
count1 = conn.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
# Re-index the same file: chunks must be replaced, not accumulated.
ri.index_file(conn, str(p), "books", ri.chunk_markdown, "key")
count2 = conn.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
assert count2 == count1
# vec rows stay in lockstep with chunk rows
assert conn.execute("SELECT COUNT(*) FROM vec_chunks").fetchone()[0] == count2
assert conn.execute(
"SELECT chunk_count FROM indexed_files WHERE file_path = ?", (str(p),)
).fetchone()[0] == stored1
def test_index_file_preserves_previous_version_on_failure(tmp_path, monkeypatch):
db = tmp_path / "db.sqlite"
conn = sqlite3.connect(str(db))
ri.init_db(conn, rebuild=True)
dim = ri.EMBEDDING_DIMS
p = tmp_path / "doc.md"
make_doc(p, paras=8)
monkeypatch.setattr(ri, "get_embeddings", fake_embeddings_ok(dim))
# First successful index — the "old good" version.
stored1, _ = ri.index_file(conn, str(p), "books", ri.chunk_markdown, "key")
assert stored1 > 0
count_old = conn.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
first_hash = conn.execute(
"SELECT file_hash FROM indexed_files WHERE file_path = ?", (str(p),)).fetchone()[0]
# Change the file (new content + hash), then make the re-index fail mid-way.
make_doc(p, paras=14)
monkeypatch.setattr(ri, "BATCH_SIZE", 1)
calls = {"n": 0}
def flaky(texts, key, retries=3):
calls["n"] += 1
if calls["n"] >= 2:
raise RuntimeError("simulated API failure")
return [[0.1] * dim for _ in texts], {"prompt_tokens": 1, "cost": 0.0}
monkeypatch.setattr(ri, "get_embeddings", flaky)
with pytest.raises(RuntimeError):
ri.index_file(conn, str(p), "books", ri.chunk_markdown, "key")
# Previous version stays fully searchable; no partial new rows linger.
assert conn.execute("SELECT COUNT(*) FROM chunks").fetchone()[0] == count_old
assert conn.execute("SELECT COUNT(*) FROM vec_chunks").fetchone()[0] == count_old
# indexed_files still points at the OLD hash, so the next run detects the change and retries.
assert conn.execute(
"SELECT file_hash FROM indexed_files WHERE file_path = ?", (str(p),)
).fetchone()[0] == first_hash
def test_index_file_empty_returns_zero(tmp_path, monkeypatch):
db = tmp_path / "db.sqlite"
conn = sqlite3.connect(str(db))
ri.init_db(conn, rebuild=True)
p = tmp_path / "empty.txt"
p.write_text("") # no chunks
monkeypatch.setattr(ri, "get_embeddings", fake_embeddings_ok(ri.EMBEDDING_DIMS))
stored, usage = ri.index_file(conn, str(p), "notes", ri.chunk_plain_text, "key")
assert stored == 0
assert conn.execute("SELECT COUNT(*) FROM chunks").fetchone()[0] == 0
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| deprecation-and-migrationaddyosmani/agent-skillsManages deprecation and migration. Use when removing old systems, APIs, or features. Use when migrating users from one implementation to another. Use when migrating a database schema in production, such as renaming or dropping a column without downtime (expand/contract). Use when deciding whether to maintain or sunset existing code. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| performance-optimizationaddyosmani/agent-skillsOptimizes application performance across frontend, backend, queries, and databases. Use when performance requirements exist, when you suspect performance regressions, when Core Web Vitals or load times need improvement, when N+1 query patterns need fixing, or when profiling reveals bottlenecks. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |
| agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |