moonlight-lupin/agent-skills / agent-ops/log-analyzer
log-analyzer
VerifiedUse when parsing agent log files to identify error patterns, rate limit hits, timeout clusters, tool failures, and component-level error counts. Produces a structured anomaly report. Cron-compatible — silent if no issues, alert digest if anomalies found.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 11.7 KB |
| references/anomaly-types.md | markdown | 4.3 KB |
| references/hermes-profile-audit.md | markdown | 3.9 KB |
| references/log-formats.md | markdown | 3.4 KB |
| references/state-failure-monitor.md | markdown | 4.5 KB |
| scripts/analyze_logs.pyexecutable | script | 23.1 KB |
| scripts/hermes_log_window.py | script | 4.2 KB |
| scripts/state_failures.pyexecutable | script | 8.8 KB |
| tests/test_analyze_logs.py | script | 8.1 KB |
| tests/test_state_failures.py | script | 9.0 KB |
tests/test_state_failures.py
View on GitHub"""Tests for log-analyzer's state.db failure-rate monitor (state_failures.py).
Hazard being defended: naive regex-over-content failure detection flags
successful tool output that merely MENTIONS error words (spike on the real
MH profile state.db: Dojo's regex claimed read_file at 70% failure, but
structured exit_code parsing shows the true rate is 0%). The monitor must
use structured signals (exit_code) for JSON-mode tools and a narrow fatal
marker list for text-mode tools.
"""
import importlib.util
import json
import sqlite3
import subprocess
import sys
import time
from pathlib import Path
SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "state_failures.py"
spec = importlib.util.spec_from_file_location("state_failures", SCRIPT)
assert spec is not None and spec.loader is not None
sf = importlib.util.module_from_spec(spec)
spec.loader.exec_module(sf)
def make_db(tmp_path, rows):
"""Build a minimal state.db with only the columns the monitor reads."""
db = tmp_path / "state.db"
con = sqlite3.connect(db)
con.execute(
"CREATE TABLE sessions (id TEXT PRIMARY KEY, started_at REAL)"
)
con.execute(
"CREATE TABLE messages (id INTEGER PRIMARY KEY, session_id TEXT, "
"role TEXT, content TEXT, tool_name TEXT, timestamp REAL)"
)
for row in rows:
con.execute(
"INSERT INTO messages (session_id, role, content, tool_name, timestamp) "
"VALUES (?,?,?,?,?)",
row,
)
con.commit()
con.close()
return str(db)
def t_row(tool, content, ts=None, role="tool", session="s1"):
return (session, role, content, tool, ts if ts is not None else time.time())
# --- classify_result: structured JSON mode -----------------------------
def test_json_exit_code_nonzero_is_failure():
mode, is_fail, _ = sf.classify_result('{"output": "boom", "exit_code": 1}')
assert mode == "json"
assert is_fail is True
def test_json_exit_code_zero_is_success():
mode, is_fail, _ = sf.classify_result('{"output": "ok", "exit_code": 0}')
assert mode == "json"
assert is_fail is False
def test_json_no_exit_code_field_is_success():
# Structured results without an exit_code carry no failure signal.
mode, is_fail, _ = sf.classify_result('{"output": "ok"}')
assert mode == "json"
assert is_fail is False
def test_json_nested_exit_code_detected():
content = '{"output": {"output": "x", "exit_code": 2}}'
mode, is_fail, _ = sf.classify_result(content)
assert mode == "json"
assert is_fail is True
# --- classify_result: text mode with narrow fatal markers --------------
def test_text_fatal_marker_is_failure():
mode, is_fail, _ = sf.classify_result("sh: 1: python3: command not found")
assert mode == "text"
assert is_fail is True
def test_text_mention_of_error_word_is_NOT_failure():
# THE spike regression: content that merely discusses error handling
# must not count as a failure.
mode, is_fail, _ = sf.classify_result(
"Error paths? Tests cover the change? Readability: another engineer"
)
assert mode == "text"
assert is_fail is False
def test_text_academic_mention_is_NOT_failure():
mode, is_fail, _ = sf.classify_result(
"the Scaled Error (Hyndman & Koehler 2006, with scored-window fallback)"
)
assert mode == "text"
assert is_fail is False
def test_malformed_json_falls_back_to_text():
mode, is_fail, _ = sf.classify_result('{"output": "unterminated')
assert mode == "text"
assert is_fail is False
def test_empty_content_is_success():
mode, is_fail, _ = sf.classify_result("")
assert mode == "empty"
assert is_fail is False
# --- categorize failure ------------------------------------------------
def test_categorize_timeout():
assert sf.categorize_failure("the command timed out after 600s") == "timeout"
def test_categorize_network():
assert sf.categorize_failure("Connection refused to 192.168.50.188:3000") == "network"
def test_categorize_command_not_found():
assert sf.categorize_failure("bash: rg: command not found") == "command_not_found"
def test_categorize_not_found():
assert sf.categorize_failure("Error: no such file or directory: /tmp/x") == "not_found"
def test_categorize_permission():
assert sf.categorize_failure("Permission denied (publickey)") == "permission"
def test_categorize_traceback():
assert sf.categorize_failure("Traceback (most recent call last):\nTypeError") == "explicit_error"
def test_categorize_other():
assert sf.categorize_failure("") == "other"
# --- analyze_state_db --------------------------------------------------
def test_analyze_totals_and_rates(tmp_path):
now = time.time()
rows = [
# terminal: 4 calls, 1 failure (exit_code 1)
t_row("terminal", '{"output": "ok", "exit_code": 0}', now),
t_row("terminal", '{"output": "ok", "exit_code": 0}', now),
t_row("terminal", '{"output": "ok", "exit_code": 0}', now),
t_row("terminal", '{"output": "boom", "exit_code": 1}', now),
# read_file: mentions error words but is a success (no exit_code)
t_row("read_file", '{"content": "error handling is hard"}', now),
# web_extract: text mode, benign
t_row("web_extract", "page about exceptions and errors", now),
# process: 2 calls, 1 failure
t_row("process", '{"output": "ok", "exit_code": 0}', now),
t_row("process", '{"output": "timed out", "exit_code": 124}', now),
]
db = make_db(tmp_path, rows)
res = sf.analyze_state_db(db, days=7, min_calls=2)
assert res["tool_calls"] == 8
assert res["failures"] == 2
tools = {t["tool"]: t for t in res["tools"]}
assert tools["terminal"]["total"] == 4
assert tools["terminal"]["failures"] == 1
assert tools["terminal"]["success_rate"] == 75.0
# read_file and web_extract must NOT count as failing
assert tools["read_file"]["failures"] == 0
assert tools["web_extract"]["failures"] == 0
# failure categories aggregated (list of {category, count, tools})
cats = {c["category"]: c for c in res["categories"]}
assert cats["timeout"]["count"] == 1
assert cats["timeout"]["tools"].get("process") == 1
# top tools ranked by failure rate, only ≥ min_calls
top = [t["tool"] for t in res["top_tools"]]
assert top == ["process", "terminal"]
def test_window_excludes_old_rows(tmp_path):
old = time.time() - 10 * 86400
rows = [
t_row("terminal", '{"output": "old fail", "exit_code": 1}', old),
t_row("terminal", '{"output": "ok", "exit_code": 0}', time.time()),
]
db = make_db(tmp_path, rows)
res = sf.analyze_state_db(db, days=7)
assert res["tool_calls"] == 1
assert res["failures"] == 0
def test_low_sample_tools_excluded_from_ranking(tmp_path):
rows = [
t_row("terminal", '{"output": "ok", "exit_code": 0}', time.time()),
# browser_navigate: a real failure but only 1 call total
t_row("browser_navigate", '{"output": "x", "exit_code": 1}', time.time()),
]
db = make_db(tmp_path, rows)
res = sf.analyze_state_db(db, days=7, min_calls=5)
tools = {t["tool"]: t for t in res["tools"]}
assert tools["browser_navigate"]["failures"] == 1
assert all(t["tool"] != "browser_navigate" for t in res["top_tools"])
def test_sessions_count_best_effort(tmp_path):
db = make_db(tmp_path, [t_row("terminal", '{"output": "ok", "exit_code": 0}')])
con = sqlite3.connect(db)
con.execute("INSERT INTO sessions (id, started_at) VALUES ('s1', ?)", (time.time(),))
con.commit()
con.close()
res = sf.analyze_state_db(db, days=7)
assert res["sessions_analyzed"] == 1
def test_missing_db_returns_error(tmp_path):
res = sf.analyze_state_db(str(tmp_path / "nope.db"), days=7)
assert "error" in res
# --- CLI / cron quiet mode ---------------------------------------------
def _run_cli(tmp_path, *args):
db = make_db(tmp_path, [])
cmd = [sys.executable, str(SCRIPT), "--db", db, *args]
return subprocess.run(cmd, capture_output=True, text=True)
def test_quiet_mode_silent_when_no_failures(tmp_path):
r = _run_cli(tmp_path, "--quiet")
assert r.returncode == 0
assert r.stdout.strip() == ""
def test_quiet_mode_reports_when_failures(tmp_path):
now = time.time()
rows = [t_row("terminal", '{"output": "boom", "exit_code": 1}', now)]
db = make_db(tmp_path, rows)
r = subprocess.run(
[sys.executable, str(SCRIPT), "--db", db, "--quiet"],
capture_output=True,
text=True,
)
assert r.returncode == 0
assert "terminal" in r.stdout
assert "8.0" not in r.stdout # sanity: real numbers, not placeholder
def test_json_output_shape(tmp_path):
now = time.time()
rows = [t_row("terminal", '{"output": "boom", "exit_code": 1}', now)]
db = make_db(tmp_path, rows)
r = subprocess.run(
[sys.executable, str(SCRIPT), "--db", db, "--json"],
capture_output=True,
text=True,
)
assert r.returncode == 0
d = json.loads(r.stdout)
assert d["failures"] == 1
assert d["tools"][0]["tool"] == "terminal"
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| brainstormingobra/superpowersYou MUST use this before any creative work - creating features, building components, adding functionality, modifying behavior, or planning anything new, in software or out of it (a talk, a business, a renovation). | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| brand-guidelinesanthropics/skillsApplies Anthropic's official brand colors and typography to any sort of artifact that may benefit from having Anthropic's look-and-feel. Use it when brand colors or style guidelines, visual formatting, or company design standards apply. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| frontend-designanthropics/skillsGuidance for distinctive, intentional visual design when building new UI or reshaping an existing one. Helps with aesthetic direction, typography, and making choices that don't read as templated defaults. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| mcp-builderanthropics/skillsGuide for creating high-quality MCP (Model Context Protocol) servers that enable LLMs to interact with external services through well-designed tools. Use when building MCP servers to integrate external APIs or services, whether in Python (FastMCP) or Node/TypeScript (MCP SDK). | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |