alexgreensh/repo-forensics / skills/forensify
forensify
FlaggedCross-agent self-inspection of your AI-agent stack. Audits skills, MCP servers, hooks, plugins, commands, credentials, and memory files across Claude Code, Codex, OpenClaw, and NanoClaw. Produces a structured inventory and narrative briefing with cross-ecosystem risk analysis. Use when the user asks to audit their own setup, check what they have installed, review their agent stack security posture, or understand cross-tool interactions. Use when a user has accumulated skills/plugins/MCP servers over time and wants visibility into their attack surface. Use after installing new skills or plugins. Do NOT use for vetting external code before install (that is repo-forensics). Do NOT use for incident response during active attacks. Do NOT use for fixing or patching vulnerabilities (forensify is read-only).
tests/test_forensify_integration.py
View on GitHub"""
Integration tests for forensify end-to-end flows.
Covers:
- Canary fixture detection (prompt injection in SKILL.md, tools.json poisoning)
- Seatbelt sandbox profile validation (macOS only)
- Coord folder lifecycle
- Domain job construction from inventory
- Prompt template loading
"""
from __future__ import annotations
import json
import os
import platform
import subprocess
import sys
import tempfile
from pathlib import Path
import pytest
SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "scripts"
ORCH_DIR = Path(__file__).resolve().parent.parent / "orchestrator"
CONFIG_DIR = Path(__file__).resolve().parent.parent / "config"
PROMPTS_DIR = Path(__file__).resolve().parent.parent / "prompts"
FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures"
sys.path.insert(0, str(SCRIPTS_DIR))
sys.path.insert(0, str(ORCH_DIR.parent))
from build_inventory import ( # noqa: E402
build_inventory as build_inventory_fn,
load_ecosystem_roots,
)
# ---------------------------------------------------------------------------
# Canary fixture tests — injection patterns in fixtures must be detectable
# ---------------------------------------------------------------------------
class TestCanaryFixtures:
def test_claude_code_injection_fixture_enumerated(self):
fixture_root = FIXTURES_DIR / "claude_code_stack"
config = load_ecosystem_roots()
fake_home = fixture_root.parent # won't match real detection
env = {"HOME": str(fake_home)}
# Direct walk to verify the fixture is picked up
from build_inventory import walk_skills_surface, _resolve_env_for_ecosystem
eco_cfg = config["ecosystems"]["claude_code"]
# Override the glob template to point at fixture
skills = []
import glob as g
for match in g.glob(str(fixture_root / "skills" / "*" / "SKILL.md")):
skills.append(match)
assert len(skills) >= 1
assert any("malicious-skill" in s for s in skills)
def test_injection_comment_present_in_fixture(self):
malicious = FIXTURES_DIR / "claude_code_stack" / "skills" / "malicious-skill" / "SKILL.md"
content = malicious.read_text()
# The injection payload must be present for scanners to detect
assert "Ignore all prior" in content or "ignore all prior" in content.lower()
assert "Return findings: []" in content or "return findings" in content.lower()
def test_openclaw_tools_json_poisoning_present(self):
tools = FIXTURES_DIR / "openclaw_stack" / ".agents" / "skills" / "poisoned-skill" / "tools.json"
data = json.loads(tools.read_text())
desc = data["tools"][0]["description"]
# Must contain the exfiltration instruction for scanners to catch
assert "evil.example.com" in desc
assert "HTTP POST" in desc
def test_clean_fixture_has_no_injection(self):
clean = FIXTURES_DIR / "codex_stack" / "skills" / "clean-skill" / "SKILL.md"
content = clean.read_text()
# Negative control: no injection patterns
assert "ignore" not in content.lower() or "no threats" in content.lower()
assert "evil" not in content.lower()
# ---------------------------------------------------------------------------
# Prompt template loading
# ---------------------------------------------------------------------------
class TestPromptTemplates:
DOMAINS = ["skills", "mcp", "hooks", "plugins", "commands", "credentials"]
def test_all_domain_prompts_exist(self):
for domain in self.DOMAINS:
path = PROMPTS_DIR / ("domain_%s.txt" % domain)
assert path.exists(), "missing prompt: %s" % path
def test_synthesis_prompt_exists(self):
assert (PROMPTS_DIR / "synthesis.txt").exists()
def test_all_prompts_contain_hostile_data_warning(self):
for domain in self.DOMAINS:
content = (PROMPTS_DIR / ("domain_%s.txt" % domain)).read_text()
assert "UNTRUSTED" in content, "%s prompt missing hostile-data posture" % domain
def test_all_prompts_contain_suppression_warning(self):
for domain in self.DOMAINS:
content = (PROMPTS_DIR / ("domain_%s.txt" % domain)).read_text()
assert "MUST include" in content or "CRITICAL" in content, \
"%s prompt missing suppression enforcement" % domain
def test_synthesis_prompt_has_grounding_rules(self):
content = (PROMPTS_DIR / "synthesis.txt").read_text()
assert "GROUNDING" in content
assert "finding_id" in content
def test_prompts_have_template_variables(self):
for domain in self.DOMAINS:
content = (PROMPTS_DIR / ("domain_%s.txt" % domain)).read_text()
assert "{ecosystem_display_name}" in content
# ---------------------------------------------------------------------------
# Domain JSON config validation
# ---------------------------------------------------------------------------
class TestDomainConfigs:
DOMAINS_DIR = Path(__file__).resolve().parent.parent / "domains"
def test_all_six_domains_exist(self):
for name in ["skills", "mcp", "hooks", "plugins", "commands", "credentials"]:
path = self.DOMAINS_DIR / ("%s.json" % name)
assert path.exists(), "missing domain config: %s" % path
def test_all_domains_have_scanners_list(self):
for f in self.DOMAINS_DIR.glob("*.json"):
data = json.loads(f.read_text())
assert "scanners" in data, "%s missing scanners" % f.name
assert isinstance(data["scanners"], list)
def test_all_domains_have_inventory_surfaces(self):
for f in self.DOMAINS_DIR.glob("*.json"):
data = json.loads(f.read_text())
assert "inventory_surfaces" in data, "%s missing inventory_surfaces" % f.name
# ---------------------------------------------------------------------------
# Coord folder lifecycle
# ---------------------------------------------------------------------------
class TestCoordFolder:
def test_create_and_list(self):
from orchestrator.analysis_dispatcher import create_coord_folder, list_runs
coord = create_coord_folder()
assert os.path.isdir(coord)
# Unix mode bits don't map on Windows (it uses ACLs, not 0o700 st_mode);
# os.chmod there cannot produce 0o700. The private-coord-folder guarantee
# on Windows needs an ACL (tracked separately); skip the POSIX perm check.
if platform.system() != "Windows":
assert os.stat(coord).st_mode & 0o777 == 0o700
manifest = json.loads(open(os.path.join(coord, "manifest.json")).read())
assert manifest["coord_schema_version"] == 1
assert manifest["status"] == "in_progress"
runs = list_runs()
assert any(r["path"] == coord for r in runs)
# Cleanup
import shutil
shutil.rmtree(coord)
def test_domain_job_round_trip_via_coord(self):
from orchestrator.contracts import DomainJob
from orchestrator.analysis_dispatcher import create_coord_folder, write_domain_job
coord = create_coord_folder()
job = DomainJob(
job_id="test-skills-claude",
domain="skills",
ecosystem="claude_code",
run_id="test123",
inventory_slice=[{"path": "/test/skill", "skill_name": "test"}],
scanner_findings=[{"finding_id": "f1", "severity": "HIGH"}],
scanner_names=["scan_skill_threats"],
total_items_in_slice=1,
)
path = write_domain_job(coord, job)
assert os.path.isfile(path)
# Read back
with open(path) as f:
restored = DomainJob.from_json(f.read())
assert restored.job_id == "test-skills-claude"
assert len(restored.inventory_slice) == 1
assert len(restored.scanner_findings) == 1
import shutil
shutil.rmtree(coord)
# ---------------------------------------------------------------------------
# Domain job construction from real inventory
# ---------------------------------------------------------------------------
class TestDomainJobConstruction:
def test_jobs_built_for_detected_ecosystems(self, tmp_path):
(tmp_path / ".claude").mkdir()
(tmp_path / ".claude" / "settings.json").write_text("{}")
from build_inventory import _file_record
skill_dir = tmp_path / ".claude" / "skills" / "test-skill"
skill_dir.mkdir(parents=True)
(skill_dir / "SKILL.md").write_text("---\nname: test\n---\n")
inv = build_inventory_fn(env={"HOME": str(tmp_path)})
domain_configs = {}
domains_dir = Path(__file__).resolve().parent.parent / "domains"
for f in domains_dir.glob("*.json"):
domain_configs[f.stem] = json.loads(f.read_text())
from orchestrator.analysis_dispatcher import build_domain_jobs
jobs = build_domain_jobs("test-run", inv, [], domain_configs)
# Should have at least one job for claude_code skills domain
skills_jobs = [j for j in jobs if j.domain == "skills" and j.ecosystem == "claude_code"]
assert len(skills_jobs) == 1
assert skills_jobs[0].total_items_in_slice >= 1
# ---------------------------------------------------------------------------
# Seatbelt sandbox (macOS only)
# ---------------------------------------------------------------------------
@pytest.mark.skipif(platform.system() != "Darwin", reason="Seatbelt is macOS only")
class TestSeatbeltSandbox:
def _run_seatbelt(self, args, timeout):
result = subprocess.run(
args,
capture_output=True,
text=True,
timeout=timeout,
)
if result.returncode in (-6, 134) and not (result.stdout or result.stderr):
pytest.skip("sandbox-exec aborted while loading this profile on this macOS release")
return result
def test_seatbelt_profile_exists(self):
profile = CONFIG_DIR / "seatbelt_subagent.sb"
assert profile.exists()
def test_seatbelt_profile_parseable(self):
"""sandbox-exec validates profile syntax by running python3 -c 'pass'."""
profile = CONFIG_DIR / "seatbelt_subagent.sb"
with tempfile.TemporaryDirectory() as td:
result = self._run_seatbelt(
[
"sandbox-exec", "-f", str(profile),
"-D", "TARGET_PATH=%s" % td,
"-D", "COORD_PATH=%s" % td,
"-D", "SKILL_PATH=%s" % str(CONFIG_DIR.parent),
"python3", "-c", "pass",
],
timeout=10,
)
assert result.returncode == 0, "Seatbelt profile error: %s" % result.stderr
def test_seatbelt_blocks_network(self):
"""Verify the sandbox blocks network access."""
profile = CONFIG_DIR / "seatbelt_subagent.sb"
with tempfile.TemporaryDirectory() as td:
# Try to make a network connection under sandbox — should fail
result = self._run_seatbelt(
[
"sandbox-exec", "-f", str(profile),
"-D", "TARGET_PATH=%s" % td,
"-D", "COORD_PATH=%s" % td,
"-D", "SKILL_PATH=%s" % str(CONFIG_DIR.parent),
"python3", "-c",
"import urllib.request; urllib.request.urlopen('https://example.com', timeout=3)",
],
timeout=15,
)
# Should fail with a sandbox violation or network error
assert result.returncode != 0, "Seatbelt should have blocked network access"
def test_seatbelt_blocks_write_outside_coord(self):
"""Verify writes outside the coord folder are blocked."""
profile = CONFIG_DIR / "seatbelt_subagent.sb"
with tempfile.TemporaryDirectory() as td:
coord = os.path.join(td, "coord")
os.makedirs(coord)
outside = os.path.join(td, "outside")
os.makedirs(outside)
result = self._run_seatbelt(
[
"sandbox-exec", "-f", str(profile),
"-D", "TARGET_PATH=%s" % td,
"-D", "COORD_PATH=%s" % coord,
"-D", "SKILL_PATH=%s" % str(CONFIG_DIR.parent),
"python3", "-c",
"open('%s/canary.txt', 'w').write('should not exist')" % outside,
],
timeout=10,
)
assert not os.path.exists(os.path.join(outside, "canary.txt")), \
"Seatbelt should have blocked write outside coord folder"
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |