alexgreensh/repo-forensics / skills/repo-forensics
repo-forensics
FlaggedSecurity forensics for git repos, AI skills, and MCP servers. Audits dependencies, detects prompt injection, credential theft, runtime dynamism, manifest drift, known CVEs, CISA KEV (actively exploited) vulns, and 2026 attack patterns. Not for fixing vulnerabilities or pentesting.
scripts/refresh_threat_dbs.py
View on GitHub#!/usr/bin/env python3
"""
refresh_threat_dbs.py - Daily background refresher for repo-forensics threat DBs.
Designed to run in the background once per day: through launchd on macOS or a
detached SessionStart kick on other platforms. Refreshes IOC + KEV caches
without making SessionStart wait on network calls.
Safety properties (post-review hardening):
- Lock file in ~/.cache (NOT /tmp) with O_NOFOLLOW (no symlink follow).
- Single-instance via fcntl.flock or msvcrt.locking.
- Cross-platform hard wall-clock cap via a supervising parent process.
- Atomic writes: ioc_manager and vuln_feed must use temp+rename internally.
- Modules loaded by absolute path via importlib (sys.path NOT polluted).
- Log inputs sanitized (no CR/LF injection from remote feed).
- Log rotation at 256KB.
- Always exits 0 (no launchd retry storms).
- Kill switch: REPO_FORENSICS_DISABLE_REFRESH=1.
- No tool calls, no scanner invocation = no recursion path.
Created by Alex Greenshpun.
"""
import errno
import importlib.util
import json
import os
import signal
import socket
import subprocess
import sys
import tempfile
import time
try:
import fcntl # POSIX
except ImportError: # pragma: no cover - exercised on Windows
fcntl = None
try:
import msvcrt # Windows
except ImportError: # pragma: no cover - POSIX
msvcrt = None
# ---------------------------------------------------------------------------
# Hard limits
# ---------------------------------------------------------------------------
REFRESH_HARD_CAP_SEC = 60
SOCKET_DEFAULT_TIMEOUT = 15
LOG_MAX_BYTES = 256 * 1024
LOG_MSG_MAX_LEN = 512
ENV_KILL_SWITCH = "REPO_FORENSICS_DISABLE_REFRESH"
CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache", "repo-forensics")
LOG_FILE = os.path.join(CACHE_DIR, "refresh.log")
LAST_RUN_MARKER = os.path.join(CACHE_DIR, ".last-refresh-v2")
LAST_ATTEMPT_MARKER = os.path.join(CACHE_DIR, ".last-refresh-attempt")
STATE_FILE = os.path.join(CACHE_DIR, "refresh-state.json")
DISABLED_MARKER = os.path.join(CACHE_DIR, "refresh.disabled")
LOCK_FILE = os.path.join(CACHE_DIR, "refresh.lock") # In CACHE_DIR, NOT /tmp
def _sanitize(s):
"""Allowlist printable ASCII + tab. Defeats log forging via terminal escapes
(ESC, BEL, BS, ANSI cursor controls) embedded in attacker-controlled feed
text that an analyst might `cat` from refresh.log."""
s = str(s)
if len(s) > LOG_MSG_MAX_LEN:
s = s[:LOG_MSG_MAX_LEN] + "...[truncated]"
out = []
for ch in s:
code = ord(ch)
if code == 9: # tab
out.append(ch)
elif 32 <= code < 127:
out.append(ch)
else:
out.append(f"\\x{code:02x}")
return "".join(out)
def _log(msg):
"""Append a timestamped line to LOG_FILE. Rotates when oversized.
Never raises."""
try:
os.makedirs(CACHE_DIR, exist_ok=True)
try:
if os.path.getsize(LOG_FILE) > LOG_MAX_BYTES:
with open(LOG_FILE, "rb") as f:
f.seek(-(LOG_MAX_BYTES // 2), os.SEEK_END)
tail = f.read()
with open(LOG_FILE, "wb") as f:
f.write(b"[truncated]\n")
f.write(tail)
except OSError:
pass
ts = time.strftime("%Y-%m-%d %H:%M:%S")
with open(LOG_FILE, "a", encoding="utf-8") as f:
f.write(f"{ts} {_sanitize(msg)}\n")
except Exception:
pass
def _alarm_handler(signum, frame):
# Async-signal-safe: only os.write to a pre-opened fd, then os._exit.
# _log() opens files / calls strftime / does malloc — none safe inside a
# signal handler. Skip logging the alarm; the launchd ExitTimeOut entry
# in the plist + the absence of a refresh marker is sufficient signal.
try:
os.write(2, b"[refresh] HARD CAP reached\n")
except OSError:
pass
os._exit(0)
def _atomic_json(path, value):
os.makedirs(os.path.dirname(path), exist_ok=True)
fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=os.path.dirname(path))
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(value, f, indent=2, sort_keys=True)
f.write("\n")
f.flush()
os.fsync(f.fileno())
os.chmod(tmp, 0o600)
os.replace(tmp, path)
except BaseException:
try:
os.unlink(tmp)
except OSError:
pass
raise
def _load_state():
try:
with open(STATE_FILE, encoding="utf-8") as f:
value = json.load(f)
return value if isinstance(value, dict) else {}
except (OSError, ValueError):
return {}
def _write_state(**updates):
try:
state = _load_state()
state.update(updates)
_atomic_json(STATE_FILE, state)
except OSError as e:
_log(f"state write failed: {e}")
def _acquire_lock():
"""Open lock file with O_NOFOLLOW (no symlink) inside CACHE_DIR.
Returns fd or None."""
try:
os.makedirs(CACHE_DIR, exist_ok=True)
except OSError as e:
_log(f"cache dir create failed: {e}")
return None
flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0)
try:
fd = os.open(LOCK_FILE, flags, 0o600)
except OSError as e:
_log(f"lock open failed: {e}")
return None
# Refuse to flock anything that's not a regular file
try:
st = os.fstat(fd)
import stat as _stat
if not _stat.S_ISREG(st.st_mode):
_log("lock file is not a regular file — aborting")
os.close(fd)
return None
except OSError as e:
_log(f"fstat failed: {e}")
try:
os.close(fd)
except OSError:
pass
return None
try:
if fcntl is not None:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
elif msvcrt is not None: # pragma: no cover - Windows
if os.fstat(fd).st_size == 0:
os.write(fd, b"\0")
os.lseek(fd, 0, os.SEEK_SET)
msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
else:
raise OSError(errno.ENOSYS, "no supported file-lock API")
return fd
except OSError as e:
if e.errno in (errno.EWOULDBLOCK, errno.EAGAIN):
_log("another refresher is running — exiting")
else:
_log(f"flock failed: {e}")
try:
os.close(fd)
except OSError:
pass
return None
def _release_lock(fd):
try:
if fcntl is not None:
fcntl.flock(fd, fcntl.LOCK_UN)
elif msvcrt is not None: # pragma: no cover - Windows
os.lseek(fd, 0, os.SEEK_SET)
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
except OSError:
pass
try:
os.close(fd)
except OSError:
pass
def _write_marker(forensics_core=None):
"""Atomic marker write. Uses forensics_core.atomic_write_text when the
helper is available; otherwise falls back to inline temp+rename. The
feature-check (hasattr) handles the case where an older forensics_core
is loaded from a stale plugin cache during version transitions."""
try:
os.makedirs(CACHE_DIR, exist_ok=True)
if forensics_core is not None and hasattr(forensics_core, "atomic_write_text"):
forensics_core.atomic_write_text(
LAST_RUN_MARKER, str(time.time()), mode=0o600
)
return True
tmp = LAST_RUN_MARKER + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
f.write(str(time.time()))
os.replace(tmp, LAST_RUN_MARKER)
os.chmod(LAST_RUN_MARKER, 0o600)
return True
except OSError as e:
_log(f"marker write failed: {e}")
return False
def _resolve_scripts_dir():
"""Use only this integrity-checked stable payload; never cross-load agents."""
here = os.path.dirname(os.path.abspath(__file__))
required = ("ioc_manager.py", "forensics_core.py", "vuln_feed.py",
"rulepack_feed.py", "rule_loader.py", "_ed25519.py")
if all(os.path.isfile(os.path.join(here, name)) for name in required):
return here
return None
def _import_module_by_path(name, path):
"""Load a module by absolute path without modifying sys.path globally.
Catches BaseException (not just Exception) so SIGALRM/KeyboardInterrupt
can't leave a half-imported module wedged in sys.modules."""
spec = importlib.util.spec_from_file_location(name, path)
if spec is None or spec.loader is None:
return None
mod = importlib.util.module_from_spec(spec)
sys.modules[name] = mod
try:
spec.loader.exec_module(mod)
except BaseException:
sys.modules.pop(name, None)
raise
return mod
_LAST_IOC_DETAIL = [""]
def _refresh_iocs(scripts_dir):
"""Refresh IOC cache. Returns True on success.
Imports under canonical name 'ioc_manager' so any internal self-imports
or module-level state stays consistent across the codebase."""
try:
ioc_path = os.path.join(scripts_dir, "ioc_manager.py")
ioc_manager = _import_module_by_path("ioc_manager", ioc_path)
if ioc_manager is None:
_log("ioc_manager module not found")
return False
ok, msg = ioc_manager.update_iocs()
_log(f"IOC: ok={ok} msg={msg}")
_LAST_IOC_DETAIL[0] = "" if ok else str(msg)
return bool(ok)
except Exception as e:
_log(f"IOC refresh exception: {type(e).__name__}: {e}")
return False
def _refresh_kev(scripts_dir):
"""Refresh KEV cache. Returns True on success."""
try:
kev_path = os.path.join(scripts_dir, "vuln_feed.py")
vuln_feed = _import_module_by_path("vuln_feed", kev_path)
if vuln_feed is None:
_log("vuln_feed module not found")
return False
ok, msg = vuln_feed.update_kev_cache()
_log(f"KEV: ok={ok} msg={msg}")
return bool(ok)
except Exception as e:
_log(f"KEV refresh exception: {type(e).__name__}: {e}")
return False
def _refresh_rulepacks(scripts_dir):
"""Fetch + verify + cache the signed rule-pack bundle (U6). Returns True on
success. Runs WITHIN the existing 60s SIGALRM hard cap and the single-
instance lock — its self-test step reuses rule_loader's save/restore SIGALRM
so it never clobbers our cap. Cross-platform updater with no scheduler
assumptions."""
try:
feed_path = os.path.join(scripts_dir, "rulepack_feed.py")
rulepack_feed = _import_module_by_path("rulepack_feed", feed_path)
if rulepack_feed is None:
_log("rulepack_feed module not found")
return False
ok, msg = rulepack_feed.update_rulepacks()
_log(f"RULEPACKS: ok={ok} msg={msg}")
return bool(ok), str(msg)
except Exception as e:
msg = f"{type(e).__name__}: {e}"
_log(f"rule-pack refresh exception: {msg}")
return False, msg
_RULEPACK_STRUCTURAL_MARKERS = (
# Inert bundle: no/insufficient pack can overlay installed versions.
"permanently unacceptable",
# Tampered or mis-signed feed; deterministic, never transient.
"signature verification failed",
# Structurally malformed bundle (schema/type/self-test rejections).
"bundle has no packs",
"schema major",
"json parse failed",
"not an object",
"rules not a list",
"rule rejected",
"self-test failed",
"pack_version not an integer",
)
def _rulepack_failure_is_critical(message):
"""Structural feed/publisher failures are critical; age and fetch blips
stay advisory. Markers mirror rulepack_feed.py's failure messages; a
cross-module consistency test guards against drift."""
text = str(message).lower()
return any(marker in text for marker in _RULEPACK_STRUCTURAL_MARKERS)
def self_check():
scripts_dir = _resolve_scripts_dir()
if scripts_dir is None:
return False
for name in ("ioc_manager.py", "forensics_core.py", "vuln_feed.py",
"rulepack_feed.py", "rule_loader.py", "_ed25519.py"):
try:
with open(os.path.join(scripts_dir, name), encoding="utf-8") as f:
compile(f.read(), name, "exec")
except (OSError, SyntaxError):
return False
return True
def _worker_main():
if (os.environ.get(ENV_KILL_SWITCH, "").lower() in ("1", "true", "yes", "on")
or os.path.exists(DISABLED_MARKER)):
_log("kill switch active — exiting")
_write_state(status="disabled")
return
# Defense in depth for hung TLS/DNS
try:
socket.setdefaulttimeout(SOCKET_DEFAULT_TIMEOUT)
except Exception:
pass
lock_fd = _acquire_lock()
if lock_fd is None:
return
# Inner POSIX hard cap. The parent supervises with subprocess timeout, but a
# worker invoked directly (tests, manual runs) would otherwise have no cap at
# all — wire the handler that was previously dead code. Windows lacks SIGALRM
# and relies solely on the parent's subprocess timeout.
_alarm_armed = hasattr(signal, "SIGALRM")
try:
# Arm the cap INSIDE the try so a failure while arming still hits the
# finally that releases the lock. Previously a raise between _acquire_lock()
# and this try leaked the flock until process exit, blocking every later
# refresh.
if _alarm_armed:
signal.signal(signal.SIGALRM, _alarm_handler)
signal.alarm(REFRESH_HARD_CAP_SEC)
started = time.time()
run_id = os.environ.get("REPO_FORENSICS_RUN_ID") or f"{int(started * 1000)}-{os.getpid()}"
try:
with open(LAST_ATTEMPT_MARKER, "w", encoding="utf-8") as f:
f.write(str(started))
os.chmod(LAST_ATTEMPT_MARKER, 0o600)
except OSError:
pass
_write_state(status="refreshing", last_attempt=started, pid=os.getpid(), run_id=run_id)
scripts_dir = _resolve_scripts_dir()
if scripts_dir is None:
_log("scripts dir not found — exiting")
_write_state(status="repair-needed", last_error="scripts dir not found")
return
_log(f"refresh start (scripts_dir={scripts_dir})")
# Pre-load forensics_core for the shared atomic-write helper used by marker.
try:
fc_path = os.path.join(scripts_dir, "forensics_core.py")
forensics_core = _import_module_by_path("forensics_core", fc_path)
except Exception:
forensics_core = None
ok_ioc = _refresh_iocs(scripts_dir)
if os.path.exists(DISABLED_MARKER):
_write_state(status="disabled", last_attempt=started, run_id=run_id,
last_error="disabled during refresh")
return
ok_kev = _refresh_kev(scripts_dir)
if os.path.exists(DISABLED_MARKER):
_write_state(status="disabled", last_attempt=started, run_id=run_id,
last_error="disabled during refresh")
return
rulepack_result = _refresh_rulepacks(scripts_dir)
if isinstance(rulepack_result, tuple):
ok_rulepacks, rulepack_detail = rulepack_result
else: # compatibility for tests and older injected helpers
ok_rulepacks, rulepack_detail = bool(rulepack_result), ""
rulepacks_critical = (not ok_rulepacks and
_rulepack_failure_is_critical(rulepack_detail))
finished = time.time()
if os.path.exists(DISABLED_MARKER):
_write_state(status="disabled", last_attempt=started, run_id=run_id,
last_error="disabled during refresh")
return
previous_feeds = _load_state().get("feeds") or {}
def feed_state(name, ok, critical=True):
previous = previous_feeds.get(name) if isinstance(previous_feeds, dict) else {}
prior_success = previous.get("last_success") if isinstance(previous, dict) else None
return {"ok": ok, "critical": critical, "last_attempt": finished,
"last_success": finished if ok else prior_success}
# IOC and KEV are the security-critical threat feeds. The rulepack BUNDLE
# is advisory freshness: a stale bundle (e.g. >30d) is a WARNING, not a
# failure, and must not block the success marker or wedge the whole
# refresh into a perpetual "degraded" state (refresh-degraded fix).
feeds = {
"ioc": feed_state("ioc", ok_ioc, critical=True),
"kev": feed_state("kev", ok_kev, critical=True),
"rulepacks": feed_state("rulepacks", ok_rulepacks,
critical=rulepacks_critical),
}
feeds["rulepacks"]["detail"] = rulepack_detail
if not ok_ioc and _LAST_IOC_DETAIL[0]:
feeds["ioc"]["detail"] = _LAST_IOC_DETAIL[0]
critical_ok = ok_ioc and ok_kev and not rulepacks_critical
advisory_failed = [n for n, r in feeds.items() if not r["ok"] and not r["critical"]]
if critical_ok:
marker_ok = _write_marker(forensics_core=forensics_core)
warn = ("advisory feed(s) stale: " + ", ".join(advisory_failed)) if advisory_failed else None
_write_state(status="healthy" if marker_ok else "degraded",
last_success=finished if marker_ok else _load_state().get("last_success"),
last_attempt=started, duration_ms=int((finished - started) * 1000),
feeds=feeds, marker_written=marker_ok,
run_id=run_id,
last_error=(warn if marker_ok else "success marker write failed"))
else:
_log("refresh incomplete — success marker not updated (critical feed failed)")
failed = [name for name, result in feeds.items() if not result["ok"] and result["critical"]]
_write_state(status="degraded", last_attempt=started,
duration_ms=int((finished - started) * 1000), feeds=feeds,
marker_written=False, run_id=run_id,
last_error="failed critical feeds: " + ", ".join(failed)
+ ((" (ioc: " + feeds["ioc"]["detail"] + ")")
if "ioc" in failed and feeds["ioc"].get("detail") else ""))
_log(f"refresh done (ioc={ok_ioc}, kev={ok_kev}, rulepacks={ok_rulepacks})")
finally:
if _alarm_armed:
signal.alarm(0)
_release_lock(lock_fd)
def main(argv=None):
argv = list(sys.argv[1:] if argv is None else argv)
if "--self-check" in argv:
return 0 if self_check() else 1
if "--worker" not in argv:
try:
completed = subprocess.run(
[sys.executable, os.path.abspath(__file__), "--worker"],
stdin=subprocess.DEVNULL, timeout=REFRESH_HARD_CAP_SEC,
check=False,
)
return completed.returncode
except subprocess.TimeoutExpired:
now = time.time()
_log("refresh timeout — worker terminated by supervisor")
_write_state(status="timeout", last_attempt=now,
last_error=f"hard cap exceeded ({REFRESH_HARD_CAP_SEC}s)")
return 0
_worker_main()
return 0
if __name__ == "__main__":
try:
exit_code = main()
except SystemExit:
raise
except BaseException as e:
_log(f"top-level exception: {type(e).__name__}: {e}")
_write_state(status="error", last_error=f"{type(e).__name__}: {e}")
exit_code = 0
sys.exit(exit_code)
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |