stark-ai-de/agent-skills / incubator/skills/engineering-workflows/hindsight-memory
hindsight-memory
VerifiedSet up, diagnose, enable or use scoped Hindsight memory without requiring Greppy. Use when the user requests approved project or workspace memory, compact Codex/pi session outcomes or source-backed reflection. Not for global memory cleanup or implicit ingestion of search-indexed repositories.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 9.4 KB |
| assets/compose.yaml | config | 2.7 KB |
| assets/parser.nix | other | 783 B |
| assets/project.example.json | config | 873 B |
| assets/workspace.example.json | config | 1.1 KB |
| diagnostics/bank_aliases.py | script | 7.0 KB |
| diagnostics/workspace_status.py | script | 2.8 KB |
| references/setup.md | markdown | 23.9 KB |
| scripts/activate.py | script | 3.2 KB |
| scripts/build_parser.py | script | 2.6 KB |
| scripts/initialize.py | script | 5.1 KB |
| scripts/mcp_launcher.py | script | 9.1 KB |
| scripts/metrics.py | script | 4.6 KB |
| scripts/private_policy.py | script | 1.4 KB |
| scripts/project_scope.py | script | 13.7 KB |
| scripts/session_hook.py | script | 6.1 KB |
| scripts/session_parser.py | script | 11.0 KB |
| scripts/weekly.py | script | 19.5 KB |
| scripts/worker.py | script | 8.8 KB |
| scripts/workspace_scope.py | script | 13.1 KB |
scripts/weekly.py
View on GitHub#!/usr/bin/env python3
"""Run the approved project import queue and weekly Hindsight reflection."""
import argparse
import datetime
import fcntl
import hashlib
import json
import os
from pathlib import Path
import re
import shlex
import stat
import subprocess
import sys
import tempfile
import time
import uuid
from zoneinfo import ZoneInfo
from project_scope import project_directory
from session_parser import SESSION_ID, parse as parse_session, read_header, snapshot, timestamp
def private_token(path):
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
with os.fdopen(fd, encoding="utf-8") as stream:
info = os.fstat(stream.fileno())
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o600:
raise ValueError("runtime environment permissions")
tokens = []
for line in stream:
key, separator, value = line.rstrip().partition("=")
if separator and key == "HINDSIGHT_API_TENANT_API_KEY":
parts = shlex.split(value)
if len(parts) != 1:
raise ValueError("runtime credential")
tokens.append(parts[0])
if len(tokens) != 1 or not tokens[0]:
raise ValueError("runtime credential")
return tokens[0]
def atomic_write(path, contents):
fd, temporary = tempfile.mkstemp(prefix=".hindsight-", dir=path.parent)
try:
with os.fdopen(fd, "w", encoding="utf-8") as stream:
stream.write(contents)
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def validate_record(record, home, cutoff, scope=None):
session_id = record["session_id"]
if not SESSION_ID.fullmatch(session_id):
raise ValueError("session id")
if scope and scope.get("kind") == "hindsight-workspace":
import workspace_scope
source = record["source"]
transcript = workspace_scope.transcript_path(scope, source, record["transcript_path"])
meta = read_header(transcript, source)
if (record.get("schemaVersion") != 2 or meta["session_id"] != session_id
or timestamp(meta["timestamp"]) < cutoff or record.get("enabled_at") != cutoff.isoformat()
or timestamp(record["checkpoint_at"]) < timestamp(meta["timestamp"])
or workspace_scope.resolve(scope, meta["cwd"]) != record.get("repository")):
raise ValueError("workspace session binding changed")
return transcript
transcript = Path(record["transcript_path"]).resolve(strict=True)
transcript.relative_to((Path(scope["sessionsRoot"]) if scope else home / ".codex/sessions").resolve(strict=True))
if not transcript.is_file():
raise ValueError("transcript")
with transcript.open(encoding="utf-8") as stream:
header = json.loads(stream.readline())
meta = header["payload"]
project = (Path(scope["projectRoot"]) if scope else home / "dev/greppy").resolve(strict=True)
project_directory(meta["cwd"], project)
started = datetime.datetime.fromisoformat(meta["timestamp"].replace("Z", "+00:00"))
if header.get("type") != "session_meta" or meta.get("id") != session_id or meta.get("source") != "cli" or started.tzinfo is None or started < cutoff:
raise ValueError("session scope")
return transcript
def call_worker(args, token, action, extra, transcript=None):
scope = getattr(args, "memory_scope", None)
state = Path(getattr(args, "memory_state", scope["stateRoot"] if scope else Path.home() / ".local/state/hindsight-greppy"))
if (state / "worker-cleanup-required.json").exists():
raise ValueError("prior owned-container cleanup requires review before dispatch")
if scope:
if scope.get("kind") == "hindsight-workspace":
from workspace_scope import load, active
policy_path = args.workspace_policy
else:
from project_scope import load, active
policy_path = args.policy
if load(policy_path) != scope:
raise ValueError("project policy changed during the batch")
if scope["consent"]["contentScope"] not in {"codex-session-content", "compact-session-outcomes"}:
raise ValueError("session content is not authorized")
if active(scope, args.docker) != args.memory_cutoff:
raise ValueError("project grant changed during the batch")
container_name = "hindsight-worker-" + uuid.uuid4().hex
command = [args.docker, "compose", "-f", args.compose, "run", "--rm", "--no-deps", "-T", "--name", container_name,
"--user", f"{os.getuid()}:{os.getgid()}",
"--volume", f"{args.worker}:/hindsight/worker.py:ro",
"--volume", f"{Path(args.worker).parent}:/hindsight/helpers:ro",
"--env", "PYTHONPATH=/hindsight/helpers",
"--env", "HINDSIGHT_API_TENANT_API_KEY"]
if getattr(args, "parser", None):
command.extend(["--volume", f"{args.parser}:/hindsight/parser.cjs:ro"])
if transcript is not None:
command.extend(["--volume", f"{transcript}:/hindsight/transcript.jsonl:ro"])
if scope:
command.extend(["--env", "HINDSIGHT_MEMORY_BANK_API"])
env = dict(os.environ)
env["HINDSIGHT_API_TENANT_API_KEY"] = token
if scope:
env["HINDSIGHT_MEMORY_BANK_API"] = scope["endpoint"].rstrip("/") + "/v1/default/banks/" + getattr(args, "memory_bank", scope.get("bankId", ""))
remaining = getattr(args, "deadline", time.monotonic() + 1800) - time.monotonic()
if remaining <= 0:
raise ValueError("batch time budget exhausted")
with tempfile.TemporaryDirectory(prefix="hindsight-job-") as directory:
job = getattr(args, "memory_job", None) if action == "ingest" else None
if job is not None:
job_path = Path(directory) / "job.json"
job_path.write_text(json.dumps(job))
job_path.chmod(0o600)
command.extend(["--volume", f"{job_path}:/hindsight/job.json:ro"])
extra = [*extra, "--job", "/hindsight/job.json"]
command.extend(["weekly", action, *extra])
try:
result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=min(1800, remaining))
except subprocess.TimeoutExpired:
cleaned = False
try:
cleanup = subprocess.run([args.docker, "rm", "--force", container_name], capture_output=True, text=True, timeout=5)
cleaned = cleanup.returncode == 0
except (OSError, subprocess.SubprocessError):
pass
if not cleaned:
state = Path(getattr(args, "memory_state", scope["stateRoot"] if scope else Path.home() / ".local/state/hindsight-greppy"))
state.mkdir(parents=True, exist_ok=True, mode=0o700)
atomic_write(state / "worker-cleanup-required.json", json.dumps({"container": container_name}) + "\n")
raise ValueError("worker timed out; owned-container cleanup requires review" if not cleaned else "worker timed out and was stopped") from None
if result.returncode:
raise ValueError("weekly worker failed")
return json.loads(result.stdout)
def render_report(start, end, patterns):
lines = [f"# Projekt-Rückblick {start} bis {end}", ""]
if not patterns:
lines.append("Für wiederkehrende Muster liegen nicht genügend zuordenbare Belege vor.")
for pattern in patterns:
finding = pattern["finding"]
lines.extend([f"## {finding['summary']}", "", f"Gegenbelege: {finding['counterevidence']}",
f"Offene Fragen: {finding['open_questions']}", "", "Belege:"])
for fact in pattern["facts"]:
metadata = fact.get("metadata") or {}
if metadata.get("session_id") in pattern["sessions"]:
lines.append(f"- Sitzung {metadata['session_id']}, Fakt {fact.get('id')}: {fact.get('text', '')}")
lines.append("")
return "\n".join(lines) + "\n"
def run(args):
home = Path.home()
scope = None
workspace_path = getattr(args, "workspace_policy", None)
if workspace_path:
from workspace_scope import load, active
if not Path(workspace_path).exists():
return {"status": "inactive", "reason": "workspace policy is absent"}
scope = load(workspace_path)
if not scope["enabled"]:
return {"status": "inactive", "reason": "workspace memory is disabled"}
cutoff = active(scope, args.docker)
args.memory_scope = scope
args.memory_cutoff = cutoff
elif getattr(args, "policy", None):
from project_scope import load, active
scope = load(args.policy)
if scope["consent"]["contentScope"] != "codex-session-content":
raise ValueError("session content is not authorized")
cutoff = active(scope, args.docker)
args.memory_scope = scope
args.memory_cutoff = cutoff
state = Path(scope["stateRoot"]) if scope else home / ".local/state/hindsight-greppy"
state.mkdir(parents=True, exist_ok=True, mode=0o700)
if (state / "worker-cleanup-required.json").exists():
raise ValueError("prior owned-container cleanup requires review before retry")
if args is not None:
args.memory_state = state
if not scope:
cutoff = datetime.datetime.fromisoformat((state / "pilot-start").read_text().strip().replace("Z", "+00:00"))
if cutoff.tzinfo is None:
raise ValueError("pilot start has no timezone")
token = private_token(Path(scope["runtimeEnv"]) if scope else home / ".config/hindsight-greppy/runtime.env")
queue = state / "queue"
queue.mkdir(exist_ok=True, mode=0o700)
ledger_path = state / "imported.json"
from project_scope import private_read
from workspace_scope import RepositoryIneligible
# Durable deduplication history grows across batches; it is not a bounded input.
ledger = json.loads(private_read(ledger_path, max_bytes=None)) if ledger_path.exists() else {}
failed = False
mode = getattr(args, "mode", "weekly")
maximum = getattr(args, "max_sessions", 8)
attempted = 0
skipped = {}
counts = {"retained_outcomes": 0, "no_new_outcomes": 0, "unchanged_sessions": 0, "skipped_ineligible": 0}
if args is not None:
args.deadline = time.monotonic() + getattr(args, "max_seconds", 240 if mode == "intake" else 1800)
for record_path in sorted(queue.glob("*.json"), key=lambda path: (path.stat().st_mtime_ns, path.name)):
if attempted >= maximum or (args is not None and time.monotonic() >= args.deadline):
break
attempted += 1
try:
original = record_path.read_bytes()
record = json.loads(original)
transcript = validate_record(record, home, cutoff, scope)
# The hook records a dirty reference, not an immutable terminal size.
# Admit the latest complete prefix, preserving an unfinished tail for retry.
raw = snapshot(transcript, complete_only=True)
digest = hashlib.sha256(raw).hexdigest()
ledger_key = record_path.stem
previous = ledger.get(ledger_key)
bank = record.get("repository", {}).get("bankId", scope.get("bankId", "greppy") if scope else "greppy")
known = previous.get("outcome_ids", []) if isinstance(previous, dict) and previous.get("bankId") == bank else []
pending = isinstance(previous, dict) and previous.get("pending_outcomes", 0) > 0
already = previous.get("sha256") if isinstance(previous, dict) else previous
checkpoint = {key: record[key] for key in ("checkpoint_at", "leaf_id", "completion_status") if key in record}
checkpoint_changed = bool(checkpoint) and (not isinstance(previous, dict) or previous.get("checkpoint") != checkpoint)
if already != digest or pending or checkpoint_changed:
if args is not None:
args.memory_bank = bank
args.memory_job = {key: record[key] for key in ("source", "leaf_id", "repository", "completion_status", "checkpoint_at") if key in record}
args.memory_job.update(byte_length=len(raw), known_outcomes=known)
if workspace_path:
from workspace_scope import git_run
root = record["repository"]["worktreeRoot"]
args.memory_job["provenance_observed_at"] = datetime.datetime.now(datetime.timezone.utc).isoformat()
for name, command in (("commit_at_intake", ["rev-parse", "--verify", "HEAD"]), ("branch_at_intake", ["symbolic-ref", "--short", "-q", "HEAD"])):
info = git_run(root, command, check=False)
if info.returncode == 0 and len(info.stdout) <= 512:
args.memory_job[name] = info.stdout.strip()
result = call_worker(args, token, "ingest", ["--transcript", "/hindsight/transcript.jsonl", "--session-id", record["session_id"]], transcript)
if result.get("session_id") != record["session_id"] or result.get("sha256") != digest:
raise ValueError("transcript changed during import")
if "outcome_ids" in result:
ids = result["outcome_ids"]
pending_count = result.get("pending_outcomes", 0)
if (not isinstance(ids, list) or len(ids) > 16 or any(not isinstance(item, str) or not re.fullmatch(r"[a-f0-9]{64}", item) for item in ids)
or type(pending_count) is not int or pending_count < 0):
raise ValueError("worker outcome acknowledgement")
ledger[ledger_key] = {"sha256": digest, "outcome_ids": sorted(set(known + ids)),
"pending_outcomes": pending_count, "bankId": bank, "checkpoint": checkpoint,
"repository": record.get("repository"), "session_id": record["session_id"]}
pending = pending_count > 0
counts["retained_outcomes"] += len(ids)
counts["no_new_outcomes"] += not ids
else:
ledger[ledger_key] = digest # Legacy worker acknowledgement compatibility.
pending = False
atomic_write(ledger_path, json.dumps(ledger, sort_keys=True) + "\n")
else:
counts["unchanged_sessions"] += 1
if pending:
# Rotate partial records behind other sessions without losing their snapshot.
os.utime(record_path, None)
continue
locks = state / "locks"
locks.mkdir(exist_ok=True, mode=0o700)
with (locks / f"{record_path.stem}.lock").open("a") as lock:
os.chmod(lock.name, 0o600)
fcntl.flock(lock, fcntl.LOCK_EX)
if record_path.read_bytes() == original and hashlib.sha256(snapshot(transcript)).hexdigest() == digest:
record_path.unlink()
else:
failed = True
except RepositoryIneligible:
# A pause preserves captured work but must not consume another bank's
# intake allowance or block its reflection. Rotate for time-budget fairness.
attempted -= 1
skipped[record_path] = original
counts["skipped_ineligible"] += 1
if record_path.exists():
os.utime(record_path, None)
except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError):
failed = True
# Preserve poison/revoked records without starving later eligible sessions.
if record_path.exists():
os.utime(record_path, None)
print("A queued session remains pending.", file=sys.stderr)
remaining = list(queue.glob("*.json"))
pending = failed or any(path not in skipped or path.read_bytes() != skipped[path] for path in remaining)
if pending or (mode == "intake" and remaining):
if mode == "intake":
return {"status": "pending", "attempted": attempted, "failed": failed, **counts}
raise ValueError("pending imports")
if mode == "intake":
return {"status": "complete", "attempted": attempted, **counts}
today = datetime.datetime.now(ZoneInfo("Europe/Berlin")).date()
end = today - datetime.timedelta(days=today.weekday())
start = end - datetime.timedelta(days=7)
year, week, _ = start.isocalendar()
reports = state / "reflections"
reports.mkdir(exist_ok=True, mode=0o700)
banks = sorted({item["bankId"] for item in ledger.values() if isinstance(item, dict)}) if workspace_path else [None]
for bank in banks:
folder = reports / bank if bank else reports
folder.mkdir(exist_ok=True, mode=0o700)
report = folder / f"{year}-W{week:02}.md"
if not report.exists():
if bank:
# Recheck current eligibility; a disabled repository cannot trigger reflection.
from workspace_scope import resolve
records = [item for item in ledger.values() if isinstance(item, dict) and item["bankId"] == bank]
try:
resolve(scope, records[0]["repository"]["projectRoot"])
except RepositoryIneligible:
continue
args.memory_bank = bank
patterns = call_worker(args, token, "reflect", ["--start", start.isoformat(), "--end", end.isoformat()])
atomic_write(report, render_report(start, end, patterns))
return {"status": "complete", "attempted": attempted, **counts}
def main():
parser = argparse.ArgumentParser()
for name in ("docker", "compose", "worker"):
parser.add_argument("--" + name, required=True)
parser.add_argument("--parser", help="Legacy compatibility; compact parsing uses the bundled Python reader")
selected = parser.add_mutually_exclusive_group()
selected.add_argument("--policy", type=Path)
selected.add_argument("--workspace-policy", type=Path)
parser.add_argument("--mode", choices=("intake", "weekly"), default="weekly")
parser.add_argument("--max-sessions", type=int, default=8)
parser.add_argument("--max-seconds", type=int, default=240)
args = parser.parse_args()
try:
if not 1 <= args.max_sessions <= 64 or not 1 <= args.max_seconds <= 3600:
raise ValueError("batch bounds")
if args.workspace_policy:
from workspace_scope import load
if not args.workspace_policy.exists():
print(json.dumps({"status": "inactive", "reason": "workspace policy is absent"}))
return 0
scope = load(args.workspace_policy)
if not scope["enabled"]:
print(json.dumps({"status": "inactive", "reason": "workspace memory is disabled"}))
return 0
state = Path(scope["stateRoot"])
elif args.policy:
from project_scope import load, active
scope = load(args.policy)
active(scope, args.docker)
state = Path(scope["stateRoot"])
else:
state = Path.home() / ".local/state/hindsight-greppy"
state.mkdir(parents=True, exist_ok=True, mode=0o700)
with (state / "weekly.lock").open("a") as lock:
os.chmod(lock.name, 0o600)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
print(json.dumps(run(args)))
except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError):
print("Hindsight weekly run did not complete; queued sessions were preserved.", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |