YujunZhou/tellonce / codex
tellonce
VerifiedUse to view, explain, correct, delete or reorganize the user's saved preferences, or to record one by hand when automatic recording is off; learning and rule injection otherwise run automatically through Codex hooks.
shared_lib/memory_upsert.py
View on GitHub#!/usr/bin/env python3
"""Non-blocking, cross-platform tellonce memory upsert entry point.
The foreground path only persists a complete user turn and starts a detached
worker. The worker performs the LLM semantic decision and SQLite transaction.
No LLM call runs on the user-response path.
"""
from __future__ import annotations
import argparse
import hashlib
import inspect as _inspect_module
import itertools
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
import threading
from contextlib import contextmanager
_LIB_DIR = os.path.dirname(os.path.abspath(__file__))
if _LIB_DIR not in sys.path:
sys.path.insert(0, _LIB_DIR)
import memory_judge
from memory_store import MemoryStore, MemoryStoreError, StaleSnapshotError
import path_config
import redaction
import transcript_adapter
from model_requests import exhausted_request, learning_recovery_delay
FINAL_STATUSES = {
"committed", "projected", "noop", "needs_user", "clarified", "dismissed",
"rejected", "failed",
}
INBOX_DIRNAME = ".tellonce-inbox"
INBOX_SUFFIX = ".json"
_INBOX_SEQUENCE = itertools.count()
TEMP_RECOVERY_AGE_SECONDS = 300
CONFIG_PATH = Path.home() / ".tellonce.config.json"
PRESENTATION_DIRNAME = ".tellonce-clarification-presentations"
MAX_TURN_ATTEMPTS = 5
def _default_turn_key() -> str:
return f"turn-{uuid.uuid4().hex}"
def hooks_enabled() -> bool:
"""Public installs learn by default; explicit opt-out always wins.
An unreadable existing config stays disabled rather than ignoring a
potentially saved opt-out. Isolated host APIs supply their own controls.
"""
value = os.environ.get("PT_MEMORY_UPSERT_ENABLED")
if value is None:
value = os.environ.get("B5_MEMORY_UPSERT_ENABLED")
if value is None:
try:
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8-sig"))
value = config.get("memory_upsert_enabled", True)
except FileNotFoundError:
value = True
except Exception:
value = False
return str(value).strip().lower() in {"1", "true", "yes", "on"}
def _presentation_path(memory_dir, presentation_key: str) -> Path:
digest = hashlib.sha256(presentation_key.encode("utf-8")).hexdigest()
return Path(memory_dir) / PRESENTATION_DIRNAME / f"{digest}.json"
def record_clarification_presentation(
memory_dir,
presentation_key: str,
turn_keys: list[str],
) -> None:
if not presentation_key:
return
path = _presentation_path(memory_dir, presentation_key)
path.parent.mkdir(parents=True, exist_ok=True)
previous = []
try:
existing = _read_json_file(str(path))
current = existing.get("current_turn_keys", existing.get("turn_keys", []))
if isinstance(current, list):
previous = [
item
for item in current
if isinstance(item, str) and item.strip()
][:3]
except (OSError, ValueError, TypeError, json.JSONDecodeError):
pass
payload = {
"presentation_key": presentation_key,
"previous_turn_keys": previous,
"current_turn_keys": list(dict.fromkeys(turn_keys))[:3],
"updated_at": time.time(),
}
tmp = path.with_name(f"{path.name}.tmp.{os.getpid()}.{uuid.uuid4().hex[:8]}")
try:
tmp.write_text(
json.dumps(payload, ensure_ascii=False, separators=(",", ":")),
encoding="utf-8",
)
os.replace(tmp, path)
finally:
try:
tmp.unlink()
except OSError:
pass
def read_clarification_presentation(
memory_dir,
presentation_key: str,
slot: str = "current",
max_age_seconds: int = 604_800,
) -> list[str]:
if not presentation_key:
return []
try:
payload = _read_json_file(str(_presentation_path(memory_dir, presentation_key)))
updated_at = float(payload.get("updated_at", 0))
key = "previous_turn_keys" if slot == "previous" else "current_turn_keys"
turn_keys = payload.get(key, payload.get("turn_keys", [])) or []
if time.time() - updated_at > max_age_seconds:
return []
if not isinstance(turn_keys, list):
return []
return list(
dict.fromkeys(
item.strip()
for item in turn_keys
if isinstance(item, str) and item.strip()
)
)[:3]
except (OSError, ValueError, TypeError, json.JSONDecodeError):
return []
def _memory_dir(memory_dir=None) -> Path:
return Path(memory_dir or path_config.get_memory_dir()).expanduser().resolve()
def _store(memory_dir=None, namespace: str = '') -> MemoryStore:
namespace = namespace or os.environ.get('PT_MEMORY_NAMESPACE', '')
if namespace and not memory_dir:
raise MemoryStoreError('namespaced memory requires an explicit directory')
resolved = _memory_dir(memory_dir)
store = MemoryStore(resolved, legacy_dirs=[])
owner = store.peek_namespace()
if namespace and owner != namespace:
raise MemoryStoreError('initialize the explicit namespaced library before enqueue/drain')
if owner:
if not memory_dir:
raise MemoryStoreError('owned experiment memory requires an explicit directory')
# Internal resolution may reopen an already owned store without
# repeating its namespace argument. No operator config is needed.
store.initialize(namespace=namespace or owner)
return store
canonical = Path(path_config.get_memory_dir()).expanduser().resolve()
legacy_dirs = path_config.get_legacy_memory_dirs() if resolved == canonical else []
store = MemoryStore(resolved, legacy_dirs=legacy_dirs)
store.initialize()
return store
def _inbox_dir(memory_dir=None) -> Path:
return _memory_dir(memory_dir) / INBOX_DIRNAME
def _worker_log_path(memory_dir=None) -> Path:
return _memory_dir(memory_dir) / ".tellonce-worker.log"
def _protect_local_store(memory_dir: Path) -> None:
memory_dir.mkdir(parents=True, exist_ok=True)
try:
os.chmod(memory_dir, 0o700)
except OSError:
pass
if memory_dir.name == "memory" and memory_dir.parent.name == ".tellonce":
try:
os.chmod(memory_dir.parent, 0o700)
except OSError:
pass
ignore_path = memory_dir.parent / ".gitignore"
try:
with ignore_path.open("x", encoding="utf-8", newline="\n") as handle:
handle.write("*\n!.gitignore\n")
handle.flush()
os.fsync(handle.fileno())
except FileExistsError:
try:
existing = ignore_path.read_text(encoding="utf-8-sig")
lines = {line.strip() for line in existing.splitlines()}
missing = [line for line in ("*", "!.gitignore") if line not in lines]
if missing:
with ignore_path.open("a", encoding="utf-8", newline="\n") as handle:
if existing and not existing.endswith(("\n", "\r")):
handle.write("\n")
handle.write("\n".join(missing) + "\n")
handle.flush()
os.fsync(handle.fileno())
except OSError:
pass
try:
os.chmod(ignore_path, 0o600)
except OSError:
pass
def _fsync_directory(path: Path) -> None:
try:
directory_fd = os.open(str(path), os.O_RDONLY)
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
except OSError:
pass
def _pending_request_paths(memory_dir=None) -> list[Path]:
inbox_dir = _inbox_dir(memory_dir)
paths = list(inbox_dir.glob(f"*{INBOX_SUFFIX}"))
cutoff = time.time() - TEMP_RECOVERY_AGE_SECONDS
for path in inbox_dir.glob(f".*{INBOX_SUFFIX}.tmp.*"):
try:
if path.stat().st_mtime <= cutoff:
paths.append(path)
except OSError:
continue
return sorted(set(paths))
def _write_inbox_request(request: dict, memory_dir=None) -> tuple[Path, bool]:
resolved_memory_dir = _memory_dir(memory_dir)
_protect_local_store(resolved_memory_dir)
inbox_dir = resolved_memory_dir / INBOX_DIRNAME
inbox_dir.mkdir(parents=True, exist_ok=True)
try:
os.chmod(inbox_dir, 0o700)
except OSError:
pass
# Pending requests are drained in filename order. A millisecond stamp let
# two requests written within the same millisecond fall back to random
# order; the nanosecond stamp plus a per-process sequence keeps
# same-process requests in enqueue order.
path = inbox_dir / f"{time.time_ns()}-{next(_INBOX_SEQUENCE):06d}-{uuid.uuid4().hex}{INBOX_SUFFIX}"
tmp = inbox_dir / f".{path.name}.tmp.{os.getpid()}"
published = False
try:
with tmp.open("x", encoding="utf-8", newline="\n") as handle:
json.dump(request, handle, ensure_ascii=False)
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
try:
os.chmod(tmp, 0o600)
except OSError:
pass
delay = 0.02
for attempt in range(6):
try:
os.replace(tmp, path)
published = True
_fsync_directory(inbox_dir)
break
except PermissionError:
if attempt == 5:
break
time.sleep(delay)
delay = min(delay * 2, 0.16)
except OSError:
break
finally:
if published:
try:
tmp.unlink()
except OSError:
pass
return (path if published else tmp), published
def _quarantine_request(request_path: Path, error: Exception) -> Path:
failed_dir = request_path.parent / "failed"
failed_dir.mkdir(parents=True, exist_ok=True)
target = failed_dir / f"{request_path.name}.{type(error).__name__}.failed"
os.replace(request_path, target)
return target
def _spawn_worker(
memory_dir,
worker_command: str = "drain",
request_file: Path | None = None,
delay_seconds: int = 0,
) -> dict:
cmd = [
sys.executable,
os.path.abspath(__file__),
worker_command,
"--memory-dir",
str(_memory_dir(memory_dir)),
]
if request_file:
cmd.extend(["--request-file", str(request_file)])
if delay_seconds:
cmd.extend(["--delay", str(delay_seconds)])
log_path = _worker_log_path(memory_dir)
log_path.parent.mkdir(parents=True, exist_ok=True)
log_handle = log_path.open("a", encoding="utf-8")
kwargs = {
"stdin": subprocess.DEVNULL,
"stdout": log_handle,
"stderr": log_handle,
"cwd": tempfile_dir(),
"env": _worker_environment(),
"close_fds": True,
}
if os.name == "nt":
kwargs["creationflags"] = (
getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
| getattr(subprocess, "DETACHED_PROCESS", 0)
)
else:
kwargs["start_new_session"] = True
try:
process = subprocess.Popen(cmd, **kwargs)
except Exception:
log_handle.close()
raise
log_handle.close()
return {"worker_pid": process.pid, "log_path": str(log_path)}
def _retry_marker(memory_dir=None) -> Path:
return _memory_dir(memory_dir) / ".tellonce-retry-scheduled"
def _schedule_retry(memory_dir, delay_seconds: int = 30) -> dict:
marker = _retry_marker(memory_dir)
expires_at = time.time() + max(5, delay_seconds) + 60
marker.parent.mkdir(parents=True, exist_ok=True)
try:
with marker.open("x", encoding="utf-8") as handle:
handle.write(str(expires_at))
except FileExistsError:
try:
existing_expiry = float(marker.read_text(encoding="utf-8").strip())
except Exception:
existing_expiry = 0
if existing_expiry > time.time():
return {"status": "already_scheduled"}
try:
marker.unlink()
except OSError:
return {"status": "already_scheduled"}
return _schedule_retry(memory_dir, delay_seconds=delay_seconds)
try:
spawned = _spawn_worker(
memory_dir,
worker_command="retry",
delay_seconds=max(5, delay_seconds),
)
return {"status": "scheduled", **spawned}
except Exception:
try:
marker.unlink()
except OSError:
pass
raise
def tempfile_dir() -> str:
return os.environ.get("TEMP") or os.environ.get("TMPDIR") or str(Path.home())
def _worker_environment() -> dict:
env = dict(os.environ)
env["PT_MEMORY_WORKER"] = "1"
env["B5_INJECT_DISABLED"] = "1"
env["B5_SHADOW_DISABLED"] = "1"
env["B5_DETERMINISTIC_DISABLED"] = "1"
return env
def enqueue(
source_text: str,
turn_key: str = "",
context: str = "",
memory_dir=None,
spawn_worker: bool = True,
force: bool = False,
clarification_candidates=None,
context_reference=None,
detect_signal: bool = False,
namespace: str = '',
) -> dict:
"""Persist one complete turn to a file inbox without opening SQLite."""
if not force and not hooks_enabled():
return {"status": "disabled", "blocking": False}
if not isinstance(source_text, str) or not source_text.strip():
raise MemoryStoreError("source_text is required")
safe_source = redaction.redact(source_text)
safe_context = redaction.redact(context or "")
turn_key = turn_key.strip() or _default_turn_key()
resolved_memory_dir = _memory_dir(memory_dir)
request_file, published = _write_inbox_request(
{
"schema_version": 1,
"turn_key": turn_key,
"source_text": safe_source,
"context": safe_context,
"context_reference": context_reference,
"detect_signal": bool(detect_signal),
"namespace": namespace,
"clarification_candidates": list(clarification_candidates or []),
"forced": bool(force),
"judge_cli": getattr(memory_judge.pt_platform, "CLI_COMMAND", ""),
"project_root": '' if namespace else path_config.get_project_root(),
"created_at": time.time(),
},
resolved_memory_dir,
)
result = {
"status": "queued" if published else "pending",
"turn_key": turn_key,
"memory_dir": str(resolved_memory_dir),
"request_file": str(request_file),
"blocking": False,
}
if not published:
result["worker_error"] = "inbox request could not be atomically published"
if spawn_worker:
result.update(
_schedule_recovery(
resolved_memory_dir,
TEMP_RECOVERY_AGE_SECONDS,
)
)
elif spawn_worker:
try:
result.update(_spawn_worker(resolved_memory_dir, worker_command="drain"))
except Exception as exc:
result["status"] = "pending"
result["worker_error"] = f"{type(exc).__name__}: {str(exc)[:300]}"
result.update(_schedule_recovery(resolved_memory_dir, 5))
return result
def _schedule_recovery(memory_dir, delay_seconds: int) -> dict:
try:
return _spawn_worker(
memory_dir,
worker_command="retry",
delay_seconds=max(1, int(delay_seconds)),
)
except Exception as exc:
return {"retry_error": f"{type(exc).__name__}: {str(exc)[:300]}"}
def _plan_contains_terminal_outcome(plan: dict) -> bool:
def contains_terminal(mutation):
if str(mutation.get("operation", "")).upper() in {
"NEEDS_USER",
"REJECT",
}:
return True
return any(
contains_terminal(child)
for child in mutation.get("children") or []
if isinstance(child, dict)
)
return any(
contains_terminal(mutation)
for mutation in plan.get("mutations", [])
if isinstance(mutation, dict)
)
def _call_judge(judge_func, source_text, active_rules, judge_context, clarification_sources, policy='general', invoke=None):
"""Invoke the judge, passing the clarification evidence corpus when the
callable supports it (test doubles may use the bare 3-arg signature)."""
try:
parameters = _inspect_module.signature(judge_func).parameters
except (TypeError, ValueError):
parameters = {}
supports_extra = "extra_evidence_sources" in parameters or any(
parameter.kind is _inspect_module.Parameter.VAR_KEYWORD
for parameter in parameters.values()
)
kwargs = {}
if supports_extra:
kwargs['extra_evidence_sources'] = clarification_sources
if 'policy' in parameters or any(p.kind is _inspect_module.Parameter.VAR_KEYWORD for p in parameters.values()):
kwargs['policy'] = policy
if invoke is not None:
if 'invoke' not in parameters and not any(p.kind is _inspect_module.Parameter.VAR_KEYWORD for p in parameters.values()):
raise MemoryStoreError('explicit transport requires a compatible semantic judge')
kwargs['invoke'] = invoke
return judge_func(source_text, active_rules, judge_context, **kwargs)
def _resolve_and_replay(store, turn_keys, resolved_by):
"""Resolve answered clarifications and replay parked committable mutations.
A needs_user turn parked under the pre-split flow can hold committable
mutations in its stored plan that were never applied (the silent-loss sink
from the v1.5.1 audit). Once the clarification is answered, those mutations
are committed against the parked turn's own registered source — their
evidence spans were validated against exactly that text at judge time.
A replay failure keeps the turn parked; success commits and resolves atomically.
"""
unique_keys = [
key
for key in dict.fromkeys(
str(item).strip() for item in turn_keys if str(item).strip()
)
]
parked_plans = {}
for key in unique_keys:
turn = store.get_turn(key)
if not turn or turn.get("status") != "needs_user":
continue
try:
parked = json.loads(turn.get("plan_json") or "{}")
except (TypeError, json.JSONDecodeError):
parked = {}
committable = [
mutation
for mutation in parked.get("mutations", [])
if isinstance(mutation, dict)
and str(mutation.get("operation", "")).upper() != "NEEDS_USER"
]
if committable and not store.turn_has_transaction(key):
parked_plans[key] = {
"mutations": committable,
"resolved_turn_keys": [],
"reason": (
"replayed after clarification resolution: "
+ str(parked.get("reason", ""))[:500]
),
"replayed_after_clarification": True,
"judged_generation": parked.get('judged_generation'),
}
resolved = []
replays = {}
replayed_any = False
for key in unique_keys:
replay_plan = parked_plans.get(key)
if not replay_plan:
resolved.extend(store.mark_clarifications_resolved([key], resolved_by=resolved_by))
continue
try:
turn = store.get_turn(key)
generation, _ = store.snapshot()
if type(replay_plan.get('judged_generation')) is not int or replay_plan['judged_generation'] != generation:
replays[key] = {'status': 'replay_needs_review',
'reason': 'Original judged generation is unavailable or stale; keep parked.'}
continue
result = store.commit_plan(
key, turn["source_text"], replay_plan, generation,
resolve_pending_by=resolved_by,
)
replays[key] = result.get('committed', result)
resolved.append(key)
replayed_any = True
except Exception as exc:
# Leave needs_user intact. A later retry can replay the same
# registered source; failure must never turn it terminal first.
replays[key] = {
"status": "replay_failed",
"error": f"{type(exc).__name__}: {str(exc)[:500]}",
}
if replayed_any:
try:
store.project()
except Exception:
pass
return resolved, replays
def _finish_clarifications(store, turn: dict, result: dict) -> dict:
try:
committed_plan = json.loads(turn.get("plan_json") or "{}")
except (TypeError, json.JSONDecodeError):
committed_plan = {}
if _plan_contains_terminal_outcome(committed_plan):
return result
candidates = set(committed_plan.get("clarification_candidates", []))
resolved, replays = _resolve_and_replay(
store,
[
turn_key
for turn_key in committed_plan.get("resolved_turn_keys", [])
if turn_key in candidates
],
resolved_by=turn["turn_key"],
)
if resolved:
result["resolved_turn_keys"] = resolved
if replays:
result["clarification_replays"] = replays
return result
def _request_store(memory_dir, namespace: str) -> MemoryStore:
"""Validate request ownership before opening any write/import path."""
if not isinstance(namespace, str) or (namespace and not namespace.strip()):
raise MemoryStoreError('invalid request namespace')
configured = os.environ.get('PT_MEMORY_NAMESPACE', '')
if configured and configured != namespace:
raise MemoryStoreError('request namespace conflicts with worker namespace')
resolved = _memory_dir(memory_dir)
owner = MemoryStore(resolved).peek_namespace() or ''
if owner != namespace:
raise MemoryStoreError('request namespace must match an initialized library')
if namespace and not memory_dir:
raise MemoryStoreError('namespaced memory requires an explicit directory')
store = _store(resolved, namespace=namespace)
if store.get_namespace() != namespace:
raise MemoryStoreError('request namespace changed during initialization')
return store
def ingest_request(request_file: str | Path, memory_dir=None, judge_func=None, *, invoke=None) -> dict:
"""Import one inbox request into SQLite, then resolve it idempotently."""
request_path = Path(request_file).expanduser().resolve()
request = _read_json_file(str(request_path))
source_text = request.get("source_text", "")
context = request.get("context", "")
clarification_candidates = request.get("clarification_candidates") or []
forced = bool(request.get("forced", False))
turn_key = str(request.get("turn_key", "")).strip()
if (
not turn_key
or not isinstance(source_text, str)
or not source_text.strip()
or not isinstance(clarification_candidates, list)
or not all(isinstance(item, str) for item in clarification_candidates)
):
raise MemoryStoreError("inbox request requires turn_key and source_text")
judge_cli = str(request.get("judge_cli", "")).strip().lower()
project_root = str(request.get("project_root", "")).strip()
request_namespace = request.get('namespace', '')
request_memory_dir = memory_dir or request_path.parent.parent
if request_namespace and project_root:
raise MemoryStoreError('experimental requests cannot change operator project configuration')
store = _request_store(request_memory_dir, request_namespace)
if project_root:
os.environ["PT_PROJECT_ROOT"] = project_root
os.environ["B5_PROJECT_ROOT"] = project_root
path_config.get_project_root.cache_clear()
path_config.get_memory_dir.cache_clear()
existing = store.ensure_turn(
turn_key,
source_text,
context,
judge_cli=judge_cli if judge_cli in {"claude", "copilot", "codex"} else "",
clarification_candidates=clarification_candidates,
expected_namespace=request_namespace,
forced=forced,
context_reference=request.get("context_reference"),
detect_signal=bool(request.get("detect_signal", False)),
)
if existing["status"] in FINAL_STATUSES and existing["result"] is not None:
result = _finish_clarifications(
store,
store.get_turn(turn_key),
existing["result"],
)
else:
result = resolve_turn(
turn_key,
memory_dir=store.memory_dir,
judge_func=judge_func,
invoke=invoke,
)
if result.get("status") in FINAL_STATUSES:
try:
request_path.unlink()
except OSError:
pass
return result
@contextmanager
def _maintain_worker_lease(store, turn_key, lease_owner, interval=30):
"""Keep ownership during bounded, potentially multi-call model work."""
stopped = threading.Event()
def heartbeat():
while not stopped.wait(interval):
try:
if not store.renew_lease(turn_key, lease_owner):
return
except Exception:
# A transient busy store may recover on the next tick. Every
# checkpoint/commit independently verifies live ownership.
continue
worker = threading.Thread(target=heartbeat, daemon=True)
worker.start()
try:
yield
finally:
stopped.set()
worker.join(timeout=6)
def resolve_turn(
turn_key: str,
memory_dir=None,
judge_func=None,
max_retries: int = 3,
invoke=None,
) -> dict:
"""Resolve one queued turn. Intended for a detached worker or tests."""
store = _store(memory_dir)
if store.get_namespace() and invoke is None and judge_func is None:
raise MemoryStoreError('experimental learning requires an explicit isolated model transport')
turn = store.get_turn(turn_key)
if not turn:
raise MemoryStoreError(f"unknown turn_key: {turn_key}")
if turn["status"] in FINAL_STATUSES and turn.get("result_json"):
return _finish_clarifications(
store,
turn,
json.loads(turn["result_json"]),
)
if callable(getattr(type(invoke),'for_turn',None)):
invoke=invoke.for_turn(dict(turn))
lease_owner = store.claim_turn(turn_key)
if not lease_owner:
current = store.get_turn(turn_key)
if current and current.get("result_json"):
return json.loads(current["result_json"])
return {"status": current["status"] if current else "missing", "turn_key": turn_key}
judge_func = judge_func or memory_judge.judge_plan
source_text = turn["source_text"]
context = turn.get("context_text") or ""
judge_cli = '' if invoke is not None else str(turn.get("judge_cli") or "").lower()
# Per-turn CLI override must not leak into the next turn of the same drain
# batch: remember the prior value and restore it on the way out.
prior_judge_cli = os.environ.get("PT_MEMORY_UPSERT_CLI")
if judge_cli in {"claude", "copilot", "codex"}:
os.environ["PT_MEMORY_UPSERT_CLI"] = judge_cli
try:
with _maintain_worker_lease(store, turn_key, lease_owner):
return _resolve_claimed_turn(
store, turn, turn_key, lease_owner, judge_func, max_retries, invoke
)
finally:
if judge_cli in {"claude", "copilot", "codex"}:
if prior_judge_cli is None:
os.environ.pop("PT_MEMORY_UPSERT_CLI", None)
else:
os.environ["PT_MEMORY_UPSERT_CLI"] = prior_judge_cli
def _resolve_claimed_turn(
store,
turn: dict,
turn_key: str,
lease_owner: str,
judge_func,
max_retries: int,
invoke=None,
) -> dict:
source_text = turn["source_text"]
context = turn.get("context_text") or ""
last_error = None
signal_state = turn.get('signal_state', 'not_required')
if signal_state == 'unchecked':
signal_generation = store.generation()
try:
has_signal = (memory_judge.classify_user_signal(source_text, invoke=invoke)
if invoke is not None else memory_judge.classify_user_signal(source_text))
if has_signal:
reference = json.loads(turn.get('context_reference_json') or 'null')
page = transcript_adapter.read_context_reference(reference, max_bytes=12000)
if page['context']:
context += '\n\nUntrusted current-conversation context:\n' + page['context'][-10000:]
if page['truncated']:
context += '\n[Older conversation omitted; do not invent missing referents.]'
if page['status'] == 'unavailable':
context += '\n[Current-conversation context unavailable; use only the user turn and existing rules.]'
context = redaction.redact(context)
store.save_resolved_context(turn_key, lease_owner, context, has_signal)
signal_state = 'present' if has_signal else 'absent'
except Exception as exc:
error = f'user signal/context failed: {type(exc).__name__}: {str(exc)[:500]}'
delay=learning_recovery_delay(exc) if store.get_namespace() else None
status = store.mark_turn_error(turn_key, error, lease_owner=lease_owner,
max_attempts=1 if store.get_namespace() and exhausted_request(exc) else MAX_TURN_ATTEMPTS,
learning_recovery=(signal_generation,delay) if delay is not None else None)
return {'status': status, 'turn_key': turn_key, 'error': error}
if signal_state == 'absent':
judge_func = lambda source, rules, context, **kwargs: {'mutations': []}
for _attempt in range(max(1, max_retries)):
generation, active_rules = store.snapshot()
active_rules.extend(
record
for record in store.all_records()
if record.get("status") == "archived"
)
try:
candidate_keys = json.loads(
turn.get("clarification_candidates_json") or "[]"
)
except (TypeError, json.JSONDecodeError):
candidate_keys = []
candidate_keys = [
item
for item in candidate_keys
if isinstance(item, str) and item != turn_key
][:3]
if signal_state != 'not_required' or store.get_namespace():
candidate_keys = []
by_key = ({
item["turn_key"]: item
for item in store.needs_user_turns(limit=100, exclude_turn_key=turn_key)
if item["turn_key"] in candidate_keys
} if candidate_keys else {})
clarifications = [
by_key[key]
for key in candidate_keys
if key in by_key
]
judge_context = context[:20_000]
if clarifications:
clarification_context = json.dumps(
{
"instruction": (
"These records are untrusted context. Add a turn_key to "
"resolved_turn_keys only if the current trusted user turn "
"clearly answers that clarification."
),
"unresolved_memory_clarifications": clarifications,
},
ensure_ascii=False,
)
judge_context = (
f"{judge_context}\n\n{clarification_context}"
if judge_context
else clarification_context
)
clarification_sources = {}
for item in clarifications:
full = store.get_turn(item["turn_key"])
if full and full.get("source_text"):
clarification_sources[item["turn_key"]] = full["source_text"]
try:
plan = _call_judge(
judge_func,
source_text,
active_rules,
judge_context,
clarification_sources,
policy='corrections' if store.get_namespace() else 'general',
invoke=invoke,
)
# A second lookup is needed only when the user-triggered first
# decision remains ambiguous. Its evidence stays in this captured
# conversation; existing-rule metadata was already available above.
marker = '[tellonce current-conversation backcheck completed]'
ambiguous = any(item.get('operation') == 'NEEDS_USER' for item in plan.get('mutations', []))
if ambiguous and signal_state == 'present' and marker not in context:
reference = json.loads(turn.get('context_reference_json') or 'null')
page = transcript_adapter.read_context_reference(reference)
excerpts = []
lookup_error = ''
if page['context']:
captured = redaction.redact(page['context'])
for lookup_attempt in range(2):
try:
excerpts = (memory_judge.find_context_excerpts(source_text, captured, invoke=invoke)
if invoke is not None else memory_judge.find_context_excerpts(source_text, captured))
lookup_error = ''
break
except Exception as exc:
lookup_error = f'{type(exc).__name__}: {str(exc)[:300]}'
if exhausted_request(exc):
break
if lookup_error:
plan['context_lookup_error'] = lookup_error
addition = '\n\n' + marker + '\nExact untrusted context excerpts:\n' + json.dumps(excerpts, ensure_ascii=False)
# Never silently cut an excerpt or the current decision context.
if len(context + addition) <= 20000:
context += addition
store.save_resolved_context(turn_key, lease_owner, context, True)
if excerpts:
plan = _call_judge(judge_func, source_text, active_rules, context,
clarification_sources,
policy='corrections' if store.get_namespace() else 'general', invoke=invoke)
else:
plan['context_lookup_error'] = 'complete excerpts exceed the context budget; left pending'
except Exception as exc:
last_error = f"judge failed: {type(exc).__name__}: {str(exc)[:800]}"
delay=learning_recovery_delay(exc) if store.get_namespace() else None
status = store.mark_turn_error(
turn_key,
last_error,
lease_owner=lease_owner,
max_attempts=1 if store.get_namespace() and exhausted_request(exc) else MAX_TURN_ATTEMPTS,
learning_recovery=(generation,delay) if delay is not None else None,
)
return {"status": status, "turn_key": turn_key, "error": last_error}
clarification_candidates = {
item["turn_key"]
for item in clarifications
}
plan = dict(plan)
plan['judged_generation'] = generation
plan["clarification_candidates"] = sorted(clarification_candidates)
plan["resolved_turn_keys"] = [
candidate
for candidate in plan.get("resolved_turn_keys", [])
if candidate in clarification_candidates
]
needs_mutations = [
mutation
for mutation in plan.get("mutations", [])
if isinstance(mutation, dict)
and str(mutation.get("operation", "")).upper() == "NEEDS_USER"
]
committable_mutations = [
mutation
for mutation in plan.get("mutations", [])
if isinstance(mutation, dict)
and str(mutation.get("operation", "")).upper() != "NEEDS_USER"
]
if needs_mutations and not committable_mutations:
plan["resolved_turn_keys"] = []
return store.mark_needs_user(turn_key, plan, lease_owner=lease_owner)
if needs_mutations:
# Mixed plan: commit the clear mutations now and park only the
# ambiguous clause. Under the old flow the whole turn was parked
# and the committable mutations were silently lost forever once
# the clarification resolved (v1.5.1 audit, finding 1).
committable_plan = dict(plan)
committable_plan["mutations"] = committable_mutations
needs_plan = dict(plan)
needs_plan["mutations"] = needs_mutations
needs_plan["resolved_turn_keys"] = []
needs_plan["partial_commit"] = True
try:
committed = store.commit_plan(
turn_key,
source_text,
committable_plan,
generation,
lease_owner=lease_owner,
pending_plan=needs_plan,
)
except StaleSnapshotError as exc:
last_error = str(exc)
continue
except Exception as exc:
last_error = f"commit failed: {type(exc).__name__}: {str(exc)[:800]}"
status = store.mark_turn_error(
turn_key,
last_error,
lease_owner=lease_owner,
max_attempts=MAX_TURN_ATTEMPTS,
)
return {"status": status, "turn_key": turn_key, "error": last_error}
try:
committed['committed']["projection"] = store.project()
except Exception as exc:
committed['committed']["projection"] = {
"status": "pending",
"error": f"{type(exc).__name__}: {str(exc)[:500]}",
}
# A plan containing NEEDS_USER never auto-resolves older
# clarifications (conservative, unchanged); resolved_turn_keys
# stayed in committable_plan only so its evidence corpus remains
# valid at store-side validation.
return committed
try:
result = store.commit_plan(
turn_key,
source_text,
plan,
generation,
lease_owner=lease_owner,
)
except StaleSnapshotError as exc:
last_error = str(exc)
continue
except Exception as exc:
last_error = f"commit failed: {type(exc).__name__}: {str(exc)[:800]}"
status = store.mark_turn_error(
turn_key,
last_error,
lease_owner=lease_owner,
max_attempts=MAX_TURN_ATTEMPTS,
)
return {"status": status, "turn_key": turn_key, "error": last_error}
if not _plan_contains_terminal_outcome(plan):
try:
resolved, replays = _resolve_and_replay(
store,
plan.get("resolved_turn_keys", []),
resolved_by=turn_key,
)
if resolved:
result["resolved_turn_keys"] = resolved
if replays:
result["clarification_replays"] = replays
except Exception as exc:
result["clarification_error"] = (
f"{type(exc).__name__}: {str(exc)[:500]}"
)
try:
projection = store.project()
result["projection"] = projection
result["status"] = "projected" if result["status"] == "committed" else result["status"]
except Exception as exc:
result["projection"] = {
"status": "pending",
"error": f"{type(exc).__name__}: {str(exc)[:500]}",
}
return result
status = store.mark_turn_error(
turn_key,
last_error or "snapshot changed repeatedly",
lease_owner=lease_owner,
max_attempts=MAX_TURN_ATTEMPTS,
)
return {
"status": status,
"turn_key": turn_key,
"error": last_error or "snapshot changed repeatedly",
}
def apply_plan(
source_text: str,
plan: dict,
turn_key: str = "",
context: str = "",
memory_dir=None,
namespace: str = '',
) -> dict:
"""Synchronous deterministic entry used by tests and manual recovery."""
store = _request_store(memory_dir, namespace)
safe_source = redaction.redact(source_text)
if not safe_source.strip():
raise ValueError("source_text is required for apply-plan")
safe_context = redaction.redact(context or "")
turn_key = turn_key.strip() or _default_turn_key()
store.ensure_turn(turn_key, safe_source, safe_context, expected_namespace=namespace)
generation, active_rules = store.snapshot()
plan = json.loads(json.dumps(plan))
validated = memory_judge.validate_plan(
plan,
safe_source,
active_rules,
strict_evidence=True,
)
result = store.commit_plan(turn_key, safe_source, validated, generation)
if result["status"] in {"committed", "noop"}:
result["projection"] = store.project()
if result["status"] == "committed":
result["status"] = "projected"
return result
def drain(
memory_dir=None,
limit: int = 20,
schedule_retry: bool = False,
forced_only: bool = False,
invoke=None,
) -> dict:
if invoke is not None and schedule_retry:
raise MemoryStoreError('explicit transport recovery must be scheduled by its owning experiment worker')
resolved_dir = _memory_dir(memory_dir)
inbox_paths = _pending_request_paths(resolved_dir)
namespaces = set()
for path in inbox_paths:
try:
request = _read_json_file(str(path))
except (ValueError, OSError):
continue # The normal drain loop quarantines malformed JSON.
namespace = request.get('namespace', '')
if not isinstance(namespace, str) or (namespace and not namespace.strip()):
raise MemoryStoreError('invalid inbox namespace')
namespaces.add(namespace)
owner = MemoryStore(resolved_dir).peek_namespace() or ''
configured = os.environ.get('PT_MEMORY_NAMESPACE', '')
expected = configured or owner or next(iter(namespaces), '')
if len(namespaces) > 1 or (namespaces and expected and namespaces != {expected}):
raise MemoryStoreError('mixed or missing inbox namespaces')
store = _store(resolved_dir, namespace=expected)
results = []
attempted_turn_keys = set()
seen_inbox_turns = {}
if forced_only:
forced_paths = []
for path in inbox_paths:
try:
if bool(_read_json_file(str(path)).get("forced", False)):
forced_paths.append(path)
except (OSError, UnicodeError, ValueError, json.JSONDecodeError):
continue
inbox_paths = forced_paths
inbox_paths = inbox_paths[: max(1, limit)]
for request_path in inbox_paths:
try:
request = _read_json_file(str(request_path))
inbox_turn_key = str(request.get("turn_key", "")).strip()
if inbox_turn_key and inbox_turn_key in seen_inbox_turns:
prior_source = seen_inbox_turns[inbox_turn_key]
current_source = str(request.get("source_text", ""))
if current_source == prior_source:
request_path.unlink()
results.append(
{
"status": "duplicate",
"turn_key": inbox_turn_key,
"request_file": str(request_path),
}
)
else:
failed_path = _quarantine_request(
request_path,
MemoryStoreError(
f"conflicting inbox requests share turn_key {inbox_turn_key}"
),
)
results.append(
{
"status": "quarantined",
"turn_key": inbox_turn_key,
"request_file": str(failed_path),
"error": "conflicting source_text for duplicate turn_key",
}
)
continue
if inbox_turn_key:
seen_inbox_turns[inbox_turn_key] = str(
request.get("source_text", "")
)
result = ingest_request(request_path, memory_dir=store.memory_dir, invoke=invoke)
results.append(result)
if result.get("turn_key"):
attempted_turn_keys.add(result["turn_key"])
except (json.JSONDecodeError, UnicodeError, ValueError, MemoryStoreError) as exc:
try:
failed_path = _quarantine_request(request_path, exc)
except OSError:
failed_path = request_path
results.append(
{
"status": "quarantined",
"request_file": str(failed_path),
"error": f"{type(exc).__name__}: {str(exc)[:500]}",
}
)
except Exception as exc:
results.append(
{
"status": "pending",
"request_file": str(request_path),
"error": f"{type(exc).__name__}: {str(exc)[:500]}",
}
)
for turn_key in store.pending_turn_keys(limit=limit, forced_only=forced_only):
if turn_key in attempted_turn_keys:
continue
try:
results.append(resolve_turn(turn_key, memory_dir=store.memory_dir, invoke=invoke))
except Exception as exc:
# resolve_turn claimed the turn (charging the attempt) before it
# raised; charging again here made one real failure count double.
status = store.mark_turn_error(
turn_key,
f"resolve failed: {type(exc).__name__}: {str(exc)[:800]}",
max_attempts=MAX_TURN_ATTEMPTS,
charge_attempt=False,
)
results.append(
{
"status": status,
"turn_key": turn_key,
"error": f"{type(exc).__name__}: {str(exc)[:500]}",
}
)
if store.projection_pending():
try:
results.append(store.project())
except Exception as exc:
results.append(
{
"status": "projection_pending",
"error": f"{type(exc).__name__}: {str(exc)[:500]}",
}
)
if forced_only:
forced_inbox_remaining = False
for path in _pending_request_paths(store.memory_dir):
try:
if bool(_read_json_file(str(path)).get("forced", False)):
forced_inbox_remaining = True
break
except (OSError, UnicodeError, ValueError, json.JSONDecodeError):
continue
remaining = bool(
forced_inbox_remaining
or store.pending_turn_keys(limit=1, forced_only=True)
or store.projection_pending()
)
else:
remaining = bool(
_pending_request_paths(store.memory_dir)
or store.pending_turn_keys(limit=1)
or store.projection_pending()
)
retry = None
if remaining and schedule_retry:
try:
retry = _schedule_retry(
store.memory_dir,
delay_seconds=store.retry_delay_seconds(),
)
except Exception as exc:
retry = {
"status": "failed",
"error": f"{type(exc).__name__}: {str(exc)[:300]}",
}
return {
"status": "drained",
"count": len(results),
"results": results,
"remaining": remaining,
"retry": retry,
}
def retry(memory_dir=None, delay_seconds: int = 30) -> dict:
time.sleep(max(0, int(delay_seconds)))
try:
_retry_marker(memory_dir).unlink()
except OSError:
pass
if not hooks_enabled():
store = _store(memory_dir)
has_forced = bool(store.pending_turn_keys(limit=1, forced_only=True))
if not has_forced:
for path in _pending_request_paths(store.memory_dir):
try:
if bool(_read_json_file(str(path)).get("forced", False)):
has_forced = True
break
except (OSError, UnicodeError, ValueError, json.JSONDecodeError):
continue
if not has_forced:
return {"status": "disabled", "remaining": False}
return drain(
memory_dir=memory_dir,
schedule_retry=True,
forced_only=not hooks_enabled(),
)
def inspect(memory_dir=None) -> dict:
store = _store(memory_dir)
generation, active = store.snapshot()
return {
"status": "ok",
"memory_dir": str(store.memory_dir),
"generation": generation,
"active": active,
"pending_turns": store.pending_turn_keys(limit=100),
"failed_turns": store.failed_turn_keys(limit=100),
"needs_user": store.needs_user_turns(limit=100),
"rejected_turns": store.rejected_turns(limit=100),
}
def dismiss(turn_key: str, memory_dir=None) -> dict:
return _store(memory_dir).dismiss_clarification(turn_key)
def _read_json_file(path: str) -> dict:
if path == "-":
value = json.load(sys.stdin)
else:
with open(path, encoding="utf-8-sig") as handle:
value = json.load(handle)
if not isinstance(value, dict):
raise ValueError("JSON input must be an object")
return value
def _print_json(value) -> None:
sys.stdout.write(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
def configure_hooks(enabled: bool | None = None) -> dict:
if not CONFIG_PATH.exists():
config = {}
else:
try:
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8-sig"))
except (OSError, json.JSONDecodeError) as exc:
raise MemoryStoreError(
f"refusing to overwrite unreadable config {CONFIG_PATH}: {exc}"
) from exc
if not isinstance(config, dict):
raise MemoryStoreError(
f"refusing to overwrite non-object config {CONFIG_PATH}"
)
if enabled is not None:
config["memory_upsert_enabled"] = bool(enabled)
CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
tmp = CONFIG_PATH.with_name(
f"{CONFIG_PATH.name}.tmp.{os.getpid()}.{uuid.uuid4().hex[:8]}"
)
try:
tmp.write_text(
json.dumps(config, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
try:
os.chmod(tmp, 0o600)
except OSError:
pass
os.replace(tmp, CONFIG_PATH)
finally:
try:
tmp.unlink()
except OSError:
pass
env_value = os.environ.get(
"PT_MEMORY_UPSERT_ENABLED",
os.environ.get("B5_MEMORY_UPSERT_ENABLED"),
)
effective = hooks_enabled()
return {
"status": "enabled" if effective else "disabled",
"configured": str(config.get("memory_upsert_enabled", True)).strip().lower() in {"1", "true", "yes", "on"},
"source": ("environment" if env_value is not None else
"config" if "memory_upsert_enabled" in config else "default"),
"config_path": str(CONFIG_PATH),
"applies_to": ["claude", "copilot", "codex"],
}
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="memory_upsert")
sub = parser.add_subparsers(dest="command", required=True)
init_parser = sub.add_parser('init')
init_parser.add_argument('--memory-dir', required=True)
init_parser.add_argument('--namespace', required=True)
enqueue_parser = sub.add_parser("enqueue")
enqueue_parser.add_argument("--request-file", default="-")
enqueue_parser.add_argument("--memory-dir", default="")
enqueue_parser.add_argument("--no-spawn", action="store_true")
enqueue_parser.add_argument("--force", action="store_true")
enqueue_parser.add_argument("--manual", action="store_true")
enqueue_parser.add_argument("--source-text", default=None)
enqueue_parser.add_argument("--turn-key", default="")
enqueue_parser.add_argument("--context", default="")
resolve_parser = sub.add_parser("resolve")
resolve_parser.add_argument("--turn-key", required=True)
resolve_parser.add_argument("--memory-dir", default="")
ingest_parser = sub.add_parser("ingest")
ingest_parser.add_argument("--request-file", required=True)
ingest_parser.add_argument("--memory-dir", default="")
drain_parser = sub.add_parser("drain")
drain_parser.add_argument("--memory-dir", default="")
drain_parser.add_argument("--limit", type=int, default=20)
retry_parser = sub.add_parser("retry")
retry_parser.add_argument("--memory-dir", default="")
retry_parser.add_argument("--delay", type=int, default=30)
apply_parser = sub.add_parser("apply-plan")
apply_parser.add_argument("--request-file", required=True)
apply_parser.add_argument("--plan-file", required=True)
apply_parser.add_argument("--memory-dir", default="")
inspect_parser = sub.add_parser("inspect")
inspect_parser.add_argument("--memory-dir", default="")
recover_parser = sub.add_parser("recover")
recover_parser.add_argument("--memory-dir", default="")
dismiss_parser = sub.add_parser("dismiss")
dismiss_parser.add_argument("--turn-key", required=True)
dismiss_parser.add_argument("--memory-dir", default="")
sub.add_parser("enable-hooks")
sub.add_parser("disable-hooks")
sub.add_parser("hook-status")
return parser
def main(argv=None) -> int:
args = build_parser().parse_args(argv)
memory_dir = getattr(args, "memory_dir", "") or None
try:
if args.command == 'init':
store = MemoryStore(memory_dir, legacy_dirs=[])
store.initialize(namespace=args.namespace)
result = {'status': 'initialized', 'memory_dir': str(store.memory_dir),
'namespace': args.namespace}
elif args.command == "enqueue":
if args.manual and hooks_enabled():
result = {
"status": "delegated_to_automatic_hook",
"blocking": False,
}
else:
request = (
{
"source_text": args.source_text,
"turn_key": args.turn_key,
"context": args.context,
}
if args.source_text is not None
else _read_json_file(args.request_file)
)
result = enqueue(
source_text=request.get("source_text", ""),
turn_key=request.get("turn_key", ""),
context=request.get("context", ""),
memory_dir=memory_dir,
spawn_worker=not args.no_spawn,
force=args.force,
context_reference=request.get("context_reference"),
detect_signal=bool(request.get("detect_signal", False)),
namespace=request.get('namespace', ''),
)
elif args.command == "resolve":
result = resolve_turn(args.turn_key, memory_dir=memory_dir)
elif args.command == "ingest":
result = ingest_request(args.request_file, memory_dir=memory_dir)
elif args.command == "drain":
result = drain(memory_dir=memory_dir, limit=args.limit, schedule_retry=True)
elif args.command == "retry":
result = retry(memory_dir=memory_dir, delay_seconds=args.delay)
elif args.command == "apply-plan":
request = _read_json_file(args.request_file)
plan = _read_json_file(args.plan_file)
result = apply_plan(
source_text=request.get("source_text", ""),
turn_key=request.get("turn_key", ""),
context=request.get("context", ""),
plan=plan,
memory_dir=memory_dir,
namespace=request.get('namespace', ''),
)
elif args.command == "inspect":
result = inspect(memory_dir=memory_dir)
elif args.command == "recover":
result = _store(memory_dir).recover()
elif args.command == "dismiss":
result = dismiss(args.turn_key, memory_dir=memory_dir)
elif args.command == "enable-hooks":
result = configure_hooks(True)
elif args.command == "disable-hooks":
result = configure_hooks(False)
elif args.command == "hook-status":
result = configure_hooks()
else:
raise MemoryStoreError(f"unsupported command: {args.command}")
_print_json(result)
return 0
except Exception as exc:
_print_json(
{
"status": "error",
"error": f"{type(exc).__name__}: {str(exc)[:1000]}",
}
)
return 1
if __name__ == "__main__":
sys.exit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |