rrrrrredy/research-toolkit / plugins/research-toolkit
research-toolkit
FlaggedUse for source-backed long-form research and writing in English or Chinese across fields, including revisions and continuation. Understand requirements, develop analysis through substantive research collaboration, and deliver a complete report after independent review, audit and acceptance.
scripts/usage_events.py
View on GitHub"""Allowlisted usage events shared by CLI, MCP and usage recording for agent tools.
Research records remain local. This module never uploads a record or exception
verbatim, and never changes the research operation's result.
"""
from __future__ import annotations
from contextlib import closing, contextmanager
from contextvars import ContextVar
from datetime import datetime, timezone
import hashlib
import hmac
import inspect
import json
import os
from pathlib import Path
import platform
import re
import sqlite3
import subprocess
import sys
import time
import uuid
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_ENDPOINT = "https://research-toolkit-usage.luosongred.workers.dev"
CONTEXT = ContextVar("research_toolkit_usage", default={})
HOSTS = {"codex", "claude", "opencode", "unknown"}
ENTRIES = {"cli", "mcp", "skill", "plugin", "adapter", "library", "unknown"}
CHANNELS = {"github-clone", "github-release", "plugin-marketplace", "manual", "unknown"}
KINDS = {"installation_seen", "call_started", "call_finished", "record_saved", "work_prepared",
"review_recorded", "host_execution", "toolkit_activated", "user_turn", "agent_input", "tool_observed",
"agent_started", "agent_finished", "session_stopped", "session_compacted", "task_linked"}
STATES = {"started", "success", "failed", "blocked", "pending", "saved", "completed", "cancelled",
"created", "prepared", "error", "waiting", "received", "unknown"}
OPERATIONS = {"start", "status", "guide", "save", "review", "finish", "check-draft", "check-reviewer",
"check-updates", "host_execution", "prepare_work", "read", "write", "edit", "search",
"browse", "shell", "skill", "delegate", "other"}
SAFE_ERRORS = {"ValueError", "OSError", "TypeError", "KeyError", "PermissionError", "TimeoutError",
"RuntimeError", "FileNotFoundError", "ToolError", "other"}
METHODS = {"skill", "research-standard", "research-workflow", "writing-style", "optional-analysis-lenses",
"subagents-and-review-loop", "quality-gates", "task-continuity", "gotchas", "postmortem-lessons"}
def home():
return Path(os.environ.get("RESEARCH_TOOLKIT_USAGE_HOME", Path.home()/".research-toolkit"/"usage"))
def disabled_by_environment():
value = os.environ.get("RESEARCH_TOOLKIT_TELEMETRY", "").lower()
return value in {"off", "0", "false"} or (bool(os.environ.get("CI")) and value != "on")
def database_path():
return home() / "usage.sqlite3"
def recording_enabled():
path = database_path()
try:
if disabled_by_environment() or not path.is_file():
return False
with closing(sqlite3.connect(path.absolute().as_uri() + "?mode=ro", uri=True, timeout=2)) as db:
return db.execute("SELECT value FROM settings WHERE key='enabled'").fetchone() == ("on",)
except (OSError, sqlite3.Error) as exc:
failure(exc)
return False
@contextmanager
def database():
path = database_path()
path.parent.mkdir(parents=True, exist_ok=True)
connection = sqlite3.connect(path, timeout=2)
connection.row_factory = sqlite3.Row
try:
connection.execute("PRAGMA foreign_keys = ON")
connection.executescript("""
CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS events (id TEXT PRIMARY KEY, created REAL NOT NULL, payload TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS sessions (id TEXT PRIMARY KEY, active INTEGER NOT NULL, task TEXT, parent TEXT);
CREATE TABLE IF NOT EXISTS aliases (alias TEXT NOT NULL, task TEXT NOT NULL, PRIMARY KEY(alias, task));
CREATE TABLE IF NOT EXISTS content (id TEXT PRIMARY KEY, created REAL NOT NULL, payload TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS content_scopes (
scope TEXT NOT NULL,
id TEXT NOT NULL REFERENCES content(id) ON DELETE CASCADE,
PRIMARY KEY(scope, id)
);
""")
# Older off settings also paused report sharing, including saved grants.
# Retain that decision while allowing explicit new content consent.
connection.execute("""
INSERT OR IGNORE INTO settings (key, value)
VALUES ('legacy_sharing_paused', CASE WHEN EXISTS (
SELECT 1 FROM settings WHERE key='enabled' AND value='off'
) THEN 'true' ELSE 'false' END)
""")
for key, value in {"installation_id": uuid.uuid4().hex, "secret": uuid.uuid4().hex+uuid.uuid4().hex,
"id_secret": uuid.uuid4().hex+uuid.uuid4().hex,
"enabled": "off", "channel": "unknown", "form": "unknown"}.items():
connection.execute("INSERT OR IGNORE INTO settings VALUES (?,?)", (key, value))
connection.commit()
yield connection
connection.commit()
finally:
connection.close()
def settings(db):
return dict(db.execute("SELECT key,value FROM settings").fetchall())
def set_setting(db, key, value):
db.execute("INSERT INTO settings VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, str(value)))
def installation_key():
return "install."+hashlib.sha256(str(ROOT).encode()).hexdigest()+"."
def installation_info(config):
prefix = installation_key()
channel = os.environ.get("RESEARCH_TOOLKIT_INSTALL_CHANNEL", config.get(prefix+"channel", "unknown"))
form = os.environ.get("RESEARCH_TOOLKIT_INSTALL_FORM", config.get(prefix+"form", "unknown"))
if form == "unknown" and (ROOT/"plugin.json").is_file():
form = "plugin"
return {"channel": channel if channel in CHANNELS else "unknown", "form": form if form in ENTRIES else "unknown"}
def opaque(config, kind, value):
if value is None or value == "":
return None
return hmac.new(config["id_secret"].encode(), (kind+":"+str(value)).encode(), hashlib.sha256).hexdigest()[:32]
def host_name():
value = os.environ.get("RESEARCH_TOOLKIT_HOST", "")
if value in HOSTS:
return value
if os.environ.get("CLAUDE_CODE_ENTRYPOINT"):
return "claude"
if os.environ.get("CODEX_THREAD_ID"):
return "codex"
if os.environ.get("OPENCODE"):
return "opencode"
return "unknown"
def notice(db, config):
if config.get("notice") == "1":
return
print(notice_text(config), file=sys.stderr)
set_setting(db, "notice", "1")
def notice_text(config, language="en"):
target = endpoint(config)
if language in {"zh", "zh-CN"}:
return (f"Research Toolkit 已开启基础使用记录,发送至 {target},由维护者在私密仓库中查看。"
"记录安装渠道、版本、操作类别和结果等元数据;不默认发送聊天、提示词、报告、来源 URL、原始路径或用户凭证。"
"报告和原始材料需要另行确认分享。可随时设置 RESEARCH_TOOLKIT_TELEMETRY=off,或运行 python scripts/usage_events.py off 关闭。")
return (f"Research Toolkit basic usage recording is on and sends metadata to {target}, viewed privately by the maintainer. "
"It records installation route, version, operation categories and outcomes. Chats, prompts, reports, source URLs, "
"raw paths and user credentials are excluded by default. Sharing reports or source materials requires separate confirmation. "
"Disable with RESEARCH_TOOLKIT_TELEMETRY=off or python scripts/usage_events.py off.")
def visible_notice(language="en"):
"""Include the first notice in the actual tool/installation return, not just stderr."""
if disabled_by_environment():
return None
try:
if not database_path().is_file():
return None
with database() as db:
config = settings(db)
if config["enabled"] != "on" or config.get("visible_notice_v2") == "1":
return None
result = {"message": notice_text(config, language), "receiver": endpoint(config),
"action": "Show this notice once in the current conversation. Keep the saved recording setting."}
set_setting(db, "visible_notice_v2", "1")
return result
except (OSError, sqlite3.Error, ValueError) as exc:
failure(exc)
return None
def enqueue(db, config, kind, *, task=None, host=None, entry=None, operation=None, outcome=None,
event_id=None, session=None, parent=None, unit=None, call=None, counts=None,
elapsed_ms=None, error=None, stage=None, revision=None, method=None):
if kind not in KINDS:
return
stamp = time.time()
version = (ROOT/"VERSION").read_text(encoding="utf-8").strip()
payload = {"schema": 1, "id": event_id or uuid.uuid4().hex,
"at": datetime.fromtimestamp(stamp, timezone.utc).isoformat(),
"installation": config["installation_id"], "version": version,
"host": host if host in HOSTS else host_name(),
"entry": entry if entry in ENTRIES else "library", "kind": kind,
**installation_info(config),
"os": {"Windows": "windows", "Darwin": "macos", "Linux": "linux"}.get(platform.system(), "other"),
"test": os.environ.get("RESEARCH_TOOLKIT_USAGE_TEST") == "1"}
for key, value in {"task": task, "session": session, "parent": parent, "unit": unit, "call": call}.items():
if value:
payload[key] = opaque(config, "session" if key == "parent" else key, value)
if operation in OPERATIONS:
payload["operation"] = operation
if outcome in STATES:
payload["outcome"] = outcome
if error:
payload["error"] = error if error in SAFE_ERRORS else "other"
if stage in {"brief", "collect", "analyze", "draft", "review", "revise", "final"}:
payload["stage"] = stage
if method in METHODS:
payload["method"] = method
if isinstance(elapsed_ms, (int, float)) and 0 <= elapsed_ms < 31_536_000_000:
payload["elapsed_ms"] = round(elapsed_ms, 3)
if isinstance(revision, str) and len(revision) == 64 and all(c in "0123456789abcdef" for c in revision):
payload["revision"] = opaque(config, "revision", revision)
if isinstance(counts, dict):
selected = {key: value for key, value in counts.items()
if key in {"sources", "claims", "requirements", "findings", "dependencies"}
and type(value) is int and 0 <= value <= 1_000_000}
if selected:
payload["counts"] = selected
db.execute("INSERT OR IGNORE INTO events VALUES (?,?,?)", (payload["id"], stamp, json.dumps(payload, ensure_ascii=False)))
db.execute("DELETE FROM events WHERE created < ?", (stamp-7*86400,))
# Bound the local queue if the receiver remains unavailable.
db.execute("DELETE FROM events WHERE id IN (SELECT id FROM events ORDER BY created DESC LIMIT -1 OFFSET 10000)")
def failure(exc):
# Observability failures must be visible locally without leaking an exception's contents.
print("Research Toolkit usage recording unavailable: "+type(exc).__name__+"; research continues.", file=sys.stderr)
def emit(kind, **fields):
if disabled_by_environment():
return
try:
if not database_path().is_file():
return
with database() as db:
config = settings(db)
if config["enabled"] != "on":
return
notice(db, config)
if config.get("seen") != "1":
enqueue(db, config, "installation_seen", entry=fields.get("entry"))
set_setting(db, "seen", "1")
enqueue(db, config, kind, **fields)
schedule_upload()
except (OSError, sqlite3.Error, ValueError) as exc:
failure(exc)
def task_identity(root):
identity = str(Path(root).resolve())
# Windows uses this spelling to access long paths, not a different task.
if identity.startswith('\\\\?\\UNC\\'):
return '\\\\' + identity[8:]
if identity.startswith('\\\\?\\'):
return identity[4:]
return identity
def remember_task(root):
if disabled_by_environment():
return
try:
if not database_path().is_file():
return
with database() as db:
if settings(db)["enabled"] == "on":
db.execute("INSERT OR IGNORE INTO aliases VALUES (?,?)", (Path(root).name, task_identity(root)))
except (OSError, sqlite3.Error) as exc:
failure(exc)
def observe_record(root, record):
context = CONTEXT.get()
operation = record.get("operation", record.get("record_type"))
kind = {"save": "record_saved", "prepare_work": "work_prepared", "host_execution": "host_execution",
"model_review": "review_recorded", "review_audit": "review_recorded",
"final_acceptance": "review_recorded"}.get(operation)
if not kind:
return
remember_task(root)
ids = record.get("ids", {})
counts = {key: len(value) for key, value in ids.items() if isinstance(value, list)} if isinstance(ids, dict) else {}
if isinstance(record.get("findings"), list):
counts["findings"] = len(record["findings"])
if isinstance(record.get("depends_on"), list):
counts["dependencies"] = len(record["depends_on"])
emit(kind, task=task_identity(root), entry=context.get("entry", "adapter" if operation == "host_execution" else "library"),
host=record.get("host"), operation=operation, call=context.get("call"),
outcome=record.get("outcome", record.get("status", "unknown")), counts=counts,
unit=record.get("unit_id"), session=record.get("host_task", record.get("execution_id")),
revision=record.get("artifact_sha256", record.get("draft_sha256")))
def observed_call(function, args, kwargs=None, *, entry):
kwargs = kwargs or {}
if not recording_enabled():
return function(*args, **kwargs)
name = function.__name__
operation = {"save_records": "save", "guidance": "guide", "check_updates": "check-updates",
"check_draft": "check-draft", "review_readiness": "check-reviewer"}.get(name, name)
bound = inspect.signature(function).bind_partial(*args, **kwargs).arguments
if name == "_dispatch":
operation = bound.get("action")
if isinstance(bound.get("request"), dict):
bound = {**bound, "task": bound["request"].get("task"), "stage": bound["request"].get("stage"),
"language": bound["request"].get("language", "en")}
root = Path(bound["workspace"])/bound["task"] if "workspace" in bound and isinstance(bound.get("task"), str) else None
if root is not None:
remember_task(root)
identity = uuid.uuid4().hex
token = CONTEXT.set({"call": identity, "entry": entry})
fields = {"task": task_identity(root) if root is not None else None, "entry": entry,
"operation": operation, "call": identity, "stage": bound.get("stage")}
started = time.monotonic()
emit("call_started", **fields, outcome="started")
try:
result = function(*args, **kwargs)
except BaseException as exc:
# Record every Python exit, including cancellation, then preserve it for
# the caller. Recording an exit must not turn it into a successful call.
outcome = "failed" if isinstance(exc, Exception) else "cancelled"
emit("call_finished", **fields, outcome=outcome, error=type(exc).__name__, elapsed_ms=(time.monotonic()-started)*1000)
raise
else:
outcome = "success"
if isinstance(result, dict):
if result.get("action_required"):
outcome = "pending"
elif result.get("ok") is False or result.get("completed") is False or result.get("reviews_complete") is False:
outcome = "blocked"
elif result.get("completed") is True:
outcome = "completed"
emit("call_finished", **fields, outcome=outcome, elapsed_ms=(time.monotonic()-started)*1000)
if isinstance(result, dict):
visible = visible_notice(bound.get("language", "en"))
if visible:
result = {**result, "usage_notice": visible}
return result
finally:
CONTEXT.reset(token)
def endpoint(config):
value = os.environ.get("RESEARCH_TOOLKIT_USAGE_ENDPOINT", config.get("endpoint", DEFAULT_ENDPOINT)).rstrip("/")
if not value:
return None
parsed = urlsplit(value)
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValueError("usage endpoint must be an origin/path without credentials, query or fragment")
if parsed.scheme != "https" and not (parsed.scheme == "http" and parsed.hostname in {"127.0.0.1", "localhost", "::1"}):
raise ValueError("usage endpoint must use HTTPS (HTTP is allowed only on loopback)")
return value
def has_pending_upload(db, config):
return db.execute("SELECT EXISTS(SELECT 1 FROM content) OR (? AND EXISTS(SELECT 1 FROM events))",
(config["enabled"] == "on",)).fetchone()[0]
def schedule_upload():
if (disabled_by_environment() or not database_path().is_file()
or os.environ.get("RESEARCH_TOOLKIT_USAGE_AUTOSEND") == "off"):
return
with database() as db:
db.execute("BEGIN IMMEDIATE")
config = settings(db)
if not has_pending_upload(db, config) or not endpoint(config) or float(config.get("next_upload", "0")) > time.time():
return
set_setting(db, "next_upload", time.time()+30)
options = {"stdin": subprocess.DEVNULL, "stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL,
"close_fds": True}
if os.name == "nt":
options["creationflags"] = subprocess.CREATE_NO_WINDOW
else:
options["start_new_session"] = True
# Reserve one delayed send for this window. Sending immediately and then
# ignoring new events for 30 seconds stranded the final batch on exit.
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "flush", "--scheduled"], **options)
def request_json(url, data, config, method="POST"):
body = json.dumps(data, ensure_ascii=False).encode() if data is not None else None
request = Request(url, data=body, method=method,
headers={"Content-Type": "application/json", "User-Agent": "Research-Toolkit-Usage/1",
"Authorization": "Bearer "+config["installation_id"]+":"+config["secret"]})
with urlopen(request, timeout=10) as response:
return json.loads(response.read(1024*1024))
def flush():
if disabled_by_environment() or not database_path().is_file():
return {"enabled": False, "sent": 0}
with database() as db:
db.execute("BEGIN IMMEDIATE")
config = settings(db)
target = endpoint(config)
if not target or not has_pending_upload(db, config):
return {"enabled": config["enabled"] == "on", "sent": 0, "receiver": bool(target)}
if float(config.get("sending_until", "0")) > time.time():
return {"sent": 0, "active_sender": True}
set_setting(db, "sending_until", time.time()+180)
sent = 0
try:
if config.get("registered_at") != target:
request_json(target+"/v1/install", {"installation": config["installation_id"]}, config)
with database() as db:
set_setting(db, "registered_at", target)
for _ in range(10):
with database() as db:
if settings(db)["enabled"] != "on" or disabled_by_environment():
break
db.execute("DELETE FROM events WHERE created < ?", (time.time()-7*86400,))
rows = db.execute("SELECT id,payload FROM events ORDER BY created LIMIT 100").fetchall()
if not rows:
break
reply = request_json(target+"/v1/events", {"events": [json.loads(row["payload"]) for row in rows]}, config)
acknowledged = set(reply.get("accepted_ids", []))
expected = {row["id"] for row in rows}
if acknowledged != expected:
raise ValueError("receiver did not acknowledge the complete batch")
with database() as db:
db.executemany("DELETE FROM events WHERE id=?", ((identity,) for identity in acknowledged))
sent += len(rows)
with database() as db:
db.execute("DELETE FROM content WHERE created < ?", (time.time()-7*86400,))
shared = db.execute("SELECT id,payload FROM content ORDER BY created LIMIT 5").fetchall()
for row in shared:
if disabled_by_environment():
break
with database() as db:
if not db.execute("SELECT 1 FROM content WHERE id=?", (row["id"],)).fetchone():
continue # A user withdrew this project grant after the batch was read.
reply = request_json(target+"/v1/content", json.loads(row["payload"]), config)
if reply.get("id") != row["id"]:
raise ValueError("receiver did not acknowledge shared content")
with database() as db:
db.execute("DELETE FROM content WHERE id=?", (row["id"],))
with database() as db:
set_setting(db, "last_upload", datetime.now(timezone.utc).isoformat())
set_setting(db, "last_error", "")
return {"sent": sent, "receiver": True}
except (HTTPError, URLError, TimeoutError, OSError, ValueError) as exc:
with database() as db:
set_setting(db, "last_error", "HTTP "+str(exc.code) if isinstance(exc, HTTPError) else type(exc).__name__)
return {"sent": sent, "pending": True, "error": "HTTP "+str(exc.code) if isinstance(exc, HTTPError) else type(exc).__name__}
finally:
with database() as db:
set_setting(db, "sending_until", "0")
def sharing_key(project):
return "content." + hashlib.sha256(task_identity(project).encode()).hexdigest()
def sharing_project(root, project, config):
"""Use the requested scope or the nearest saved choice containing this task."""
if project is not None:
project = Path(task_identity(project))
root.relative_to(project)
return project
offered = None
for candidate in (root, *root.parents):
key = sharing_key(candidate)
choice = config.get(key)
if choice == "offered" or (choice is None and config.get(key + ".offered") == "true"):
if offered is None:
offered = candidate
elif choice is not None:
return candidate
return offered or root
def check_shared_file(relative, text):
parts = [part.lower() for part in relative.parts]
if any(part in {".git", "credentials", "auth.json", ".aws", ".ssh", ".npmrc", ".netrc"}
or re.fullmatch(r"(?:credentials|secrets?|client[_-]secret(?:[._-][\w.-]+)?)\.(?:json|ya?ml|toml|ini|cfg)", part)
or part == ".env" or part.startswith(".env.")
or part in {"id_rsa", "id_ed25519", "id_ecdsa"} for part in parts):
raise ValueError(f"{relative.as_posix()}: credential/configuration files cannot be shared")
patterns = [r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |ENCRYPTED )?PRIVATE KEY-----",
r"\bsk-(?:proj-|ant-api\d+-)?[A-Za-z0-9_-]{20,}\b",
r"\b(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{40,}|AKIA[0-9A-Z]{16})\b",
r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b",
r"\bAIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])",
r"\beyJ[A-Za-z0-9_-]{5,}\.eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{10,}\b"]
# Match assigned credentials, not mentions of password/Bearer in prose.
values = [r'''(?i)\bauthorization["']?\s*[:=]\s*["']?Bearer\s+([^\s"'`,;]+)''',
r'''(?i)\b(?:password|passwd|pwd|client_secret|api_key)["']?\s*[:=]\s*["']?([^\s"'`,;]+)''',
r'''(?i)\b(?:postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|rediss?|mssql)://[^\s/:@]+:([^\s/@]+)@''']
def assigned_secret(value):
# Keep clearly redacted values and variable references usable in reports.
value = value.rstrip(".")
return not (value.lower() in {"", "redacted", "[redacted]", "<redacted>", "xxx", "xxxx"}
or re.fullmatch(r"<[^>]+>|\$\{[^}]+\}|\$[A-Za-z_][A-Za-z0-9_]*|%[A-Za-z_][A-Za-z0-9_]*%|\*+", value)
or re.fullmatch(r"(?:YOUR|EXAMPLE|PLACEHOLDER)_[A-Z0-9_]+", value))
if any(re.search(pattern, text) for pattern in patterns) or any(
assigned_secret(match.group(1)) for pattern in values for match in re.finditer(pattern, text)):
raise ValueError(f"{relative.as_posix()}: selected content contains a credential pattern; share a redacted copy")
def queue_content(db, config, root, project, paths):
"""Queue only the selected UTF-8 files, through the existing content receiver."""
root = Path(task_identity(root))
if disabled_by_environment():
raise ValueError("content sharing is disabled by the process environment")
files = []
for index, path in enumerate(paths):
source = (root / path).resolve()
relative = source.relative_to(root)
if source.stat().st_size > 1024 * 1024:
raise ValueError("selected files must each be at most 1 MiB")
text = source.read_text(encoding="utf-8")
check_shared_file(relative, text)
files.append({"name": f"file-{index+1}.txt", "text": text})
if not files:
raise ValueError("select at least one file")
packet = {"id": uuid.uuid4().hex, "task": opaque(config, "task", task_identity(root)),
"consent": True, "files": files, "at": datetime.now(timezone.utc).isoformat()}
if len(json.dumps(packet).encode()) > 1_900_000:
raise ValueError("selected content packet exceeds 1,900,000 bytes")
db.execute("INSERT INTO content VALUES (?,?,?)", (packet["id"], time.time(), json.dumps(packet)))
scopes = (directory for directory in (root, *root.parents) if directory.is_relative_to(project))
db.executemany("INSERT INTO content_scopes VALUES (?,?)",
((sharing_key(directory), packet["id"]) for directory in scopes))
return {"queued": packet["id"], "files": len(files)}
def withdraw_content(db, config, root, project):
key = sharing_key(project)
db.execute("DELETE FROM content WHERE id IN (SELECT id FROM content_scopes WHERE scope=?)", (key,))
legacy = db.execute("SELECT id,payload FROM content WHERE id NOT IN (SELECT id FROM content_scopes)").fetchall()
if not legacy:
return
# Older queues recorded a task-to-grant link but no containing directories
# per packet. Use known task paths locally and drop unlocatable pending
# packets under a containing grant when the user narrows consent.
in_scope = {value for name, value in config.items() if name.startswith(key + ".task.")}
in_scope.add(opaque(config, "task", task_identity(root)))
ancestors = tuple(sharing_key(parent) + ".task." for parent in project.parents)
ancestor_tasks = {value for name, value in config.items() if name.startswith(ancestors)}
known_paths = {opaque(config, "task", row["task"]): Path(row["task"])
for row in db.execute("SELECT DISTINCT task FROM aliases")}
for row in legacy:
task = json.loads(row["payload"]).get("task")
path = known_paths.get(task)
affected_ancestor = task in ancestor_tasks and (path is None or path.is_relative_to(project))
if task in in_scope or affected_ancestor:
db.execute("DELETE FROM content WHERE id=?", (row["id"],))
def content_offer(root, artifact, project=None, language="en"):
"""Offer once after delivery, or queue reports within an explicit project grant."""
if disabled_by_environment() or not database_path().is_file():
return {"status": "disabled"}
root = Path(task_identity(root))
artifact = (root / artifact).resolve().relative_to(root).as_posix()
with database() as db:
db.execute("BEGIN IMMEDIATE")
config = settings(db)
project = sharing_project(root, project, config)
key = sharing_key(project)
choice = config.get(key, "ask")
if (choice in {"ask", "reports"} and config.get("legacy_sharing_paused") == "true"
and config.get(key + ".sharing_resumed") != "true"):
return {"status": "disabled"}
if choice == "ask" and key not in config and config["enabled"] != "on":
return {"status": "disabled"}
confirm_scope = choice == "reports" and config.get(key + ".scope_confirmed") != "true"
if confirm_scope:
# Old grants do not record whether the user saw the actual scope.
# Reuse the existing invitation; do not guess a narrower directory.
choice = "ask"
set_setting(db, key, choice)
db.execute("DELETE FROM settings WHERE key=?", (key + ".offered",))
if choice == "reports":
checksum = hashlib.sha256((root / artifact).read_bytes()).hexdigest()
sent_key = key + ".report." + hashlib.sha256(task_identity(root / artifact).encode()).hexdigest()
if config.get(sent_key) == checksum:
return {"status": "already_queued"}
result = {"status": "queued", **queue_content(db, config, root, project, [artifact])}
set_setting(db, sent_key, checksum)
elif choice != "ask":
return {"status": choice}
else:
if not confirm_scope and config.get(key + ".offered") == "true":
return {"status": "offered"}
recipient = "rrrrrredy/research-toolkit-usage (private)"
prompt = (f"报告已交付。愿意将 {artifact} 分享给 Research Toolkit 维护者,"
f"用于改进研究方法和产品吗?文件会发送到私密仓库 {recipient}。"
"本次不包含聊天记录或原始材料。可选择:仅分享这次报告;"
f"允许今后自动分享目录 {project} 及其子目录下所有研究任务的最终报告;不分享并不再提示。"
"未回复不会上传,也不影响报告使用。以后可随时关闭。"
if language in {"zh", "zh-CN"} else
f"Your report is ready. May we share {artifact} with the Research Toolkit maintainer "
f"in {recipient} to improve research methods and the product? Chat and source materials "
"are excluded. Choose: share this report once; automatically share future final reports "
f"for all research tasks in {project} and its subdirectories; or decline and stop asking. No reply means no upload. "
"Your report remains available; you can turn sharing off later.")
if confirm_scope:
prompt = ("旧版保存的持续分享授权需要确认实际目录范围。" if language in {"zh", "zh-CN"}
else "Please confirm the actual directory scope of your previously saved report-sharing grant. ") + prompt
result = {"status": "ask", "prompt": prompt, "files": [artifact],
"recipient": recipient, "project_root": str(project), "task_root": str(root),
"choices": ["once", "reports", "decline"]}
# Displaying a prompt must not replace an explicit child choice.
# Historical `offered` values remain readable invitation markers.
set_setting(db, key + ".offered", "true")
if result["status"] == "queued":
schedule_upload()
return result
def delivery_sharing(root, artifact, language="en"):
# Sharing is separate from research completion. Keep a local storage failure
# visible without changing a successful delivery into a research failure.
try:
return content_offer(root, artifact, language=language)
except (OSError, ValueError, sqlite3.Error) as exc:
return {"status": "error", "error": type(exc).__name__, "next_action": "Report delivered; retry content sharing separately."}
def main():
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=["status", "on", "off", "flush", "delete", "share", "share-offer"])
parser.add_argument("--task-root", type=Path)
parser.add_argument("--project-root", type=Path, help="Directory whose descendant task reports the user authorized; otherwise reuse a saved explicit scope, or this task only")
parser.add_argument("--file", type=Path, action="append", default=[])
parser.add_argument("--decision", choices=["once", "reports", "decline", "ask"], default="once")
parser.add_argument("--language", choices=["en", "zh", "zh-CN"], default="en")
parser.add_argument("--confirm-content", action="store_true", help="The user explicitly authorized sharing these selected files")
parser.add_argument("--scheduled", action="store_true", help=argparse.SUPPRESS)
args = parser.parse_args()
if args.action == "share-offer":
if not args.task_root or len(args.file) != 1:
parser.error("share-offer needs --task-root and one delivered report --file")
print(json.dumps(content_offer(args.task_root, args.file[0], args.project_root, args.language), ensure_ascii=False))
return
if args.action == "flush":
if args.scheduled:
time.sleep(30)
result = flush()
print(json.dumps(result))
if args.scheduled and result.get("receiver") and not result.get("pending"):
with database() as db:
remaining = has_pending_upload(db, settings(db))
if remaining:
schedule_upload()
return
if args.action == "status" and not database_path().is_file():
print(json.dumps({"enabled": False, "installation": None, "receiver": endpoint({}),
**installation_info({}), "pending_events": 0, "pending_content": 0,
"last_upload": None, "last_error": None}))
return
if args.action == "delete" and not database_path().is_file():
parser.error("no local usage installation to delete")
if args.action == "share":
if not args.task_root:
parser.error("share requires --task-root")
if args.decision in {"once", "reports"}:
if not args.confirm_content or not args.file:
parser.error("share requires selected --file paths and --confirm-content after the user's agreement")
if args.decision == "reports" and len(args.file) != 1:
parser.error("reports authorizes final reports only; use once for separately selected chat or source files")
if disabled_by_environment():
parser.error("content sharing is disabled by the process environment")
with database() as db:
if args.action in {"on", "off", "share"}:
db.execute("BEGIN IMMEDIATE")
config = settings(db)
if args.action in {"off", "on"}:
set_setting(db, "enabled", args.action)
if args.action == "off":
db.execute("DELETE FROM events")
db.execute("DELETE FROM sessions")
print(json.dumps({"enabled": args.action == "on", "queued_cleared": args.action == "off",
"scope": "basic_usage"}))
elif args.action == "status":
print(json.dumps({"enabled": config["enabled"] == "on" and not disabled_by_environment(),
"installation": config["installation_id"], "receiver": endpoint(config),
**installation_info(config),
"pending_events": db.execute("SELECT count(*) FROM events").fetchone()[0],
"pending_content": db.execute("SELECT count(*) FROM content").fetchone()[0],
"last_upload": config.get("last_upload"), "last_error": config.get("last_error")}))
elif args.action == "share":
root = Path(task_identity(args.task_root))
project = sharing_project(root, args.project_root, config)
key = sharing_key(project)
if args.decision in {"decline", "ask"}:
withdraw_content(db, config, root, project)
set_setting(db, key, args.decision)
if args.decision == "ask":
set_setting(db, key + ".sharing_resumed", "true")
db.execute("DELETE FROM settings WHERE key=?", (key + ".offered",))
print(json.dumps({"decision": args.decision, "project_root": str(project)}))
else:
result = queue_content(db, config, root, project, args.file)
if args.decision == "reports" or config.get(key) not in {"reports", "decline", "ask"}:
set_setting(db, key, args.decision)
if args.decision == "reports":
set_setting(db, key + ".scope_confirmed", "true")
set_setting(db, key + ".sharing_resumed", "true")
# The user selected this containing project after an invitation.
# Retire only prompt markers on that task's path, not explicit choices.
for candidate in (root, *root.parents):
if candidate == project:
break
offered_key = sharing_key(candidate)
if config.get(offered_key) == "offered":
db.execute("DELETE FROM settings WHERE key=?", (offered_key,))
if config.get(offered_key) in {None, "offered"}:
db.execute("DELETE FROM settings WHERE key=?", (offered_key + ".offered",))
source = (root / args.file[0]).resolve()
sent_key = key + ".report." + hashlib.sha256(task_identity(source).encode()).hexdigest()
set_setting(db, sent_key, hashlib.sha256(source.read_bytes()).hexdigest())
print(json.dumps(result))
elif args.action == "delete":
target = endpoint(config)
if not target:
parser.error("no receiver configured")
identities = {opaque(config, "task", task_identity(args.task_root)),
opaque({**config, "id_secret": config["secret"]}, "task", task_identity(args.task_root))} if args.task_root else set()
requests = [{"task": identity} for identity in sorted(identities)] if identities else [{}]
responses = [request_json(target+"/v1/delete", data, config) for data in requests]
print(json.dumps(responses[0] if len(responses) == 1 else {"results": responses}))
for table in ("content", "events"):
if identities:
matching = [(row["id"],) for row in db.execute(f"SELECT id,payload FROM {table}")
if json.loads(row["payload"]).get("task") in identities]
db.executemany(f"DELETE FROM {table} WHERE id=?", matching)
else:
db.execute(f"DELETE FROM {table}")
if args.action == "share":
schedule_upload()
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |