rrrrrredy/research-toolkit / plugins/research-toolkit
research-toolkit
FlaggedUse for source-backed long-form research and writing in English or Chinese across fields, including revisions and continuation. Understand requirements, develop analysis through substantive research collaboration, and deliver a complete report after independent review, audit and acceptance.
scripts/check_delivery.py
View on GitHub#!/usr/bin/env python3
"""Validate that a user-visible delivery claim matches current research state."""
from __future__ import annotations
import argparse
import csv
import hashlib
import io
import json
import re
from pathlib import Path
from typing import Any
from check_review_completion import content_review_rows, inspect_review_completion
CANONICAL_STAGES = {"brief", "collect", "analyze", "draft", "review", "revise", "final"}
PROGRESS_STATUSES = {"in_progress", "paused", "blocked", "complete"}
OPEN_ISSUE_KEYS = {
"open_issues",
"unresolved_issues",
"pending_issues",
"remaining_issues",
"blockers",
"explicitly_not_completed",
}
CLOSED_ISSUE_TERMS = {"closed", "resolved", "handled", "accepted_limitation", "waived"}
OPEN_ISSUE_TERMS = {
"open", "unresolved", "pending", "blocking", "incomplete", "not_completed",
"in_progress", "needs_revision", "fail", "failed", "blocked", "blocking_issue",
}
RESOLVED_REQUIREMENT_STATUSES = {"satisfied", "accepted_limitation", "waived", "out_of_scope"}
DELIVERY_CONTRACT_VERSION = 3
REQUIREMENT_DECISION_STATUSES = {"accepted_limitation", "waived", "out_of_scope"}
READING_REQUIREMENTS = {"full_text", "relevant_sections"}
COMPLETION_PATTERNS = [
r"(?:终稿|最终稿|完整稿)(?:已经|已|现已)?(?:完成|形成|交付|可交付)",
r"(?:任务|研究|报告|成稿)(?:已经|已|现已)?(?:完成|交付)",
r"(?:都|全都|全部(?:内容)?)(?:已经|已|现已)?(?:搞定|完成|就绪)",
r"(?:任务|研究|报告|成稿|终稿|最终稿|完整稿)(?:已经|已|现已)?(?:写好|做好|搞定|就绪)",
r"(?:已经|已|现已|全部|正式)(?:完成|交付).{0,6}(?:任务|研究|报告|终稿|最终稿|完整稿)",
r"\b(?:final report|final delivery|final answer) (?:is )?(?:complete|completed|ready|delivered)\b",
r"\b(?:task|research|report|work) (?:is |has been )?(?:complete|completed|done|delivered)\b",
r"\b(?:everything|all work|all content) (?:is |has been )?(?:done|complete|completed|ready|all set)\b",
r"\b(?:ready to (?:publish|ship|deliver|submit)|good to go)\b",
]
NEGATED_COMPLETION_PATTERNS = [
r"(?:尚|还|仍|并)?未.{0,5}(?:完成|交付|形成)",
r"(?:没有|并没有).{0,5}(?:完成|交付|形成)",
r"(?:不是|并非|非|不构成|不能视为).{0,5}(?:终稿|最终稿|完整稿|完成|交付)",
r"(?:还不能|不能|无法|不可以|不可).{0,5}(?:发布|提交|交付)",
r"(?:阶段稿|草稿|中间稿)",
r"\b(?:not final|not complete|not completed|not ready|not ready to publish|unfinished|incomplete|draft|work in progress)\b",
]
LIMITATION_DISCLOSURE_TERMS = [
"已知限制",
"限制",
"不确定",
"未能",
"无法确认",
"公开信息不足",
"accepted limitation",
"known limitation",
"limitation",
]
REQUIRED_HASH_INPUTS = [
"state/task_spec.md",
"state/progress.json",
"data/source_registry.csv",
"data/claims_registry.csv",
"logs/review.jsonl",
"delivery_message.md",
]
OPTIONAL_HASH_INPUTS = [
"state/requirements.jsonl",
"data/uncertainty_registry.csv",
]
CANONICAL_TEXT_HASH_SUFFIXES = {".csv", ".html", ".json", ".jsonl", ".md", ".py", ".txt", ".yaml", ".yml"}
GLOBAL_REVIEW_SCOPES = {
"global_final_delivery",
"full_report",
"full report",
"full_draft",
"full draft",
"global",
"全文",
"全稿",
}
PASS_REVIEW_TERMS = {"pass", "passed", "complete", "approved"}
REVIEW_STATUS_KEYS = {"result", "status", "decision", "verdict", "finding_type", "outcome"}
FAIL_REVIEW_TERMS = {"fail", "failed", "needs_revision", "unresolved", "open", "blocking_issue", "blocked"}
REVIEW_ISSUE_KEYS = {
"issues",
"findings",
"open_issues",
"unresolved_issues",
"blockers",
}
def read_text(path: Path) -> str:
if not path.exists():
return ""
return path.read_text(encoding="utf-8")
def read_json_or_none(path: Path) -> Any | None:
if not path.exists():
return None
try:
return json.loads(path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError, UnicodeError):
return None
def inspect_jsonl(path: Path, label: str) -> tuple[list[dict[str, Any]], list[str]]:
try:
content = read_text(path)
except (OSError, UnicodeError):
return [], [f"{label} log is unreadable or is not valid UTF-8."]
return inspect_jsonl_text(content, label)
def inspect_jsonl_text(content: str, label: str) -> tuple[list[dict[str, Any]], list[str]]:
rows: list[dict[str, Any]] = []
findings: list[str] = []
for line_number, line in enumerate(content.splitlines(), start=1):
line = line.strip()
if not line:
continue
try:
value = json.loads(line)
except json.JSONDecodeError:
findings.append(f"{label} row {line_number} is not valid JSON.")
continue
if isinstance(value, dict):
rows.append(value)
else:
findings.append(f"{label} row {line_number} is not an object.")
return rows, findings
def inspect_csv_text(
content: str, label: str, required_columns: tuple[str, ...] = (),
unique_id: str | None = None,
) -> tuple[list[dict[str, str]], list[str]]:
"""Parse logical CSV records and retain structural errors."""
rows: list[dict[str, str]] = []
findings: list[str] = []
reader = csv.DictReader(io.StringIO(content, newline=""), strict=True)
try:
columns = reader.fieldnames or []
missing = set(required_columns) - set(columns)
if missing:
findings.append(f"{label} is missing columns: {', '.join(sorted(missing))}.")
if not columns or any(not column.strip() for column in columns) or len(set(columns)) != len(columns):
findings.append(f"{label} needs distinct, non-empty column names.")
seen: set[str] = set()
for row in reader:
if None in row or any(value is None for value in row.values()):
findings.append(f"{label} record ending at line {reader.line_num} has the wrong field count.")
continue
if unique_id:
identity = row.get(unique_id, "").strip()
if not identity or identity in seen:
findings.append(f"{label} record ending at line {reader.line_num} has a missing or duplicate {unique_id}.")
seen.add(identity)
rows.append(row)
except csv.Error as exc:
findings.append(f"{label} CSV error at line {reader.line_num}: {exc}.")
return rows, findings
def nonempty_text(value: Any) -> bool:
return isinstance(value, str) and bool(value.strip())
def has_evidence(value: Any) -> bool:
return nonempty_text(value) or (
isinstance(value, list) and bool(value) and all(nonempty_text(item) for item in value)
)
def reading_evidence_target(value: Any) -> str | None:
"""Extract a local path without treating page locators or URLs as files."""
if not nonempty_text(value):
return None
locator = value.strip().strip("`")
link = re.fullmatch(r"\[[^]]+\]\(([^)]+)\)", locator)
if link:
locator = link[1]
if re.match(r"https?://", locator, re.IGNORECASE):
return None
if re.match(r"file://|[A-Za-z]:", locator, re.IGNORECASE) or locator.startswith(("/", "\\")):
raise ValueError("Local reading evidence must use a task-relative file path.")
file_ref = re.match(
r"(.+?\.(?:md|txt|pdf|html?|jsonl?|csv|tsv|docx|xlsx|pptx|rst|log|ya?ml|png|jpe?g|webp))"
r"(?=$|[#:,,]|\s+(?:pp?\.|pages?\b|paragraphs?\b|sections?\b|第))", locator, re.IGNORECASE)
target = file_ref[1] if file_ref else locator.split("#", 1)[0]
path_shaped = file_ref or target.startswith(("./", "../")) or (
not any(c.isspace() for c in target) and ("/" in target or "\\" in target))
if not path_shaped:
return None
return target.replace("\\", "/")
def inspect_reading_evidence(root: Path, value: Any, *, local_path=False) -> list[str]:
"""Check clear local file references; other locators still need content review."""
if not nonempty_text(value):
return ["Reading evidence needs a nonempty reference."]
try:
target = reading_evidence_target(value)
if local_path:
target = value.strip()
except ValueError as exc:
return [str(exc)]
if target is None:
return []
path = resolve_inside(root, target.replace("\\", "/"))
if path is None or not path.is_file():
return [f"Local reading evidence is missing, not a file, or outside the task: {target}"]
try:
with path.open("rb") as stream:
populated = any(chunk.strip() for chunk in iter(lambda: stream.read(65536), b""))
except OSError:
return [f"Local reading evidence is unreadable: {target}"]
return [] if populated else [f"Local reading evidence is empty: {target}"]
def inspect_source_reading_evidence(root: Path, source: dict[str, Any]) -> list[str]:
"""Use the same local-source checks before drafting and binding delivery inputs."""
findings = []
source_id = source.get("source_id", "unknown")
for field in ("url", "read_evidence", "evidence_path"):
value = source.get(field)
if field != "read_evidence" and not nonempty_text(value):
continue
for error in inspect_reading_evidence(root, value, local_path=field == "evidence_path"):
findings.append(
f"data/source_registry.csv row {source_id}, {field}: {error.rstrip('.')}. "
f"For a local file, use a task-relative path to an existing nonempty file, "
f"such as sources/{source_id}.md; put reading-location explanations in locator.")
return findings
def inspect_required_reading(row: dict[str, Any], registry_path: Path) -> list[str]:
"""Check declared reading records, not actual comprehension or source truth."""
if "reading_requirement" not in row and "required_source_ids" not in row:
return []
required = row.get("reading_requirement")
source_ids = row.get("required_source_ids")
if not isinstance(required, str) or required not in READING_REQUIREMENTS:
return ["reading_requirement must be full_text or relevant_sections."]
if not (isinstance(source_ids, list) and source_ids
and all(nonempty_text(value) for value in source_ids)
and len(set(source_ids)) == len(source_ids)):
return ["required_source_ids must be a non-empty list of distinct source ids."]
try:
rows, findings = inspect_csv_text(
read_text(registry_path), "Source registry", ("source_id", "title"), "source_id"
)
except (OSError, UnicodeError):
return ["Required reading has no readable UTF-8 source registry."]
if findings:
return findings
for source_id in source_ids:
matches = [source for source in rows if source.get("source_id") == source_id]
if len(matches) != 1:
findings.append(f"Required source {source_id} must have exactly one registry row.")
continue
source = matches[0]
allowed = {"full_text"} if required == "full_text" else {"full_text", "relevant_sections"}
if source.get("read_scope") not in allowed:
findings.append(f"Required source {source_id} has not met its {required} reading requirement.")
findings.extend(f"Required source {source_id}: {finding}" for finding in
inspect_reading_evidence(registry_path.parent.parent, source.get("read_evidence")))
return findings
def inspect_requirements(
path: Path, contract_version: int = DELIVERY_CONTRACT_VERSION, *,
parsed_records: tuple[list[dict[str, Any]], list[str]] | None = None,
) -> tuple[list[str], list[str]]:
"""Return terminal-blocking requirement findings and accepted limitations."""
findings: list[str] = []
accepted_limitations: list[str] = []
if not path.exists():
return findings, accepted_limitations
rows, parse_findings = parsed_records if parsed_records is not None else inspect_jsonl(path, "Requirement")
findings.extend(parse_findings)
for record_number, row in enumerate(rows, start=1):
requirement_id = str(row.get("requirement_id") or row.get("id") or "").strip()
status = str(row.get("status", "")).strip().lower()
label = requirement_id or f"record {record_number}"
if not requirement_id:
findings.append(f"Requirement record {record_number} has no stable id.")
if status not in RESOLVED_REQUIREMENT_STATUSES:
findings.append(
f"Requirement {label} is not terminally resolved (status: {status or '<missing>'})."
)
if type(contract_version) is int and contract_version >= 2 and status in RESOLVED_REQUIREMENT_STATUSES:
if status in REQUIREMENT_DECISION_STATUSES:
decision = row.get("user_decision")
if not (isinstance(decision, dict)
and nonempty_text(decision.get("source_turn"))
and nonempty_text(decision.get("quote"))):
findings.append(
f"Requirement {label} needs a specific user_decision source_turn and quote "
f"before it can be {status}; disclosure alone is not permission."
)
else:
if not has_evidence(row.get("evidence")):
findings.append(f"Requirement {label} is satisfied without evidence.")
findings.extend(
f"Requirement {label}: {finding}" for finding in inspect_required_reading(
row, path.parent.parent / "data" / "source_registry.csv"
)
)
if status == "accepted_limitation":
limitation = str(
row.get("summary") or row.get("description") or row.get("evidence") or label
).strip()
accepted_limitations.append(limitation)
return findings, accepted_limitations
def length_constraint_errors(rule) -> list[str]:
"""Explain invalid declared bounds without guessing units or changing the contract."""
if not isinstance(rule, dict):
return ['must be a JSON object, for example {"unit":"nonwhitespace_characters", '
'"scope":"whole_artifact","min":800,"max":1200}']
problems = []
unknown = sorted(set(rule) - {"unit", "scope", "min", "max"})
if unknown:
problems.append(f"has unknown fields: {', '.join(unknown)}; use min/max for bounds and unit for the counting method")
if rule.get("unit") not in ("whitespace_words", "nonwhitespace_characters"):
problems.append("unit must be whitespace_words or nonwhitespace_characters")
if rule.get("scope") != "whole_artifact":
problems.append("scope must be whole_artifact (includes headings, Markdown, citations and references)")
if not ({"min", "max"} & set(rule)):
problems.append("needs at least one bound: min and/or max, for example min: 800, max: 1200")
for key in ("min", "max"):
if key in rule and (type(rule[key]) is not int or rule[key] < 0):
problems.append(f"{key} must be a nonnegative JSON integer, not text, a boolean or a decimal")
if (type(rule.get("min")) is int and type(rule.get("max")) is int
and rule["min"] > rule["max"]):
problems.append(f"min ({rule['min']}) must not exceed max ({rule['max']})")
return problems
def inspect_length_constraints(root: Path, artifact: str) -> tuple[list[dict], list[str]]:
"""Measure explicitly declared bounds; never infer a limit from prose or certify depth."""
rows, errors = inspect_jsonl(root / "state/requirements.jsonl", "Requirement")
constraints = [row for row in rows if "length_constraint" in row]
if not constraints:
return [], errors
path = resolve_inside(root, artifact)
try:
if path is None or not path.is_file():
raise ValueError("Report path must name a file inside the task.")
report = path.read_text(encoding="utf-8")
except (OSError, ValueError) as exc:
return [], [*errors, f"Cannot measure report length: {exc}"]
counts = {"whitespace_words": len(report.split()),
"nonwhitespace_characters": sum(not char.isspace() for char in report)}
checks, seen = [], set()
for number, row in enumerate(constraints, 1):
identifier = row.get("requirement_id", row.get("id"))
label = identifier if nonempty_text(identifier) else f"length record {number}"
rule = row["length_constraint"]
if not nonempty_text(identifier) or identifier in seen:
errors.append(f"{label}: length requirements need distinct nonempty IDs.")
continue
seen.add(identifier)
if not nonempty_text(row.get("summary")):
errors.append(f"{label}: summary must state the agreed length requirement and counting scope.")
rule_errors = length_constraint_errors(rule)
if rule_errors:
errors.extend(f"{label}: length_constraint {problem}." for problem in rule_errors)
continue
actual = counts[rule["unit"]]
met = actual >= rule.get("min", 0) and ("max" not in rule or actual <= rule["max"])
status = str(row.get("status", "")).strip().lower()
check = {"requirement_id": identifier, "unit": rule["unit"], "scope": rule["scope"],
"min": rule.get("min"), "max": rule.get("max"), "actual": actual,
"bounds_met": met, "status": status}
checks.append(check)
if status in REQUIREMENT_DECISION_STATUSES:
decision = row.get("user_decision")
if not (isinstance(decision, dict) and all(nonempty_text(decision.get(key))
for key in ("source_turn", "quote"))):
errors.append(f"{label}: {status} needs its existing user_decision source_turn and quote.")
elif status != "satisfied":
errors.append(f"{label}: resolve the length requirement before delivery (status: {status or '<missing>'}).")
else:
if not has_evidence(row.get("evidence")):
errors.append(f"{label}: satisfied length requirement needs an evidence locator.")
if not met:
errors.append(f"{label}: measured {actual} {rule['unit']} over whole_artifact, "
f"outside declared bounds min={rule.get('min')}, max={rule.get('max')}.")
return checks, errors
def length_receipt_matches(receipt: dict, checks: list[dict]) -> bool:
# JSON comparison keeps booleans distinct from integer measurements.
return json.dumps(receipt.get("length_checks", []), sort_keys=True) == json.dumps(checks, sort_keys=True)
def claims_completion(message: str, artifact: str = "final.md") -> bool:
"""Return True only for an affirmative user-visible completion claim."""
# Retain partial-work qualifiers and scope them to the matched subject.
partial_subject = r"(?:\b(?:draft|outline|chapter|section|partial|interim|preliminary|working)\s+(?:(?:of|the|a|an)\s+)*|(?:阶段稿|草稿|中间稿|初稿|提纲|章节|部分)(?:的)?\s*)$"
for clause in re.split(r"[。!?!?;;\n]|[.](?=\s|$)", message.lower()):
negated_spans = [match.span() for pattern in NEGATED_COMPLETION_PATTERNS[:4]
for match in re.finditer(pattern, clause, flags=re.IGNORECASE)]
for pattern in COMPLETION_PATTERNS:
for match in re.finditer(pattern, clause, flags=re.IGNORECASE):
if any(start < match.end() and match.start() < end for start, end in negated_spans):
continue
prefix = clause[:match.start()]
if re.search(partial_subject, prefix, flags=re.IGNORECASE):
continue
if re.search(r"(?:\bnot\s+|(?:不是|并非|不能视为)\s*)$", prefix):
continue
# Generic readiness also needs to keep an explicitly named draft.
if re.match(r"(?:ready to|good to go)", match.group()):
if re.search(r"\b(?:draft|outline|chapter|section)\b|阶段稿|草稿|中间稿|提纲", prefix):
continue
return True
# Runtime replies often name the primary file instead of saying "the report".
# Only its visible filename counts; unrelated links and link targets do not.
name = re.escape(Path(artifact.replace("\\", "/")).name)
if not name:
return False
reference = rf"(?<!!)\[\s*`?{name}`?\s*\]\([^\n)]+\)|`{name}`"
linked = re.sub(reference, " irf_primary_artifact ", message.lower(), flags=re.IGNORECASE)
linked = re.sub(
r"(?:\b(?:draft|outline|chapter|section)\b|阶段稿|草稿|中间稿|提纲)\s*irf_primary_artifact",
" partial_artifact ", linked, flags=re.IGNORECASE,
)
# Keep partial-work nouns in place here, so "completed a draft of file"
# cannot become "completed file" by deleting its qualification.
for pattern in NEGATED_COMPLETION_PATTERNS[:4]:
linked = re.sub(pattern, " ", linked, flags=re.IGNORECASE)
linked = re.sub(
r"\b(?:not(?: yet)?|never)\s+(?:complete|completed|done|finished|delivered)\b",
" ", linked, flags=re.IGNORECASE,
)
patterns = (
r"(?:已经|现已|已)\s*(?:完成|交付|写好|做好)\s*(?:了\s*)?irf_primary_artifact\b(?!\s*(?:的|中|['’]s\b))",
r"\birf_primary_artifact\s*(?:已经|现已|已)\s*(?:完成|交付|写好|做好)",
r"\b(?:completed|finished|delivered)\s+(?:the\s+)?irf_primary_artifact\b(?!\s*(?:的|中|['’]s\b))",
r"\birf_primary_artifact\s+(?:(?:is|has been)\s+)?(?:complete|completed|done|delivered)\b",
)
return any(re.search(pattern, linked, flags=re.IGNORECASE) for pattern in patterns)
def issue_is_open(issue: Any) -> bool:
if issue in (None, "", [], {}):
return False
if isinstance(issue, str):
return True
if not isinstance(issue, dict):
return bool(issue)
status = str(issue.get("status", "")).strip().lower()
if status in CLOSED_ISSUE_TERMS:
return False
if status in OPEN_ISSUE_TERMS:
return True
if any(issue.get(key) for key in ("resolution", "handling", "accepted_as_limitation")):
return False
return True
def collect_open_issues(data: Any) -> list[Any]:
if not isinstance(data, dict):
return []
issues: list[Any] = []
for key in OPEN_ISSUE_KEYS:
value = data.get(key)
if isinstance(value, list):
issues.extend(item for item in value if issue_is_open(item))
elif issue_is_open(value):
issues.append(value)
return issues
def collect_accepted_limitations(data: Any) -> list[str]:
if not isinstance(data, dict):
return []
value = data.get("accepted_limitations", [])
if isinstance(value, str):
return [value] if value.strip() else []
if not isinstance(value, list):
return []
limitations: list[str] = []
for item in value:
if isinstance(item, str) and item.strip():
limitations.append(item.strip())
elif isinstance(item, dict):
text = str(item.get("description") or item.get("limitation") or item.get("issue") or "").strip()
if text:
limitations.append(text)
return limitations
def sha256_file(path: Path) -> str:
"""Hash binary files byte-for-byte and text files with canonical LF newlines."""
digest = hashlib.sha256()
canonical_text = path.suffix.lower() in CANONICAL_TEXT_HASH_SUFFIXES
pending_cr = b""
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
if not canonical_text:
digest.update(chunk)
continue
chunk = pending_cr + chunk
pending_cr = b""
if chunk.endswith(b"\r"):
pending_cr = b"\r"
chunk = chunk[:-1]
digest.update(chunk.replace(b"\r\n", b"\n").replace(b"\r", b"\n"))
if pending_cr:
digest.update(b"\n")
return digest.hexdigest()
def resolve_inside(root: Path, relative_path: str) -> Path | None:
if not isinstance(relative_path, str) or not relative_path.strip() or "\x00" in relative_path:
return None
try:
candidate = (root / relative_path).resolve()
candidate.relative_to(root.resolve())
except (OSError, ValueError, RuntimeError):
return None
return candidate
def review_row_has_open_issues(row: dict[str, Any]) -> bool:
for key in REVIEW_ISSUE_KEYS:
value = row.get(key)
if isinstance(value, list) and any(issue_is_open(issue) for issue in value):
return True
if not isinstance(value, list) and issue_is_open(value):
return True
return False
def latest_global_review_passes(rows: list[dict[str, Any]]) -> bool:
return any(scope_key(str(row.get("scope", ""))) == "__global__" for row in rows) and not review_has_unresolved_findings(rows)
def review_row_is_blocking(row: dict[str, Any]) -> bool:
return review_row_has_open_issues(row) or any(
re.search(rf"\b{re.escape(term)}\b", str(row.get(key, "")), flags=re.IGNORECASE)
for key in REVIEW_STATUS_KEYS for term in FAIL_REVIEW_TERMS
)
def review_has_unresolved_findings(rows: list[dict[str, Any]]) -> bool:
"""A clean global review covers prior ordinary reviews, not later blockers.
Later local findings need a clean review of the same scope or a new global
review. Task-declared required scopes remain independently checked by the
delivery contract; this helper does not erase them or require new artifacts.
"""
rows = content_review_rows(rows)
global_indices = [i for i, row in enumerate(rows)
if scope_key(str(row.get("scope", ""))) == "__global__"]
start = 0
if global_indices:
latest = global_indices[-1]
if not review_passes(rows[latest]):
return True
start = latest + 1
pending: set[str] = set()
for row in rows[start:]:
scope = scope_key(str(row.get("scope") or row.get("review_type") or "__unspecified__"))
if review_row_is_blocking(row):
pending.add(scope)
elif review_passes(row):
pending.discard(scope)
return bool(pending)
def review_passes(row: dict[str, Any]) -> bool:
result = str(
row.get("result")
or row.get("status")
or row.get("decision")
or row.get("verdict")
or ""
).strip().lower()
return result in PASS_REVIEW_TERMS and not review_row_is_blocking(row)
def scope_key(value: str) -> str:
normalized = value.strip().lower()
return "__global__" if normalized in GLOBAL_REVIEW_SCOPES else normalized
def assess_limitation_disclosure(message: str, limitations: list[str]) -> dict[str, Any]:
"""Observe literal coverage and known contradictions, never certify meaning.
Unmatched paraphrases require review; they are not automatically false. This
does not add a compulsory wording, declaration file or model/API dependency.
"""
normalize = lambda value: re.sub(r"[\W_]+", "", value.casefold())
normalized_message = normalize(message)
unmatched = [value for value in limitations
if not normalize(value) or normalize(value) not in normalized_message]
scrubbed = re.sub(
r"(?:并非|并不是|不是|不能说|不应说|不得宣称)\s*(?:没有|不存在|无)(?:任何|已知)?(?:限制|局限|不确定性)",
"", message, flags=re.IGNORECASE,
)
scrubbed = re.sub(
r"\bnot\s+(?:without|free\s+of)\s+(?:any\s+|known\s+)?(?:limitations?|uncertainties)\b",
"", scrubbed, flags=re.IGNORECASE,
)
denial = any(re.search(pattern, scrubbed, flags=re.IGNORECASE) for pattern in (
r"(?:没有|不存在|无)(?:任何|已知|什么)?(?:限制|局限|不确定性)",
r"\b(?:no|without)\s+(?:any\s+|known\s+)?(?:limitations?|uncertainties)\b",
r"\bfree\s+of\s+(?:any\s+|known\s+)?(?:limitations?|uncertainties)\b",
))
if not limitations:
status = "not_applicable"
elif denial:
status = "contradiction"
elif not unmatched:
status = "text_covered"
elif len(unmatched) == len(limitations) and not any(
term.casefold() in message.casefold() for term in LIMITATION_DISCLOSURE_TERMS
):
status = "absent"
else:
status = "needs_review"
return {"status": status, "unmatched_limitations": unmatched,
"semantic_verification": False}
def inspect_actual_delivery(actual_message: Path | None, intended_message: str | None):
"""Compare an optional caller capture against the recorded delivery."""
if actual_message is None:
return "not_provided", None
try:
captured = Path(actual_message).read_text(encoding="utf-8")
except (OSError, UnicodeError):
return "unreadable", ("missing_actual_delivery",
"The caller-provided actual delivery capture is missing or unreadable.")
normalize = lambda text: text.replace("\r\n", "\n").replace("\r", "\n").strip()
if intended_message is None or normalize(captured) != normalize(intended_message):
return "mismatch", ("actual_delivery_mismatch",
"The captured actual reply differs from the receipt-bound intended message.")
return "matched", None
def evaluate_delivery(
project_root: Path,
artifact: str = "final.md",
delivery_message: str = "delivery_message.md",
receipt: str = "state/final_delivery.json",
actual_message: Path | None = None,
contract_version: int = DELIVERY_CONTRACT_VERSION,
require_complete: bool = False,
) -> dict[str, Any]:
root = project_root.resolve()
flags: list[str] = []
findings: list[str] = []
def add(flag: str, finding: str) -> None:
if flag not in flags:
flags.append(flag)
findings.append(finding)
else:
index = flags.index(flag)
if finding not in findings[index].split("\n"):
findings[index] += "\n" + finding
if type(contract_version) is not int or contract_version not in {1, 2, 3}:
add("invalid_delivery_contract_version", "Delivery contract version must be 1, 2 or 3.")
if require_complete and contract_version != DELIVERY_CONTRACT_VERSION:
add("invalid_delivery_contract_version", "Completion checking requires the current delivery contract; omit --contract-version or use 3.")
progress_path = root / "state" / "progress.json"
progress = read_json_or_none(progress_path)
if not isinstance(progress, dict):
progress = {}
stage = str(progress.get("stage", "")).strip().lower()
status = str(progress.get("status", "")).strip().lower()
if require_complete and (stage != "final" or status != "complete"):
add("incomplete_task", f"Task is not final/complete (stage={stage or '<missing>'}, status={status or '<missing>'}). Complete the outstanding work and call finish.")
if stage not in CANONICAL_STAGES:
add("invalid_progress_stage", f"Progress stage is not canonical: {stage or '<missing>'}.")
if status not in PROGRESS_STATUSES:
add("invalid_progress_status", f"Progress status is not canonical: {status or '<missing>'}.")
if stage == "final" and status != "complete":
add("invalid_completion_status", "A final stage requires status 'complete'.")
elif status == "complete" and stage != "final":
add("invalid_completion_status", "Status 'complete' requires stage 'final'.")
message_path = resolve_inside(root, delivery_message)
receipt_path = resolve_inside(root, receipt)
artifact_path = resolve_inside(root, artifact)
for label, path in (("delivery message", message_path), ("receipt", receipt_path), ("artifact", artifact_path)):
if path is None:
add("invalid_delivery_path", f"The {label} path must stay inside the task directory.")
message_exists = message_path is not None and message_path.is_file()
try:
message = read_text(message_path) if message_exists else ""
except (OSError, UnicodeError):
message = ""
add("invalid_delivery_message", "The intended delivery message is unreadable or is not valid UTF-8.")
completion_claim = claims_completion(message, artifact=artifact)
terminal_state = stage == "final" and status == "complete"
terminal_intent = completion_claim or terminal_state or require_complete
if terminal_intent and not message_exists:
add("missing_delivery_message", "Terminal delivery has no user-visible delivery message to validate.")
if completion_claim and not terminal_state:
add(
"completion_claim_without_terminal_state",
"The user-visible message claims completion while progress is not final/complete.",
)
open_progress_issues = collect_open_issues(progress)
if completion_claim and open_progress_issues:
add(
"completion_claim_with_open_blockers",
"The user-visible message claims completion while progress still records open blockers.",
)
elif terminal_state and open_progress_issues:
add(
"terminal_state_with_open_blockers",
"Progress is final/complete while open blockers or explicitly incomplete units remain.",
)
requirement_findings, requirement_limitations = inspect_requirements(
root / "state" / "requirements.jsonl", contract_version=contract_version
)
if terminal_intent:
for finding in requirement_findings:
add("unresolved_required_corrections", finding)
receipt_exists = receipt_path is not None and receipt_path.is_file()
receipt_data = read_json_or_none(receipt_path) if receipt_exists else None
if terminal_intent and not receipt_exists:
add("missing_delivery_receipt", "Terminal delivery is missing state/final_delivery.json.")
elif receipt_exists and not isinstance(receipt_data, dict):
add("invalid_delivery_receipt", "The delivery receipt is not valid JSON object data.")
length_checks = []
if terminal_intent or isinstance(receipt_data, dict):
length_checks, length_errors = inspect_length_constraints(root, artifact)
for finding in length_errors:
add("invalid_length_requirement", finding)
if isinstance(receipt_data, dict) and not length_receipt_matches(receipt_data, length_checks):
add("stale_length_check", "Receipt length measurements differ from the current report and requirements.")
accepted_limitations = collect_accepted_limitations(progress)
accepted_limitations.extend(requirement_limitations)
if isinstance(receipt_data, dict):
accepted_limitations.extend(collect_accepted_limitations(receipt_data))
if receipt_data.get("schema_version") != 1 or receipt_data.get("status") != "pass":
add("invalid_delivery_receipt", "The delivery receipt must use schema_version 1 and status 'pass'.")
if receipt_data.get("scope") != "global_final_delivery":
add(
"insufficient_final_review_scope",
"The delivery receipt does not cover the global final delivery.",
)
if receipt_data.get("artifact") != artifact:
add("delivery_receipt_artifact_mismatch", "The receipt names a different primary artifact.")
if collect_open_issues(receipt_data):
add("invalid_delivery_receipt", "The delivery receipt records unresolved issues despite PASS status.")
expected_inputs = [
artifact, *[path for path in REQUIRED_HASH_INPUTS if path != "delivery_message.md"],
delivery_message,
]
expected_inputs.extend(path for path in OPTIONAL_HASH_INPUTS if (root / path).exists())
hashes = receipt_data.get("artifacts")
if not isinstance(hashes, dict):
add("incomplete_delivery_receipt", "The delivery receipt has no artifact hash map.")
else:
# A binding remains required even when its file has disappeared.
expected_inputs = list(dict.fromkeys([*expected_inputs, *hashes]))
for relative in expected_inputs:
path = resolve_inside(root, relative)
if path is None or not path.is_file():
add("missing_delivery_inputs", f"Required delivery input is missing or outside the project: {relative}.")
continue
recorded_hash = str(hashes.get(relative, "")).strip().lower()
if not recorded_hash:
add("incomplete_delivery_receipt", f"The delivery receipt omits a hash for {relative}.")
else:
try:
current_hash = sha256_file(path).lower()
except OSError:
add("missing_delivery_inputs", f"Bound delivery input is unreadable: {relative}.")
continue
if recorded_hash != current_hash:
add("stale_delivery_receipt", f"The delivery receipt is stale for {relative}.")
review_rows, review_findings = inspect_jsonl(root / "logs" / "review.jsonl", "Review")
review_completion = None
if contract_version == 3 and terminal_intent:
review_completion = inspect_review_completion(
root, progress, artifact, review_rows, hash_file=sha256_file, resolve_path=resolve_inside
)
for flag, finding in zip(review_completion["flags"], review_completion["findings"]):
add(flag, finding)
plan = progress.get("review_plan")
if isinstance(plan, dict) and plan.get("purpose") != "report_delivery":
add("invalid_review_purpose", "Use check_review_completion.py for evaluation completion; report delivery needs report_delivery.")
review_rows = content_review_rows(review_rows)
required_scopes = progress.get("required_review_scopes", [])
valid_scopes = (
isinstance(required_scopes, list)
and all(isinstance(value, str) and value.strip() for value in required_scopes)
)
if valid_scopes:
keys = [scope_key(value) for value in required_scopes]
valid_scopes = len(keys) == len(set(keys))
if not valid_scopes:
add("invalid_required_review_scopes", "required_review_scopes must be a list of distinct, non-empty scope names.")
if terminal_intent:
for finding in review_findings:
add("invalid_review_log", finding)
if not latest_global_review_passes(review_rows):
add(
"insufficient_final_review_scope",
"The latest full-report or global-final review is missing, non-PASS, or records open issues.",
)
if valid_scopes:
latest_by_scope = {
scope_key(str(row.get("scope", ""))): row for row in review_rows
}
for scope in required_scopes:
latest = latest_by_scope.get(scope_key(scope))
if latest is None:
add("missing_required_review", f"No review exists for required scope {scope!r}.")
elif not review_passes(latest):
add("failed_required_review", f"The latest review for required scope {scope!r} is not a clean PASS.")
if type(contract_version) is int and contract_version >= 2 and artifact_path is not None and artifact_path.is_file():
latest_by_scope = {scope_key(str(row.get("scope", ""))): row for row in review_rows}
scopes_to_bind = {"__global__"}
if valid_scopes:
scopes_to_bind.update(scope_key(value) for value in required_scopes)
current_hash = sha256_file(artifact_path)
for scope in sorted(scopes_to_bind):
row = latest_by_scope.get(scope)
if row is None or not review_passes(row):
continue # Missing/failed reviews already have their own findings.
bound_hash = row.get("artifact_sha256")
if not isinstance(bound_hash, str) or not re.fullmatch(r"[0-9a-fA-F]{64}", bound_hash):
add("missing_review_artifact_binding", f"Latest {scope!r} review has no valid artifact_sha256.")
elif bound_hash.lower() != current_hash:
add("stale_review_artifact", f"Latest {scope!r} review does not cover the current artifact.")
accepted_limitations = list(dict.fromkeys(accepted_limitations))
disclosure = assess_limitation_disclosure(message, accepted_limitations)
warnings: list[str] = []
if type(contract_version) is int and contract_version == 1:
warnings.append(
"Legacy v1 record check only: user-decision evidence, declared reading depth and "
"review-to-artifact binding were not checked. This is not current-contract acceptance."
)
if type(contract_version) is int and contract_version == 2:
warnings.append(
"Legacy v2 record check only: model-review completion, validity audits and sampling were not checked. "
"This is not current-contract acceptance."
)
if terminal_intent and disclosure["status"] in {"absent", "contradiction"}:
add(
"undisclosed_accepted_limitations",
"Accepted limitations exist backstage but are absent from the user-visible delivery message."
if disclosure["status"] == "absent" else
"Accepted limitations exist backstage but the delivery message explicitly denies them.",
)
elif terminal_intent and disclosure["status"] == "needs_review":
warnings.append(
"Specific limitation coverage requires semantic review; a generic keyword or unmatched "
"paraphrase is not verification. Inspect limitation_disclosure.unmatched_limitations."
)
delivery_observation, issue = inspect_actual_delivery(actual_message, message if message_exists else None)
if issue:
add(*issue)
return {
"ok": not flags,
"delivery_contract_version": contract_version,
**({"length_checks": length_checks} if length_checks else {}),
"current_contract_checked": type(contract_version) is int and contract_version == DELIVERY_CONTRACT_VERSION,
"semantic_verification": False,
"review_completion": review_completion,
"flags": flags,
"findings": findings,
"completion_claim": completion_claim,
"terminal_state": terminal_state,
"stage": stage,
"status": status,
"accepted_limitations": accepted_limitations,
"limitation_disclosure": disclosure,
"warnings": warnings,
"delivery_observation": delivery_observation,
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("project_root", nargs="?", default=".")
parser.add_argument("--artifact", default="final.md")
parser.add_argument("--delivery-message", default="delivery_message.md")
parser.add_argument("--receipt", default="state/final_delivery.json")
parser.add_argument(
"--actual-message", type=Path,
help="Compare a caller-provided runtime capture with the receipt-bound message; no capture is inferred.",
)
parser.add_argument("--json", action="store_true")
parser.add_argument(
"--require-complete", action="store_true",
help="Read-only completion check: require final/complete and all current delivery checks; do not finish or review the task.",
)
parser.add_argument(
"--contract-version", type=int, choices=(1, 2, 3), default=DELIVERY_CONTRACT_VERSION,
help="Default 3 checks current records. Use 1 or 2 only for labelled historical diagnostics.",
)
args = parser.parse_args()
result = evaluate_delivery(
Path(args.project_root),
artifact=args.artifact,
delivery_message=args.delivery_message,
receipt=args.receipt,
actual_message=args.actual_message,
contract_version=args.contract_version,
require_complete=args.require_complete,
)
if args.json:
print(json.dumps(result, ensure_ascii=False, indent=2))
elif result["ok"]:
label = "current task is final/complete and delivery checks pass" if args.require_complete else (
f"legacy v{args.contract_version} record checks, not current-contract acceptance" if args.contract_version < 3 else "mechanical delivery checks")
print(f"PASS: {label}; not a semantic quality verdict.")
else:
print("FAIL: current task completion is not confirmed." if args.require_complete else "FAIL: delivery claim is not safe.")
for finding in result["findings"]:
print(f"- {finding}")
if not args.json:
for warning in result["warnings"]:
print(f"REVIEW: {warning}")
return 0 if result["ok"] else 1
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |