K-Dense-AI/scientific-agent-skills / skills/peer-review
peer-review
VerifiedPrepares evidence-bounded, constructive peer-review drafts and structured manuscript assessments. Supports authorized review of scientific manuscripts, protocols, preprints, or research proposals; reporting-guideline selection; claim–evidence checks; methods, statistics, reproducibility, ethics, figure/table, and citation critique; or revision-response planning.
scripts/validate_claim_evidence.py
View on GitHub#!/usr/bin/env python3
"""Validate a bounded claim-evidence alignment matrix without echoing prose."""
from __future__ import annotations
import argparse
from collections import Counter
from typing import Any
from _common import (
ValidationError,
error_exit,
issue,
read_csv_records,
require_enum,
require_identifier,
require_text,
require_unique,
split_identifiers,
write_json_report,
)
FIELDS = (
"claim_id",
"location",
"claim_type",
"claim_summary",
"evidence_ids",
"support_level",
"alignment_issue",
"limitation",
"requested_action",
)
CLAIM_TYPES = {
"primary_outcome",
"secondary_outcome",
"causal",
"mechanistic",
"diagnostic",
"prediction",
"safety",
"generalization",
"methods",
"other",
}
SUPPORT_LEVELS = {
"supported",
"partly_supported",
"unsupported",
"not_assessed",
}
ALIGNMENT_ISSUES = {
"none",
"direction",
"magnitude",
"population",
"outcome",
"timepoint",
"causal_language",
"scope",
"uncertainty",
"selective_reporting",
"other",
}
def load_matrix(raw_path: str) -> list[dict[str, Any]]:
rows = read_csv_records(
raw_path,
required_fields=FIELDS,
allowed_fields=FIELDS,
)
parsed: list[dict[str, Any]] = []
claim_ids: list[str] = []
for line_number, row in enumerate(rows, start=2):
context = f"matrix row {line_number}"
claim_id = require_identifier(row["claim_id"], f"{context}.claim_id")
claim_ids.append(claim_id)
parsed.append(
{
"claim_id": claim_id,
"location": require_text(
row["location"], f"{context}.location", maximum=500
),
"claim_type": require_enum(
row["claim_type"], CLAIM_TYPES, f"{context}.claim_type"
),
"claim_summary": require_text(
row["claim_summary"],
f"{context}.claim_summary",
minimum=10,
maximum=2_000,
),
"evidence_ids": split_identifiers(
row["evidence_ids"],
f"{context}.evidence_ids",
allow_empty=True,
),
"support_level": require_enum(
row["support_level"],
SUPPORT_LEVELS,
f"{context}.support_level",
),
"alignment_issue": require_enum(
row["alignment_issue"],
ALIGNMENT_ISSUES,
f"{context}.alignment_issue",
),
"limitation": require_text(
row["limitation"],
f"{context}.limitation",
allow_empty=True,
maximum=2_000,
),
"requested_action": require_text(
row["requested_action"],
f"{context}.requested_action",
allow_empty=True,
maximum=2_000,
),
}
)
require_unique(claim_ids, "claim-evidence matrix")
return parsed
def validate_matrix(rows: list[dict[str, Any]]) -> dict[str, Any]:
errors: list[dict[str, str]] = []
warnings: list[dict[str, str]] = []
support_counts = Counter()
issue_counts = Counter()
gap_ids: list[str] = []
action_missing_ids: list[str] = []
for row in rows:
claim_id = row["claim_id"]
support = row["support_level"]
alignment_issue = row["alignment_issue"]
evidence_ids = row["evidence_ids"]
support_counts[support] += 1
issue_counts[alignment_issue] += 1
if support == "supported":
if not evidence_ids:
errors.append(issue("SUPPORTED_CLAIM_HAS_NO_EVIDENCE", claim_id))
if alignment_issue != "none":
errors.append(issue("SUPPORTED_CLAIM_HAS_ALIGNMENT_ISSUE", claim_id))
elif support == "partly_supported":
gap_ids.append(claim_id)
if not evidence_ids:
errors.append(issue("PARTIAL_CLAIM_HAS_NO_EVIDENCE", claim_id))
if alignment_issue == "none":
errors.append(issue("PARTIAL_CLAIM_NEEDS_ISSUE_CODE", claim_id))
elif support in {"unsupported", "not_assessed"}:
gap_ids.append(claim_id)
if alignment_issue == "none" and support == "unsupported":
errors.append(issue("UNSUPPORTED_CLAIM_NEEDS_ISSUE_CODE", claim_id))
if support != "supported" and not row["requested_action"]:
action_missing_ids.append(claim_id)
warnings.append(issue("REQUESTED_ACTION_MISSING", claim_id))
if alignment_issue != "none" and not row["limitation"]:
warnings.append(issue("LIMITATION_CONTEXT_MISSING", claim_id))
if row["claim_type"] in {"causal", "mechanistic"} and support == "supported":
warnings.append(
issue("CAUSAL_OR_MECHANISTIC_SUPPORT_REQUIRES_EXPERT_REVIEW", claim_id)
)
return {
"schema_version": "2.0",
"valid": not errors,
"status": (
"INVALID_MATRIX"
if errors
else "VALID_WITH_ALIGNMENT_GAPS"
if gap_ids
else "VALID_NO_RECORDED_GAPS"
),
"errors": errors,
"warnings": warnings,
"claim_count": len(rows),
"support_counts": dict(sorted(support_counts.items())),
"alignment_issue_counts": dict(sorted(issue_counts.items())),
"claim_ids_requiring_resolution": sorted(gap_ids),
"claim_ids_missing_requested_action": sorted(action_missing_ids),
"notice": (
"This report checks matrix structure and declared claim-evidence "
"alignment only. It does not verify evidence truth, reproduce analyses, "
"or determine manuscript merit. Reports contain identifiers, not claim "
"or manuscript text."
),
}
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description=(
"Validate a local claim-evidence CSV and emit only identifiers and "
"counts; raw claim text is never echoed."
)
)
parser.add_argument("matrix", help="Local claim-evidence matrix CSV")
parser.add_argument("-o", "--output", help="Optional local JSON report")
parser.add_argument(
"--force", action="store_true", help="Replace an existing output file"
)
return parser
def main() -> int:
args = build_parser().parse_args()
try:
report = validate_matrix(load_matrix(args.matrix))
write_json_report(report, args.output, force=args.force)
return 0 if report["valid"] else 1
except ValidationError as exc:
return error_exit(exc)
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| add-product-e2e-evalpaperclipai/paperclipAdd or extend a Paperclip full-stack runner E2E workflow, fixture, matcher, or report evidence path for local or Daytona execution. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| add-runner-evalpaperclipai/paperclipAdd or extend a Paperclip Runner protocol evaluation definition, roster, assertion, or report fixture with provenance and narrow validation. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| diagnose-why-work-stoppedpaperclipai/paperclipCompatibility entry point for failed, stalled, looping, or over-recovered Paperclip work. Use fix-failed-paperclip-run for the full diagnosis, regression, repair, real verification, and green-PR workflow; honor an explicitly requested diagnosis-only scope. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| fix-failed-paperclip-runpaperclipai/paperclipDiagnose and fix failed, stuck, slow, or incorrectly blocked Paperclip tasks and runs. Use with a task/run URL or failure report to trace the root cause, prove a regression red then green, check second-order effects, verify the real workflow, and deliver a PR with passing checks. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |