Skip to content

dimagi-internal/ace / skills/eoi-round-review

eoi-round-review

Verified

AI-review a marketplace EOI round's responses on its labs criteria and write a shortlist doc. Needs the round slug.

FileKindSize
SKILL.mdskill md61.8 KB
check_claims.pyscript13.0 KB
rubric_health.pyscript24.2 KB
vocabulary.pyscript3.7 KB

check_claims.py

View on GitHub
#!/usr/bin/env python3
"""Check an eoi-round-review batch's reviews before they go to labs.

Usage:
    python3 check_claims.py <batch.json> <out.json> <criteria.json>

batch.json    the batch the scorer was given:
              [{response_id, org_name, org_slug, earlier_submissions,
                labs_history, answers: [{question_id, question, answer}]}]
out.json      the scorer's reviews for that batch:
              [{response_id, org_name, criteria_scores, claims, outcome, role,
                cost_per_site_usd, cost_note, ask_next, flags, one_line}]
criteria.json the round's saved criteria list ([{id, name, weight, ...}]);
              {"criteria": [...]} or a round with "evaluation_criteria" also works.

Prints every problem found, or "OK: every claim checks". Exits 1 on problems,
2 when a file cannot be read.

Quote matching mirrors labs' save-time check: lowercase, map curly quotes and
en/em dashes to their plain forms, collapse runs of whitespace, then a plain
substring test against the answer the cite's question_id names. A quote that
passes here can still be refused by labs if labs changes its rule; one that
fails here would be refused.

When a criterion's scoring_guide is in the anchored shape eoi-criteria-proposal
writes (anchors at 1/3/5/7/10 and a Silent rule; see
skills/eoi-criteria-proposal/check_rubric.py), each criterion:<id> claim must
also say which anchor the score was read against: its text starts
"Anchor <n>: ", where <n> is the anchor nearest the score (either one on a tie),
or "Silent: " when the answer said nothing and the Silent rule set the score.
Older banded guides ("1-3: ...") are not checked for this.

Flags must be keys of the defined list in vocabulary.py (labs refuses any other
flag), each at most once, and each needs a `flag` claim whose text starts with
that flag's label or key ("Long but thin: ..." or "padded: ...") saying what in
the response raised it. A `flag` claim that names no flag on the review is a
problem too.

Standard library only, so it runs anywhere python3 does.
"""

import json
import os
import re
import sys

sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "eoi-criteria-proposal"))
from check_rubric import is_anchored, nearest_anchors  # noqa: E402

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from vocabulary import FLAGS, OUTCOMES  # noqa: E402

ABOUT_FIXED = ("strength", "concern", "flag", "outcome")
BASES = ("quoted", "synthesized")

_CHAR_MAP = str.maketrans({
    "‘": "'", "’": "'",
    "“": '"', "”": '"',
    "–": "-", "—": "-",
})
_WS = re.compile(r"\s+")
_ANCHOR_CITE = re.compile(r"^\s*(?:anchor\s+(\d{1,2})|(silent))\s*:", re.IGNORECASE)


def normalise(text):
    """Labs' quote normalisation: lowercase, plain quotes and dashes, one space."""
    return _WS.sub(" ", str(text).translate(_CHAR_MAP).lower()).strip()


def load(path, what):
    try:
        with open(path, encoding="utf-8") as fh:
            return json.load(fh)
    except (OSError, ValueError) as exc:
        print(f"cannot read {what} {path}: {exc}", file=sys.stderr)
        sys.exit(2)


def criteria_list(raw):
    if isinstance(raw, dict):
        # marketplace_round_responses nests the criteria in "round"; marketplace_round_scores
        # has a "round" header beside them, and its "criteria" are per-criterion statistics.
        inner = raw.get("round")
        if isinstance(inner, dict) and ("evaluation_criteria" in inner or "criteria" in inner):
            raw = {**raw, **inner}
        raw = raw.get("evaluation_criteria") or raw.get("criteria") or []
    return [c for c in raw or [] if isinstance(c, dict) and c.get("id")]


def criteria_ids(raw):
    return [c["id"] for c in criteria_list(raw)]


def anchored_ids(raw):
    """Ids of criteria whose scoring_guide is in the anchored shape."""
    return {c["id"] for c in criteria_list(raw) if is_anchored(c.get("scoring_guide"))}


def check_anchor_cite(where, text, score):
    """An anchored criterion's reason names the anchor its score was read against."""
    m = _ANCHOR_CITE.match(text or "")
    if not m:
        return [f"{where}: text must start 'Anchor <n>: ' (or 'Silent: ') naming the anchor scored against"]
    if m.group(2) or not isinstance(score, int) or isinstance(score, bool):
        return []
    anchor = int(m.group(1))
    allowed = nearest_anchors(score)
    if anchor not in allowed:
        return [f"{where}: cites anchor {anchor} for a score of {score}; the nearest anchor is "
                f"{' or '.join(map(str, allowed))}"]
    return []


def flag_named(text):
    """The flag a flag claim's text starts with ("<label>: " or "<key>: "), or None."""
    head = (text or "").split(":", 1)
    if len(head) != 2:
        return None
    name = head[0].strip().lower()
    for key, meaning in FLAGS.items():
        if name in (key, meaning["label"].lower()):
            return key
    return None


def check_flags(rid, flags, flag_claims):
    """Flags are defined keys, each explained by a flag claim that names it."""
    problems = []
    if not isinstance(flags, list):
        return [f"{rid}: flags is not a list"]
    seen = []
    for flag in flags:
        if not isinstance(flag, str) or flag not in FLAGS:
            problems.append(f"{rid}: flag {flag!r} is not a defined flag (labs refuses it); use one of: "
                            f"{', '.join(FLAGS)}")
        elif flag in seen:
            problems.append(f"{rid}: flag {flag!r} is listed twice")
        else:
            seen.append(flag)
    named = {}
    for where, text in flag_claims:
        key = flag_named(text)
        if key is None:
            problems.append(f"{where}: a flag claim's text must start with the flag it explains "
                            f"('<label>: ' or '<key>: ', e.g. 'Long but thin: ')")
        elif key not in seen:
            problems.append(f"{where}: explains flag {key!r}, which is not in this review's flags")
        else:
            named[key] = True
    for flag in seen:
        if flag not in named:
            problems.append(f"{rid}: flag {flag!r} has no flag claim saying what raised it")
    return problems


def check_cite(where, cite, answers):
    problems = []
    if not isinstance(cite, dict):
        return [f"{where}: cite is not an object"]
    if "question" in cite:
        qid = cite["question"]
        if qid not in answers:
            return [f"{where}: question_id {qid!r} is not a question of this response"]
        if "quote" in cite:
            quote = cite["quote"]
            if not isinstance(quote, str) or not normalise(quote):
                problems.append(f"{where}: empty quote")
            elif normalise(quote) not in normalise(answers[qid]):
                problems.append(f"{where}: quote not found in answer {qid}: {quote!r}")
        elif not (cite.get("absent") is True or cite.get("whole") is True):
            problems.append(f"{where}: a question cite needs a quote, absent: true or whole: true")
    elif "source" in cite:
        source = cite["source"]
        if source == "web":
            if not cite.get("url"):
                problems.append(f"{where}: web cite has no url")
        elif source != "labs":
            problems.append(f"{where}: unknown source {source!r} (labs or web)")
        if not cite.get("detail"):
            problems.append(f"{where}: {source} cite has no detail")
    else:
        problems.append(f"{where}: cite names neither a question nor a source")
    return problems


def check_review(review, response, crit_ids, anchored=frozenset()):
    rid = review.get("response_id")
    problems = []
    answers = {a.get("question_id"): a.get("answer") or "" for a in response.get("answers", [])}

    scores = review.get("criteria_scores")
    if not isinstance(scores, dict):
        problems.append(f"{rid}: criteria_scores is missing")
        scores = {}
    for cid in crit_ids:
        score = scores.get(cid)
        if score is None:
            problems.append(f"{rid}: no score for criterion {cid}")
        elif isinstance(score, bool) or not isinstance(score, int) or not 1 <= score <= 10:
            problems.append(f"{rid}: score for {cid} is {score!r}, not a whole number 1-10")
    for cid in scores:
        if cid not in crit_ids:
            problems.append(f"{rid}: scores criterion {cid!r}, which is not on the round")

    outcome = review.get("outcome")
    if outcome not in OUTCOMES:
        problems.append(f"{rid}: outcome is {outcome!r}, not one of {'/'.join(OUTCOMES)} "
                        f"({' / '.join(o['label'] for o in OUTCOMES.values())})")

    claims = review.get("claims")
    if not isinstance(claims, list) or not claims:
        problems.append(f"{rid}: no claims")
        claims = []
    seen_ids = set()
    abouts = set()
    flag_claims = []
    for n, claim in enumerate(claims):
        if not isinstance(claim, dict):
            problems.append(f"{rid} claim #{n + 1}: not an object")
            continue
        cl_id = claim.get("id")
        where = f"{rid} {cl_id or f'claim #{n + 1}'}"
        if not cl_id:
            problems.append(f"{where}: claim has no id")
        elif cl_id in seen_ids:
            problems.append(f"{where}: claim id is used twice")
        seen_ids.add(cl_id)
        about = claim.get("about")
        if about in ABOUT_FIXED:
            abouts.add(about)
            if about == "flag":
                flag_claims.append((where, claim.get("text")))
        elif isinstance(about, str) and about.startswith("criterion:"):
            cid = about[len("criterion:"):]
            if cid not in crit_ids:
                problems.append(f"{where}: about {about!r} names a criterion not on the round")
            elif cid in anchored:
                problems.extend(check_anchor_cite(where, claim.get("text"), scores.get(cid)))
            abouts.add(about)
        else:
            problems.append(f"{where}: about is {about!r}")
        if not (claim.get("text") or "").strip():
            problems.append(f"{where}: claim has no text")
        elif _BATCH_TALK.search(claim["text"]):
            problems.append(f"{where}: compares within the scoring batch ({_BATCH_TALK.search(claim['text']).group(0)!r}); "
                            "people read this in labs, where there are no batches: compare with the whole pool or not at all")
        if claim.get("basis") not in BASES:
            problems.append(f"{where}: basis is {claim.get('basis')!r}, not quoted or synthesized")
        cites = claim.get("cites", [])
        if not isinstance(cites, list):
            problems.append(f"{where}: cites is not a list")
            continue
        for k, cite in enumerate(cites):
            problems.extend(check_cite(f"{where} cite {k + 1}", cite, answers))

    for cid in crit_ids:
        if f"criterion:{cid}" not in abouts:
            problems.append(f"{rid}: no criterion:{cid} claim giving the reason for its score")
    if "outcome" not in abouts:
        problems.append(f"{rid}: no outcome claim")
    problems.extend(check_flags(rid, review.get("flags") or [], flag_claims))
    return problems


# A batch is how ACE splits the scoring, not something a labs reader can see. "The only
# applicant in this batch with a sampling frame" is false of the pool (ULAB v5, 2026-11-08:
# nine such claims reached labs past calibration).
_BATCH_TALK = re.compile(r"\b(?:this|the|my|our|each|that) batch(?:es)?\b|\bbatch \d+\b", re.I)


def check(batch, out, crit_ids, anchored=frozenset()):
    problems = []
    if not crit_ids:
        problems.append("criteria file has no criteria with an id")
    if not isinstance(out, list):
        return problems + ["out.json is not a list of reviews"]
    responses = {r.get("response_id"): r for r in batch}
    reviewed = {}
    for n, review in enumerate(out):
        if not isinstance(review, dict):
            problems.append(f"review #{n + 1}: not an object")
            continue
        rid = review.get("response_id")
        if rid not in responses:
            problems.append(f"review #{n + 1}: response_id {rid!r} is not in the batch")
            continue
        if rid in reviewed:
            problems.append(f"{rid}: reviewed twice")
            continue
        reviewed[rid] = review
        problems.extend(check_review(review, responses[rid], crit_ids, anchored))
    for rid in responses:
        if rid not in reviewed:
            problems.append(f"{rid}: in the batch but not reviewed")
    return problems


def main(argv):
    if len(argv) != 4:
        print("usage: python3 check_claims.py <batch.json> <out.json> <criteria.json>", file=sys.stderr)
        return 2
    batch = load(argv[1], "batch")
    out = load(argv[2], "reviews")
    crit_raw = load(argv[3], "criteria")
    crit_ids = criteria_ids(crit_raw)
    if not isinstance(batch, list):
        print("batch.json is not a list of responses", file=sys.stderr)
        return 2
    problems = check(batch, out, crit_ids, anchored_ids(crit_raw))
    if problems:
        for p in problems:
            print(p)
        print(f"{len(problems)} problem(s)")
        return 1
    print("OK: every claim checks")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))
receiving-code-reviewobra/superpowersUse when receiving code review feedback, before implementing suggestions, especially if feedback seems unclear or technically questionable - requires technical rigor and verification, not performative agreement or blind implementationStars: 322kUpdated Claude CodeCodexVerifiedInstall
requesting-code-reviewobra/superpowersUse when completing tasks, implementing major features, or before merging to verify work meets requirementsStars: 322kUpdated Claude CodeCodexVerifiedInstall
using-git-worktreesobra/superpowersUse when starting feature work that needs isolation from current workspace or before executing implementation plans - ensures an isolated workspace exists via native tools or git worktree fallbackStars: 322kUpdated Claude CodeCodexVerifiedInstall
verification-before-completionobra/superpowersUse when about to claim work is complete, fixed, or passing, before committing or creating PRs - requires running verification commands and confirming output before making any success claims; evidence before assertions alwaysStars: 322kUpdated Claude CodeCodexVerifiedInstall