borghei/Claude-Skills / research-ops/product-research
product-research
VerifiedContinuous product discovery research operations — picking the method that fits the question, recruiting and screening participants, building interview guides, and scoring evidence into insights. Use when planning or running discovery.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 13.6 KB |
| assets/interview-guide-template.md | markdown | 4.6 KB |
| assets/sample_evidence.json | config | 2.8 KB |
| assets/sample_research_question.json | config | 394 B |
| assets/sample_screener.json | config | 2.4 KB |
| references/interview-craft.md | markdown | 17.1 KB |
| references/method-selection-guide.md | markdown | 16.7 KB |
| scripts/insight_confidence_scorer.py | script | 9.9 KB |
| scripts/method_recommender.py | script | 11.7 KB |
| scripts/screener_validator.py | script | 12.3 KB |
scripts/screener_validator.py
View on GitHub#!/usr/bin/env python3
"""
screener_validator.py — Validate a recruiting screener for transparency,
disqualification logic, quota coverage, and professional-respondent exposure.
Reads a JSON screener (criteria, items, quotas, incentive) and reports every
criterion no item tests, every item whose qualifying answer is obvious to a
motivated respondent, and every gap in the quota plan.
Stdlib only. Deterministic. Text (default) or JSON output.
Usage:
python3 screener_validator.py --input screener.json
python3 screener_validator.py --input screener.json --format json
python3 screener_validator.py --input screener.json --strict
Input schema:
{
"study": "Ticket triage discovery",
"target_sessions": 8,
"incentive_value": 120,
"recruiting_source": "existing_customers",
"criteria": [
{"id": "C1", "description": "Handles inbound tickets daily",
"kind": "qualify", "tested_by": ["Q2"]},
{"id": "C2", "description": "Works at a competitor",
"kind": "disqualify", "tested_by": ["Q4"]}
],
"items": [
{"id": "Q2", "text": "In a typical week, which of these tasks do you
personally perform?",
"options": ["Triaging inbound tickets", "Writing documentation",
"Managing billing", "None of these"],
"qualifying_options": ["Triaging inbound tickets"]}
],
"quotas": [{"segment": "high_volume", "target": 4},
{"segment": "low_volume", "target": 4}]
}
"""
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
# Above this, an incentive starts attracting people who answer for the money.
HIGH_INCENTIVE = 200
MIN_DISTRACTOR_OPTIONS = 2
TRAP_HINT_WORDS = ("trap", "decoy", "fictional", "does not exist", "invented")
@dataclass
class Finding:
"""A defect in the screener design."""
severity: str
ref: str
message: str
def check_criteria_coverage(criteria: list[dict[str, Any]],
item_ids: set[str]) -> list[Finding]:
"""Flag criteria with no testing item, and items referenced but not defined."""
findings: list[Finding] = []
if not criteria:
findings.append(Finding("fail", "screener",
"no criteria defined — a screener without explicit "
"qualifying criteria cannot be validated or audited"))
return findings
has_disqualifier = any(c.get("kind") == "disqualify" for c in criteria)
if not has_disqualifier:
findings.append(Finding("fail", "screener",
"no disqualifying criteria — every screener needs explicit "
"exclusions (competitors, employees, prior participants), "
"not just inclusions"))
for crit in criteria:
cid = str(crit.get("id", "?"))
tested_by = crit.get("tested_by") or []
if not tested_by:
findings.append(Finding("fail", cid,
f"criterion '{crit.get('description', '')}' is not tested "
"by any item — it will not be enforced"))
continue
for ref in tested_by:
if ref not in item_ids:
findings.append(Finding("fail", cid,
f"references item '{ref}', which is not defined"))
return findings
def check_item(item: dict[str, Any]) -> list[Finding]:
"""Check one screener item for transparency and option-set quality."""
item_id = str(item.get("id", "?"))
findings: list[Finding] = []
options = [str(o) for o in (item.get("options") or [])]
qualifying = [str(o) for o in (item.get("qualifying_options") or [])]
if not str(item.get("text", "")).strip():
findings.append(Finding("fail", item_id, "item has no text"))
if not options:
# Open-numeric items are fine, but they cannot be scored automatically.
if item.get("type") not in ("numeric", "open"):
findings.append(Finding("warn", item_id,
"no options and not marked type 'numeric' or 'open' — "
"qualification cannot be applied consistently"))
return findings
if qualifying:
unknown = [q for q in qualifying if q not in options]
if unknown:
findings.append(Finding("fail", item_id,
f"qualifying option(s) {unknown} are not in the option list"))
if len(qualifying) == len(options):
findings.append(Finding("fail", item_id,
"every option qualifies — this item screens nobody out"))
distractors = len(options) - len(qualifying)
# Trap items are meant to have a single disqualifying option.
if 0 < distractors < MIN_DISTRACTOR_OPTIONS and not item.get("is_trap"):
findings.append(Finding("warn", item_id,
f"only {distractors} non-qualifying option(s) — with so few "
"distractors the qualifying answer is guessable; add more"))
if len(qualifying) == 1 and options and options[0] == qualifying[0]:
findings.append(Finding("warn", item_id,
"the qualifying option is listed first — move it into the "
"middle of the list so it does not read as the intended answer"))
else:
findings.append(Finding("warn", item_id,
"no qualifying_options declared — reviewers cannot tell what "
"this item screens on"))
lowered_text = str(item.get("text", "")).lower()
if any(word in lowered_text for word in TRAP_HINT_WORDS):
findings.append(Finding("fail", item_id,
"item text reveals that it is a trap item — remove the hint"))
seen = {o.strip().lower() for o in options}
if len(seen) != len(options):
findings.append(Finding("fail", item_id, "duplicate response options"))
return findings
def check_recruiting(screener: dict[str, Any], items: list[dict[str, Any]]) -> list[Finding]:
"""Check incentive level, trap coverage, and quota arithmetic."""
findings: list[Finding] = []
incentive = float(screener.get("incentive_value", 0) or 0)
source = str(screener.get("recruiting_source", "")).lower()
target = int(screener.get("target_sessions", 0) or 0)
has_trap = any(item.get("is_trap") for item in items)
if source in ("panel", "public_list") and not has_trap:
findings.append(Finding("fail", "recruiting",
"panel recruiting without a trap item — professional "
"respondents will qualify themselves in; add an item with "
"an invented option that disqualifies on selection"))
elif not has_trap:
findings.append(Finding("warn", "recruiting",
"no trap item — add one if the incentive is meaningful "
"relative to the participant's time"))
if incentive >= HIGH_INCENTIVE and not has_trap:
findings.append(Finding("fail", "recruiting",
f"incentive of {incentive:g} is high enough to attract "
"answer-for-money respondents and there is no trap item"))
quotas = screener.get("quotas") or []
if not quotas:
findings.append(Finding("warn", "recruiting",
"no quotas — recruiting without quotas produces a sample of "
"convenience skewed toward your most engaged users"))
else:
quota_total = sum(int(q.get("target", 0)) for q in quotas)
if target and quota_total != target:
findings.append(Finding("warn", "recruiting",
f"quota targets sum to {quota_total} but target_sessions "
f"is {target}"))
thin = [q["segment"] for q in quotas if int(q.get("target", 0)) < 3]
if thin:
findings.append(Finding("warn", "recruiting",
f"segment(s) {thin} have fewer than 3 sessions — that is "
"anecdote, not a segment finding"))
if target and target < 5:
findings.append(Finding("warn", "recruiting",
f"{target} sessions is below the 5-6 needed to see a pattern "
"in a single segment"))
return findings
def validate(screener: dict[str, Any]) -> dict[str, Any]:
"""Run every screener check and aggregate the findings."""
items = screener.get("items") or []
if not items:
raise ValueError("screener must contain a non-empty 'items' list")
item_ids = {str(i.get("id", "")) for i in items}
findings: list[Finding] = []
findings.extend(check_criteria_coverage(screener.get("criteria") or [], item_ids))
for item in items:
findings.extend(check_item(item))
findings.extend(check_recruiting(screener, items))
counts = {"fail": 0, "warn": 0}
for f in findings:
counts[f.severity] = counts.get(f.severity, 0) + 1
return {
"study": screener.get("study", ""),
"item_count": len(items),
"criteria_count": len(screener.get("criteria") or []),
"target_sessions": screener.get("target_sessions", 0),
"verdict": "not ready to field" if counts["fail"] else "ready to field",
"severity_counts": counts,
"findings": [{"severity": f.severity, "ref": f.ref, "message": f.message}
for f in findings],
}
def render_text(report: dict[str, Any]) -> str:
"""Render the validation as human-readable text."""
lines = [f"Screener validation — {report['study']}",
f"Criteria: {report['criteria_count']} | Items: {report['item_count']} | "
f"Target sessions: {report['target_sessions']}",
f"Verdict: {report['verdict'].upper()}", ""]
counts = report["severity_counts"]
lines.append(f"Findings: {counts.get('fail', 0)} fail, {counts.get('warn', 0)} warn")
if not report["findings"]:
lines.append(" none — screener is clean")
order = {"fail": 0, "warn": 1}
for f in sorted(report["findings"], key=lambda x: order.get(x["severity"], 9)):
lines.append(f" [{f['severity'].upper():4}] {f['ref']}: {f['message']}")
return "\n".join(lines)
def parse_args() -> argparse.Namespace:
"""Parse command-line arguments."""
parser = argparse.ArgumentParser(
description="Validate a recruiting screener before recruiting opens.",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
parser.add_argument("--input", required=True, help="Path to the JSON screener")
parser.add_argument("--format", choices=["text", "json"], default="text",
help="Output format; text is the default")
parser.add_argument("--output", help="Write output to this file instead of stdout")
parser.add_argument("--strict", action="store_true",
help="Exit non-zero if any finding has severity 'fail'")
return parser.parse_args()
def main() -> int:
"""CLI entry point."""
args = parse_args()
try:
screener = json.loads(Path(args.input).read_text(encoding="utf-8"))
report = validate(screener)
out = json.dumps(report, indent=2) if args.format == "json" else render_text(report)
if args.output:
Path(args.output).write_text(out, encoding="utf-8")
print(f"wrote {args.output}", file=sys.stderr)
else:
print(out)
except OSError as exc:
print(f"error: cannot read or write file: {exc}", file=sys.stderr)
return 1
except json.JSONDecodeError as exc:
print(f"error: --input is not valid JSON: {exc}", file=sys.stderr)
return 1
except (ValueError, KeyError, TypeError, ZeroDivisionError) as exc:
print(f"error: invalid screener: {exc}", file=sys.stderr)
return 1
if args.strict and report["severity_counts"].get("fail", 0) > 0:
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| add-product-e2e-evalpaperclipai/paperclipAdd or extend a Paperclip full-stack runner E2E workflow, fixture, matcher, or report evidence path for local or Daytona execution. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| add-runner-evalpaperclipai/paperclipAdd or extend a Paperclip Runner protocol evaluation definition, roster, assertion, or report fixture with provenance and narrow validation. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| diagnose-why-work-stoppedpaperclipai/paperclipCompatibility entry point for failed, stalled, looping, or over-recovered Paperclip work. Use fix-failed-paperclip-run for the full diagnosis, regression, repair, real verification, and green-PR workflow; honor an explicitly requested diagnosis-only scope. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| fix-failed-paperclip-runpaperclipai/paperclipDiagnose and fix failed, stuck, slow, or incorrectly blocked Paperclip tasks and runs. Use with a task/run URL or failure report to trace the root cause, prove a regression red then green, check second-order effects, verify the real workflow, and deliver a PR with passing checks. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |