borghei/Claude-Skills / engineering/senior-data-scientist
senior-data-scientist
VerifiedExperiment sizing, feature profiling with leakage checks, and model evaluation from saved predictions, plus causal-inference workflows. Use when sizing an A/B test, vetting features, or scoring a model before release.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 9.9 KB |
| assets/customers_sample.csv | other | 3.3 KB |
| assets/eval_thresholds.json | config | 114 B |
| assets/experiment_design.json | config | 235 B |
| assets/predictions_sample.csv | other | 1.7 KB |
| assets/regression_sample.csv | other | 210 B |
| references/ds-operations.md | markdown | 18.8 KB |
| references/ds-workflows.md | markdown | 8.4 KB |
| references/experiment_design_frameworks.md | markdown | 1.4 KB |
| references/feature_engineering_patterns.md | markdown | 1.4 KB |
| references/statistical_methods_advanced.md | markdown | 1.4 KB |
| scripts/experiment_designer.pyexecutable | script | 13.2 KB |
| scripts/feature_engineering_pipeline.pyexecutable | script | 14.3 KB |
| scripts/model_evaluation_suite.pyexecutable | script | 16.0 KB |
scripts/model_evaluation_suite.py
View on GitHub#!/usr/bin/env python3
"""Score saved predictions, compare models, and give an offline readiness verdict.
Reports binary-classification or regression metrics side by side, fold stability, and
pass/fail against config thresholds. Does not train models. Standard library only.
Usage: python3 model_evaluation_suite.py --input preds.csv --folds 5 --config thresholds.json
"""
import argparse
import csv
import json
import math
import statistics
import sys
from collections import Counter
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
NULLS = {"", "na", "n/a", "null", "none", "nan"}
fail = lambda msg: (print(msg, file=sys.stderr), sys.exit(2)) # bad input exits 2
CHECKS = {"min_accuracy": "accuracy", "min_precision": "precision", "min_recall": "recall",
"min_f1": "f1", "min_auc": "auc", "max_log_loss": "log_loss", "max_brier": "brier",
"min_r2": "r2", "max_rmse": "rmse", "max_mae": "mae", "max_mape": "mape"}
DEFAULT_CONFIG = {"classification": {"min_auc": 0.85, "max_brier": 0.15}, "regression": {}}
EXAMPLE = ("id,actual,model_a,model_b\nr01,1,0.91,0.70\nr02,0,0.12,0.40\n"
"r03,1,0.78,0.45\nr04,0,0.35,0.62")
def load_csv(path: str) -> Tuple[List[str], List[Dict[str, str]]]:
"""Read the CSV into a header and rows, exiting on a ragged row or bad file."""
try:
with Path(path).open(newline="", encoding="utf-8-sig") as handle:
reader = csv.reader(handle)
header, rows = next(reader, []), []
for rec in reader:
if rec and len(rec) != len(header):
fail(f"ERROR: {path} line {reader.line_num} has {len(rec)} field(s) "
f"but the header has {len(header)}")
rows += [dict(zip(header, rec))] if rec else []
except FileNotFoundError:
fail(f"ERROR: file not found: {path}")
except (UnicodeDecodeError, csv.Error) as exc:
fail(f"ERROR: {path} is not a readable UTF-8 CSV: {exc}")
if not header or not rows:
fail(f"ERROR: {path} needs a header row and at least one data row")
return header, rows
def numeric(values: List[str]) -> Optional[List[float]]:
"""All values as floats, or None if any is missing or not a number."""
try:
out = [float(v) for v in values]
except (TypeError, ValueError):
return None
return out if all(math.isfinite(x) for x in out) else None
def why_unusable(values: List[str]) -> Optional[str]:
"""Why a column cannot be scored, or None when every value is a finite number."""
if any((v or "").strip().lower() in NULLS for v in values):
return "has missing values"
return None if numeric(values) is not None else "has non-numeric values"
def r6(m: Dict[str, Any]) -> Dict[str, Any]:
"""Round the float values of a metrics dict to 6 places."""
return {k: round(v, 6) if isinstance(v, float) else v for k, v in m.items()}
def auc(y: List[int], s: List[float]) -> Optional[float]:
"""ROC AUC by the rank-sum (Mann-Whitney) method, ties get average ranks."""
n1, n0 = sum(y), len(y) - sum(y)
if n1 == 0 or n0 == 0:
return None
counts = Counter(s)
rank, below = {}, 0
for v in sorted(counts): # tied scores share the average of their 1-based ranks
rank[v], below = below + (counts[v] + 1) / 2, below + counts[v]
pos_rank_sum = sum(rank[v] for v, label in zip(s, y) if label == 1)
return (pos_rank_sum - n1 * (n1 + 1) / 2) / (n1 * n0)
def classify(y: List[int], p: List[float], kind: str, threshold: float) -> Dict[str, Any]:
"""Binary metrics: confusion matrix, rates, AUC and probability scores."""
m: Dict[str, Any] = {}
if kind != "score":
pred = [1 if v >= (0.5 if kind == "label" else threshold) else 0 for v in p]
pairs = list(zip(y, pred))
tp, fp, fn, tn = (pairs.count(cell) for cell in ((1, 1), (0, 1), (1, 0), (0, 0)))
prec = tp / (tp + fp) if tp + fp else 0.0
rec = tp / (tp + fn) if tp + fn else 0.0
m.update(confusion={"tp": tp, "fp": fp, "fn": fn, "tn": tn},
accuracy=(tp + tn) / len(y), precision=prec, recall=rec,
f1=2 * prec * rec / (prec + rec) if prec + rec else 0.0,
specificity=tn / (tn + fp) if tn + fp else 0.0)
if kind != "label":
m["auc"] = auc(y, p)
if kind == "probability":
eps = 1e-15
m["log_loss"] = -statistics.fmean(
a * math.log(max(v, eps)) + (1 - a) * math.log(max(1 - v, eps)) for a, v in zip(y, p))
m["brier"] = statistics.fmean((v - a) ** 2 for a, v in zip(y, p))
return r6(m)
def regress(y: List[float], p: List[float]) -> Dict[str, Any]:
"""Regression metrics. MAPE skips rows where the actual is zero."""
err = [b - a for a, b in zip(y, p)]
mean_y = statistics.fmean(y)
sst = sum((a - mean_y) ** 2 for a in y)
nonzero = [(a, b) for a, b in zip(y, p) if a != 0]
return r6({"mae": statistics.fmean(abs(e) for e in err),
"rmse": math.sqrt(statistics.fmean(e ** 2 for e in err)),
"r2": 1 - sum(e ** 2 for e in err) / sst if sst else None,
"mape": statistics.fmean(abs((b - a) / a) for a, b in nonzero) if nonzero else None,
"mape_rows_skipped": len(y) - len(nonzero), "bias": statistics.fmean(err)})
def evaluate(path: str, target: str, cols: Optional[List[str]], task: str, positive: Optional[str],
threshold: float, folds: int, config: Optional[Dict[str, Any]]) -> Dict[str, Any]:
"""Score every prediction column, run fold stability and readiness checks."""
header, rows = load_csv(path)
if target not in header:
fail(f"ERROR: target column '{target}' not in {path}. Columns: {', '.join(header)}")
rows, total = [r for r in rows if (r.get(target) or "").strip().lower() not in NULLS], len(rows)
if not rows:
fail(f"ERROR: every '{target}' value in {path} is missing")
actual = [(r.get(target) or "").strip() for r in rows]
auto = cols is None
cols = [h for h in header if h != target] if auto else cols
if [c for c in cols if c not in header] or not cols:
fail(f"ERROR: prediction column(s) not found: "
f"{', '.join(c for c in cols if c not in header) or 'none besides the target'}")
skipped = {c: why for c in cols if (why := why_unusable([r.get(c) for r in rows]))}
if skipped and (not auto or len(skipped) == len(cols)):
fail(("ERROR: no usable prediction column: " if auto else "ERROR: ")
+ "; ".join(f"column '{c}' {why}" for c, why in skipped.items()))
cols = [c for c in cols if c not in skipped]
if folds == 1 or folds < 0 or folds > len(rows):
fail(f"ERROR: --folds must be 0 (off) or between 2 and the row count ({len(rows)})")
labels = sorted(set(actual))
if len(labels) < 2:
fail(f"ERROR: '{target}' has a single value ({labels[0]!r}); "
"metrics need at least two classes or values")
if task == "auto":
task = "classification" if len(labels) == 2 else "regression"
if task == "classification":
if len(labels) != 2:
fail(f"ERROR: binary classification needs 2 classes in '{target}', found {len(labels)}")
pos = positive if positive is not None else ("1" if "1" in labels else labels[1])
if pos not in labels:
fail(f"ERROR: positive class '{pos}' not among {labels}")
y: List[Any] = [1 if a == pos else 0 for a in actual]
else:
y = numeric(actual) or fail(f"ERROR: '{target}' must be numeric for regression")
models: Dict[str, Any] = {}
for c in cols:
p: List[float] = numeric([r.get(c) for r in rows]) or []
kind = ("regression" if task == "regression" else "label" if set(p) <= {0.0, 1.0}
else "probability" if all(0 <= v <= 1 for v in p) else "score")
score = (lambda a, b: classify(a, b, kind, threshold)) if task == "classification" else regress
primary = "rmse" if task == "regression" else ("f1" if kind == "label" else "auc")
res: Dict[str, Any] = {"kind": kind, "metrics": score(y, p), "primary_metric": primary}
if folds >= 2:
vals = [v for v in (score(y[f::folds], p[f::folds]).get(primary) for f in range(folds))
if v is not None]
res["folds"] = r6({"k": folds, "metric": primary, "values": vals,
"mean": statistics.fmean(vals) if vals else None,
"std": statistics.stdev(vals) if len(vals) > 1 else None})
models[c] = res
cfg = config if config is not None else DEFAULT_CONFIG[task]
known = [k for k in cfg if k in CHECKS or k == "max_fold_std"]
for res in models.values():
checks = []
for key, limit in ((k, cfg[k]) for k in known):
if isinstance(limit, bool) or not isinstance(limit, (int, float)):
fail(f"ERROR: threshold {key} must be a number, got {limit!r}")
value = (res.get("folds", {}).get("std") if key == "max_fold_std"
else res["metrics"].get(CHECKS[key]))
ok = None if value is None else (value >= limit if key.startswith("min_") else value <= limit)
checks.append({"check": key, "limit": limit, "value": value, "pass": ok})
res["checks"] = checks
res["not_applicable"] = [ch["check"] for ch in checks if ch["pass"] is None]
res["ready"] = all(m) if (m := [c["pass"] for c in checks if c["pass"] is not None]) else None
ranked = sorted((n for n in models if models[n]["metrics"].get(models[n]["primary_metric"])
is not None), key=lambda n: models[n]["metrics"][models[n]["primary_metric"]],
reverse=task != "regression")
is_cls = task == "classification"
return {"input": path, "rows": len(rows), "rows_skipped_missing_actual": total - len(rows),
"task": task, "target": target, "positive_class": pos if is_cls else None,
"threshold": threshold if is_cls else None, "thresholds": cfg,
"ignored_config_keys": [k for k in cfg if k not in known], "models": models,
"skipped_columns": skipped, "best_model": ranked[0] if ranked else None,
"note": "Verdicts use only the checks a model can be measured on. Offline check "
"only: latency, error rate and drift need serving metrics."}
def fmt_num(v: Any) -> str:
"""Format a metric value for the text table."""
return "n/a" if v is None else f"{v:.4f}" if isinstance(v, float) else str(v)
def output(rep: Dict[str, Any], fmt: str) -> None:
"""Print the report as JSON or as a side-by-side text table."""
if fmt == "json":
print(json.dumps(rep, indent=2))
return
names = list(rep["models"])
extra = f", positive class '{rep['positive_class']}'" if rep["positive_class"] else ""
if rep["rows_skipped_missing_actual"]:
extra += f", {rep['rows_skipped_missing_actual']} row(s) with no actual skipped"
print(f"Model evaluation: {rep['input']} ({rep['rows']} rows, {rep['task']}{extra})\n")
print(f" {'metric':<18}" + "".join(f"{n:>14}" for n in names))
print(f" {'kind':<18}" + "".join(f"{rep['models'][n]['kind']:>14}" for n in names))
keys = [k for k in ("accuracy", "precision", "recall", "f1", "specificity", "auc", "log_loss",
"brier", "mae", "rmse", "r2", "mape", "mape_rows_skipped", "bias")
if any(k in rep["models"][n]["metrics"] for n in names)]
for k in keys:
print(f" {k:<18}" + "".join(f"{fmt_num(rep['models'][n]['metrics'].get(k)):>14}" for n in names))
for n in names:
cm = rep["models"][n]["metrics"].get("confusion")
if cm:
print(f"\n {n} confusion (threshold {rep['threshold']}): "
+ " ".join(f"{k.upper()} {v}" for k, v in cm.items()))
fo = rep["models"][n].get("folds")
if fo and fo["values"]:
print(f" {n} {fo['k']}-fold {fo['metric']}: mean {fmt_num(fo['mean'])}, std "
f"{fmt_num(fo['std'])} ({', '.join(f'{v:.3f}' for v in fo['values'])})")
if rep["skipped_columns"]:
print("\n Skipped: " + "; ".join(f"'{c}' {w}" for c, w in rep["skipped_columns"].items()))
print(f"\nBest by primary metric: {rep['best_model']}")
print("\nReadiness (offline thresholds):")
for n in names:
res = rep["models"][n]
detail = ", ".join(f"{c['check']} {fmt_num(c['value'])} vs {c['limit']} "
f"{'PASS' if c['pass'] else 'FAIL'}" for c in res["checks"]
if c["pass"] is not None)
if res["not_applicable"]:
detail += f"{'; ' if detail else ''}not applicable: {', '.join(res['not_applicable'])}"
detail += "" if res["ready"] is not None else " (set a measurable one, e.g. min_f1)"
verdict = {True: "READY", False: "NOT READY", None: "NO VERDICT"}[res["ready"]]
print(f" {n:<12} {verdict:<11} {detail or 'no thresholds set (pass --config)'}")
if rep["ignored_config_keys"]:
print(f" Ignored (not measurable from predictions): {', '.join(rep['ignored_config_keys'])}")
print(f" {rep['note']}")
def main() -> None:
"""Parse arguments, evaluate, print, and optionally write the report."""
p = argparse.ArgumentParser(description="Evaluate saved predictions: metrics, folds, readiness.")
p.add_argument("--input", "-i", help="CSV with an actual column and prediction columns")
p.add_argument("--target", "-t", default="actual", help="actual/label column (default actual)")
p.add_argument("--predictions", "-p", help="comma-separated prediction columns (default: all usable)")
p.add_argument("--task", choices=["auto", "classification", "regression"], default="auto",
help="auto: classification when the target has exactly 2 values")
p.add_argument("--positive", help="positive class label (default 1, else the last sorted label)")
p.add_argument("--threshold", type=float, default=0.5, help="probability cut-off (0.5)")
p.add_argument("--folds", "-k", type=int, default=0, help="round-robin folds for stability")
p.add_argument("--config", "-c", help="thresholds JSON, e.g. {\"min_auc\": 0.85}")
p.add_argument("--output", "-o", help="directory to write evaluation_report.json into")
p.add_argument("--force", action="store_true", help="overwrite an existing file in --output")
p.add_argument("--format", choices=["text", "json"], default="text", help="output format")
p.add_argument("--example", action="store_true", help="print a sample predictions CSV and exit")
args = p.parse_args()
if args.example:
print(EXAMPLE)
return
if not args.input:
p.error("--input is required (or use --example)")
config = None
if args.config:
try:
config = json.loads(Path(args.config).read_text(encoding="utf-8"))
except FileNotFoundError:
fail(f"ERROR: file not found: {args.config}")
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
fail(f"ERROR: {args.config} is not valid JSON: {exc}")
if not isinstance(config, dict):
fail(f"ERROR: {args.config} must hold a JSON object of thresholds")
cols = [c.strip() for c in args.predictions.split(",")] if args.predictions else None
target = Path(args.output) / "evaluation_report.json" if args.output else None
if target and target.exists() and not args.force:
fail(f"ERROR: {target} already exists; pass --force to overwrite")
try:
rep = evaluate(args.input, args.target, cols, args.task, args.positive, args.threshold,
args.folds, config)
except OverflowError:
fail(f"ERROR: {args.input} has values too large to score (overflow); rescale them first")
output(rep, args.format)
if target:
try:
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(rep, indent=2) + "\n", encoding="utf-8")
except OSError as exc:
fail(f"ERROR: cannot write {target}: {exc}")
print(f"Wrote {target}", file=sys.stderr)
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |