borghei/Claude-Skills / personal-productivity/reflect
reflect
VerifiedTurn reflection into decisions by scoring predictions against outcomes and tracking whether commitments held. Use when running a quarterly review, scoring forecast calibration, or reflection keeps producing notes instead of change.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 15.5 KB |
| assets/prediction-log-template.md | markdown | 3.3 KB |
| assets/quarterly-reflection-template.md | markdown | 3.6 KB |
| assets/sample_commitments.json | config | 1.3 KB |
| assets/sample_commitments_quarterly.json | config | 2.5 KB |
| assets/sample_predictions.json | config | 4.7 KB |
| references/calibration-and-forecasting.md | markdown | 18.7 KB |
| references/reflection-cadences.md | markdown | 17.5 KB |
| scripts/calibration_core.py | script | 7.4 KB |
| scripts/calibration_scorer.py | script | 9.0 KB |
| scripts/reflection_prompt_generator.py | script | 11.0 KB |
scripts/calibration_core.py
View on GitHub#!/usr/bin/env python3
"""Scoring internals for calibration_scorer.py — loading, statistics, reading.
Everything here turns raw prediction records into numbers, plus the thresholds
that say when a number means something. The companion CLI (calibration_scorer.py)
orchestrates these, renders them, and verifies them. Helper module: imported by
its sibling, not run directly.
Deterministic: no clock reads, no network, no randomness.
"""
import json
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
# A gap this large between stated confidence and observed hit rate is a real bias,
# not sampling noise, once a bucket holds enough predictions.
MEANINGFUL_GAP = 0.10
MIN_BUCKET_N = 5
# Below this many resolved predictions, calibration estimates are too noisy to act on.
MIN_TOTAL_N = 20
def load_predictions(path: Path) -> List[Dict[str, Any]]:
"""Load prediction records from a JSON list or an object with a 'predictions' key."""
data = json.loads(path.read_text(encoding="utf-8"))
if isinstance(data, dict):
if "predictions" not in data:
raise ValueError(
f"{path.name} is a JSON object with no 'predictions' key "
f"(found: {', '.join(sorted(data)) or 'nothing'}) — expected a prediction log")
preds = data["predictions"]
else:
preds = data
if not isinstance(preds, list):
raise ValueError("log must be a JSON list or an object with a 'predictions' list")
if not preds:
raise ValueError("prediction log is empty — nothing to score")
return preds
def normalise(pred: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Validate one prediction; return None when it is not yet resolved."""
if "confidence" not in pred:
raise KeyError(f"prediction missing 'confidence': {pred}")
confidence = float(pred["confidence"])
if confidence > 1:
confidence /= 100.0
if not 0.0 <= confidence <= 1.0:
raise ValueError(f"confidence must be 0-1 or 0-100: {pred}")
outcome = pred.get("outcome")
if outcome is None or (isinstance(outcome, str) and outcome.lower() in {"unresolved", "pending", ""}):
return None
if isinstance(outcome, str):
outcome = outcome.lower() in {"true", "yes", "hit", "correct", "1"}
return {
"id": str(pred.get("id", "?")),
"statement": str(pred.get("statement", "")),
"domain": str(pred.get("domain", "general")),
"confidence": confidence,
"outcome": 1.0 if outcome else 0.0,
# Unrounded: this feeds the aggregate Brier and therefore the identity.
"brier": (confidence - (1.0 if outcome else 0.0)) ** 2,
}
def bucket_index(confidence: float, buckets: int) -> int:
"""Map a confidence value to a bucket index, clamping the top edge."""
return min(int(confidence * buckets), buckets - 1)
def calibration_table(preds: List[Dict[str, Any]], buckets: int) -> List[Dict[str, Any]]:
"""Group predictions by stated confidence and compare to observed hit rate."""
grouped: Dict[int, List[Dict[str, Any]]] = {}
for pred in preds:
grouped.setdefault(bucket_index(pred["confidence"], buckets), []).append(pred)
rows: List[Dict[str, Any]] = []
width = 1.0 / buckets
for index in sorted(grouped):
group = grouped[index]
mean_conf = sum(p["confidence"] for p in group) / len(group)
observed = sum(p["outcome"] for p in group) / len(group)
gap = mean_conf - observed
rows.append({
"range": f"{index * width:.0%}-{(index + 1) * width:.0%}",
"n": len(group),
"mean_confidence": round(mean_conf, 3),
"observed_rate": round(observed, 3),
"gap": round(gap, 3),
"verdict": bucket_verdict(gap, len(group)),
})
return rows
def bucket_verdict(gap: float, n: int) -> str:
"""Label a bucket as over/under-confident, calibrated, or too small to judge."""
if n < MIN_BUCKET_N:
return "too-few"
if gap > MEANINGFUL_GAP:
return "overconfident"
if gap < -MEANINGFUL_GAP:
return "underconfident"
return "calibrated"
def murphy_decomposition(preds: List[Dict[str, Any]]) -> Tuple[float, float, float]:
"""Split the Brier score into reliability, resolution, and uncertainty.
Groups by distinct forecast value, not by display bucket. The identity
BS = REL - RES + UNC holds exactly only under a partition by distinct
confidence; coarser bins discard within-bin variance and the identity
stops closing. This is deliberately independent of --buckets, so a
readability setting cannot perturb the statistics.
"""
total = len(preds)
base_rate = sum(p["outcome"] for p in preds) / total
grouped: Dict[float, List[float]] = {}
for pred in preds:
grouped.setdefault(pred["confidence"], []).append(pred["outcome"])
reliability = 0.0
resolution = 0.0
for confidence, outcomes in grouped.items():
weight = len(outcomes) / total
observed = sum(outcomes) / len(outcomes)
reliability += weight * (confidence - observed) ** 2
resolution += weight * (observed - base_rate) ** 2
return reliability, resolution, base_rate * (1 - base_rate)
def domain_breakdown(preds: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Brier score and confidence gap per prediction domain."""
grouped: Dict[str, List[Dict[str, Any]]] = {}
for pred in preds:
grouped.setdefault(pred["domain"], []).append(pred)
rows = []
for domain, group in grouped.items():
mean_conf = sum(p["confidence"] for p in group) / len(group)
observed = sum(p["outcome"] for p in group) / len(group)
rows.append({
"domain": domain,
"n": len(group),
"brier": round(sum(p["brier"] for p in group) / len(group), 4),
"gap": round(mean_conf - observed, 3),
})
return sorted(rows, key=lambda r: -r["brier"])
def interpret(brier: float, gap: float, total: int, skill: float) -> List[str]:
"""Turn the numbers into statements about what to change."""
notes: List[str] = []
if total < MIN_TOTAL_N:
notes.append(
f"Only {total} resolved predictions — below {MIN_TOTAL_N}, calibration estimates are "
"too noisy to act on. Keep logging before drawing conclusions.")
if gap > MEANINGFUL_GAP:
notes.append(
f"Overconfident by {gap:.0%} overall. The standard correction is to shade every stated "
"confidence toward 50% by roughly that margin, then re-measure in a quarter.")
elif gap < -MEANINGFUL_GAP:
notes.append(
f"Underconfident by {abs(gap):.0%} overall — you know more than you claim. This costs "
"you decisiveness; state the confidence you actually hold.")
else:
notes.append(f"Well calibrated overall (gap {gap:+.0%}). Stated confidence can be trusted as an input to decisions.")
if skill <= 0:
notes.append(
"Skill score at or below zero: these forecasts carry no more information than always "
"predicting the base rate. Check whether the predictions are specific enough to be wrong.")
elif skill < 0.15:
notes.append(f"Skill score {skill:.2f} — barely beating the base rate. Predictions may be too safe to be useful.")
if brier < 0.15:
notes.append(f"Brier {brier:.3f} is strong; the main gains left are in resolution, not reliability.")
return notes
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| receiving-code-reviewobra/superpowersUse when receiving code review feedback, before implementing suggestions, especially if feedback seems unclear or technically questionable - requires technical rigor and verification, not performative agreement or blind implementation | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| requesting-code-reviewobra/superpowersUse when completing tasks, implementing major features, or before merging to verify work meets requirements | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| using-git-worktreesobra/superpowersUse when starting feature work that needs isolation from current workspace or before executing implementation plans - ensures an isolated workspace exists via native tools or git worktree fallback | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| verification-before-completionobra/superpowersUse when about to claim work is complete, fixed, or passing, before committing or creating PRs - requires running verification commands and confirming output before making any success claims; evidence before assertions always | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |