brycewang-stanford/Auto-Empirical-Research-Skills / skills/23-Learning-Bayesian-Statistics-baygent-skills/bayesian-workflow
bayesian-workflow
VerifiedOpinionated Bayesian modeling workflow with PyMC and ArviZ. Contains critical guardrails (nutpie sampler, prior/posterior predictive checks, LOO-PIT calibration, prior sensitivity checks, 94% HDI, non-centered parameterizations, reproducible seeds) that agents won't apply unprompted — always consult before writing Bayesian model code. Trigger on: building probabilistic/Bayesian models, prior elicitation, MCMC inference, convergence diagnostics (divergences, R-hat, ESS), model comparison (LOO-CV, ELPD, stacking weights), hierarchical/multilevel models, count regressions, logistic regression with uncertainty, prior sensitivity analysis, reporting Bayesian results, or mentions of PyMC, ArviZ, InferenceData, credible intervals, posterior distributions, shrinkage, uncertainty quantification. Also trigger for model comparison, diagnosing sampling problems, choosing priors, or presenting stats to non-technical audiences.
| File | Kind | Size |
|---|---|---|
| README.md | markdown | 3.9 KB |
| SKILL.md | skill md | 13.9 KB |
| main.py | script | 95 B |
| pyproject.toml | manifest | 181 B |
| references/diagnostics.md | markdown | 5.6 KB |
| references/hierarchical.md | markdown | 7.8 KB |
| references/model-comparison.md | markdown | 3.9 KB |
| references/model-criticism.md | markdown | 11.4 KB |
| references/priors.md | markdown | 7.8 KB |
| references/reporting.md | markdown | 8.8 KB |
| references/sensitivity.md | markdown | 4.5 KB |
| scripts/calibration_check.py | script | 9.8 KB |
| scripts/diagnose_model.py | script | 6.1 KB |
scripts/diagnose_model.py
View on GitHub"""
Automated Bayesian model diagnostics.
Runs convergence checks, posterior predictive checks, and produces
a structured report.
Usage:
python diagnose_model.py --idata path/to/inference_data.nc
python diagnose_model.py --idata path/to/inference_data.nc --output report.json
"""
import argparse
import json
import sys
import warnings
import arviz as az
import numpy as np
import pymc as pm
warnings.filterwarnings("ignore", category=FutureWarning)
# arviz_stats >= 1.0.0 provides diagnose() for one-call diagnostics
try:
import arviz_stats as azs
HAS_DIAGNOSE = hasattr(azs, "diagnose")
except ImportError:
HAS_DIAGNOSE = False
def check_convergence(idata):
"""Run all convergence diagnostics. Returns structured results."""
# Use arviz_stats.diagnose() if available — it covers R-hat, ESS,
# divergences, tree depth saturation, and E-BFMI in one call.
if HAS_DIAGNOSE:
has_errors, diagnostics = azs.diagnose(
idata, return_diagnostics=True, show_diagnostics=True
)
return {
"all_ok": not has_errors,
"diagnostics": diagnostics,
"method": "arviz_stats.diagnose",
}
# Fallback for arviz-stats < 1.0.0
summary = az.summary(idata)
num_chains = idata.chains.size
results = {
"rhat": {
"max": float(summary["r_hat"].max()),
"ok": bool((summary["r_hat"] <= 1.01).all()),
"problematic_params": list(summary[summary["r_hat"] > 1.01].index),
},
"ess_bulk": {
"min": int(summary["ess_bulk"].min()),
"ok": bool((summary["ess_bulk"] >= 100 * num_chains).all()),
"problematic_params": list(
summary[summary["ess_bulk"] < 100 * num_chains].index
),
},
"ess_tail": {
"min": int(summary["ess_tail"].min()),
"ok": bool((summary["ess_tail"] >= 100 * num_chains).all()),
"problematic_params": list(
summary[summary["ess_tail"] < 100 * num_chains].index
),
},
}
# Divergences
if "diverging" in idata.sample_stats:
n_div = int(idata.sample_stats["diverging"].sum())
total_samples = int(idata.sample_stats["diverging"].size)
results["divergences"] = {
"count": n_div,
"pct": round(100 * n_div / total_samples, 2),
"ok": n_div == 0,
}
else:
results["divergences"] = {"count": 0, "pct": 0.0, "ok": True}
results["all_ok"] = all(
results[k]["ok"] for k in ["rhat", "ess_bulk", "ess_tail", "divergences"]
)
results["method"] = "manual"
return results
def check_loo(idata, model=None):
"""Run LOO-CV and check Pareto k diagnostics."""
try:
if "log_likelihood" not in idata.groups():
if model is not None:
pm.compute_log_likelihood(idata, model=model)
else:
return {"computed": False, "error": "No log_likelihood group and no model provided. Pass --model or call pm.compute_log_likelihood(idata, model=model) before saving."}
loo = az.loo(idata, pointwise=True)
pareto_k = loo.pareto_k.values
results = {
"elpd": float(loo.elpd_loo),
"se": float(loo.se),
"p_loo": float(loo.p_loo),
"pareto_k": {
"max": float(pareto_k.max()),
"n_bad": int(np.sum(pareto_k > 0.7)),
"n_marginal": int(np.sum((pareto_k > 0.5) & (pareto_k <= 0.7))),
"ok": bool(np.all(pareto_k <= 0.7)),
},
"computed": True,
}
except Exception as e:
results = {"computed": False, "error": str(e)}
return results
def check_posterior_predictive(idata):
"""Check if posterior predictive data exists and basic stats."""
if not hasattr(idata, "posterior_predictive"):
return {
"available": False,
"message": "No posterior predictive samples found. Run pm.sample_posterior_predictive().",
}
pp_vars = list(idata.posterior_predictive.data_vars)
results = {"available": True, "variables": pp_vars}
if hasattr(idata, "observed_data"):
obs_vars = list(idata.observed_data.data_vars)
results["observed_variables"] = obs_vars
return results
def generate_report(idata, model=None):
"""Generate complete diagnostics report."""
report = {
"convergence": check_convergence(idata),
"loo": check_loo(idata, model=model),
"posterior_predictive": check_posterior_predictive(idata),
}
# Overall assessment
issues = []
if not report["convergence"]["all_ok"]:
issues.append("Convergence issues detected — results may be unreliable")
if report["loo"].get("computed") and not report["loo"]["pareto_k"]["ok"]:
issues.append(
f"{report['loo']['pareto_k']['n_bad']} observations with Pareto k > 0.7"
)
if not report["posterior_predictive"]["available"]:
issues.append("No posterior predictive checks available")
report["overall"] = {
"ok": len(issues) == 0,
"issues": issues,
"recommendation": "Model is ready for interpretation."
if len(issues) == 0
else "Address the following issues before interpreting results: "
+ "; ".join(issues),
}
return report
def main():
parser = argparse.ArgumentParser(description="Bayesian model diagnostics")
parser.add_argument(
"--idata", required=True, help="Path to InferenceData (.nc file)"
)
parser.add_argument(
"--output",
default=None,
help="Path to save JSON report (default: print to stdout)",
)
args = parser.parse_args()
try:
idata = az.from_netcdf(args.idata)
except Exception as e:
print(json.dumps({"error": f"Could not load InferenceData: {e}"}))
sys.exit(1)
report = generate_report(idata)
output = json.dumps(report, indent=2)
if args.output:
with open(args.output, "w") as f:
f.write(output)
print(f"Report saved to {args.output}")
else:
print(output)
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |