Skip to content

brycewang-stanford/Auto-Empirical-Research-Skills / skills/23-Learning-Bayesian-Statistics-baygent-skills/bayesian-workflow

bayesian-workflow

Verified

Opinionated Bayesian modeling workflow with PyMC and ArviZ. Contains critical guardrails (nutpie sampler, prior/posterior predictive checks, LOO-PIT calibration, prior sensitivity checks, 94% HDI, non-centered parameterizations, reproducible seeds) that agents won't apply unprompted — always consult before writing Bayesian model code. Trigger on: building probabilistic/Bayesian models, prior elicitation, MCMC inference, convergence diagnostics (divergences, R-hat, ESS), model comparison (LOO-CV, ELPD, stacking weights), hierarchical/multilevel models, count regressions, logistic regression with uncertainty, prior sensitivity analysis, reporting Bayesian results, or mentions of PyMC, ArviZ, InferenceData, credible intervals, posterior distributions, shrinkage, uncertainty quantification. Also trigger for model comparison, diagnosing sampling problems, choosing priors, or presenting stats to non-technical audiences.

FileKindSize
README.mdmarkdown3.9 KB
SKILL.mdskill md13.9 KB
main.pyscript95 B
pyproject.tomlmanifest181 B
references/diagnostics.mdmarkdown5.6 KB
references/hierarchical.mdmarkdown7.8 KB
references/model-comparison.mdmarkdown3.9 KB
references/model-criticism.mdmarkdown11.4 KB
references/priors.mdmarkdown7.8 KB
references/reporting.mdmarkdown8.8 KB
references/sensitivity.mdmarkdown4.5 KB
scripts/calibration_check.pyscript9.8 KB
scripts/diagnose_model.pyscript6.1 KB

scripts/diagnose_model.py

View on GitHub
"""
Automated Bayesian model diagnostics.

Runs convergence checks, posterior predictive checks, and produces
a structured report.

Usage:
    python diagnose_model.py --idata path/to/inference_data.nc
    python diagnose_model.py --idata path/to/inference_data.nc --output report.json
"""

import argparse
import json
import sys
import warnings

import arviz as az
import numpy as np
import pymc as pm

warnings.filterwarnings("ignore", category=FutureWarning)

# arviz_stats >= 1.0.0 provides diagnose() for one-call diagnostics
try:
    import arviz_stats as azs

    HAS_DIAGNOSE = hasattr(azs, "diagnose")
except ImportError:
    HAS_DIAGNOSE = False


def check_convergence(idata):
    """Run all convergence diagnostics. Returns structured results."""
    # Use arviz_stats.diagnose() if available — it covers R-hat, ESS,
    # divergences, tree depth saturation, and E-BFMI in one call.
    if HAS_DIAGNOSE:
        has_errors, diagnostics = azs.diagnose(
            idata, return_diagnostics=True, show_diagnostics=True
        )
        return {
            "all_ok": not has_errors,
            "diagnostics": diagnostics,
            "method": "arviz_stats.diagnose",
        }

    # Fallback for arviz-stats < 1.0.0
    summary = az.summary(idata)
    num_chains = idata.chains.size

    results = {
        "rhat": {
            "max": float(summary["r_hat"].max()),
            "ok": bool((summary["r_hat"] <= 1.01).all()),
            "problematic_params": list(summary[summary["r_hat"] > 1.01].index),
        },
        "ess_bulk": {
            "min": int(summary["ess_bulk"].min()),
            "ok": bool((summary["ess_bulk"] >= 100 * num_chains).all()),
            "problematic_params": list(
                summary[summary["ess_bulk"] < 100 * num_chains].index
            ),
        },
        "ess_tail": {
            "min": int(summary["ess_tail"].min()),
            "ok": bool((summary["ess_tail"] >= 100 * num_chains).all()),
            "problematic_params": list(
                summary[summary["ess_tail"] < 100 * num_chains].index
            ),
        },
    }

    # Divergences
    if "diverging" in idata.sample_stats:
        n_div = int(idata.sample_stats["diverging"].sum())
        total_samples = int(idata.sample_stats["diverging"].size)
        results["divergences"] = {
            "count": n_div,
            "pct": round(100 * n_div / total_samples, 2),
            "ok": n_div == 0,
        }
    else:
        results["divergences"] = {"count": 0, "pct": 0.0, "ok": True}

    results["all_ok"] = all(
        results[k]["ok"] for k in ["rhat", "ess_bulk", "ess_tail", "divergences"]
    )
    results["method"] = "manual"

    return results


def check_loo(idata, model=None):
    """Run LOO-CV and check Pareto k diagnostics."""
    try:
        if "log_likelihood" not in idata.groups():
            if model is not None:
                pm.compute_log_likelihood(idata, model=model)
            else:
                return {"computed": False, "error": "No log_likelihood group and no model provided. Pass --model or call pm.compute_log_likelihood(idata, model=model) before saving."}
        loo = az.loo(idata, pointwise=True)
        pareto_k = loo.pareto_k.values

        results = {
            "elpd": float(loo.elpd_loo),
            "se": float(loo.se),
            "p_loo": float(loo.p_loo),
            "pareto_k": {
                "max": float(pareto_k.max()),
                "n_bad": int(np.sum(pareto_k > 0.7)),
                "n_marginal": int(np.sum((pareto_k > 0.5) & (pareto_k <= 0.7))),
                "ok": bool(np.all(pareto_k <= 0.7)),
            },
            "computed": True,
        }
    except Exception as e:
        results = {"computed": False, "error": str(e)}

    return results


def check_posterior_predictive(idata):
    """Check if posterior predictive data exists and basic stats."""
    if not hasattr(idata, "posterior_predictive"):
        return {
            "available": False,
            "message": "No posterior predictive samples found. Run pm.sample_posterior_predictive().",
        }

    pp_vars = list(idata.posterior_predictive.data_vars)
    results = {"available": True, "variables": pp_vars}

    if hasattr(idata, "observed_data"):
        obs_vars = list(idata.observed_data.data_vars)
        results["observed_variables"] = obs_vars

    return results


def generate_report(idata, model=None):
    """Generate complete diagnostics report."""
    report = {
        "convergence": check_convergence(idata),
        "loo": check_loo(idata, model=model),
        "posterior_predictive": check_posterior_predictive(idata),
    }

    # Overall assessment
    issues = []
    if not report["convergence"]["all_ok"]:
        issues.append("Convergence issues detected — results may be unreliable")
    if report["loo"].get("computed") and not report["loo"]["pareto_k"]["ok"]:
        issues.append(
            f"{report['loo']['pareto_k']['n_bad']} observations with Pareto k > 0.7"
        )
    if not report["posterior_predictive"]["available"]:
        issues.append("No posterior predictive checks available")

    report["overall"] = {
        "ok": len(issues) == 0,
        "issues": issues,
        "recommendation": "Model is ready for interpretation."
        if len(issues) == 0
        else "Address the following issues before interpreting results: "
        + "; ".join(issues),
    }

    return report


def main():
    parser = argparse.ArgumentParser(description="Bayesian model diagnostics")
    parser.add_argument(
        "--idata", required=True, help="Path to InferenceData (.nc file)"
    )
    parser.add_argument(
        "--output",
        default=None,
        help="Path to save JSON report (default: print to stdout)",
    )
    args = parser.parse_args()

    try:
        idata = az.from_netcdf(args.idata)
    except Exception as e:
        print(json.dumps({"error": f"Could not load InferenceData: {e}"}))
        sys.exit(1)

    report = generate_report(idata)

    output = json.dumps(report, indent=2)
    if args.output:
        with open(args.output, "w") as f:
            f.write(output)
        print(f"Report saved to {args.output}")
    else:
        print(output)


if __name__ == "__main__":
    main()
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall