Skip to content

wandb/senpai / plugins/senpai/skills/wandb-primary

wandb-primary

Verified

Comprehensive primary skill for agents working with Weights & Biases. Covers both the W&B SDK (training runs, metrics, artifacts, sweeps) and the Weave SDK (GenAI traces, evaluations, scorers). Includes helper libraries, gotcha tables, and data analysis patterns. Use this skill whenever the user asks about W&B runs, Weave traces, evaluations, training metrics, loss curves, model comparisons, or any Weights & Biases data — even if they don't say "W&B" explicitly.

scripts/wandb_helpers.py

View on GitHub
# SPDX-FileCopyrightText: 2026 CoreWeave, Inc.
# SPDX-License-Identifier: Apache-2.0
# SPDX-PackageName: skills

"""Helpers for working with W&B (Weights & Biases) training data.

The wandb SDK is for the W&B Models product — training runs, metrics history,
hyperparameter sweeps, artifacts, and system metrics. These helpers convert
run data into pandas-friendly structures for analysis.

Usage (in sandbox):
    import os
    import sys
    sys.path.insert(0, f"{os.environ['SENPAI_PLUGIN']}/skills/wandb-primary/scripts")
    from wandb_helpers import (
        runs_to_dataframe,   # Convert runs to a clean pandas DataFrame
        diagnose_run,        # Quick diagnostic summary of a training run
        compare_configs,     # Side-by-side config diff between two runs
    )
"""

from __future__ import annotations

from typing import Any, Callable, Iterator


# ---------------------------------------------------------------------------
# Fast history scan (beta_scan_history with fallback)
# ---------------------------------------------------------------------------

def _scan_history_kwargs(
    keys: list[str] | None,
    min_step: int,
    max_step: int | None,
) -> dict[str, Any]:
    kwargs: dict[str, Any] = {}
    if keys is not None:
        kwargs["keys"] = keys
    if min_step:
        kwargs["min_step"] = min_step
    if max_step is not None:
        kwargs["max_step"] = max_step
    return kwargs

def fast_scan_history(
    run: Any,
    keys: list[str] | None = None,
    page_size: int = 1000,
    min_step: int = 0,
    max_step: int | None = None,
) -> Iterator[dict[str, Any]]:
    """Iterate a run's full history using the fastest available method.

    Prefers `run.beta_scan_history` (reads locally-cached parquet, no API
    round-trips) and falls back to `run.scan_history` if the beta API isn't
    present or raises. Signature matches the common subset of both.

    W&B marks `beta_scan_history` as "still in development" — the fallback
    exists so callers don't have to branch.

    Args:
        run: A W&B Run object.
        keys: Metrics to fetch; None returns all.
        page_size: Rows per batch (beta only).
        min_step: Inclusive lower bound.
        max_step: Exclusive upper bound.

    Yields:
        dict rows, same shape as `scan_history`.
    """
    fallback_kwargs = _scan_history_kwargs(keys, min_step, max_step)
    beta = getattr(run, "beta_scan_history", None)
    if beta is not None:
        yielded = 0
        try:
            for row in beta(
                keys=keys,
                page_size=page_size,
                min_step=min_step,
                max_step=max_step,
            ):
                yielded += 1
                yield row
            return
        except Exception:
            if yielded:
                skipped = 0
                for row in run.scan_history(**fallback_kwargs):
                    if skipped < yielded:
                        skipped += 1
                        continue
                    yield row
                return
    yield from run.scan_history(**fallback_kwargs)


def discover_history_keys(
    run: Any,
    predicate: Callable[[str, Any], bool],
    max_rows: int = 500,
) -> list[str]:
    """Find history keys that may appear only after sparse logging intervals."""
    found: set[str] = set()
    for i, row in enumerate(fast_scan_history(run)):
        for key, value in row.items():
            if predicate(key, value):
                found.add(key)
        if found or i + 1 >= max_rows:
            break
    return sorted(found)


# ---------------------------------------------------------------------------
# Runs -> DataFrame
# ---------------------------------------------------------------------------

def runs_to_dataframe(
    runs: Any,
    limit: int = 200,
    metric_keys: list[str] | None = None,
) -> list[dict[str, Any]]:
    """Convert W&B runs to a list of flat dicts (ready for pd.DataFrame).

    Always slices runs to avoid loading entire projects into memory.

    Args:
        runs: W&B Runs object from api.runs().
        limit: Max runs to process (default 200).
        metric_keys: Summary metric keys to include. If None, includes
                     "loss", "val_loss", "accuracy".

    Returns:
        List of dicts with run metadata + config + selected metrics.
    """
    if metric_keys is None:
        metric_keys = ["loss", "val_loss", "accuracy"]

    rows = []
    for run in runs[:limit]:
        row = {
            "id": run.id,
            "name": run.name,
            "state": run.state,
            "created_at": run.created_at,
        }
        # Config (skip internal keys)
        for k, v in run.config.items():
            if not k.startswith("_"):
                row[f"config.{k}"] = v
        # Summary metrics
        for key in metric_keys:
            row[key] = run.summary_metrics.get(key)
        rows.append(row)
    return rows


# ---------------------------------------------------------------------------
# Run diagnostics
# ---------------------------------------------------------------------------

def diagnose_run(run: Any) -> dict[str, Any]:
    """Quick diagnostic summary of a training run.

    Loads the full loss history and checks for convergence, overfitting,
    NaN values, and other common training issues.

    Args:
        run: A W&B Run object from api.run().

    Returns:
        Dict with diagnostic keys: total_steps, final_loss, min_loss,
        min_loss_step, has_nan, final_10pct_mean, train_val_gap,
        likely_overfit, converged.
    """
    import pandas as pd

    df = pd.DataFrame(list(run.scan_history(keys=["loss", "val_loss"])))
    loss = df["loss"].dropna()

    diagnostics: dict[str, Any] = {
        "total_steps": len(loss),
        "final_loss": loss.iloc[-1] if len(loss) else None,
        "min_loss": loss.min() if len(loss) else None,
        "min_loss_step": int(loss.idxmin()) if len(loss) else None,
        "has_nan": bool(loss.isna().any()),
        "final_10pct_mean": float(loss.tail(max(1, len(loss) // 10)).mean())
        if len(loss)
        else None,
    }

    # Overfitting check (val_loss diverging from train loss)
    if "val_loss" in df.columns:
        val = df["val_loss"].dropna()
        if len(val) > 10:
            tail_size = max(1, len(val) // 5)
            train_tail = float(loss.tail(tail_size).mean())
            val_tail = float(val.tail(tail_size).mean())
            diagnostics["train_val_gap"] = round(val_tail - train_tail, 6)
            diagnostics["likely_overfit"] = val_tail > train_tail * 1.2

    # Convergence check
    if len(loss) > 100:
        last_pct = loss.tail(max(1, len(loss) // 10))
        diagnostics["converged"] = bool(last_pct.std() < last_pct.mean() * 0.01)

    return diagnostics


# ---------------------------------------------------------------------------
# Config comparison
# ---------------------------------------------------------------------------

def compare_configs(run_a: Any, run_b: Any) -> list[dict[str, Any]]:
    """Side-by-side config comparison between two W&B runs.

    Returns only the keys that differ between the two runs.

    Args:
        run_a: First W&B Run object.
        run_b: Second W&B Run object.

    Returns:
        List of dicts with: key, run_a_name, run_a_value, run_b_name, run_b_value
    """
    config_a = {k: v for k, v in run_a.config.items() if not k.startswith("_")}
    config_b = {k: v for k, v in run_b.config.items() if not k.startswith("_")}

    all_keys = sorted(set(config_a) | set(config_b))
    diffs = []
    for k in all_keys:
        val_a = config_a.get(k)
        val_b = config_b.get(k)
        if val_a != val_b:
            diffs.append({
                "key": k,
                run_a.name: val_a,
                run_b.name: val_b,
            })
    return diffs
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall