wandb/senpai / plugins/senpai/skills/wandb-primary
wandb-primary
VerifiedComprehensive primary skill for agents working with Weights & Biases. Covers both the W&B SDK (training runs, metrics, artifacts, sweeps) and the Weave SDK (GenAI traces, evaluations, scorers). Includes helper libraries, gotcha tables, and data analysis patterns. Use this skill whenever the user asks about W&B runs, Weave traces, evaluations, training metrics, loss curves, model comparisons, or any Weights & Biases data — even if they don't say "W&B" explicitly.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 16.6 KB |
| references/TRAINING_DIAGNOSTICS.md | markdown | 6.6 KB |
| references/WANDB_SDK.md | markdown | 12.9 KB |
| references/WEAVE_SDK.md | markdown | 6.6 KB |
| scripts/curve_plots.py | script | 16.1 KB |
| scripts/step_axis.py | script | 5.8 KB |
| scripts/training_diagnostics.py | script | 18.6 KB |
| scripts/wandb_helpers.py | script | 7.7 KB |
| scripts/weave_helpers.py | script | 12.4 KB |
scripts/wandb_helpers.py
View on GitHub# SPDX-FileCopyrightText: 2026 CoreWeave, Inc.
# SPDX-License-Identifier: Apache-2.0
# SPDX-PackageName: skills
"""Helpers for working with W&B (Weights & Biases) training data.
The wandb SDK is for the W&B Models product — training runs, metrics history,
hyperparameter sweeps, artifacts, and system metrics. These helpers convert
run data into pandas-friendly structures for analysis.
Usage (in sandbox):
import os
import sys
sys.path.insert(0, f"{os.environ['SENPAI_PLUGIN']}/skills/wandb-primary/scripts")
from wandb_helpers import (
runs_to_dataframe, # Convert runs to a clean pandas DataFrame
diagnose_run, # Quick diagnostic summary of a training run
compare_configs, # Side-by-side config diff between two runs
)
"""
from __future__ import annotations
from typing import Any, Callable, Iterator
# ---------------------------------------------------------------------------
# Fast history scan (beta_scan_history with fallback)
# ---------------------------------------------------------------------------
def _scan_history_kwargs(
keys: list[str] | None,
min_step: int,
max_step: int | None,
) -> dict[str, Any]:
kwargs: dict[str, Any] = {}
if keys is not None:
kwargs["keys"] = keys
if min_step:
kwargs["min_step"] = min_step
if max_step is not None:
kwargs["max_step"] = max_step
return kwargs
def fast_scan_history(
run: Any,
keys: list[str] | None = None,
page_size: int = 1000,
min_step: int = 0,
max_step: int | None = None,
) -> Iterator[dict[str, Any]]:
"""Iterate a run's full history using the fastest available method.
Prefers `run.beta_scan_history` (reads locally-cached parquet, no API
round-trips) and falls back to `run.scan_history` if the beta API isn't
present or raises. Signature matches the common subset of both.
W&B marks `beta_scan_history` as "still in development" — the fallback
exists so callers don't have to branch.
Args:
run: A W&B Run object.
keys: Metrics to fetch; None returns all.
page_size: Rows per batch (beta only).
min_step: Inclusive lower bound.
max_step: Exclusive upper bound.
Yields:
dict rows, same shape as `scan_history`.
"""
fallback_kwargs = _scan_history_kwargs(keys, min_step, max_step)
beta = getattr(run, "beta_scan_history", None)
if beta is not None:
yielded = 0
try:
for row in beta(
keys=keys,
page_size=page_size,
min_step=min_step,
max_step=max_step,
):
yielded += 1
yield row
return
except Exception:
if yielded:
skipped = 0
for row in run.scan_history(**fallback_kwargs):
if skipped < yielded:
skipped += 1
continue
yield row
return
yield from run.scan_history(**fallback_kwargs)
def discover_history_keys(
run: Any,
predicate: Callable[[str, Any], bool],
max_rows: int = 500,
) -> list[str]:
"""Find history keys that may appear only after sparse logging intervals."""
found: set[str] = set()
for i, row in enumerate(fast_scan_history(run)):
for key, value in row.items():
if predicate(key, value):
found.add(key)
if found or i + 1 >= max_rows:
break
return sorted(found)
# ---------------------------------------------------------------------------
# Runs -> DataFrame
# ---------------------------------------------------------------------------
def runs_to_dataframe(
runs: Any,
limit: int = 200,
metric_keys: list[str] | None = None,
) -> list[dict[str, Any]]:
"""Convert W&B runs to a list of flat dicts (ready for pd.DataFrame).
Always slices runs to avoid loading entire projects into memory.
Args:
runs: W&B Runs object from api.runs().
limit: Max runs to process (default 200).
metric_keys: Summary metric keys to include. If None, includes
"loss", "val_loss", "accuracy".
Returns:
List of dicts with run metadata + config + selected metrics.
"""
if metric_keys is None:
metric_keys = ["loss", "val_loss", "accuracy"]
rows = []
for run in runs[:limit]:
row = {
"id": run.id,
"name": run.name,
"state": run.state,
"created_at": run.created_at,
}
# Config (skip internal keys)
for k, v in run.config.items():
if not k.startswith("_"):
row[f"config.{k}"] = v
# Summary metrics
for key in metric_keys:
row[key] = run.summary_metrics.get(key)
rows.append(row)
return rows
# ---------------------------------------------------------------------------
# Run diagnostics
# ---------------------------------------------------------------------------
def diagnose_run(run: Any) -> dict[str, Any]:
"""Quick diagnostic summary of a training run.
Loads the full loss history and checks for convergence, overfitting,
NaN values, and other common training issues.
Args:
run: A W&B Run object from api.run().
Returns:
Dict with diagnostic keys: total_steps, final_loss, min_loss,
min_loss_step, has_nan, final_10pct_mean, train_val_gap,
likely_overfit, converged.
"""
import pandas as pd
df = pd.DataFrame(list(run.scan_history(keys=["loss", "val_loss"])))
loss = df["loss"].dropna()
diagnostics: dict[str, Any] = {
"total_steps": len(loss),
"final_loss": loss.iloc[-1] if len(loss) else None,
"min_loss": loss.min() if len(loss) else None,
"min_loss_step": int(loss.idxmin()) if len(loss) else None,
"has_nan": bool(loss.isna().any()),
"final_10pct_mean": float(loss.tail(max(1, len(loss) // 10)).mean())
if len(loss)
else None,
}
# Overfitting check (val_loss diverging from train loss)
if "val_loss" in df.columns:
val = df["val_loss"].dropna()
if len(val) > 10:
tail_size = max(1, len(val) // 5)
train_tail = float(loss.tail(tail_size).mean())
val_tail = float(val.tail(tail_size).mean())
diagnostics["train_val_gap"] = round(val_tail - train_tail, 6)
diagnostics["likely_overfit"] = val_tail > train_tail * 1.2
# Convergence check
if len(loss) > 100:
last_pct = loss.tail(max(1, len(loss) // 10))
diagnostics["converged"] = bool(last_pct.std() < last_pct.mean() * 0.01)
return diagnostics
# ---------------------------------------------------------------------------
# Config comparison
# ---------------------------------------------------------------------------
def compare_configs(run_a: Any, run_b: Any) -> list[dict[str, Any]]:
"""Side-by-side config comparison between two W&B runs.
Returns only the keys that differ between the two runs.
Args:
run_a: First W&B Run object.
run_b: Second W&B Run object.
Returns:
List of dicts with: key, run_a_name, run_a_value, run_b_name, run_b_value
"""
config_a = {k: v for k, v in run_a.config.items() if not k.startswith("_")}
config_b = {k: v for k, v in run_b.config.items() if not k.startswith("_")}
all_keys = sorted(set(config_a) | set(config_b))
diffs = []
for k in all_keys:
val_a = config_a.get(k)
val_b = config_b.get(k)
if val_a != val_b:
diffs.append({
"key": k,
run_a.name: val_a,
run_b.name: val_b,
})
return diffs
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |