borghei/Claude-Skills / marketing/ab-test-setup
ab-test-setup
VerifiedDesign and analyze A/B tests: sample size, test duration, and statistical significance for conversion experiments. Use when setting up an A/B test, calculating sample size, designing an experiment, or analyzing results.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 6.6 KB |
| examples/test_results.csv | other | 3.7 KB |
| references/ab-testing-guide.md | markdown | 5.5 KB |
| scripts/results_analyzer.py | script | 11.7 KB |
| scripts/sample_size_calculator.py | script | 10.8 KB |
| scripts/test_designer.py | script | 12.7 KB |
scripts/results_analyzer.py
View on GitHub#!/usr/bin/env python3
"""
A/B Test Results Analyzer
Analyzes A/B test results with statistical significance testing, confidence
intervals, effect size calculation, and actionable recommendations.
Supports two-proportion z-tests for conversion rate experiments.
Expected JSON input: {"test_name", "variants": {"control": {"visitors", "conversions"},
"treatment": {"visitors", "conversions"}}, "significance_level"}
Usage:
python results_analyzer.py results.json
python results_analyzer.py results.json --format json
python results_analyzer.py results.json --batch
python results_analyzer.py results.json --bayesian
"""
import argparse
import json
import math
import sys
from typing import Any, Dict, List, Optional, Tuple
def norm_ppf(p: float) -> float:
"""Inverse normal CDF using rational approximation."""
if p <= 0 or p >= 1:
raise ValueError("p must be between 0 and 1")
if p < 0.5:
return -norm_ppf(1 - p)
t = math.sqrt(-2 * math.log(1 - p))
c0, c1, c2 = 2.515517, 0.802853, 0.010328
d1, d2, d3 = 1.432788, 0.189269, 0.001308
return t - (c0 + c1 * t + c2 * t**2) / (1 + d1 * t + d2 * t**2 + d3 * t**3)
def norm_cdf(x: float) -> float:
"""Standard normal CDF."""
return 0.5 * (1 + math.erf(x / math.sqrt(2)))
def two_proportion_z_test(
n1: int, x1: int, n2: int, x2: int
) -> Dict[str, float]:
"""
Perform a two-proportion z-test.
Args:
n1: Sample size of control
x1: Conversions in control
n2: Sample size of treatment
x2: Conversions in treatment
Returns:
z_statistic, p_value (two-sided)
"""
p1 = x1 / n1 if n1 > 0 else 0
p2 = x2 / n2 if n2 > 0 else 0
p_pooled = (x1 + x2) / (n1 + n2) if (n1 + n2) > 0 else 0
se = math.sqrt(p_pooled * (1 - p_pooled) * (1/n1 + 1/n2)) if n1 > 0 and n2 > 0 else 0
if se == 0:
return {"z_statistic": 0, "p_value": 1.0, "se": 0}
z = (p2 - p1) / se
p_value = 2 * (1 - norm_cdf(abs(z)))
return {"z_statistic": round(z, 4), "p_value": round(p_value, 6), "se": round(se, 6)}
def confidence_interval(
n1: int, x1: int, n2: int, x2: int, alpha: float = 0.05
) -> Dict[str, float]:
"""Calculate confidence interval for difference in proportions."""
p1 = x1 / n1 if n1 > 0 else 0
p2 = x2 / n2 if n2 > 0 else 0
diff = p2 - p1
se = math.sqrt(p1 * (1 - p1) / n1 + p2 * (1 - p2) / n2) if n1 > 0 and n2 > 0 else 0
z = norm_ppf(1 - alpha / 2)
margin = z * se
return {
"point_estimate": round(diff, 6),
"lower_bound": round(diff - margin, 6),
"upper_bound": round(diff + margin, 6),
"margin_of_error": round(margin, 6),
"confidence_level": round((1 - alpha) * 100, 0),
}
def cohens_h(p1: float, p2: float) -> float:
"""Calculate Cohen's h effect size for proportions."""
return 2 * math.asin(math.sqrt(p2)) - 2 * math.asin(math.sqrt(p1))
def bayesian_analysis(n1: int, x1: int, n2: int, x2: int,
n_samples: int = 100000) -> Dict[str, Any]:
"""Simple Bayesian analysis using Beta-Binomial model with grid approximation.
Uses a uniform Beta(1,1) prior."""
# Beta posterior parameters
alpha1, beta1 = x1 + 1, n1 - x1 + 1
alpha2, beta2 = x2 + 1, n2 - x2 + 1
# Compute probability that treatment > control using numerical integration
# Grid-based approximation
grid_size = 1000
prob_b_better = 0.0
step = 1.0 / grid_size
for i in range(grid_size):
p = (i + 0.5) * step
# Beta PDF approximation using Stirling's for large factorials
# Use log-space for numerical stability
# For simplicity, use the analytical result for Beta distributions
pass
# Simpler approximation: normal approximation to beta posterior
mean1 = alpha1 / (alpha1 + beta1)
var1 = (alpha1 * beta1) / ((alpha1 + beta1) ** 2 * (alpha1 + beta1 + 1))
mean2 = alpha2 / (alpha2 + beta2)
var2 = (alpha2 * beta2) / ((alpha2 + beta2) ** 2 * (alpha2 + beta2 + 1))
diff_mean = mean2 - mean1
diff_std = math.sqrt(var1 + var2)
if diff_std > 0:
prob_b_better = norm_cdf(diff_mean / diff_std)
else:
prob_b_better = 0.5
# Credible interval for the difference
z95 = 1.96
ci_lower = diff_mean - z95 * diff_std
ci_upper = diff_mean + z95 * diff_std
return {
"probability_treatment_better": round(prob_b_better, 4),
"expected_difference": round(diff_mean, 6),
"credible_interval_95": {
"lower": round(ci_lower, 6),
"upper": round(ci_upper, 6),
},
"control_posterior_mean": round(mean1, 6),
"treatment_posterior_mean": round(mean2, 6),
}
def analyze_test(data: Dict[str, Any], include_bayesian: bool = False) -> Dict[str, Any]:
"""Analyze a single A/B test."""
test_name = data.get("test_name", "Unnamed Test")
metric = data.get("metric", "conversion_rate")
alpha = data.get("significance_level", 0.05)
variants = data.get("variants", {})
# Support both dict and list format for variants
if isinstance(variants, list):
if len(variants) >= 2:
control = variants[0]
treatment = variants[1]
else:
return {"error": "Need at least 2 variants"}
else:
control = variants.get("control", {})
treatment = variants.get("treatment", {})
n1 = control.get("visitors", 0)
x1 = control.get("conversions", 0)
n2 = treatment.get("visitors", 0)
x2 = treatment.get("conversions", 0)
if n1 == 0 or n2 == 0:
return {"error": "Both variants must have visitors > 0", "test_name": test_name}
p1 = x1 / n1
p2 = x2 / n2
relative_change = (p2 - p1) / p1 if p1 > 0 else 0
# Statistical test
z_test = two_proportion_z_test(n1, x1, n2, x2)
ci = confidence_interval(n1, x1, n2, x2, alpha)
effect_size = cohens_h(p1, p2)
# Significance determination
is_significant = z_test["p_value"] < alpha
direction = "positive" if p2 > p1 else "negative" if p2 < p1 else "neutral"
# Effect size interpretation
abs_h = abs(effect_size)
if abs_h < 0.2:
effect_label = "negligible"
elif abs_h < 0.5:
effect_label = "small"
elif abs_h < 0.8:
effect_label = "medium"
else:
effect_label = "large"
# Recommendation
if is_significant and direction == "positive":
recommendation = "SHIP - Statistically significant positive result"
confidence = "high"
elif is_significant and direction == "negative":
recommendation = "REVERT - Statistically significant negative result"
confidence = "high"
elif not is_significant and abs(relative_change) > 0.05:
recommendation = "EXTEND - Trending but not yet significant. Consider running longer."
confidence = "low"
else:
recommendation = "NO EFFECT - No meaningful difference detected"
confidence = "medium"
result = {
"test_name": test_name,
"metric": metric,
"control": {
"visitors": n1,
"conversions": x1,
"rate": round(p1, 6),
"rate_pct": round(p1 * 100, 3),
},
"treatment": {
"visitors": n2,
"conversions": x2,
"rate": round(p2, 6),
"rate_pct": round(p2 * 100, 3),
},
"difference": {
"absolute": round(p2 - p1, 6),
"absolute_pct": round((p2 - p1) * 100, 3),
"relative_pct": round(relative_change * 100, 2),
"direction": direction,
},
"statistical_test": {
"method": "Two-proportion z-test (two-sided)",
"z_statistic": z_test["z_statistic"],
"p_value": z_test["p_value"],
"significance_level": alpha,
"is_significant": is_significant,
},
"confidence_interval": ci,
"effect_size": {
"cohens_h": round(effect_size, 4),
"interpretation": effect_label,
},
"recommendation": recommendation,
"confidence": confidence,
}
if include_bayesian:
result["bayesian"] = bayesian_analysis(n1, x1, n2, x2)
return result
def print_human(results: List[Dict[str, Any]]) -> None:
"""Print analysis in human-readable format."""
for r in results:
if "error" in r:
print(f"\n Error in {r.get('test_name', 'unknown')}: {r['error']}")
continue
print("=" * 65)
print(f" A/B Test Results: {r['test_name']}")
print("=" * 65)
c = r["control"]
t = r["treatment"]
print(f"\n --- Variant Performance ({r['metric']}) ---")
print(f" {'Variant':<12} {'Visitors':>10} {'Conversions':>12} {'Rate':>10}")
print(f" {'-'*12} {'-'*10} {'-'*12} {'-'*10}")
print(f" {'Control':<12} {c['visitors']:>10,} {c['conversions']:>12,} {c['rate_pct']:>9.3f}%")
print(f" {'Treatment':<12} {t['visitors']:>10,} {t['conversions']:>12,} {t['rate_pct']:>9.3f}%")
d = r["difference"]
sign = "+" if d["relative_pct"] >= 0 else ""
print(f"\n --- Effect ---")
print(f" Absolute Change: {sign}{d['absolute_pct']:.3f}pp")
print(f" Relative Change: {sign}{d['relative_pct']:.2f}%")
print(f" Direction: {d['direction']}")
s = r["statistical_test"]
print(f"\n --- Statistical Significance ---")
print(f" Z-statistic: {s['z_statistic']:.4f}")
print(f" P-value: {s['p_value']:.6f}")
print(f" Alpha: {s['significance_level']}")
sig_label = "YES - Statistically significant" if s["is_significant"] else "NO - Not statistically significant"
print(f" Significant: {sig_label}")
ci = r["confidence_interval"]
print(f"\n --- {ci['confidence_level']:.0f}% Confidence Interval ---")
print(f" [{ci['lower_bound']*100:+.3f}pp, {ci['upper_bound']*100:+.3f}pp]")
print(f" Margin of Error: +/-{ci['margin_of_error']*100:.3f}pp")
e = r["effect_size"]
print(f"\n --- Effect Size ---")
print(f" Cohen's h: {e['cohens_h']:.4f} ({e['interpretation']})")
if "bayesian" in r:
b = r["bayesian"]
print(f"\n --- Bayesian Analysis ---")
print(f" P(Treatment > Control): {b['probability_treatment_better']*100:.1f}%")
print(f" Expected Difference: {b['expected_difference']*100:.3f}pp")
ci_b = b["credible_interval_95"]
print(f" 95% Credible Interval: [{ci_b['lower']*100:.3f}pp, {ci_b['upper']*100:.3f}pp]")
print(f"\n {'=' * 61}")
print(f" RECOMMENDATION: {r['recommendation']}")
print(f" Confidence: {r['confidence']}")
print()
def main():
parser = argparse.ArgumentParser(
description="Analyze A/B test results with statistical significance testing"
)
parser.add_argument("file", help="JSON file with test results")
parser.add_argument("--format", choices=["human", "json"], default="human", help="Output format")
parser.add_argument("--batch", action="store_true",
help="Process multiple tests (expects 'tests' array)")
parser.add_argument("--bayesian", action="store_true",
help="Include Bayesian analysis alongside frequentist")
args = parser.parse_args()
with open(args.file, "r", encoding="utf-8") as f:
data = json.load(f)
if args.batch:
tests = data.get("tests", [])
else:
tests = [data]
results = [analyze_test(t, args.bayesian) for t in tests]
if args.format == "json":
output = {"results": results} if len(results) > 1 else results[0]
print(json.dumps(output, indent=2))
else:
print_human(results)
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |