Skip to content

getaero-io/gtm-eng-skills / skills/deepline-scoring

deepline-scoring

Verified

Use when discovering niche signals, auditing ICP or won/lost evidence, rescoring accounts, or building account and lead scoring Plays. Triggers on fit scoring, engagement scoring, backtesting, holdout evaluation, predictive lift, external proxies, and scoring leakage. Skip pure outreach copy or contributor skill installation tasks.

FileKindSize
.gitignoreother26 B
SKILL.mdskill md12.4 KB
package.jsonmanifest114 B
references/account-website-features.mdmarkdown10.3 KB
references/backtesting-task.mdmarkdown10.0 KB
references/buyer-language-research.mdmarkdown8.6 KB
references/capacity-evidence.mdmarkdown4.4 KB
references/dedupe.mdmarkdown1.5 KB
references/jev-feature-extraction.mdmarkdown8.2 KB
references/keyword-catalog.mdmarkdown4.9 KB
references/pitfalls.mdmarkdown1.6 KB
references/proven-signals.mdmarkdown1.1 KB
references/quality-gate.mdmarkdown2.8 KB
references/report-template.mdmarkdown6.2 KB
references/scorecard-creation.mdmarkdown3.3 KB
references/scorecard-training.mdmarkdown6.5 KB
references/scoring-delivery.mdmarkdown8.3 KB
references/scoring-diagnostics.mdmarkdown8.9 KB
references/scoring-pitfalls.mdmarkdown2.8 KB
references/signal-interpretation.mdmarkdown2.3 KB
references/step-7-prospects.mdmarkdown1.6 KB
references/technology-evidence.mdmarkdown9.4 KB
references/testing-and-evaluation.mdmarkdown6.8 KB
scripts/analyze_signals.pyscript16.9 KB
scripts/analyze_signals_v2.pyscript28.3 KB
scripts/build_feature_matrix.pyscript4.7 KB
scripts/collect_sources.pyscript16.1 KB
scripts/dedupe_utils.pyscript15.6 KB
scripts/dedupe_utils_v2.pyscript15.7 KB
scripts/find_contacts_v2.pyscript1.9 KB
scripts/hubspot_accounts.pyscript7.1 KB
scripts/jev_features.pyscript7.1 KB
scripts/replay-score.play.tsscript2.3 KB
scripts/requirements-scorecard.txtmarkdown263 B
scripts/scorecard/__init__.pyscript514 B
scripts/scorecard/config.pyscript5.0 KB
scripts/scorecard/explainer_artifacts.pyscript8.4 KB
scripts/scorecard/pipeline.pyscript38.9 KB
scripts/scorecard/report.pyscript30.4 KB
scripts/scorecard/scorer.pyscript17.3 KB
scripts/scorecard/specs.pyscript1.3 KB
scripts/scoring_contract.tsscript7.5 KB
scripts/technology_evidence.pyscript3.8 KB
scripts/tests/contract_check.tsscript1.9 KB
scripts/tests/test_jev_features.pyscript3.4 KB
scripts/tests/test_scorecard.pyscript9.5 KB
scripts/train_scorecard.pyscript20.0 KB
skill-metadata.jsonconfig3.8 KB

scripts/scorecard/report.py

View on GitHub
# --- scorecard_report.py ---
from pathlib import Path
import json
import pandas as pd
import numpy as np
try:
    from tabulate import tabulate as _TAB
except ImportError:
    _TAB = None

from .pipeline import ScorecardPipeline
from .config import RunConfig

def build_top_contributors(breakdown_df: pd.DataFrame, raw_df: pd.DataFrame, features_config: object, top_n: int = 5) -> pd.Series:
    out_to_in = {}
    if features_config is not None:
        if isinstance(features_config, dict) or isinstance(features_config, list):
            feats_list = features_config.get("features", features_config) if isinstance(features_config, dict) else features_config
            for spec in (list(feats_list) if feats_list is not None else []):
                col_in = spec.get("input")
                col_out = spec.get("output", col_in)
                if col_in and col_out:
                    out_to_in[str(col_out)] = str(col_in)
        else:
            raise ValueError("features_config must be a dict or list if provided")

    def _display_val(idx, feat_name: str):
        base_in = out_to_in.get(str(feat_name), str(feat_name))
        if base_in in raw_df.columns:
            return raw_df.loc[idx, base_in]
        if feat_name in raw_df.columns:
            return raw_df.loc[idx, feat_name]
        return None

    def _row_explain(idx):
        pts = breakdown_df.loc[idx]
        pos = pts[pts > 0].sort_values(ascending=False)
        if pos.empty:
            return ""
        items = []
        for fname, val in pos.iloc[:top_n].items():
            v_raw = _display_val(idx, fname)
            disp_name = out_to_in.get(str(fname), str(fname))
            # Display integers without trailing .0 for stability
            if v_raw is None:
                v_str = ""
            else:
                try:
                    import numpy as _np
                    if isinstance(v_raw, (int, _np.integer)):
                        v_str = str(int(v_raw))
                    elif isinstance(v_raw, (float, _np.floating)) and float(v_raw).is_integer():
                        v_str = str(int(v_raw))
                    else:
                        v_str = str(v_raw)
                except Exception:
                    v_str = str(v_raw)
            items.append(f"(+{int(val)}) {disp_name}: {v_str}")
        return "\n".join(items)

    s = pd.Series(index=breakdown_df.index, dtype=object)
    for idx in breakdown_df.index:
        s.loc[idx] = _row_explain(idx)
    return s

def _feature_summary_from_pipe(pipe) -> pd.DataFrame:
    assert pipe.model_ is not None and pipe._validation_split_ is not None, "Fit first."
    model = pipe.model_
    X_tr, X_va, y_tr, y_va = pipe._validation_split_
    points_va, breakdown_va = model.score_points(X_va)
    rows = []
    abs_total = breakdown_va.abs().sum(axis=1).replace(0, np.nan)
    denom = abs_total.sum()
    for fname, fb in model.features_.items():
        beta = float(model.coefs_.get(fname, 0.0))
        iv = float(fb.iv) if fb.iv is not None else np.nan
        beta_iv = abs(beta) * (iv if iv == iv else 0.0)
        mean_abs = float(breakdown_va[fname].abs().mean())
        share_abs = float(breakdown_va[fname].abs().sum() / denom) if denom and denom > 0 else 0.0
        rows.append({"feature": fname, "kind": fb.kind, "beta": beta, "IV": iv, "|beta|*IV": beta_iv,
                     "mean_abs_points_on_valid": mean_abs, "share_abs_points": share_abs})
    return pd.DataFrame(rows).sort_values(["|beta|*IV", "mean_abs_points_on_valid", "share_abs_points"], ascending=False).reset_index(drop=True)

def eval_performance_split(model, X_in: pd.DataFrame, y_in: pd.Series, top_share: float) -> dict:
    from math import ceil as _ceil
    from sklearn.metrics import roc_auc_score as _auc_fn, average_precision_score as _ap_fn
    raw_pts, _ = model.score_points(X_in)
    k = max(1, int(_ceil(len(raw_pts) * top_share))) if len(raw_pts) else 0
    order = raw_pts.sort_values(ascending=False)
    top_idx = order.index[:k]
    base = float(y_in.mean()) if len(y_in) else float('nan')
    prec_top = float(y_in.loc[top_idx].mean()) if len(top_idx) else float('nan')
    lift = float(prec_top / base) if base and base > 0 else float('nan')
    auc = float(_auc_fn(y_in, model.predict_proba(X_in))) if len(pd.unique(y_in)) > 1 else float('nan')
    ap = float(_ap_fn(y_in, model.predict_proba(X_in))) if len(y_in) > 0 else float('nan')
    return {
        "rows": int(len(y_in)),
        "positives": int(y_in.sum()),
        "baseline_rate": base,
        "top_share": float(top_share),
        "precision_top_share": prec_top,
        "lift_top_share": lift,
        "auc": auc,
        "average_precision": ap,
    }

def _build_value_points_table(pipe, features_config, raw_df: pd.DataFrame = None, y_col: str = None) -> pd.DataFrame:
    """Produce per-category points for all features that use categorical mapping.

    Returns a DataFrame with columns: feature, kind, input_value, mapped_bin, points, count, label_rate, lift (optional)
    """
    rows = []
    if features_config is None:
        return pd.DataFrame(rows)
    feats = features_config.get("features", features_config) if isinstance(features_config, dict) else features_config
    feats = list(feats or [])
    model = pipe.model_
    # Compute base rate for lift if labels are available
    base_rate = None
    if raw_df is not None and (y_col is not None) and (y_col in raw_df.columns):
        try:
            base_rate = float(raw_df[y_col].astype(int).mean())
        except Exception:
            base_rate = None

    for spec in feats:
        col_in = spec.get("input")
        col_out = spec.get("output", col_in)
        t = spec.get("transform") or {}
        mapping = t.get("mapping") if isinstance(t, dict) else None
        if not mapping:
            continue
        unknown_label = t.get("unknown_label", "Other")
        fb = model.features_.get(col_out)
        if fb is None:
            continue
        points_map = getattr(fb, "points_map", {}) or {}
        feature_kind = getattr(fb, "kind", "")
        # Iterate over declared categories in mapping
        for cat, bin_label in mapping.items():
            mapped_bin = str(bin_label) if bin_label is not None else str(unknown_label)
            pts = int(points_map.get(mapped_bin, 0))
            count = None
            rate = None
            if raw_df is not None and (col_in in raw_df.columns):
                mask = raw_df[col_in].astype("object") == cat
                count = int(mask.sum())
                if (y_col is not None) and (y_col in raw_df.columns) and count > 0:
                    rate = float(raw_df.loc[mask, y_col].astype(int).mean())
            rows.append({
                "feature": col_in,
                "kind": feature_kind,
                "input_value": cat,
                "mapped_bin": mapped_bin,
                "points": pts,
                **({"count": count} if count is not None else {}),
                **({"label_rate": rate} if rate is not None else {}),
                **({"lift": (float(rate) / float(base_rate)) if (rate is not None and base_rate and base_rate > 0) else None} if rate is not None else {}),
            })
    return pd.DataFrame(rows)


def generate_reports(pipe, out_dir: str = "/mnt/data/reports", features_config: object = None, raw_df: pd.DataFrame = None, y_col: str = None):
    Path(out_dir).mkdir(parents=True, exist_ok=True)
    assert pipe.model_ is not None, "Fit the pipeline first."
    points = pipe.export_points_tables(); p_points = f"{out_dir}/points_table.csv"; points.to_csv(p_points, index=False)
    report_model = pipe.validation_bucket_report(top_share=0.10, use_policy=False)
    p_report_model = f"{out_dir}/bucket_report_model.csv"; report_model.to_csv(p_report_model, index=False)
    report_policy = pipe.validation_bucket_report(top_share=0.10, use_policy=True)
    p_report_policy = f"{out_dir}/bucket_report_policy.csv"; report_policy.to_csv(p_report_policy, index=False)
    feat_summary = _feature_summary_from_pipe(pipe); p_feat = f"{out_dir}/feature_summary.csv"; feat_summary.to_csv(p_feat, index=False)
    meta = {"gate": pipe.gate_, "tier_cuts": pipe.tier_cuts_, "validation_model": pipe.validation_report()}
    p_meta = f"{out_dir}/gate_and_tier_meta.json"; Path(p_meta).write_text(json.dumps(meta, indent=2))
    points_ext = points.copy()
    points_ext["predicate"] = points_ext.apply(lambda r: f"{r['feature']}=={r['bin']}", axis=1)
    storytelling = points_ext.merge(report_model, on="predicate", how="left")
    cols_order = [
        "feature", "kind", "bin", "predicate", "points",
        "prob_lift", "bucket_rate", "baseline_rate",
        "WOE", "beta", "model_odds_multiplier",
        "n", "share", "top_share_presence", "intrusion_ratio",
        "top_share_precision_segment", "top_share_precision_overall", "precision_gap",
        "is_reference", "iv",
    ]
    storytelling = storytelling.reindex(columns=[c for c in cols_order if c in storytelling.columns])
    p_story = f"{out_dir}/storytelling.csv"; storytelling.to_csv(p_story, index=False)
    p_model_lift = None
    if pipe._validation_split_ is not None:
        X_tr, X_va, y_tr, y_va = pipe._validation_split_
        raw_points, _ = pipe.model_.score_points(X_va)
        k = max(1, int(len(raw_points) * 0.10))
        order = raw_points.sort_values(ascending=False)
        top_idx = order.index[:k]
        prec_top = float(y_va.loc[top_idx].mean()) if len(top_idx) else float('nan')
        base_rate = float(y_va.mean()) if len(y_va) else float('nan')
        model_lift = float(prec_top / base_rate) if base_rate and base_rate > 0 else float('nan')
        vr = pipe.validation_report()
        model_rows = [{
            "top_share": 0.10,
            "precision_top_share": prec_top,
            "baseline_rate": base_rate,
            "lift_top_share": model_lift,
            "auc": float(vr.get("auc", float('nan'))),
            "average_precision": float(vr.get("average_precision", float('nan'))),
            "n_valid": int(vr.get("n_valid", 0)),
        }]
        model_df = pd.DataFrame(model_rows)
        p_model_lift = f"{out_dir}/model_lift.csv"; model_df.to_csv(p_model_lift, index=False)
    p_tier = None
    if pipe.tier_cuts_ is not None and pipe._validation_split_ is not None:
        X_tr, X_va, y_tr, y_va = pipe._validation_split_
        applied = pipe.score_and_assign_tiers(X_va)
        tiers = applied["tier"]
        base_rate = float(y_va.mean()) if len(y_va) else np.nan
        rows = []
        for t in ["A", "B", "C"]:
            mask = tiers == t
            if mask.any():
                rate = float(y_va.loc[mask].mean())
                lift = float(rate / base_rate) if base_rate and base_rate > 0 else np.nan
                rows.append({"tier": t, "rate": rate, "baseline_rate": base_rate, "lift": lift, "count": int(mask.sum())})
        tier_df = pd.DataFrame(rows)
        p_tier = f"{out_dir}/tier_lifts.csv"; tier_df.to_csv(p_tier, index=False)
    # Optional: emit value-level points for mapped categoricals
    p_value_points = None
    try:
        vp = _build_value_points_table(pipe, features_config, raw_df=raw_df, y_col=y_col)
        if isinstance(vp, pd.DataFrame) and not vp.empty:
            p_value_points = f"{out_dir}/value_points.csv"; vp.to_csv(p_value_points, index=False)
    except Exception:
        p_value_points = None

    # Print a concise value-level preview at the very end to ensure it appears in logs
    try:
        if isinstance(vp, pd.DataFrame) and not vp.empty:
            feat_to_show = "INDUSTRY" if "INDUSTRY" in vp["feature"].unique().tolist() else vp["feature"].iloc[0]
            sub = vp.loc[vp["feature"] == feat_to_show].copy()
            sub = sub.sort_values(["points", "count" if "count" in sub.columns else "points"], ascending=[False, False]).head(20)
            cols = [c for c in ["feature","kind","input_value","mapped_bin","points","count","lift"] if c in sub.columns]
            print("\n=== Value-level Points (preview) ===")
            if _TAB is not None:
                print(_TAB(sub[cols], headers='keys', tablefmt='github', floatfmt='.6f', showindex=False))
            else:
                print(sub[cols].to_string(index=False))
    except Exception:
        pass

    return {
        "points_table_csv": p_points,
        "bucket_report_model_csv": p_report_model,
        "bucket_report_policy_csv": p_report_policy,
        "feature_summary_csv": p_feat,
        "storytelling_csv": p_story,
        **({"model_lift_csv": p_model_lift} if p_model_lift else {}),
        **({"tier_lifts_csv": p_tier} if p_tier else {}),
        "gate_and_tier_meta_json": p_meta,
        **({"value_points_csv": p_value_points} if p_value_points else {}),
    }

def train_scorecard(
    X_or_df: pd.DataFrame,
    y: pd.Series = None,
    config: RunConfig = None,
    raw_df: pd.DataFrame = None,
    features_config: object = None,
) -> dict:
    """
    Fit the scorecard end-to-end and return in-memory artifacts.

    Config keys (examples):
      - y_col (required)
      - test_size, random_state, do_calibration
      - n_bins_numeric, n_bins_cat_high, max_low_card
      - l1_C, scale
      - top_share
      - gate: {enable: bool, top_share: float, min_intrusion: float, min_prec_gain: float}
      - tiers: {enable: bool, top_share_hint: float, precA_min: float, covA_max: float}
    """
    assert config is not None, "config is required"
    y_col = config.y_col
    if y is not None:
        df = X_or_df.copy()
        df[y_col] = pd.Series(y).astype(int).values
    else:
        df = X_or_df
    assert y_col in df.columns, f"Missing target column {y_col}"

    # Dataset summary (pre-split)
    y_all = df[y_col].astype(int)
    n_rows = int(len(y_all))
    n_pos = int(y_all.sum())
    base_rate = float(y_all.mean()) if n_rows else float('nan')
    print("\n=== Dataset Summary ===")
    print(f"rows={n_rows}, positives={n_pos}, base_rate={base_rate:.6f}")

    pipe = ScorecardPipeline(
        y_col=y_col,
        test_size=config.test_size,
        random_state=config.random_state,
        do_calibration=config.do_calibration,
        n_bins_numeric=config.n_bins_numeric,
        n_bins_cat_high=config.n_bins_cat_high,
        max_low_card=config.max_low_card,
        l1_C=config.l1_C,
        scale=config.scale,
        method=getattr(config, "method", "logistic"),
        woe_prior=getattr(config, "woe_prior", 0.5),
        max_features=getattr(config, "max_features", None),
        max_abs_points=getattr(config, "max_abs_points", None),
    )

    df_fit = df.copy()
    raw_helper_cols = [c for c in df_fit.columns if c != y_col and (str(c).endswith("_RAW"))]
    if raw_helper_cols:
        df_fit = df_fit.drop(columns=raw_helper_cols)
    # Drop known passthrough/non-feature columns from modeling frame
    passthrough_cols = []
    if "account_id" in df_fit.columns:
        passthrough_cols.append("account_id")
    if passthrough_cols:
        df_fit = df_fit.drop(columns=passthrough_cols)

    model = pipe.fit(df_fit)

    points = pipe.export_points_tables().copy()
    points["predicate"] = points.apply(lambda r: f"{r['feature']}=={r['bin']}", axis=1)
    top_share = float(config.top_share)
    report_model = pipe.validation_bucket_report(top_share=top_share, use_policy=False)
    scorecard = points.merge(report_model, on="predicate", how="left")
    feature_summary = _feature_summary_from_pipe(pipe)

    X_tr, X_va, y_tr, y_va = pipe._validation_split_
    raw_points, _ = model.score_points(X_va)
    k = max(1, int(len(raw_points) * top_share))
    order = raw_points.sort_values(ascending=False)
    top_idx = order.index[:k]
    prec_top = float(y_va.loc[top_idx].mean()) if len(top_idx) else float("nan")
    base_rate = float(y_va.mean()) if len(y_va) else float("nan")
    lift_top = float(prec_top / base_rate) if base_rate and base_rate > 0 else float("nan")
    vr = pipe.validation_report()
    model_lift_rows = [{
        "variant": "model",
        "top_share": top_share,
        "precision_top_share": prec_top,
        "baseline_rate": base_rate,
        "lift_top_share": lift_top,
        "auc": float(vr.get("auc", float('nan'))),
        "average_precision": float(vr.get("average_precision", float('nan'))),
        "n_valid": int(vr.get("n_valid", 0)),
    }]

    decision_points_policy = None
    policy_cfg = config.policy
    if policy_cfg and getattr(policy_cfg, "rules", None):
            decision_points_policy = raw_points.copy()
            overrides = pd.Series(0, index=decision_points_policy.index, dtype=int)
            import re
            def _match(rule, row):
                v = row.get(rule.target, None)
                if rule.source == "bin":
                    v = str(v) if v is not None else None
                    if rule.op == "lte":
                        return (float(v) <= float(rule.value)) if v is not None else False
                    if rule.op == "gte":
                        return (float(v) >= float(rule.value)) if v is not None else False
                    if rule.op == "eq":
                        return v == rule.value
                    if rule.op == "in":
                        return v in (rule.values or [])
                    if rule.op == "between":
                        lo, hi = rule.values[0], rule.values[1]
                        return (float(v) >= float(lo)) and (float(v) <= float(hi)) if v is not None else False
                    if rule.op == "regex":
                        return bool(re.search(str(rule.value), str(v or "")))
                return False
            for rule in sorted([r for r in policy_cfg.rules if r.enabled], key=lambda r: int(r.priority)):
                mask = X_va.apply(lambda r: _match(rule, r), axis=1)
                if policy_cfg.combine == "sum":
                    overrides.loc[mask] = overrides.loc[mask] + int(rule.override_points)
                else:
                    overrides.loc[mask] = np.maximum(overrides.loc[mask].astype(int), int(rule.override_points))
            decision_points_policy = decision_points_policy + overrides
            order_p = decision_points_policy.sort_values(ascending=False)
            top_idx_p = order_p.index[:k]
            prec_top_p = float(y_va.loc[top_idx_p].mean()) if len(top_idx_p) else float("nan")
            lift_top_p = float(prec_top_p / base_rate) if base_rate and base_rate > 0 else float("nan")
            model_lift_rows.append({
                "variant": "policy",
                "top_share": top_share,
                "precision_top_share": prec_top_p,
                "baseline_rate": base_rate,
                "lift_top_share": lift_top_p,
                "auc": float(vr.get("auc", float('nan'))),
                "average_precision": float(vr.get("average_precision", float('nan'))),
                "n_valid": int(vr.get("n_valid", 0)),
            })

    model_lift = pd.DataFrame(model_lift_rows)

    tier_cuts = None
    tier_lifts = None
    policy_tier_lifts = None
    tiers_cfg = config.tiers
    if tiers_cfg.enable:
        pipe.learn_tier_cutoffs(
            top_share_hint=top_share,
            precA_min=tiers_cfg.precA_min,
            covA_max=tiers_cfg.covA_max,
        )
        tier_cuts = dict(pipe.tier_cuts_) if pipe.tier_cuts_ is not None else None
        applied = pipe.score_and_assign_tiers(X_va)
        tiers = applied["tier"]
        rows = []
        for t in ["A", "B", "C"]:
            mask = tiers == t
            if mask.any():
                rate = float(y_va.loc[mask].mean())
                lift = float(rate / base_rate) if base_rate and base_rate > 0 else float('nan')
                rows.append({"variant": "policy" if pipe.gate_ else "model", "tier": t, "rate": rate, "baseline_rate": base_rate, "lift": lift, "count": int(mask.sum())})
        tier_lifts = pd.DataFrame(rows) if rows else None
        if decision_points_policy is not None and pipe.tier_cuts_ is not None:
            A_cut = pipe.tier_cuts_.get("cut_A"); B_cut = pipe.tier_cuts_.get("cut_B")
            def _tier_from_pts(v: float):
                x = float(v)
                if A_cut is not None and x >= A_cut:
                    return "A"
                if B_cut is not None and x >= B_cut:
                    return "B"
                return "C"
            tiers_p = decision_points_policy.apply(_tier_from_pts)
            rows_p = []
            for t in ["A","B","C"]:
                mk = (tiers_p == t)
                if mk.any():
                    rate = float(y_va.loc[mk].mean())
                    lift = float(rate / base_rate) if base_rate and base_rate > 0 else float('nan')
                    rows_p.append({"tier": t, "rate": rate, "baseline_rate": base_rate, "lift": lift, "count": int(mk.sum())})
            policy_tier_lifts = pd.DataFrame(rows_p) if rows_p else None

    # Scoring outputs
    scores_va = pipe.score(X_va)
    validation_scoring = pd.DataFrame({
        "raw_points": scores_va["raw_points"].astype(int).values,
        "model_prob": scores_va["model_prob"].astype(float).values,
    }, index=X_va.index)

    X_all = df.drop(columns=[y_col]).copy()
    scores_all = pipe.score(X_all)
    all_scoring = pd.DataFrame({
        "raw_points": scores_all["raw_points"].astype(int).values,
        "model_prob": scores_all["model_prob"].astype(float).values,
    }, index=X_all.index)

    # Explainer omitted in training persistence; scoring/inference should call build_top_contributors

    # Overall calibration on full dataset

    # Print condensed scorecard on every run
    print("\n=== Scorecard (Condensed: feature, bin, points, lift_total, n_total, support_valid, lift_valid, share_total) ===")
    y_all_full = df[y_col].astype(int).copy()
    # Now that y_all_full is defined, print calibration
    n_all = int(len(y_all_full))
    actual_rate_all = float(y_all_full.mean()) if n_all else float('nan')
    avg_pred_all = float(scores_all["model_prob"].mean()) if n_all else float('nan')
    ratio_pa = float(avg_pred_all / actual_rate_all) if (actual_rate_all and actual_rate_all > 0) else float('nan')
    print("\n=== Overall Calibration (full dataset) ===")
    print(f"n={n_all}, actual_rate={actual_rate_all:.6f}, avg_pred={avg_pred_all:.6f}, pred/actual={ratio_pa:.3f}")
    base_rate_total = float(y_all_full.mean()) if len(y_all_full) else float('nan')
    rows_bins = []
    for fname, fb in pipe.model_.features_.items():
        if str(fname).lower() == "account_id":
            continue
        if fname not in X_all.columns:
            continue
        bins_all = X_all[fname].astype("object").astype(str)
        grp = pd.DataFrame({"bin": bins_all, "y": y_all_full}).groupby("bin").agg(n_total=("y","size"), rate_total=("y","mean")).reset_index()
        grp["feature"] = fname
        grp["kind"] = getattr(fb, "kind", "")
        grp["points"] = grp["bin"].map((getattr(fb, "points_map", {}) or {})).fillna(0).astype(int)
        grp["share_total"] = grp["n_total"] / float(len(y_all_full) or 1)
        grp["lift_total"] = grp.apply(lambda r: float(r["rate_total"]/base_rate_total) if base_rate_total and base_rate_total>0 else float('nan'), axis=1)
        rows_bins.append(grp)
    bins_total_df = pd.concat(rows_bins, ignore_index=True) if rows_bins else pd.DataFrame()
    sc_small_cols = [c for c in ["feature","kind","bin","points","prob_lift","n"] if c in scorecard.columns]
    sc_small_val = scorecard.reindex(columns=sc_small_cols).copy()
    sc_small_val = sc_small_val.rename(columns={"prob_lift":"lift_valid","n":"support_valid"})
    condensed = pd.merge(
        bins_total_df,
        sc_small_val,
        on=["feature","kind","bin","points"], how="left"
    )
    condensed = condensed[["feature","kind","bin","points","lift_total","n_total","support_valid","lift_valid","share_total"]]
    condensed = condensed.sort_values(["feature","points"], ascending=[True, False])
    try:
        from tabulate import tabulate
        print(tabulate(condensed, headers='keys', tablefmt='github', floatfmt='.6f', showindex=False))
    except ImportError:
        print(condensed.to_string(index=False))

    # Print value-level points for mapped categoricals (brief preview)
    if features_config is not None:
        vp_df = _build_value_points_table(pipe, features_config, raw_df=raw_df, y_col=y_col)
        if isinstance(vp_df, pd.DataFrame) and not vp_df.empty:
            print("\n=== Value-level Points (preview, top by feature) ===")
            # Limit rows per feature to keep output readable
            for feat in vp_df["feature"].unique():
                sub = vp_df.loc[vp_df["feature"] == feat].copy()
                sub = sub.sort_values(["points", "count" if "count" in sub.columns else "points"], ascending=[False, False]).head(10)
                cols = [c for c in ["feature","kind","input_value","mapped_bin","points","count","lift"] if c in sub.columns]
                if _TAB is not None:
                    print(_TAB(sub[cols], headers='keys', tablefmt='github', floatfmt='.6f', showindex=False))
                else:
                    print(sub[cols].to_string(index=False))

    # Build per-row results for full dataset (points, probability, tier when available)
    tiers_series = None
    if pipe.tier_cuts_ is not None:
        policy = pipe.apply_policy_and_tiers(df=X_all, raw_points=scores_all["raw_points"])
        tiers_series = policy["tier"].astype(str)
    all_results_cols = {
        "points": scores_all["raw_points"].astype(int).values,
        "probability": scores_all["model_prob"].astype(float).values,
    }
    if tiers_series is not None:
        all_results_cols["tier"] = tiers_series.values
    # Pass through id column if present
    if "account_id" in df.columns:
        all_results_cols["account_id"] = df["account_id"].values
    # Do not persist explainer from training path
    all_results = pd.DataFrame(all_results_cols, index=X_all.index)

    model_meta = {
        "intercept": model.intercept_,
        "scale": model.scale_,
        "coefs": model.coefs_,
        "has_calibrator": model.calibrator_ is not None,
        "validation": pipe.validation_report(),
        "feature_kinds": getattr(pipe, "_kinds_", {}),
    }

    # Print per-row scoring results (sample)
    cols = [c for c in ["account_id","points","probability","tier"] if c in all_results.columns]
    if cols:
        print("\n=== Scoring results (account_id, points, probability, tier) ===")
        if _TAB is not None:
            print(_TAB(all_results[cols].head(25), headers='keys', tablefmt='github', floatfmt='.6f', showindex=False))
        else:
            print(all_results[cols].head(25).to_string(index=False))

    # Print performance metrics: model lift and tier lifts, plus by-split summary
    X_tr, X_va, y_tr, y_va = pipe._validation_split_
    X_all_perf = df.drop(columns=[y_col]).copy(); y_all_perf = df[y_col].astype(int).copy()
    split_rows = [
        {"split": "train", **eval_performance_split(model, X_tr, y_tr, top_share)},
        {"split": "valid", **eval_performance_split(model, X_va, y_va, top_share)},
        {"split": "all", **eval_performance_split(model, X_all_perf, y_all_perf, top_share)},
    ]
    print("\n=== Performance (by split) ===")
    df_splits = pd.DataFrame(split_rows)
    if _TAB is not None:
        print(_TAB(df_splits, headers='keys', tablefmt='github', floatfmt='.6f', showindex=False))
    else:
        print(df_splits.to_string(index=False))
    if isinstance(model_lift, pd.DataFrame) and not model_lift.empty:
        print("\n=== Performance (Model Lift) ===")
        cols_ml = [c for c in ["top_share","precision_top_share","baseline_rate","lift_top_share","auc","average_precision","n_valid"] if c in model_lift.columns]
        if _TAB is not None:
            print(_TAB(model_lift[cols_ml], headers='keys', tablefmt='github', floatfmt='.6f', showindex=False))
        else:
            print(model_lift[cols_ml].to_string(index=False))
    if isinstance(tier_lifts, pd.DataFrame) and not tier_lifts.empty:
        print("\n=== Performance (Tier Lifts) ===")
        cols_tl = [c for c in ["tier","rate","baseline_rate","lift","count"] if c in tier_lifts.columns]
        if _TAB is not None:
            print(_TAB(tier_lifts[cols_tl], headers='keys', tablefmt='github', floatfmt='.6f', showindex=False))
        else:
            print(tier_lifts[cols_tl].to_string(index=False))

    # Decile lift by actual outcomes on the full dataset
    decile_lift_df = None
    try:
        probs_all = scores_all["model_prob"] if isinstance(scores_all, dict) else None
    except Exception:
        probs_all = None
    if probs_all is not None and len(probs_all) > 0:
        n_all = len(probs_all)
        ranks = probs_all.rank(method="first", ascending=False)
        deciles = np.ceil(ranks / float(max(1, n_all)) * 10).astype(int).clip(1, 10)
        base_rate_all = float(y_all_full.mean()) if len(y_all_full) else float('nan')
        df_dec = pd.DataFrame({"decile": deciles.astype(int), "prob": probs_all.astype(float).values, "y": y_all_full.astype(int).values})
        agg = df_dec.groupby("decile").agg(n=("y","size"), actual_rate=("y","mean"), avg_prob=("prob","mean")).reset_index().sort_values("decile")
        agg["actual_lift"] = agg.apply(lambda r: (float(r["actual_rate"]) / float(base_rate_all)) if base_rate_all and base_rate_all > 0 else float('nan'), axis=1)
        decile_lift_df = agg
        print("\n=== Decile Lift (actuals over full dataset) ===")
        cols_dl = [c for c in ["decile","n","actual_rate","avg_prob","actual_lift"] if c in decile_lift_df.columns]
        if _TAB is not None:
            print(_TAB(decile_lift_df[cols_dl], headers='keys', tablefmt='github', floatfmt='.6f', showindex=False))
        else:
            print(decile_lift_df[cols_dl].to_string(index=False))

    return {
        "scorecard": scorecard.sort_values(["feature", "points"], ascending=[True, False]).reset_index(drop=True),
        "condensed_scorecard": condensed.sort_values(["feature","points"], ascending=[True, False]).reset_index(drop=True),
        "feature_summary": feature_summary.reset_index(drop=True),
        "model_lift": model_lift.reset_index(drop=True),
        **({"tier_lifts": tier_lifts.reset_index(drop=True)} if tier_lifts is not None else {}),
        **({"policy_tier_lifts": policy_tier_lifts.reset_index(drop=True)} if policy_tier_lifts is not None else {}),
        **({"decile_lift": decile_lift_df.reset_index(drop=True)} if decile_lift_df is not None else {}),
        **({"tier_cuts": tier_cuts} if tier_cuts is not None else {}),
        **({"validation_scoring": validation_scoring} if validation_scoring is not None else {}),
        **({"all_scoring": all_scoring} if all_scoring is not None else {}),
        **({"all_results": all_results} if all_results is not None else {}),
        "pipeline": pipe,
        "model_meta": model_meta,
    }
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall