Skip to content

kali20gakki/msAgent / skills/profiler/ascend-cluster-fast-slow-rank-detector

ascend-cluster-fast-slow-rank-detector

Verified

专门用于 Ascend 集群 Profiling 性能数据的“快慢卡”诊断专家技能。当用户提供【集群性能数据目录/路径】并要求分析【快慢卡】、【慢节点】、【负载不均衡】或【集群瓶颈】时,必须触发此技能。该技能会自动接收集群路径,调度相关工具输出快慢卡的宏观定性与微观根因(如 Host 下发瓶颈、算子计算劣化)。

FileKindSize
SKILL.mdskill md8.0 KB
scripts/compare_api_stats.pyscript8.7 KB
scripts/compare_op_stats.pyscript8.9 KB
scripts/rank_data_finder.pyscript6.0 KB

scripts/compare_op_stats.py

View on GitHub
# -------------------------------------------------------------------------
# This file is part of the MindStudio project.
# Copyright (c) 2026 Huawei Technologies Co.,Ltd.
#
# MindStudio is licensed under Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#
#          http://license.coscl.org.cn/MulanPSL2
#
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.
# -------------------------------------------------------------------------

#!/usr/bin/env python3
"""
对比慢卡与快卡的算子统计(op_statistic),找出慢卡上耗时异常或调用异常的算子,作为计算型瓶颈证据。

支持:
- text:从 ASCEND_PROFILER_OUTPUT/op_statistic.csv 读取(通过 profiler_info_{rank}.json 定位目录)
- db:从 ascend_pytorch_profiler_{rank}.db 的 TASK + COMPUTE_TASK_INFO 聚合
"""
from __future__ import annotations

import argparse
import json
import os
import sqlite3
import sys
from typing import Any, Dict, List, Optional

# Allow importing from same package when run as script
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
if SCRIPT_DIR not in sys.path:
    sys.path.insert(0, SCRIPT_DIR)
from rank_data_finder import resolve_rank_paths

# op_statistic.csv 列名多种写法
OP_FIELD_MAPPINGS = {
    "op_type": ["OP Type", "Op Type", "op_type", "OpType", "Type"],
    "total_time": ["Total Time(us)", "Total Time (us)", "total_time", "TotalTime"],
    "count": ["Count", "count", "Call Count"],
    "avg_time": ["Avg Time(us)", "Avg Time (us)", "avg_time", "AvgTime"],
}


def _find_col(df, field_key: str):
    import pandas as pd
    for name in OP_FIELD_MAPPINGS.get(field_key, []):
        if name in df.columns:
            return df[name]
    return None


def load_op_stats_csv(path: str) -> List[Dict[str, Any]]:
    import pandas as pd
    df = pd.read_csv(path)
    op_col = _find_col(df, "op_type")
    time_col = _find_col(df, "total_time")
    count_col = _find_col(df, "count")
    if op_col is None or time_col is None:
        raise ValueError(f"CSV 缺少 op_type 或 total_time 列: {path}")
    df = df.copy()
    df["_total_time_us"] = pd.to_numeric(time_col, errors="coerce").fillna(0)
    df["_count"] = pd.to_numeric(count_col, errors="coerce").fillna(1) if count_col is not None else 1
    op_name = op_col.name if hasattr(op_col, "name") else op_col
    agg = df.groupby(op_name).agg(total_time_us=("_total_time_us", "sum"), count=("_count", "sum")).reset_index()
    agg = agg.rename(columns={op_name: "op_type"})
    return agg.to_dict("records")


def load_op_stats_db(db_path: str) -> List[Dict[str, Any]]:
    """从 ascend_pytorch_profiler_{rank}.db 聚合算子耗时与次数。TASK 表有 startNs/endNs,COMPUTE_TASK_INFO 有 name(STRING_IDS 映射)。"""
    conn = sqlite3.connect(db_path)
    try:
        # 检查表是否存在
        cur = conn.execute(
            "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('TASK','COMPUTE_TASK_INFO','STRING_IDS')"
        )
        tables = {r[0] for r in cur.fetchall()}
        if "TASK" not in tables or "COMPUTE_TASK_INFO" not in tables:
            raise ValueError(f"DB 中缺少 TASK 或 COMPUTE_TASK_INFO 表: {db_path}")

        # TASK.endNs - TASK.startNs = duration_ns; COMPUTE_TASK_INFO.name 可能是 INTEGER 需 JOIN STRING_IDS
        sql = """
        SELECT
            COALESCE(s.value, CAST(c.name AS TEXT)) AS op_type,
            SUM(t.endNs - t.startNs) / 1000.0 AS total_time_us,
            COUNT(*) AS count
        FROM TASK t
        JOIN COMPUTE_TASK_INFO c ON t.globalTaskId = c.globalTaskId
        LEFT JOIN STRING_IDS s ON s.id = c.name
        GROUP BY COALESCE(s.value, CAST(c.name AS TEXT))
        """
        cur = conn.execute(sql)
        rows = cur.fetchall()
        return [
            {"op_type": r[0], "total_time_us": r[1], "count": int(r[2])}
            for r in rows
        ]
    finally:
        conn.close()


def compare_op_stats(
    slow_data: List[Dict[str, Any]],
    fast_data: List[Dict[str, Any]],
    top_n: int = 20,
) -> Dict[str, Any]:
    """按 op_type 对齐,计算耗时差、比值,按差异从大到小排序,取 top_n。"""
    fast_by_op = {(r["op_type"]): r for r in fast_data}
    results = []
    for s in slow_data:
        op = s["op_type"]
        slow_time = float(s.get("total_time_us", 0))
        slow_count = int(s.get("count", 0))
        f = fast_by_op.get(op)
        if not f:
            diff_time = slow_time
            ratio = float("inf") if slow_time else 1.0
            fast_time = 0.0
            fast_count = 0
            slow_avg = (slow_time / slow_count) if slow_count else 0.0
            fast_avg = 0.0
        else:
            fast_time = float(f.get("total_time_us", 0))
            fast_count = int(f.get("count", 0))
            diff_time = slow_time - fast_time
            ratio = (slow_time / fast_time) if fast_time else (float("inf") if slow_time else 1.0)
            slow_avg = (slow_time / slow_count) if slow_count else 0.0
        fast_avg = (fast_time / fast_count) if fast_count else 0.0
        avg_time_diff = slow_avg - fast_avg
        results.append({
            "op_type": str(op),
            "slow_total_time_us": slow_time,
            "slow_count": slow_count,
            "fast_total_time_us": fast_time,
            "fast_count": fast_count,
            "diff_total_time_us": diff_time,
            "total_time_ratio_slow_vs_fast": ratio,
            "slow_avg_time_us": slow_avg,
            "fast_avg_time_us": fast_avg,
            "avg_time_diff_us": avg_time_diff,
        })
    # 按 diff 绝对值降序,取 top_n
    results.sort(key=lambda x: abs(x["diff_total_time_us"]), reverse=True)
    top = results[:top_n]
    return {
        "summary": {
            "total_ops_slow": len(slow_data),
            "total_ops_fast": len(fast_data),
            "top_n_by_abs_diff": top_n,
        },
        "top_differences": top,
        "conclusion": "以下为慢卡相对快卡差异最大的算子(按 |diff_total_time_us| 排序),可用于定位计算型慢卡瓶颈。",
    }


def main() -> int:
    parser = argparse.ArgumentParser(
        description="对比慢卡与快卡的算子统计,输出差异最大的算子列表。",
    )
    parser.add_argument("cluster_path", type=str, help="集群性能数据根目录")
    parser.add_argument("slow_rank", type=int, help="慢卡 Rank 号")
    parser.add_argument("fast_rank", type=int, help="快卡 Rank 号(作为基准)")
    parser.add_argument("--slow-path", type=str, default=None, help="慢卡 op_statistic.csv 或 .db 路径(可选,覆盖自动发现)")
    parser.add_argument("--fast-path", type=str, default=None, help="快卡 op_statistic.csv 或 .db 路径(可选)")
    parser.add_argument("--top", type=int, default=20, help="输出差异最大的前 N 条算子,默认 20")
    parser.add_argument("--json", action="store_true", help="以 JSON 输出")
    args = parser.parse_args()

    slow_path, fast_path, data_type, err = resolve_rank_paths(
        args.cluster_path, args.slow_rank, args.fast_rank, "op",
        slow_path=args.slow_path, fast_path=args.fast_path,
    )
    if err:
        print(err, file=sys.stderr)
        return 1

    try:
        if data_type == "text":
            slow_data = load_op_stats_csv(slow_path)
            fast_data = load_op_stats_csv(fast_path)
        else:
            slow_data = load_op_stats_db(slow_path)
            fast_data = load_op_stats_db(fast_path)
    except Exception as e:
        print(f"加载数据失败: {e}", file=sys.stderr)
        return 2

    result = compare_op_stats(slow_data, fast_data, top_n=args.top)
    result["meta"] = {
        "slow_rank": args.slow_rank,
        "fast_rank": args.fast_rank,
        "slow_path": slow_path,
        "fast_path": fast_path,
        "data_type": data_type,
    }

    if args.json:
        print(json.dumps(result, ensure_ascii=False, indent=2))
    else:
        print("=== 慢卡 vs 快卡 算子统计对比 ===")
        print(f"慢卡 Rank {args.slow_rank},快卡 Rank {args.fast_rank},数据来源: {data_type}")
        print(f"差异最大的前 {args.top} 个算子(按 |diff_total_time_us| 排序):\n")
        for i, row in enumerate(result["top_differences"], 1):
            print(f"{i}. {row['op_type']}")
            print(f"   慢卡: total_time_us={row['slow_total_time_us']:.0f}, count={row['slow_count']}")
            print(f"   快卡: total_time_us={row['fast_total_time_us']:.0f}, count={row['fast_count']}")
            print(f"   diff_total_time_us={row['diff_total_time_us']:.0f}, ratio(slow/fast)={row['total_time_ratio_slow_vs_fast']}, avg_time_diff_us={row['avg_time_diff_us']:.2f}")
            print()
        print(result["conclusion"])
    return 0


if __name__ == "__main__":
    sys.exit(main())
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall