kali20gakki/msAgent / skills/profiler/ascend-cluster-fast-slow-rank-detector
ascend-cluster-fast-slow-rank-detector
Verified专门用于 Ascend 集群 Profiling 性能数据的“快慢卡”诊断专家技能。当用户提供【集群性能数据目录/路径】并要求分析【快慢卡】、【慢节点】、【负载不均衡】或【集群瓶颈】时,必须触发此技能。该技能会自动接收集群路径,调度相关工具输出快慢卡的宏观定性与微观根因(如 Host 下发瓶颈、算子计算劣化)。
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 8.0 KB |
| scripts/compare_api_stats.py | script | 8.7 KB |
| scripts/compare_op_stats.py | script | 8.9 KB |
| scripts/rank_data_finder.py | script | 6.0 KB |
scripts/compare_op_stats.py
View on GitHub# -------------------------------------------------------------------------
# This file is part of the MindStudio project.
# Copyright (c) 2026 Huawei Technologies Co.,Ltd.
#
# MindStudio is licensed under Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#
# http://license.coscl.org.cn/MulanPSL2
#
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.
# -------------------------------------------------------------------------
#!/usr/bin/env python3
"""
对比慢卡与快卡的算子统计(op_statistic),找出慢卡上耗时异常或调用异常的算子,作为计算型瓶颈证据。
支持:
- text:从 ASCEND_PROFILER_OUTPUT/op_statistic.csv 读取(通过 profiler_info_{rank}.json 定位目录)
- db:从 ascend_pytorch_profiler_{rank}.db 的 TASK + COMPUTE_TASK_INFO 聚合
"""
from __future__ import annotations
import argparse
import json
import os
import sqlite3
import sys
from typing import Any, Dict, List, Optional
# Allow importing from same package when run as script
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
if SCRIPT_DIR not in sys.path:
sys.path.insert(0, SCRIPT_DIR)
from rank_data_finder import resolve_rank_paths
# op_statistic.csv 列名多种写法
OP_FIELD_MAPPINGS = {
"op_type": ["OP Type", "Op Type", "op_type", "OpType", "Type"],
"total_time": ["Total Time(us)", "Total Time (us)", "total_time", "TotalTime"],
"count": ["Count", "count", "Call Count"],
"avg_time": ["Avg Time(us)", "Avg Time (us)", "avg_time", "AvgTime"],
}
def _find_col(df, field_key: str):
import pandas as pd
for name in OP_FIELD_MAPPINGS.get(field_key, []):
if name in df.columns:
return df[name]
return None
def load_op_stats_csv(path: str) -> List[Dict[str, Any]]:
import pandas as pd
df = pd.read_csv(path)
op_col = _find_col(df, "op_type")
time_col = _find_col(df, "total_time")
count_col = _find_col(df, "count")
if op_col is None or time_col is None:
raise ValueError(f"CSV 缺少 op_type 或 total_time 列: {path}")
df = df.copy()
df["_total_time_us"] = pd.to_numeric(time_col, errors="coerce").fillna(0)
df["_count"] = pd.to_numeric(count_col, errors="coerce").fillna(1) if count_col is not None else 1
op_name = op_col.name if hasattr(op_col, "name") else op_col
agg = df.groupby(op_name).agg(total_time_us=("_total_time_us", "sum"), count=("_count", "sum")).reset_index()
agg = agg.rename(columns={op_name: "op_type"})
return agg.to_dict("records")
def load_op_stats_db(db_path: str) -> List[Dict[str, Any]]:
"""从 ascend_pytorch_profiler_{rank}.db 聚合算子耗时与次数。TASK 表有 startNs/endNs,COMPUTE_TASK_INFO 有 name(STRING_IDS 映射)。"""
conn = sqlite3.connect(db_path)
try:
# 检查表是否存在
cur = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name IN ('TASK','COMPUTE_TASK_INFO','STRING_IDS')"
)
tables = {r[0] for r in cur.fetchall()}
if "TASK" not in tables or "COMPUTE_TASK_INFO" not in tables:
raise ValueError(f"DB 中缺少 TASK 或 COMPUTE_TASK_INFO 表: {db_path}")
# TASK.endNs - TASK.startNs = duration_ns; COMPUTE_TASK_INFO.name 可能是 INTEGER 需 JOIN STRING_IDS
sql = """
SELECT
COALESCE(s.value, CAST(c.name AS TEXT)) AS op_type,
SUM(t.endNs - t.startNs) / 1000.0 AS total_time_us,
COUNT(*) AS count
FROM TASK t
JOIN COMPUTE_TASK_INFO c ON t.globalTaskId = c.globalTaskId
LEFT JOIN STRING_IDS s ON s.id = c.name
GROUP BY COALESCE(s.value, CAST(c.name AS TEXT))
"""
cur = conn.execute(sql)
rows = cur.fetchall()
return [
{"op_type": r[0], "total_time_us": r[1], "count": int(r[2])}
for r in rows
]
finally:
conn.close()
def compare_op_stats(
slow_data: List[Dict[str, Any]],
fast_data: List[Dict[str, Any]],
top_n: int = 20,
) -> Dict[str, Any]:
"""按 op_type 对齐,计算耗时差、比值,按差异从大到小排序,取 top_n。"""
fast_by_op = {(r["op_type"]): r for r in fast_data}
results = []
for s in slow_data:
op = s["op_type"]
slow_time = float(s.get("total_time_us", 0))
slow_count = int(s.get("count", 0))
f = fast_by_op.get(op)
if not f:
diff_time = slow_time
ratio = float("inf") if slow_time else 1.0
fast_time = 0.0
fast_count = 0
slow_avg = (slow_time / slow_count) if slow_count else 0.0
fast_avg = 0.0
else:
fast_time = float(f.get("total_time_us", 0))
fast_count = int(f.get("count", 0))
diff_time = slow_time - fast_time
ratio = (slow_time / fast_time) if fast_time else (float("inf") if slow_time else 1.0)
slow_avg = (slow_time / slow_count) if slow_count else 0.0
fast_avg = (fast_time / fast_count) if fast_count else 0.0
avg_time_diff = slow_avg - fast_avg
results.append({
"op_type": str(op),
"slow_total_time_us": slow_time,
"slow_count": slow_count,
"fast_total_time_us": fast_time,
"fast_count": fast_count,
"diff_total_time_us": diff_time,
"total_time_ratio_slow_vs_fast": ratio,
"slow_avg_time_us": slow_avg,
"fast_avg_time_us": fast_avg,
"avg_time_diff_us": avg_time_diff,
})
# 按 diff 绝对值降序,取 top_n
results.sort(key=lambda x: abs(x["diff_total_time_us"]), reverse=True)
top = results[:top_n]
return {
"summary": {
"total_ops_slow": len(slow_data),
"total_ops_fast": len(fast_data),
"top_n_by_abs_diff": top_n,
},
"top_differences": top,
"conclusion": "以下为慢卡相对快卡差异最大的算子(按 |diff_total_time_us| 排序),可用于定位计算型慢卡瓶颈。",
}
def main() -> int:
parser = argparse.ArgumentParser(
description="对比慢卡与快卡的算子统计,输出差异最大的算子列表。",
)
parser.add_argument("cluster_path", type=str, help="集群性能数据根目录")
parser.add_argument("slow_rank", type=int, help="慢卡 Rank 号")
parser.add_argument("fast_rank", type=int, help="快卡 Rank 号(作为基准)")
parser.add_argument("--slow-path", type=str, default=None, help="慢卡 op_statistic.csv 或 .db 路径(可选,覆盖自动发现)")
parser.add_argument("--fast-path", type=str, default=None, help="快卡 op_statistic.csv 或 .db 路径(可选)")
parser.add_argument("--top", type=int, default=20, help="输出差异最大的前 N 条算子,默认 20")
parser.add_argument("--json", action="store_true", help="以 JSON 输出")
args = parser.parse_args()
slow_path, fast_path, data_type, err = resolve_rank_paths(
args.cluster_path, args.slow_rank, args.fast_rank, "op",
slow_path=args.slow_path, fast_path=args.fast_path,
)
if err:
print(err, file=sys.stderr)
return 1
try:
if data_type == "text":
slow_data = load_op_stats_csv(slow_path)
fast_data = load_op_stats_csv(fast_path)
else:
slow_data = load_op_stats_db(slow_path)
fast_data = load_op_stats_db(fast_path)
except Exception as e:
print(f"加载数据失败: {e}", file=sys.stderr)
return 2
result = compare_op_stats(slow_data, fast_data, top_n=args.top)
result["meta"] = {
"slow_rank": args.slow_rank,
"fast_rank": args.fast_rank,
"slow_path": slow_path,
"fast_path": fast_path,
"data_type": data_type,
}
if args.json:
print(json.dumps(result, ensure_ascii=False, indent=2))
else:
print("=== 慢卡 vs 快卡 算子统计对比 ===")
print(f"慢卡 Rank {args.slow_rank},快卡 Rank {args.fast_rank},数据来源: {data_type}")
print(f"差异最大的前 {args.top} 个算子(按 |diff_total_time_us| 排序):\n")
for i, row in enumerate(result["top_differences"], 1):
print(f"{i}. {row['op_type']}")
print(f" 慢卡: total_time_us={row['slow_total_time_us']:.0f}, count={row['slow_count']}")
print(f" 快卡: total_time_us={row['fast_total_time_us']:.0f}, count={row['fast_count']}")
print(f" diff_total_time_us={row['diff_total_time_us']:.0f}, ratio(slow/fast)={row['total_time_ratio_slow_vs_fast']}, avg_time_diff_us={row['avg_time_diff_us']:.2f}")
print()
print(result["conclusion"])
return 0
if __name__ == "__main__":
sys.exit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |