Ascend/msagent / skills/profiler/ascend-cluster-compare
ascend-cluster-compare
VerifiedAscend cluster comparison tool. Invoke when user asks to compare two cluster datasets (DB or TEXT), or to generate cluster_analysis_output from raw profiling data via msprof-analyze and then compare.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 26.1 KB |
| references/db_schema.md | markdown | 8.2 KB |
| references/text_schema.md | markdown | 9.0 KB |
| scripts/advanced_insights.py | script | 15.2 KB |
| scripts/cluster_data_extractor.py | script | 36.0 KB |
| scripts/generate_cluster_md.py | script | 28.0 KB |
| scripts/generate_cluster_report.py | script | 46.3 KB |
| scripts/run_advanced_analysis.py | script | 20.9 KB |
| scripts/verify_extracted_metrics.py | script | 7.2 KB |
| templates/cluster_compare_report.html | other | 15.3 KB |
scripts/run_advanced_analysis.py
View on GitHub#!/usr/bin/env python3
"""
run_advanced_analysis.py
msprof-analyze 进阶分析编排脚本。
职责:
1. 检测集群数据格式(DB/TEXT)与 msprof-analyze 工具可用性;
2. 判断哪些进阶 feature 可运行, 必要时自动补跑前置 cluster_time_summary;
3. 执行进阶 feature, 已有输出时直接复用(--force 可强制重跑);
4. 解析输出(CSV / SQLite 表)为结构化数据, 输出 advanced_analysis.json,
供 generate_cluster_report.py --advanced 注入 HTML 报告「进阶分析」章节。
进阶分析能力对照(源自 msprof-analyze advanced_features 文档):
- cluster_time_compare_summary : 双集群 rank/step 级时间拆解对比, 需两侧 DB 均含
ClusterTimeSummary 表(缺失时本脚本自动补跑); 纯 TEXT 模式数据不支持。
- free_analysis : 单集群空闲原因分析, 支持 --export_type text 输出 CSV
(cluster_analysis_output/FreeAnalysis/free_analysis.csv)。
- communication_bottleneck : 单集群通信瓶颈定位(慢/快 rank + 原因), 支持
--export_type text(cluster_analysis_output/CommunicationBottleneckAnalysis/
communication_bottleneck.csv)。
其余 feature(freq_analysis / ep_load_balance / slow_rank / slow_link / hccl_sum /
communication_matrix_sum / mstx_sum / cann_api_sum / export_summary 等)可按相同
模式扩展: 补一个 run_xxx + parse_xxx 即可。
用法:
python run_advanced_analysis.py --cluster-a <dirA> [--cluster-b <dirB>] --output <advanced.json>
python run_advanced_analysis.py --cluster-a <dirA> --cluster-b <dirB> --output adv.json --dry-run
注意:
单集群 feature(free_analysis / communication_bottleneck)会对 A、B 两个集群分别执行,
msprof 输出目录按集群隔离(<work-dir>/cluster_A、<work-dir>/cluster_B), 结果记录带
cluster 标注; 若提取器(cluster_data_extractor.py)已在各集群数据目录生成过进阶分析
输出, 本脚本会自动复用(status=cached), 不会重复执行。
"""
import argparse
import csv
import json
import os
import shutil
import sqlite3
import subprocess
from datetime import datetime
FEAT_TIME_CMP = "cluster_time_compare_summary"
FEAT_FREE = "free_analysis"
FEAT_COMM_BN = "communication_bottleneck"
TIME_SUMMARY_TABLE = "ClusterTimeSummary"
COMPARE_TABLE = "ClusterTimeCompareSummary"
CSV_FREE = "free_analysis.csv"
CSV_COMM_BN = "communication_bottleneck.csv"
DEFAULT_TIMEOUT = 7200 # 秒, 集群级分析可能较慢
def log(msg):
print(msg, flush=True)
def _to_float(v):
try:
return float(v)
except (TypeError, ValueError):
return 0.0
def _pick(row, *names):
"""大小写不敏感、忽略空格/下划线差异地从 DictReader 行中取列值。"""
norm = {}
for k, v in row.items():
if k is None:
continue
norm[k.strip().lower().replace(" ", "").replace("_", "")] = (v or "").strip()
for n in names:
v = norm.get(n.strip().lower().replace(" ", "").replace("_", ""))
if v:
return v
return ""
# ==================== 环境与数据探测 ====================
def find_msprof(bin_name):
return shutil.which(bin_name)
def get_tool_version(msprof):
try:
p = subprocess.run([msprof, "--version"], capture_output=True, text=True, timeout=60)
out = (p.stdout or "").strip() or (p.stderr or "").strip()
return out.splitlines()[0][:120] if out else ""
except Exception:
return ""
def detect_db(data_dir):
for cand in (
os.path.join(data_dir, "cluster_analysis_output", "cluster_analysis.db"),
os.path.join(data_dir, "cluster_analysis.db"),
os.path.join(data_dir, "cluster.db"),
):
if os.path.isfile(cand):
return cand
return None
def detect_text_dir(data_dir):
for sub in (os.path.join(data_dir, "cluster_analysis_output"), data_dir):
if sub and os.path.isfile(os.path.join(sub, "cluster_step_trace_time.csv")):
return sub
return None
def table_exists(db_path, table):
try:
con = sqlite3.connect(db_path)
try:
row = con.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,)
).fetchone()
return row is not None
finally:
con.close()
except Exception:
return False
def detect_cluster(data_dir):
info = {
"data_dir": data_dir,
"db_path": None,
"db_kind": None,
"text_dir": None,
"format": "unknown",
"has_time_summary_table": False,
}
if not data_dir or not os.path.isdir(data_dir):
info["format"] = "not_found"
return info
db = detect_db(data_dir)
if db:
info["db_path"] = db
info["db_kind"] = "db_new" if os.path.basename(db) == "cluster_analysis.db" else "db_old"
info["format"] = "db"
info["has_time_summary_table"] = table_exists(db, TIME_SUMMARY_TABLE)
txt = detect_text_dir(data_dir)
if txt:
info["text_dir"] = txt
info["format"] = info["format"] if info["db_path"] else "text"
return info
def run_tool(msprof, args_list, timeout=DEFAULT_TIMEOUT):
cmd = [msprof] + args_list
log(f" [exec] {' '.join(cmd)}")
try:
p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
return p.returncode, (p.stdout or ""), (p.stderr or "")
except FileNotFoundError:
return 127, "", f"命令不存在: {msprof}"
except subprocess.TimeoutExpired:
return 124, "", "执行超时"
except Exception as e:
return 1, "", str(e)
def find_output_file(roots, filename, max_depth=4):
"""在候选根目录下限深递归搜索文件, 返回第一个命中路径。"""
for root in roots:
if not root or not os.path.isdir(root):
continue
for dirpath, dirnames, filenames in os.walk(root):
if dirpath[len(root):].count(os.sep) >= max_depth:
dirnames[:] = []
continue
if filename in filenames:
return os.path.join(dirpath, filename)
return None
def find_db_with_table(roots, table, max_depth=4):
"""搜索包含指定表的 cluster_analysis.db。"""
for root in roots:
if not root or not os.path.isdir(root):
continue
for dirpath, dirnames, filenames in os.walk(root):
if dirpath[len(root):].count(os.sep) >= max_depth:
dirnames[:] = []
continue
if "cluster_analysis.db" in filenames:
p = os.path.join(dirpath, "cluster_analysis.db")
if table_exists(p, table):
return p
return None
# ==================== 输出解析 ====================
def parse_free_csv(csv_path, top_n=10):
"""free_analysis text 输出解析: 列含 Rank ID/Start Time(us)/End Time(us)/
Duration(us)/Pytorch Idle Time(us)/Cann Idle Time(us)/Reason。按 Reason 聚合。"""
rows = []
with open(csv_path, "r", encoding="utf-8-sig", newline="") as f:
for row in csv.DictReader(f):
if row:
rows.append(row)
agg = {}
total_us = 0.0
for r in rows:
reason = _pick(r, "Reason") or "未标注原因"
dur_us = _to_float(_pick(r, "Duration(us)", "Duration"))
rank = _pick(r, "Rank ID", "Rank")
total_us += dur_us
a = agg.setdefault(reason, {"count": 0, "duration_us": 0.0, "ranks": set()})
a["count"] += 1
a["duration_us"] += dur_us
if rank:
a["ranks"].add(rank)
reasons = []
for reason, a in agg.items():
reasons.append({
"reason": reason[:200],
"count": a["count"],
"total_ms": round(a["duration_us"] / 1000.0, 2),
"pct": round(a["duration_us"] / total_us * 100, 1) if total_us > 0 else 0,
"rank_count": len(a["ranks"]),
"sample_ranks": sorted(a["ranks"])[:8],
})
reasons.sort(key=lambda x: x["total_ms"], reverse=True)
return {
"rows_total": len(rows),
"total_ms": round(total_us / 1000.0, 2),
"reasons": reasons[:top_n],
}
def parse_comm_bn_csv(csv_path, top_n=20):
"""communication_bottleneck text 输出解析: 列含 Communication Op/
Slow Rank ID/Fast Rank ID/Reason 等(列名以实际文件为准)。"""
with open(csv_path, "r", encoding="utf-8-sig", newline="") as f:
rows = [
{k.strip(): (v or "").strip() for k, v in r.items() if k}
for r in csv.DictReader(f) if r
]
items = []
for r in rows:
op = _pick(r, "Communication Op", "CommunicationOp", "Op Name", "Op")
slow = _pick(r, "Slow Rank ID", "SlowRankID", "Slow Rank")
fast = _pick(r, "Fast Rank ID", "FastRankID", "Fast Rank")
reason = _pick(r, "Reason")
dur_us = _to_float(_pick(r, "Duration(us)", "Elapsed(us)", "Time(us)", "Transit Time(us)"))
items.append({
"op": op[:120],
"slow_rank": slow,
"fast_rank": fast,
"reason": reason[:300],
"duration_ms": round(dur_us / 1000.0, 2),
})
items.sort(key=lambda x: x["duration_ms"], reverse=True)
return {
"rows_total": len(rows),
"columns": list(rows[0].keys()) if rows else [],
"items": items[:top_n],
}
def parse_compare_summary(db_path, top_n=10):
"""ClusterTimeCompareSummary 表解析: 字段 rank/step/{metrics}/{metrics}Base/
{metrics}Diff, 时间单位 μs → 统一转 ms。"""
con = sqlite3.connect(db_path)
try:
cur = con.execute(f"SELECT * FROM {COMPARE_TABLE}")
cols = [d[0] for d in cur.description]
rows = [dict(zip(cols, r)) for r in cur.fetchall()]
finally:
con.close()
if not rows:
return {"rows_total": 0}
diff_cols = [c for c in cols if c.endswith("Diff")]
metrics = [c[:-4] for c in diff_cols]
key_cols = [c for c in cols if not c.endswith("Base") and not c.endswith("Diff")]
def _nums(r, col):
v = r.get(col)
return v if isinstance(v, (int, float)) else None
overall = {}
for m in metrics:
b = [v for r in rows if (v := _nums(r, m + "Base")) is not None]
c = [v for r in rows if (v := _nums(r, m)) is not None]
d = [v for r in rows if (v := _nums(r, m + "Diff")) is not None]
overall[m] = {
"base_avg_ms": round(sum(b) / len(b) / 1000.0, 2) if b else None,
"cur_avg_ms": round(sum(c) / len(c) / 1000.0, 2) if c else None,
"diff_avg_ms": round(sum(d) / len(d) / 1000.0, 2) if d else None,
}
rank_col = next(
(c for c in key_cols if c.strip().lower() in ("rank", "rank_id", "rankid", "index")),
None,
)
top_ranks = []
if rank_col and metrics:
stage_metric = next((m for m in metrics if "stage" in m.lower()), metrics[0])
by_rank = {}
for r in rows:
by_rank.setdefault(r.get(rank_col), []).append(r)
for rk, rs in by_rank.items():
item = {"rank": rk, "step_count": len(rs)}
for m in metrics:
vals = [v for x in rs if (v := _nums(x, m + "Diff")) is not None]
item[m + "_diff_ms"] = round(sum(vals) / len(vals) / 1000.0, 2) if vals else None
top_ranks.append(item)
top_ranks.sort(key=lambda x: (x.get(stage_metric + "_diff_ms") or 0), reverse=True)
top_ranks = top_ranks[:top_n]
return {
"rows_total": len(rows),
"key_cols": key_cols,
"metrics": metrics,
"overall": overall,
"rank_col": rank_col,
"top_ranks": top_ranks,
}
# ==================== feature 编排 ====================
def feature_availability(name, tool_available, ca, cb):
"""返回 (available, reason)。"""
if not tool_available:
return False, "msprof-analyze 工具不可用(未安装或不在 PATH)"
if name == FEAT_TIME_CMP:
if not (ca and cb):
return False, "cluster_time_compare_summary 需要提供两个集群目录(缺少 --cluster-b)"
for tag, c in (("集群A", ca), ("集群B", cb)):
if not c["db_path"]:
return False, f"{tag} 无 DB 格式数据(当前格式: {c['format']}),该 feature 仅支持 DB 输入"
return True, ""
target = cb or ca
if not target or target["format"] in ("unknown", "not_found"):
return False, "目标集群目录下未识别到 cluster_analysis_output / profiling 数据"
return True, ""
def ensure_time_summary(msprof, c, tag):
"""确保集群 DB 含 ClusterTimeSummary 表; 缺失时自动补跑。返回 (ok, msg)。"""
if c["has_time_summary_table"]:
return True, ""
log(f" [prep] {tag} 缺少 {TIME_SUMMARY_TABLE} 表, 自动补跑 cluster_time_summary ...")
rc, out, err = run_tool(msprof, ["-m", "cluster_time_summary", "-d", c["data_dir"]])
if rc != 0:
tail = (err or out or "").strip().splitlines()[-5:]
return False, f"{tag} 补跑 cluster_time_summary 失败(rc={rc}): " + " | ".join(tail)[-400:]
c["has_time_summary_table"] = True
return True, ""
def run_feature_time_cmp(msprof, ca, cb, work_dir, top_num, force):
feat = {
"name": FEAT_TIME_CMP,
"label": "集群时间拆解对比",
"desc": "双集群 rank/step 级时间指标对比(基准/当前/差值),需两侧 DB 含 ClusterTimeSummary 表",
}
ok, reason = feature_availability(FEAT_TIME_CMP, bool(msprof), ca, cb)
feat["available"], feat["reason"] = ok, reason
if not ok:
feat["status"] = "skipped" if not (ca and cb) else "not_available"
return feat
ok1, msg1 = ensure_time_summary(msprof, ca, "集群A(基准)")
ok2, msg2 = ensure_time_summary(msprof, cb, "集群B(对比)")
if not (ok1 and ok2):
feat["status"] = "failed"
feat["reason"] = msg1 or msg2
return feat
existing = find_db_with_table([work_dir, cb["data_dir"], ca["data_dir"]], COMPARE_TABLE)
if existing and not force:
feat["status"] = "cached"
feat["data"] = parse_compare_summary(existing, top_num)
feat["data"]["source_db"] = existing
return feat
cmd = ["-m", FEAT_TIME_CMP, "-d", cb["data_dir"], "--bp", ca["data_dir"], "-o", work_dir]
feat["command"] = " ".join(cmd)
rc, out, err = run_tool(msprof, cmd)
db_hit = find_db_with_table([work_dir, cb["data_dir"]], COMPARE_TABLE) if rc == 0 else None
if rc == 0 and db_hit:
feat["status"] = "success"
feat["data"] = parse_compare_summary(db_hit, top_num)
feat["data"]["source_db"] = db_hit
else:
tail = (err or out or "").strip().splitlines()[-5:]
feat["status"] = "failed"
feat["reason"] = (f"执行失败(rc={rc}): " + " | ".join(tail))[-500:] if tail else "执行完成但未找到结果 DB"
return feat
def run_feature_on_target(msprof, target, work_dir, top_num, force,
feat_name, feat_label, feat_desc, csv_name,
cluster_tag=None):
"""单集群 feature(free_analysis / communication_bottleneck)通用编排。
cluster_tag: 集群标识("A"/"B")。提供时 msprof 输出目录改为 <work_dir>/cluster_<tag>,
避免多集群共享同一 work_dir 时输出互相覆盖; 缓存搜索同时覆盖该子目录与集群数据目录。
"""
feat = {"name": feat_name, "label": feat_label, "desc": feat_desc}
if cluster_tag:
feat["cluster"] = cluster_tag
ok, reason = feature_availability(feat_name, bool(msprof), target, None)
feat["available"], feat["reason"] = ok, reason
if not ok:
feat["status"] = "not_available"
return feat
out_dir = os.path.join(work_dir, f"cluster_{cluster_tag}") if cluster_tag else work_dir
os.makedirs(out_dir, exist_ok=True)
roots = [out_dir, work_dir, target["data_dir"], target.get("text_dir")]
existing = find_output_file(roots, csv_name)
if existing and not force:
feat["status"] = "cached"
feat["data"] = (parse_free_csv if feat_name == FEAT_FREE else parse_comm_bn_csv)(existing, top_num)
feat["data"]["source_csv"] = existing
return feat
cmd = ["-m", feat_name, "-d", target["data_dir"],
"--export_type", "text", "--top_num", str(top_num), "-o", out_dir]
feat["command"] = " ".join(cmd)
rc, out, err = run_tool(msprof, cmd)
csv_hit = find_output_file([out_dir, work_dir, target["data_dir"]], csv_name) if rc == 0 else None
if rc == 0 and csv_hit:
feat["status"] = "success"
feat["data"] = (parse_free_csv if feat_name == FEAT_FREE else parse_comm_bn_csv)(csv_hit, top_num)
feat["data"]["source_csv"] = csv_hit
else:
tail = (err or out or "").strip().splitlines()[-5:]
feat["status"] = "failed"
feat["reason"] = (f"执行失败(rc={rc}): " + " | ".join(tail))[-500:] if tail else "执行完成但未找到输出 CSV"
return feat
# ==================== 主流程 ====================
def main():
ap = argparse.ArgumentParser(description="msprof-analyze 进阶分析编排(供集群比对报告注入)")
ap.add_argument("--cluster-a", required=True, help="集群A目录(基准; 单集群模式下同时作为分析目标)")
ap.add_argument("--cluster-b", default=None, help="集群B目录(对比; 提供后单集群 feature 优先分析 B)")
ap.add_argument("--output", required=True, help="输出 advanced_analysis.json 路径")
ap.add_argument("--msprof", default="msprof-analyze", help="msprof-analyze 可执行文件名/路径")
ap.add_argument("--top-num", type=int, default=10, help="各 feature 结果 Top N")
ap.add_argument("--work-dir", default=None, help="msprof 输出目录(默认: <output 同目录>/advanced_work)")
ap.add_argument("--dry-run", action="store_true", help="仅检测可用性,不执行任何分析")
ap.add_argument("--force", action="store_true", help="忽略已有输出,强制重新执行")
args = ap.parse_args()
out_path = os.path.abspath(args.output)
out_dir = os.path.dirname(out_path) or "."
os.makedirs(out_dir, exist_ok=True)
work_dir = os.path.abspath(args.work_dir or os.path.join(out_dir, "advanced_work"))
if not args.dry_run:
os.makedirs(work_dir, exist_ok=True)
msprof = find_msprof(args.msprof)
tool = {
"bin": args.msprof,
"available": msprof is not None,
"path": msprof or "",
"version": "",
}
if tool["available"]:
tool["version"] = get_tool_version(msprof)
ca = detect_cluster(os.path.abspath(args.cluster_a))
cb = detect_cluster(os.path.abspath(args.cluster_b)) if args.cluster_b else None
log(f"[detect] 集群A: format={ca['format']}, db={ca['db_path']}, "
f"time_summary={ca['has_time_summary_table']}")
if cb:
log(f"[detect] 集群B: format={cb['format']}, db={cb['db_path']}, "
f"time_summary={cb['has_time_summary_table']}")
result = {
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"dry_run": bool(args.dry_run),
"tool": tool,
"clusters": {"A": ca, "B": cb},
"features": [],
}
if not tool["available"]:
log("[warn] msprof-analyze 不可用, 所有 feature 标记为 not_available")
# 1) 双集群时间拆解对比(DB only)
result["features"].append(run_feature_time_cmp(msprof, ca, cb, work_dir, args.top_num, args.force))
# 2/3) 单集群 feature: free_analysis / communication_bottleneck(对 A、B 分别执行)
for tag, cluster in (("A", ca), ("B", cb)):
if cluster is None:
continue
tag_label = "基准" if tag == "A" else "对比"
for feat_name, feat_label, feat_desc, csv_name in (
(FEAT_FREE, "空闲时间原因分析",
"定位单集群空闲(Free)时间的成因分类与耗时排名, 辅助 Host 下发瓶颈定位",
CSV_FREE),
(FEAT_COMM_BN, "通信瓶颈定位",
"定位单集群慢卡/快卡与通信瓶颈原因, 辅助链路劣化排查",
CSV_COMM_BN),
):
feat = run_feature_on_target(
msprof, cluster, work_dir, args.top_num, args.force,
feat_name, feat_label, feat_desc, csv_name, cluster_tag=tag)
result["features"].append(feat)
log(f"[feat] 集群{tag}({tag_label}) {feat_name}: {feat['status']}"
+ (f"({feat['reason']})" if feat.get("reason") and feat["status"] != "success" else ""))
if args.dry_run:
for f in result["features"]:
f["status"] = "not_available" if not f.get("available") else "pending"
with open(out_path, "w", encoding="utf-8") as f:
json.dump(result, f, ensure_ascii=False, indent=2)
log(f"[done] 进阶分析结果已写入: {out_path}")
for f in result["features"]:
log(f" - {f['name']}: {f['status']}" + (f"({f['reason']})" if f.get("reason") and f["status"] != "success" else ""))
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 197k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |