Skip to content

kali20gakki/msAgent / skills/profiler/ascend-cluster-fast-slow-rank-detector

ascend-cluster-fast-slow-rank-detector

Verified

专门用于 Ascend 集群 Profiling 性能数据的“快慢卡”诊断专家技能。当用户提供【集群性能数据目录/路径】并要求分析【快慢卡】、【慢节点】、【负载不均衡】或【集群瓶颈】时,必须触发此技能。该技能会自动接收集群路径,调度相关工具输出快慢卡的宏观定性与微观根因(如 Host 下发瓶颈、算子计算劣化)。

FileKindSize
SKILL.mdskill md8.0 KB
scripts/compare_api_stats.pyscript8.7 KB
scripts/compare_op_stats.pyscript8.9 KB
scripts/rank_data_finder.pyscript6.0 KB

scripts/rank_data_finder.py

View on GitHub
# -------------------------------------------------------------------------
# This file is part of the MindStudio project.
# Copyright (c) 2026 Huawei Technologies Co.,Ltd.
#
# MindStudio is licensed under Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#
#          http://license.coscl.org.cn/MulanPSL2
#
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.
# -------------------------------------------------------------------------

"""
Discover slow/fast rank profiler data paths under a cluster root.
Supports text (op_statistic.csv / api_statistic.csv) and db (ascend_pytorch_profiler_{rank}.db).
"""
from __future__ import annotations

import json
import os
from typing import Tuple, Optional, Literal

DataType = Literal["text", "db"]


# 限制从 cluster_path 起向下遍历的最大深度,避免误入深层无关目录
DEFAULT_MAX_WALK_DEPTH = 6


def find_profiler_info_dirs(
    cluster_path: str,
    max_depth: int = DEFAULT_MAX_WALK_DEPTH,
) -> dict[int, str]:
    """
    Find all profiler_info_{rank_id}.json under cluster_path and return
    rank_id -> directory (parent of the json file).
    Traversal is limited to max_depth levels below cluster_path.
    """
    result = {}
    cluster_path = os.path.abspath(cluster_path)
    if not os.path.isdir(cluster_path):
        return result
    cluster_path = os.path.normpath(cluster_path)
    for root, dirs, files in os.walk(cluster_path):
        try:
            rel = os.path.relpath(root, cluster_path)
        except ValueError:
            continue
        depth = len(rel.split(os.sep)) if rel != "." else 0
        if depth > max_depth:
            dirs.clear()
            continue
        for f in files:
            if f.startswith("profiler_info_") and f.endswith(".json"):
                try:
                    rank_str = f.replace("profiler_info_", "").replace(".json", "")
                    rank_id = int(rank_str)
                    result[rank_id] = root
                except ValueError:
                    continue
    return result


def find_rank_csv_path(
    cluster_path: str, rank_id: int, csv_name: str
) -> Optional[str]:
    """
    Find path to csv_name (op_statistic.csv or api_statistic.csv) for the given rank.
    csv_name must be 'op_statistic.csv' or 'api_statistic.csv'.
    """
    dirs = find_profiler_info_dirs(cluster_path)
    rank_dir = dirs.get(rank_id)
    if not rank_dir:
        return None
    candidate = os.path.join(rank_dir, "ASCEND_PROFILER_OUTPUT", csv_name)
    return candidate if os.path.isfile(candidate) else None


def find_rank_db_path(cluster_path: str, rank_id: int) -> Optional[str]:
    """Find path to ascend_pytorch_profiler_{rank_id}.db for the given rank."""
    dirs = find_profiler_info_dirs(cluster_path)
    rank_dir = dirs.get(rank_id)
    if not rank_dir:
        return None
    candidate = os.path.join(
        rank_dir, "ASCEND_PROFILER_OUTPUT", f"ascend_pytorch_profiler_{rank_id}.db"
    )
    return candidate if os.path.isfile(candidate) else None


def _data_type_from_path(path: str) -> DataType:
    return "db" if path.lower().endswith(".db") else "text"


def resolve_rank_paths(
    cluster_path: str,
    slow_rank: int,
    fast_rank: int,
    mode: Literal["op", "api"],
    slow_path: Optional[str] = None,
    fast_path: Optional[str] = None,
) -> Tuple[Optional[str], Optional[str], DataType, str]:
    """
    Resolve paths for slow and fast rank data.

    mode: 'op' -> op_statistic.csv or db for op; 'api' -> api_statistic.csv or db for api.

    Returns:
        (slow_path, fast_path, data_type, error_message)
        If error_message is non-empty, path resolution failed.
    """
    csv_name = "op_statistic.csv" if mode == "op" else "api_statistic.csv"

    if slow_path and fast_path:
        if not os.path.isfile(slow_path):
            return None, None, "text", f"慢卡路径不存在或不是文件: {slow_path}"
        if not os.path.isfile(fast_path):
            return None, None, "text", f"快卡路径不存在或不是文件: {fast_path}"
        dt = _data_type_from_path(slow_path)
        return slow_path, fast_path, dt, ""

    cluster_path = os.path.abspath(cluster_path)
    if not os.path.isdir(cluster_path):
        return None, None, "text", f"集群数据路径不存在或不是目录: {cluster_path}"

    # Resolve slow/fast paths (explicit or discovered)
    def resolve_one(rank_id: int, explicit: Optional[str], get_csv=None, get_db=None):
        if explicit and os.path.isfile(explicit):
            if explicit.lower().endswith(".csv"):
                return explicit, None
            if explicit.lower().endswith(".db"):
                return None, explicit
        return (get_csv() if get_csv else None), (get_db() if get_db else None)

    slow_csv, slow_db = resolve_one(
        slow_rank, slow_path,
        get_csv=lambda: find_rank_csv_path(cluster_path, slow_rank, csv_name),
        get_db=lambda: find_rank_db_path(cluster_path, slow_rank),
    )
    fast_csv, fast_db = resolve_one(
        fast_rank, fast_path,
        get_csv=lambda: find_rank_csv_path(cluster_path, fast_rank, csv_name),
        get_db=lambda: find_rank_db_path(cluster_path, fast_rank),
    )

    if slow_csv and fast_csv:
        return slow_csv, fast_csv, "text", ""
    if slow_db and fast_db:
        return slow_db, fast_db, "db", ""

    if slow_db and fast_db:
        return slow_db, fast_db, "db", ""

    missing = []
    if not slow_csv and not slow_db:
        missing.append(f"Rank {slow_rank}")
    if not fast_csv and not fast_db:
        missing.append(f"Rank {fast_rank}")
    return (
        None,
        None,
        "text",
        f"未找到以下卡的数据(未找到 {csv_name} 或 ascend_pytorch_profiler_{{rank}}.db):{', '.join(missing)}。请确认集群路径或手动指定 --slow-path / --fast-path。",
    )
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall