Skip to content

kali20gakki/msAgent / skills/profiler/ascend-npu-snapshot-analyzer

ascend-npu-snapshot-analyzer

Unverified

分析 PyTorch memory snapshot pickle 文件(_dump_snapshot 导出的内存快照),提供内存峰值、碎片、泄漏、OOM 检测与交互式 HTML 报告。仅当用户明确提及 memory snapshot、内存快照、_dump_snapshot、pickle 内存文件 或需要对 snapshot pickle 做内存分析时才触发,不处理一般的 NPU 性能分析、SQLite 查询或通用内存问题。

scripts/snapshot_analyze.py

View on GitHub
# -------------------------------------------------------------------------
# This file is part of the MindStudio project.
# Copyright (c) 2026 Huawei Technologies Co.,Ltd.
#
# MindStudio is licensed under Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#
#          http://license.coscl.org.cn/MulanPSL2
#
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.
# -------------------------------------------------------------------------

#!/usr/bin/env python3
"""
Ascend NPU Memory Snapshot 高层分析工具

支持六种分析模式:
    overview  - 总体概览
    peak      - 峰值分析
    fragment  - 碎片分析
    leak      - 泄漏检测
    oom       - OOM 分析
    compare   - 跨快照对比
    all       - 全模式分析 + HTML 报告

用法:
    python snapshot_analyze.py snapshot.db --mode overview
    python snapshot_analyze.py snapshot.db --mode peak
    python snapshot_analyze.py snapshot.db --mode fragment
    python snapshot_analyze.py snapshot.db --mode leak
    python snapshot_analyze.py snapshot.db --mode oom
    python snapshot_analyze.py snapshot.db --mode compare --ref other.db
    python snapshot_analyze.py snapshot.db --mode all -o report.html
"""

import argparse
import html
import json
import os
import sys
from typing import Any, Dict, List, Optional

import snapshot_queries as q


def _format_bytes(size_bytes: int) -> str:
    if size_bytes < 1024:
        return f"{size_bytes} B"
    elif size_bytes < 1024 ** 2:
        return f"{size_bytes / 1024:.1f} KB"
    elif size_bytes < 1024 ** 3:
        return f"{size_bytes / (1024 ** 2):.1f} MB"
    else:
        return f"{size_bytes / (1024 ** 3):.2f} GB"


def _health_status(frag_pct: float, segment_count: int, has_oom: bool) -> Dict[str, str]:
    if has_oom or frag_pct > 15:
        return {"icon": "[ERR]", "level": "严重", "text": "严重 (OOM 或碎片率 >15%)"}
    elif frag_pct > 5 or segment_count > 100:
        return {"icon": "[WARN]", "level": "需关注", "text": "需关注 (碎片率偏高或 Segment 偏多)"}
    else:
        return {"icon": "[OK]", "level": "健康", "text": "健康"}


def _metric_status(value: float, thresholds: tuple) -> str:
    low, high = thresholds
    if value < low:
        return "[OK]"
    elif value <= high:
        return "[WARN]"
    return "[ERR]"


def analyze_overview(db_path: str) -> Dict[str, Any]:
    overview = q.get_device_overview(db_path)
    device_count = q.get_device_count(db_path)
    oom_events = q.get_oom_events(db_path)

    total_reserved = sum(d["reserved_bytes"] for d in overview)
    total_allocated = sum(d["allocated_bytes"] for d in overview)
    total_segments = sum(d["segment_count"] for d in overview)
    frag_pct = round((total_reserved - total_allocated) / total_reserved * 100, 1) if total_reserved > 0 else 0

    health = _health_status(frag_pct, total_segments, len(oom_events) > 0)

    return {
        "mode": "overview",
        "health": health,
        "device_count": device_count,
        "oom_count": len(oom_events),
        "summary": {
            "reserved": total_reserved,
            "reserved_fmt": _format_bytes(total_reserved),
            "allocated": total_allocated,
            "allocated_fmt": _format_bytes(total_allocated),
            "frag_pct": frag_pct,
            "segment_count": total_segments,
        },
        "devices": overview,
    }


def _format_frames(frames_json: Optional[str], max_frames: int = 5) -> str:
    """将 frames_json 格式化为可读的调用路径,跳过 boilerplate 帧并截断超长函数名"""
    if not frames_json:
        return "(无堆栈信息)"
    try:
        frames_list = json.loads(frames_json)
        if not frames_list:
            return "(空堆栈)"

        skip_prefixes = ("torch::unwind::", "torch::CapturedTraceback::", "std::promise", "std::error_code")

        parts = []
        for f in frames_list:
            if isinstance(f, dict):
                name = f.get("name", f.get("filename", "??"))
                if any(name.startswith(p) for p in skip_prefixes):
                    continue
                if len(name) > 80:
                    name = name[:77] + "..."
                line = f.get("line", 0)
                parts.append(f"{name}:{line}" if line else name)
            elif isinstance(f, str):
                if any(f.startswith(p) for p in skip_prefixes):
                    continue
                if len(f) > 80:
                    f = f[:77] + "..."
                parts.append(f)
            else:
                parts.append(str(f))
            if len(parts) >= max_frames:
                break
        return html.escape(" → ".join(parts) if parts else "(无有效堆栈帧)")
    except (json.JSONDecodeError, TypeError):
        return html.escape(str(frames_json)[:120])


def analyze_peak(db_path: str, device_index: Optional[int] = None) -> Dict[str, Any]:
    timeline = q.get_peak_timeline(db_path, device_index)

    if not timeline:
        return {"mode": "peak", "error": "无时序数据"}

    per_device = {}
    current_dev = None
    reserved = 0
    device_start_idx = {}

    for i, event in enumerate(timeline):
        dev = event.get("device_index", 0)
        if current_dev is None:
            current_dev = dev
        if dev != current_dev:
            current_dev = dev
            reserved = 0

        if dev not in device_start_idx:
            device_start_idx[dev] = i

        action = event["action"]
        size = event.get("size", 0)

        if action in ("segment_alloc", "segment_map"):
            reserved += size
        elif action in ("segment_free", "segment_unmap"):
            reserved -= size

        if dev not in per_device:
            per_device[dev] = {
                "peak_reserved": 0,
                "peak_index": i,
                "peak_event": event,
                "baseline_reserved": 0,
                "baseline_index": 0,
            }
        pd = per_device[dev]

        if reserved > pd["peak_reserved"]:
            pd["peak_reserved"] = reserved
            pd["peak_index"] = i
            pd["peak_event"] = event

    for dev, pd in per_device.items():
        dev_timeline = [e for e in timeline if e.get("device_index", 0) == dev]
        cutoff = max(1, len(dev_timeline) // 10)
        if cutoff < len(dev_timeline):
            pd["baseline_index"] = dev_timeline[cutoff].get("trace_index", 0)
            baseline_r = 0
            for j in range(cutoff + 1):
                event = dev_timeline[j]
                action = event["action"]
                size = event.get("size", 0)
                if action in ("segment_alloc", "segment_map"):
                    baseline_r += size
                elif action in ("segment_free", "segment_unmap"):
                    baseline_r -= size
            pd["baseline_reserved"] = baseline_r

    alloc_events = q.get_peak_alloc_events(db_path, device_index, limit=10)
    peak_contributors = []
    for evt in alloc_events:
        peak_contributors.append({
            "trace_index": evt["trace_index"],
            "action": evt["action"],
            "size": evt["size"],
            "size_fmt": _format_bytes(evt["size"]),
            "addr": hex(evt["addr"]) if evt.get("addr") else "N/A",
            "call_path": _format_frames(evt.get("frames_json")),
        })

    top_blocks = q.get_top_blocks_with_stack(db_path, device_index, limit=10)
    peak_blocks = []
    for blk in top_blocks:
        peak_blocks.append({
            "size": blk["size"],
            "size_fmt": _format_bytes(blk["size"]),
            "requested_size": blk["requested_size"],
            "requested_fmt": _format_bytes(blk["requested_size"]),
            "state": blk["state"],
            "segment_addr": hex(blk["segment_address"]) if blk.get("segment_address") else "N/A",
            "call_path": _format_frames(blk.get("frames_json")),
        })

    devices = []
    for dev, pd in sorted(per_device.items()):
        peak_r = pd["peak_reserved"]
        base_r = pd["baseline_reserved"]
        devices.append({
            "device_index": dev,
            "peak_trace_index": pd["peak_event"].get("trace_index"),
            "peak": {
                "reserved": peak_r,
                "reserved_fmt": _format_bytes(peak_r),
            },
            "baseline": {
                "description": "基线 (该设备时间线前 10% 位置)",
                "trace_index": pd["baseline_index"],
                "reserved": base_r,
                "reserved_fmt": _format_bytes(base_r),
            },
            "deltas": {
                "description": "该设备从基线到峰值的增长量",
                "reserved": peak_r - base_r,
                "reserved_fmt": _format_bytes(peak_r - base_r),
                "reserved_pct": round((peak_r - base_r) / base_r * 100, 1) if base_r > 0 else None,
            },
        })

    overall_peak = sum(pd["peak_reserved"] for pd in per_device.values())
    overall_baseline = sum(pd["baseline_reserved"] for pd in per_device.values())

    return {
        "mode": "peak",
        "device": device_index,
        "devices": devices,
        "peak": {
            "reserved": overall_peak,
            "reserved_fmt": _format_bytes(overall_peak),
        },
        "baseline": {
            "description": "基线 (各设备时间线前 10% 位置)",
            "reserved": overall_baseline,
            "reserved_fmt": _format_bytes(overall_baseline),
        },
        "deltas": {
            "description": "从基线到峰值的增长量",
            "reserved": overall_peak - overall_baseline,
            "reserved_fmt": _format_bytes(overall_peak - overall_baseline),
            "reserved_pct": round((overall_peak - overall_baseline) / overall_baseline * 100, 1) if overall_baseline > 0 else None,
        },
        "peak_alloc_events": {
            "description": "峰值归因: 最大的 segment_alloc / segment_map 事件 (按 size 降序),这些操作的代码路径是峰值的主要贡献者",
            "top": peak_contributors,
        },
        "peak_blocks": {
            "description": "峰值归因: 最大的 active_allocated block (按 size 降序),展示占用峰值内存的具体 tensor 分配",
            "top": peak_blocks,
        },
    }


def analyze_fragment(db_path: str, device_index: Optional[int] = None) -> Dict[str, Any]:
    overview = q.get_device_overview(db_path)
    frag_detail = q.get_fragmentation_detail(db_path, device_index)
    block_dist = q.get_block_state_dist(db_path, device_index)

    total_reserved = sum(d["reserved_bytes"] for d in overview)
    total_allocated = sum(d["allocated_bytes"] for d in overview)
    frag_pct = round((total_reserved - total_allocated) / total_reserved * 100, 1) if total_reserved > 0 else 0

    free_bytes = total_reserved - total_allocated
    pending_bytes = sum(b["total_size"] for b in block_dist if b["state"] == "active_pending_free")
    pending_pct = round(pending_bytes / free_bytes * 100, 1) if free_bytes > 0 else 0

    is_pseudo = pending_pct > 50

    top_fragmented = sorted(frag_detail, key=lambda x: x["frag_pct"], reverse=True)[:5]

    return {
        "mode": "fragment",
        "device": device_index,
        "overall": {
            "frag_pct": frag_pct,
            "status": _metric_status(frag_pct, (5, 15)),
            "free_bytes": free_bytes,
            "free_fmt": _format_bytes(free_bytes),
            "pending_bytes": pending_bytes,
            "pending_fmt": _format_bytes(pending_bytes),
            "pending_pct": pending_pct,
            "is_pseudo": is_pseudo,
        },
        "top_fragmented": top_fragmented,
    }


def analyze_leak(db_path: str, device_index: Optional[int] = None) -> Dict[str, Any]:
    trend = q.get_net_segment_trend(db_path, device_index)
    long_blocks = q.get_long_lived_blocks(db_path, device_index)
    stack_attr = q.get_stack_attribution(db_path, device_index)

    is_monotonic = False
    if len(trend) >= 2:
        values = [t["net_segments"] for t in trend]
        is_monotonic = all(values[i] <= values[i + 1] for i in range(len(values) - 1)) and values[-1] > values[0]

    long_lived_count = len(long_blocks)

    if is_monotonic and long_lived_count >= 5:
        risk = {"icon": "[ERR]", "level": "高风险"}
    elif is_monotonic or long_lived_count >= 3:
        risk = {"icon": "[WARN]", "level": "中风险"}
    else:
        risk = {"icon": "[OK]", "level": "低风险"}

    trend_start = trend[0]["net_segments"] if trend else 0
    trend_end = trend[-1]["net_segments"] if trend else 0

    suspects = []
    for attr in stack_attr[:3]:
        key_path = _format_frames(attr.get("frames_json"))

        suspects.append({
            "stack_id": attr["stack_id"],
            "total_size": attr["total_size"],
            "total_size_fmt": _format_bytes(attr["total_size"]),
            "block_count": attr["block_count"],
            "pct": attr["pct"],
            "key_path": key_path,
        })

    return {
        "mode": "leak",
        "device": device_index,
        "risk": risk,
        "monotonic_growth": {
            "detected": is_monotonic,
            "start": trend_start,
            "end": trend_end,
        },
        "long_lived_blocks": long_lived_count,
        "suspects": suspects,
    }


def analyze_oom(db_path: str, device_index: Optional[int] = None) -> Dict[str, Any]:
    oom_events = q.get_oom_events(db_path, device_index)

    if not oom_events:
        return {"mode": "oom", "detected": False, "events": []}

    results = []
    for oom in oom_events:
        dev = oom["device_index"]
        idx = oom["trace_index"]
        device_free = oom.get("device_free", 0)

        start_idx = max(0, idx - 50)
        pre_events = q.get_trace_range(db_path, dev, start_idx, idx)

        pre_allocs = [e for e in pre_events if e["action"] == "alloc"]
        pre_allocs.sort(key=lambda x: x.get("size", 0), reverse=True)
        last_5 = pre_allocs[:5] if len(pre_allocs) >= 5 else pre_allocs

        last_5_total = sum(e.get("size", 0) for e in last_5)

        if len(last_5) >= 3:
            stack_counts: Dict[str, int] = {}
            for e in last_5:
                frames = e.get("frames_json", "")
                key = str(frames)[:60] if frames else "(无堆栈)"
                stack_counts[key] = stack_counts.get(key, 0) + 1
            max_repeat = max(stack_counts.values())
            if max_repeat >= 3:
                root_cause = "同一堆栈重复分配"
            elif last_5_total > device_free * 3:
                root_cause = "一连串大分配"
            else:
                root_cause = "单次过大分配请求"
        else:
            root_cause = "数据不足,无法推断"

        results.append({
            "device": dev,
            "oom_index": idx,
            "device_free": device_free,
            "device_free_fmt": _format_bytes(device_free),
            "last_5_allocations": [
                {
                    "size": e.get("size", 0),
                    "size_fmt": _format_bytes(e.get("size", 0)),
                    "action": e.get("action"),
                    "frames": e.get("frames_json"),
                }
                for e in reversed(last_5)
            ],
            "root_cause": root_cause,
        })

    return {"mode": "oom", "detected": True, "events": results}


def analyze_compare(db_path: str, ref_path: str) -> Dict[str, Any]:
    if not os.path.exists(ref_path):
        return {"mode": "compare", "error": f"参考文件不存在: {ref_path}"}

    a_overview = q.get_device_overview(db_path)
    b_overview = q.get_device_overview(ref_path)

    a_reserved = sum(d["reserved_bytes"] for d in a_overview)
    a_allocated = sum(d["allocated_bytes"] for d in a_overview)
    a_segments = sum(d["segment_count"] for d in a_overview)
    a_frag = round((a_reserved - a_allocated) / a_reserved * 100, 1) if a_reserved > 0 else 0

    b_reserved = sum(d["reserved_bytes"] for d in b_overview)
    b_allocated = sum(d["allocated_bytes"] for d in b_overview)
    b_segments = sum(d["segment_count"] for d in b_overview)
    b_frag = round((b_reserved - b_allocated) / b_reserved * 100, 1) if b_reserved > 0 else 0

    a_oom = len(q.get_oom_events(db_path))
    b_oom = len(q.get_oom_events(ref_path))

    def _diff(a_val, b_val):
        return b_val - a_val

    def _trend(diff):
        if diff > 0:
            return "[+]"
        elif diff < 0:
            return "[-]"
        return "[=]"

    metrics = [
        {
            "name": "Reserved",
            "a": a_reserved, "a_fmt": _format_bytes(a_reserved),
            "b": b_reserved, "b_fmt": _format_bytes(b_reserved),
            "diff": _diff(a_reserved, b_reserved),
            "diff_fmt": _format_bytes(abs(_diff(a_reserved, b_reserved))),
            "trend": _trend(_diff(a_reserved, b_reserved)),
        },
        {
            "name": "Allocated",
            "a": a_allocated, "a_fmt": _format_bytes(a_allocated),
            "b": b_allocated, "b_fmt": _format_bytes(b_allocated),
            "diff": _diff(a_allocated, b_allocated),
            "diff_fmt": _format_bytes(abs(_diff(a_allocated, b_allocated))),
            "trend": _trend(_diff(a_allocated, b_allocated)),
        },
        {
            "name": "碎片率",
            "a": a_frag, "a_fmt": f"{a_frag}%",
            "b": b_frag, "b_fmt": f"{b_frag}%",
            "diff": round(b_frag - a_frag, 1),
            "diff_fmt": f"{abs(round(b_frag - a_frag, 1))}%",
            "trend": _trend(b_frag - a_frag),
        },
        {
            "name": "Segment 数",
            "a": a_segments, "a_fmt": str(a_segments),
            "b": b_segments, "b_fmt": str(b_segments),
            "diff": _diff(a_segments, b_segments),
            "diff_fmt": str(abs(_diff(a_segments, b_segments))),
            "trend": _trend(_diff(a_segments, b_segments)),
        },
        {
            "name": "OOM 事件",
            "a": a_oom, "a_fmt": str(a_oom),
            "b": b_oom, "b_fmt": str(b_oom),
            "diff": _diff(a_oom, b_oom),
            "diff_fmt": str(abs(_diff(a_oom, b_oom))),
            "trend": _trend(_diff(a_oom, b_oom)),
        },
    ]

    conn_a = q._connect(db_path)
    try:
        conn_b = q._connect(ref_path)
        try:
            a_addrs = set(r[0] for r in conn_a.execute("SELECT address FROM segments").fetchall())
            b_addrs = set(r[0] for r in conn_b.execute("SELECT address FROM segments").fetchall())
            new_addrs = b_addrs - a_addrs

            new_segments = []
            for addr in list(new_addrs)[:5]:
                row = conn_b.execute(
                    "SELECT address, total_size, cs.frames_json FROM segments s "
                    "LEFT JOIN call_stacks cs ON s.stack_id = cs.id WHERE s.address = ?",
                    (addr,),
                ).fetchone()
                if row:
                    new_segments.append({
                        "address": hex(row[0]),
                        "size": row[1],
                        "size_fmt": _format_bytes(row[1]),
                        "frames": row[2],
                    })

            grown_segments = []
            for addr in a_addrs & b_addrs:
                a_row = conn_a.execute("SELECT total_size FROM segments WHERE address = ?", (addr,)).fetchone()
                b_row = conn_b.execute("SELECT total_size FROM segments WHERE address = ?", (addr,)).fetchone()
                if a_row and b_row and b_row[0] > a_row[0]:
                    grown_segments.append({
                        "address": hex(addr),
                        "size_a": a_row[0],
                        "size_a_fmt": _format_bytes(a_row[0]),
                        "size_b": b_row[0],
                        "size_b_fmt": _format_bytes(b_row[0]),
                        "growth": b_row[0] - a_row[0],
                        "growth_fmt": _format_bytes(b_row[0] - a_row[0]),
                    })

            grown_segments.sort(key=lambda x: x["growth"], reverse=True)
            grown_segments = grown_segments[:3]
        finally:
            conn_b.close()
    finally:
        conn_a.close()

    one_line = (
        f"Reserved {_trend(_diff(a_reserved, b_reserved))} "
        f"{_format_bytes(abs(_diff(a_reserved, b_reserved)))} "
        f"({round(_diff(a_reserved, b_reserved) / a_reserved * 100, 1) if a_reserved > 0 else 0}%), "
        f"碎片率 {_trend(b_frag - a_frag)} {abs(round(b_frag - a_frag, 1))}%"
    )

    return {
        "mode": "compare",
        "one_line": one_line,
        "metrics": metrics,
        "new_segments": new_segments,
        "new_segment_count": len(new_addrs),
        "grown_segments": grown_segments,
    }


def generate_html_report(results: Dict[str, Any], db_path: str, output_path: str):
    """生成自包含 HTML 报告"""
    overview = results.get("overview", {})
    devices = overview.get("devices", [])
    summary = overview.get("summary", {})
    health = overview.get("health", {})

    peak = results.get("peak", {})
    fragment = results.get("fragment", {})
    leak = results.get("leak", {})
    oom = results.get("oom", {})
    compare = results.get("compare", {})

    _script_dir = os.path.dirname(os.path.abspath(__file__))
    _echarts_path = os.path.join(_script_dir, "echarts.min.js")
    if os.path.exists(_echarts_path):
        with open(_echarts_path, "r", encoding="utf-8") as _f:
            _echarts_js = _f.read()
        _echarts_tag = f"<script>{_echarts_js}</script>"
    else:
        _echarts_tag = '<script src="https://cdn.jsdelivr.net/npm/echarts@5.5.0/dist/echarts.min.js"></script>'

    devices_json = json.dumps(devices, ensure_ascii=False, default=str).replace("</", "<\\/")
    peak_json = json.dumps(peak, ensure_ascii=False, default=str).replace("</", "<\\/")
    fragment_json = json.dumps(fragment, ensure_ascii=False, default=str).replace("</", "<\\/")
    leak_json = json.dumps(leak, ensure_ascii=False, default=str).replace("</", "<\\/")
    oom_json = json.dumps(oom, ensure_ascii=False, default=str).replace("</", "<\\/")
    compare_json = json.dumps(compare, ensure_ascii=False, default=str).replace("</", "<\\/")

    html_template = f"""<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Ascend NPU Memory Snapshot 分析报告</title>
{_echarts_tag}
<style>
* {{ margin: 0; padding: 0; box-sizing: border-box; }}
body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: #f5f7fa; color: #333; }}
.nav {{ position: sticky; top: 0; background: #1a1a2e; padding: 12px 24px; z-index: 100; display: flex; gap: 20px; flex-wrap: wrap; }}
.nav a {{ color: #ccc; text-decoration: none; font-size: 14px; }}
.nav a:hover {{ color: #fff; }}
.container {{ max-width: 1400px; margin: 0 auto; padding: 24px; }}
.header {{ background: #fff; border-radius: 8px; padding: 24px; margin-bottom: 24px; box-shadow: 0 1px 3px rgba(0,0,0,0.08); }}
.header h1 {{ font-size: 22px; margin-bottom: 8px; }}
.header .meta {{ color: #888; font-size: 13px; }}
.health-card {{ display: inline-block; padding: 8px 16px; border-radius: 6px; font-weight: 600; margin: 12px 0; }}
.health-ok {{ background: #e8f5e9; color: #2e7d32; }}
.health-warn {{ background: #fff3e0; color: #e65100; }}
.health-err {{ background: #ffebee; color: #c62828; }}
.cards {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 16px; margin-bottom: 24px; }}
.card {{ background: #fff; border-radius: 8px; padding: 20px; box-shadow: 0 1px 3px rgba(0,0,0,0.08); text-align: center; }}
.card .label {{ font-size: 13px; color: #888; margin-bottom: 8px; }}
.card .value {{ font-size: 24px; font-weight: 700; }}
.card .status {{ font-size: 12px; margin-top: 4px; }}
.section {{ background: #fff; border-radius: 8px; padding: 24px; margin-bottom: 24px; box-shadow: 0 1px 3px rgba(0,0,0,0.08); }}
.section h2 {{ font-size: 18px; margin-bottom: 16px; padding-bottom: 8px; border-bottom: 1px solid #eee; }}
.chart {{ width: 100%; height: 450px; }}
table {{ width: 100%; border-collapse: collapse; }}
th, td {{ padding: 10px 14px; text-align: left; border-bottom: 1px solid #eee; font-size: 14px; }}
th {{ background: #f8f9fa; font-weight: 600; }}
tr:hover {{ background: #f8f9fa; }}
.insight {{ background: #f0f7ff; border-left: 3px solid #1976d2; padding: 12px 16px; margin-top: 16px; border-radius: 0 6px 6px 0; font-size: 14px; }}
.suggestions {{ display: grid; gap: 12px; }}
.suggestion {{ border: 1px solid #e0e0e0; border-radius: 8px; padding: 16px; }}
.suggestion.high {{ border-left: 4px solid #c62828; }}
.suggestion.medium {{ border-left: 4px solid #e65100; }}
.suggestion .tag {{ display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 12px; margin-right: 8px; }}
.tag-high {{ background: #ffebee; color: #c62828; }}
.tag-medium {{ background: #fff3e0; color: #e65100; }}
</style>
</head>
<body>

<div class="nav">
    <a href="#overview">总览</a>
    <a href="#devices">设备对比</a>
    <a href="#peak">时序曲线</a>
    <a href="#fragment">碎片分析</a>
    <a href="#leak">泄漏检测</a>
    <a href="#oom">OOM</a>
    <a href="#attribution">堆栈归因</a>
    <a href="#suggestions">建议</a>
</div>

<div class="container">

<div class="header" id="overview">
    <h1>Ascend NPU Memory Snapshot 分析报告</h1>
    <div class="meta">文件: {html.escape(os.path.basename(db_path))} | 分析时间: <span id="analysis-time"></span></div>
    <div class="health-card health-{health.get('level', 'warn').lower().replace('需关注', 'warn').replace('严重', 'err').replace('健康', 'ok')}">
        {health.get('icon', '?')} 健康状态: {health.get('text', '未知')}
    </div>
</div>

<div class="cards" id="cards">
    <div class="card"><div class="label">Reserved</div><div class="value">{summary.get('reserved_fmt', '-')}</div></div>
    <div class="card"><div class="label">Allocated</div><div class="value">{summary.get('allocated_fmt', '-')}</div></div>
    <div class="card"><div class="label">碎片率</div><div class="value">{summary.get('frag_pct', '-')}%</div></div>
    <div class="card"><div class="label">Segment 数</div><div class="value">{summary.get('segment_count', '-')}</div></div>
    <div class="card"><div class="label">OOM 事件</div><div class="value">{overview.get('oom_count', '-')}</div></div>
</div>

<div class="section" id="devices">
    <h2>设备内存概览</h2>
    <div class="chart" id="chart-devices"></div>
</div>

<div class="section" id="peak">
    <h2>内存时序曲线 &amp; 峰值分析</h2>
    <div class="chart" id="chart-peak"></div>
    <div id="peak-content"></div>
</div>

<div class="section" id="fragment">
    <h2>碎片分析</h2>
    <div class="chart" id="chart-fragment"></div>
</div>

<div class="section" id="leak">
    <h2>泄漏检测</h2>
    <div id="leak-content"></div>
</div>

<div class="section" id="oom">
    <h2>OOM 分析</h2>
    <div id="oom-content"></div>
</div>

<div class="section" id="attribution">
    <h2>堆栈归因</h2>
    <div class="chart" id="chart-attribution"></div>
</div>

<div class="section" id="suggestions">
    <h2>优化建议</h2>
    <div class="suggestions" id="suggestions-content"></div>
</div>

</div>

<script>
document.getElementById('analysis-time').textContent = new Date().toLocaleString();

var devicesData = {devices_json};
var peakData = {peak_json};
var fragmentData = {fragment_json};
var leakData = {leak_json};
var oomData = {oom_json};
var compareData = {compare_json};

(function() {{
    var devNames = devicesData.map(function(d) {{ return 'Device ' + d.device_index; }});
    var reserved = devicesData.map(function(d) {{ return (d.reserved_bytes / (1024*1024*1024)).toFixed(2); }});
    var allocated = devicesData.map(function(d) {{ return (d.allocated_bytes / (1024*1024*1024)).toFixed(2); }});
    var active = devicesData.map(function(d) {{ return (d.active_bytes / (1024*1024*1024)).toFixed(2); }});

    var chart = echarts.init(document.getElementById('chart-devices'));
    chart.setOption({{
        tooltip: {{ trigger: 'axis' }},
        legend: {{ data: ['Reserved', 'Allocated', 'Active'] }},
        xAxis: {{ type: 'category', data: devNames }},
        yAxis: {{ type: 'value', name: 'GB' }},
        series: [
            {{ name: 'Reserved', type: 'bar', data: reserved, itemStyle: {{ color: '#5470c6' }} }},
            {{ name: 'Allocated', type: 'bar', data: allocated, itemStyle: {{ color: '#91cc75' }} }},
            {{ name: 'Active', type: 'bar', data: active, itemStyle: {{ color: '#fac858' }} }}
        ]
    }});
}})();

(function() {{
    var peakDevices = peakData.devices || [];
    var devNames = peakDevices.map(function(d) {{ return 'Device ' + d.device_index; }});
    var peakReserved = peakDevices.map(function(d) {{ return (d.peak?.reserved || 0) / (1024*1024*1024); }});
    var baseReserved = peakDevices.map(function(d) {{ return (d.baseline?.reserved || 0) / (1024*1024*1024); }});

    var chart = echarts.init(document.getElementById('chart-peak'));
    chart.setOption({{
        title: {{ text: '内存峰值分析 (按设备)', subtext: '基线: 各设备时间线前 10% 位置' }},
        tooltip: {{ trigger: 'axis' }},
        legend: {{ data: ['峰值 Reserved', '基线 Reserved'] }},
        xAxis: {{ type: 'category', data: devNames }},
        yAxis: {{ type: 'value', name: 'GB' }},
        series: [
            {{ name: '峰值 Reserved', type: 'bar', data: peakReserved, itemStyle: {{ color: '#c62828' }} }},
            {{ name: '基线 Reserved', type: 'bar', data: baseReserved, itemStyle: {{ color: '#5470c6' }} }}
        ]
    }});
}})();

(function() {{
    var peakHtml = '';
    var deltas = peakData.deltas || {{}};
    peakHtml += '<div class="insight" style="margin-bottom:16px"><strong>总体增幅</strong>: ';
    peakHtml += '全局峰值 ' + (peakData.peak?.reserved_fmt || 'N/A') + ' ';
    peakHtml += '(<span style="color:#c62828;font-weight:700">+' + (deltas.reserved_fmt || 'N/A') + (deltas.reserved_pct != null ? ', +' + deltas.reserved_pct + '%' : '') + '</span>)</div>';

    var peakDevices = peakData.devices || [];
    if (peakDevices.length > 0) {{
        peakHtml += '<h3>各设备峰值详情</h3>';
        peakHtml += '<table><tr><th>设备</th><th>峰值 Reserved</th><th>基线 Reserved</th><th>增幅</th><th>峰值 trace_index</th></tr>';
        peakDevices.forEach(function(d) {{
            var deltas = d.deltas || {{}};
            peakHtml += '<tr><td>Device ' + d.device_index + '</td><td style="font-weight:600">' + (d.peak?.reserved_fmt || 'N/A') + '</td><td>' + (d.baseline?.reserved_fmt || 'N/A') + '</td><td style="color:#c62828;font-weight:600">+' + (deltas.reserved_fmt || 'N/A') + (deltas.reserved_pct != null ? ' (+' + deltas.reserved_pct + '%)' : '') + '</td><td>' + (d.peak_trace_index || 'N/A') + '</td></tr>';
        }});
        peakHtml += '</table>';
    }}

    if (peakData.peak_alloc_events && peakData.peak_alloc_events.top && peakData.peak_alloc_events.top.length > 0) {{
        peakHtml += '<h3 style="margin-top:16px">峰值归因: 最大的 Segment 分配操作 (segment_alloc / segment_map)</h3>';
        peakHtml += '<p style="color:#888;font-size:13px;margin-bottom:8px">以下操作分配了最大的 segment,是峰值内存的主要贡献者。调用路径指向触发这些分配的代码。</p>';
        peakHtml += '<table><tr><th>#</th><th>大小</th><th>操作</th><th>地址</th><th>调用路径</th></tr>';
        peakData.peak_alloc_events.top.forEach(function(evt, i) {{
            peakHtml += '<tr><td>' + (i+1) + '</td><td style="font-weight:600">' + (evt.size_fmt || 'N/A') + '</td><td>' + (evt.action || '') + '</td><td style="font-family:monospace;font-size:12px">' + (evt.addr || 'N/A') + '</td><td style="font-size:12px;color:#555;max-width:400px;word-break:break-all">' + (evt.call_path || '') + '</td></tr>';
        }});
        peakHtml += '</table>';
    }}

    if (peakData.peak_blocks && peakData.peak_blocks.top && peakData.peak_blocks.top.length > 0) {{
        peakHtml += '<h3 style="margin-top:16px">峰值归因: 最大的活跃 Block (active_allocated)</h3>';
        peakHtml += '<p style="color:#888;font-size:13px;margin-bottom:8px">以下 block 是峰值时刻占用内存最大的 tensor 分配。调用路径指向创建这些 tensor 的代码。</p>';
        peakHtml += '<table><tr><th>#</th><th>大小</th><th>请求大小</th><th>调用路径</th></tr>';
        peakData.peak_blocks.top.forEach(function(blk, i) {{
            peakHtml += '<tr><td>' + (i+1) + '</td><td style="font-weight:600">' + (blk.size_fmt || 'N/A') + '</td><td>' + (blk.requested_fmt || 'N/A') + '</td><td style="font-size:12px;color:#555;max-width:400px;word-break:break-all">' + (blk.call_path || '') + '</td></tr>';
        }});
        peakHtml += '</table>';
    }}

    if (!peakData.peak_alloc_events && !peakData.peak_blocks) {{
        peakHtml += '<p>无峰值归因数据</p>';
    }}
    document.getElementById('peak-content').innerHTML = peakHtml;
}})();

(function() {{
    var topFrag = fragmentData.top_fragmented || [];
    var names = topFrag.map(function(s) {{ return s.address ? '0x' + s.address.toString(16).substring(0, 8) + '...' : 'N/A'; }});
    var used = topFrag.map(function(s) {{ return (100 - (s.frag_pct || 0)).toFixed(1); }});
    var waste = topFrag.map(function(s) {{ return (s.frag_pct || 0).toFixed(1); }});

    var chart = echarts.init(document.getElementById('chart-fragment'));
    chart.setOption({{
        title: {{ text: '碎片化最严重的 Segment TOP 5', subtext: '整体碎片率: ' + (fragmentData.overall?.frag_pct || 'N/A') + '%' }},
        tooltip: {{ trigger: 'axis' }},
        legend: {{ data: ['使用率', '碎片率'] }},
        xAxis: {{ type: 'category', data: names }},
        yAxis: {{ type: 'value', name: '%', max: 100 }},
        series: [
            {{ name: '使用率', type: 'bar', stack: 'total', data: used, itemStyle: {{ color: '#91cc75' }} }},
            {{ name: '碎片率', type: 'bar', stack: 'total', data: waste, itemStyle: {{ color: '#ee6666' }} }}
        ]
    }});
}})();

(function() {{
    var leakHtml = '';
    if (leakData.risk) {{
        leakHtml += '<div class="health-card health-' + (leakData.risk.level === '高风险' ? 'err' : leakData.risk.level === '中风险' ? 'warn' : 'ok') + '">';
        leakHtml += leakData.risk.icon + ' 风险等级: ' + leakData.risk.level + '</div>';
        leakHtml += '<p style="margin-top:12px">单调增长: ' + (leakData.monotonic_growth?.detected ? '是' : '否') + '</p>';
        leakHtml += '<p>长生命周期 Block: ' + (leakData.long_lived_blocks || 0) + ' 个</p>';
    }}
    if (leakData.suspects && leakData.suspects.length > 0) {{
        leakHtml += '<h3 style="margin-top:16px">泄漏嫌疑 TOP 3</h3>';
        leakData.suspects.forEach(function(s, i) {{
            leakHtml += '<div class="suggestion high" style="margin-top:8px"><strong>#' + (i+1) + '</strong> ';
            leakHtml += '累计: ' + (s.total_size_fmt || 'N/A') + ' | ' + (s.block_count || 0) + ' 个 block';
            leakHtml += '<br><span style="color:#888;font-size:13px">' + (s.key_path || '') + '</span></div>';
        }});
    }}
    if (!leakData.risk) {{
        leakHtml = '<p>无泄漏检测数据</p>';
    }}
    document.getElementById('leak-content').innerHTML = leakHtml;
}})();

(function() {{
    var oomHtml = '';
    if (oomData.detected && oomData.events) {{
        oomData.events.forEach(function(evt) {{
            oomHtml += '<div class="health-card health-err" style="margin-bottom:12px">OOM 事件: Device ' + evt.device + ', trace_index = ' + evt.oom_index + '</div>';
            oomHtml += '<p>OOM 时可用内存: ' + (evt.device_free_fmt || 'N/A') + '</p>';
            oomHtml += '<p>根因推断: ' + (evt.root_cause || 'N/A') + '</p>';
            if (evt.last_5_allocations && evt.last_5_allocations.length > 0) {{
                oomHtml += '<h4 style="margin-top:12px">OOM 前最后分配:</h4><table><tr><th>大小</th><th>类型</th></tr>';
                evt.last_5_allocations.forEach(function(a) {{
                    oomHtml += '<tr><td>' + (a.size_fmt || 'N/A') + '</td><td>' + (a.action || '') + '</td></tr>';
                }});
                oomHtml += '</table>';
            }}
        }});
    }} else {{
        oomHtml = '<p>未检测到 OOM 事件</p>';
    }}
    document.getElementById('oom-content').innerHTML = oomHtml;
}})();

(function() {{
    var chart = echarts.init(document.getElementById('chart-attribution'));
    chart.setOption({{
        title: {{ text: '堆栈归因 TOP 10' }},
        tooltip: {{ trigger: 'axis' }},
        xAxis: {{ type: 'value', name: 'GB' }},
        yAxis: {{ type: 'category', data: ['数据加载中...'] }},
        series: [{{ type: 'bar', data: [0] }}]
    }});
}})();

(function() {{
    var suggestions = [];
    var fragPct = fragmentData.overall?.frag_pct || 0;
    var leakRisk = leakData.risk?.level || '';
    var hasOom = oomData.detected || false;

    if (hasOom) {{
        suggestions.push({{ level: 'high', tag: 'OOM', text: '减小 batch_size 或启用 gradient checkpointing 以降低峰值内存。', ref: '' }});
    }}
    if (fragPct > 15) {{
        suggestions.push({{ level: 'high', tag: '碎片', text: '碎片率 ' + fragPct + '% 偏高,建议定期调用 torch.npu.empty_cache() 释放空闲 segment。', ref: '' }});
    }}
    if (leakRisk === '高风险') {{
        suggestions.push({{ level: 'high', tag: '泄漏', text: '检测到高风险疑似泄漏,建议检查 DDP/FSDP 的 gradient bucket 配置。', ref: 'https://pytorch.org/docs/stable/ddp.html' }});
    }}
    if (fragPct > 5 && fragPct <= 15) {{
        suggestions.push({{ level: 'medium', tag: '碎片', text: '碎片率 ' + fragPct + '% 略高,可考虑增大 PYTORCH_NPU_ALLOC_CONF 中的 expandable_segments 参数。', ref: '' }});
    }}
    if (leakRisk === '中风险') {{
        suggestions.push({{ level: 'medium', tag: '泄漏', text: '检测到中风险疑似泄漏,建议检查循环中 tensor 是否正确 detach 或使用 torch.no_grad()。', ref: '' }});
    }}
    if (suggestions.length === 0) {{
        suggestions.push({{ level: 'medium', tag: '监控', text: '内存状态正常,建议在训练脚本中增加周期性 snapshot 采集以便后续趋势对比。', ref: '' }});
    }}

    var html = '';
    suggestions.forEach(function(s) {{
        html += '<div class="suggestion ' + s.level + '"><span class="tag tag-' + s.level + '">' + s.tag + '</span>' + s.text;
        if (s.ref) html += ' <a href="' + s.ref + '" target="_blank">参考</a>';
        html += '</div>';
    }});
    document.getElementById('suggestions-content').innerHTML = html;
}})();
</script>
</body>
</html>"""

    with open(output_path, "w", encoding="utf-8") as f:
        f.write(html_template)

    return output_path


def main():
    parser = argparse.ArgumentParser(
        description="Ascend NPU Memory Snapshot 高层分析工具",
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument("db_path", help="SQLite 数据库路径")
    parser.add_argument("--mode", "-m", required=True,
                        choices=["overview", "peak", "fragment", "leak", "oom", "compare", "all"],
                        help="分析模式")
    parser.add_argument("--ref", help="对比参考 DB 路径 (compare 模式)")
    parser.add_argument("--device", "-d", type=int, help="指定分析的 device")
    parser.add_argument("--output", "-o", help="HTML 报告输出路径 (all 模式)")
    parser.add_argument("--json", action="store_true", help="输出 JSON 格式")

    args = parser.parse_args()

    if not os.path.exists(args.db_path):
        print(f"错误: 文件不存在: {args.db_path}", file=sys.stderr)
        sys.exit(1)

    results: Dict[str, Any] = {}

    try:
        if args.mode == "overview":
            results["overview"] = analyze_overview(args.db_path)

        elif args.mode == "peak":
            results["peak"] = analyze_peak(args.db_path, args.device)

        elif args.mode == "fragment":
            results["fragment"] = analyze_fragment(args.db_path, args.device)

        elif args.mode == "leak":
            results["leak"] = analyze_leak(args.db_path, args.device)

        elif args.mode == "oom":
            results["oom"] = analyze_oom(args.db_path, args.device)

        elif args.mode == "compare":
            if not args.ref:
                print("错误: compare 模式需要 --ref 参数", file=sys.stderr)
                sys.exit(1)
            results["compare"] = analyze_compare(args.db_path, args.ref)

        elif args.mode == "all":
            results["overview"] = analyze_overview(args.db_path)
            results["peak"] = analyze_peak(args.db_path, args.device)
            results["fragment"] = analyze_fragment(args.db_path, args.device)
            results["leak"] = analyze_leak(args.db_path, args.device)
            results["oom"] = analyze_oom(args.db_path, args.device)

            output_path = args.output or (os.path.splitext(args.db_path)[0] + "_report.html")
            generate_html_report(results, args.db_path, output_path)
            results["report_path"] = output_path

        if args.json:
            print(json.dumps(results, indent=2, ensure_ascii=False, default=str))
        elif args.mode != "all":
            print(json.dumps(results, indent=2, ensure_ascii=False, default=str))
        else:
            report_path = results.get("report_path", "")
            print(f"报告已生成: {report_path}")
            overview = results.get("overview", {})
            summary = overview.get("summary", {})
            health = overview.get("health", {})
            print(f"\n{health.get('icon', '?')} 健康状态: {health.get('text', '未知')}")
            print(f"Reserved: {summary.get('reserved_fmt', '-')}")
            print(f"Allocated: {summary.get('allocated_fmt', '-')}")
            print(f"碎片率: {summary.get('frag_pct', '-')}%")
            print(f"Segment 数: {summary.get('segment_count', '-')}")

    except Exception as e:
        print(f"错误: {e}", file=sys.stderr)
        import traceback
        traceback.print_exc()
        sys.exit(1)


if __name__ == "__main__":
    main()
deprecation-and-migrationaddyosmani/agent-skillsManages deprecation and migration. Use when removing old systems, APIs, or features. Use when migrating users from one implementation to another. Use when migrating a database schema in production, such as renaming or dropping a column without downtime (expand/contract). Use when deciding whether to maintain or sunset existing code.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
performance-optimizationaddyosmani/agent-skillsOptimizes application performance across frontend, backend, queries, and databases. Use when performance requirements exist, when you suspect performance regressions, when Core Web Vitals or load times need improvement, when N+1 query patterns need fixing, or when profiling reveals bottlenecks.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json.Stars: 118kUpdated Claude CodeCodexVerifiedInstall
agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json.Stars: 118kUpdated Claude CodeCodexVerifiedInstall