kali20gakki/msAgent / skills/host-cpu/msagent-perf-cpu-bind-tune
msagent-perf-cpu-bind-tune
Flagged昇腾 NPU 多卡任务的 Host CPU 绑核方案求解器:覆盖 PyTorch 训练/离线推理与 vLLM-Ascend、SGLang 推理服务的 rank/worker/engine 分核,基于 NUMA 拓扑与 卡↔进程映射输出可执行、可回滚、可验证的绑核计划(taskset 前缀 / cpuset / OMP 环境变量),自动判定事前(env 片段)与事中(在线 sched_setaffinity)双形态。 触发:绑核方案、核怎么分、per-rank 绑核、NUMA 绑定方案、cpuset 划分、多卡分核、 CPU range 冲突、TTFT 或吞吐受 CPU 影响。 不用于:绑核没生效/绑核异常诊断(→ perf-cpu-config-audit);运行期是否落核 (→ perf-cpu-tracing);变慢根因归因(→ perf-cpu-perf-triage)。
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 10.1 KB |
| references/checks.md | markdown | 1.4 KB |
| references/framework-profiles.md | markdown | 5.1 KB |
| references/mechanisms.md | markdown | 2.8 KB |
| references/permissions.md | markdown | 5.4 KB |
| references/process-roles.md | markdown | 2.8 KB |
| resources/plan_template.json | config | 4.7 KB |
| resources/report.css | other | 7.6 KB |
| resources/report_template.html | other | 3.9 KB |
| scripts/apply_plan.py | script | 11.8 KB |
| scripts/cpu_list.py | script | 1.4 KB |
| scripts/discover.py | script | 16.8 KB |
| scripts/emit_plan.py | script | 19.8 KB |
| scripts/feasibility.py | script | 4.7 KB |
| scripts/model.py | script | 9.5 KB |
| scripts/npu_query.py | script | 16.2 KB |
| scripts/plan.py | script | 10.7 KB |
| scripts/preflight.py | script | 5.0 KB |
| scripts/procfs.py | script | 9.0 KB |
| scripts/report.py | script | 31.0 KB |
| scripts/roles.py | script | 1.5 KB |
| scripts/topology.py | script | 16.8 KB |
| scripts/unit_alloc.py | script | 8.2 KB |
| scripts/verify_config.py | script | 5.7 KB |
| scripts/workspace.py | script | 2.5 KB |
| scripts/zone_alloc.py | script | 3.7 KB |
scripts/emit_plan.py
View on GitHub# -------------------------------------------------------------------------
# This file is part of the MindStudio project.
# Copyright (c) 2026 Huawei Technologies Co.,Ltd.
#
# MindStudio is licensed under Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#
# http://license.coscl.org.cn/MulanPSL2
#
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.
# -------------------------------------------------------------------------
"""plan 组装:命令、回滚、验证脚本、收益预估。
只产出动作契约,不执行 taskset;apply 由主会话确认后执行。
"""
import hashlib
import json
import logging
import os
import re
import shlex
import socket
import sys
import tempfile
import time
import topology
_LOG = logging.getLogger(__name__)
def _ts():
return time.strftime("%Y%m%d-%H%M%S", time.localtime())
def _hash(obj):
raw = json.dumps(obj, ensure_ascii=False, sort_keys=True).encode("utf-8")
return hashlib.sha256(raw).hexdigest()[:16]
def _script(name):
return os.path.join(os.path.dirname(os.path.abspath(__file__)), name)
def _apply_command(plan_path, extra):
return "%s %s %s%s" % (
shlex.quote(sys.executable or "python3"),
shlex.quote(_script("apply_plan.py")),
shlex.quote(plan_path),
extra,
)
def _unit_commands(unit, plan_path, form):
"""执行命令:执行时枚举线程,不落 tid 快照。"""
if form != "in_run" or not unit.get("pid"):
return [], []
return (
[_apply_command(plan_path, " --roles comm,io")],
[_apply_command(plan_path, " --rollback")],
)
def _build_unit_output(units, plan_path, form):
"""组装 units 输出字段,并收集各 unit 原始 cgroup cpuset。"""
unit_out = []
all_before = []
for unit in units:
commands, rollback = _unit_commands(unit, plan_path, form)
all_before.append((unit.get("before") or {}).get("cgroup_cpuset"))
unit_out.append({
"unit_id": unit.get("unit_id"),
"unit_type": unit.get("unit_type"),
"framework": unit.get("framework"),
"pid": unit.get("pid"),
"pids": unit.get("pids"),
"process_name": unit.get("process_name"),
"primary_npu": unit.get("primary_npu"),
"npus": unit.get("npus"),
"npus_significant": unit.get("npus_significant"),
"numa_node": unit.get("target_node"),
"target_node_source": unit.get("target_node_source"),
"npu_affinity": unit.get("npu_affinity"),
"current_node": unit.get("current_node"),
"current_affinity": unit.get("current_affinity"),
"cpuset": unit.get("cpuset"),
"main_cpuset": unit.get("main_cpuset"),
"compute_cpuset": unit.get("compute_cpuset"),
"work_cpuset": unit.get("work_cpuset"),
"thread_binding": unit.get("thread_binding"),
"roles": unit.get("roles"),
"env": unit.get("env"),
"mechanism": "sched_setaffinity" if form == "in_run" else "launch_wrapper",
"command_source": "runtime_task_scan" if form == "in_run" else "env_only",
"commands": commands,
"before": unit.get("before"),
"rollback": rollback,
"target_uid": unit.get("target_uid"),
"is_own_process": unit.get("is_own_process"),
"requires_privilege": bool(unit.get("pid")) and not bool(unit.get("is_own_process")),
"evidence_level": unit.get("evidence_level"),
"last_cpu": unit.get("last_cpu"),
"last_cpu_node": unit.get("last_cpu_node"),
"mems_allowed": unit.get("mems_allowed"),
"cgroup_mems": unit.get("cgroup_mems"),
"deviations": unit.get("deviations"),
"capacity_gap": unit.get("capacity_gap", 0),
"thread_roles": unit.get("thread_roles"),
})
return unit_out, all_before
def _common_container_cpuset(all_before):
"""所有 unit 的 cgroup cpuset 一致时视为容器级 cpuset。"""
unique = {x for x in all_before if x}
if len(unique) == 1:
return unique.pop()
return None
def _privilege_profile(form, units, preflight_info):
"""推导执行主体、权限需求与 numa_maps 可读性。"""
target_uids = sorted({u.get("target_uid") for u in units if u.get("target_uid") is not None})
has_pid = any(u.get("pid") for u in units)
own_processes = bool(has_pid) and all(u.get("is_own_process") for u in units if u.get("pid"))
is_root = bool(preflight_info.get("is_root"))
can_execute = bool(preflight_info.get("has_taskset")) and (own_processes or is_root)
if form == "pre_run":
execution_mode = "plan_only"
elif can_execute:
execution_mode = "self"
else:
execution_mode = "delegated"
# root 具备 CAP_SYS_NICE,跨用户目标进程也不额外需要提权
requires_privilege = form == "in_run" and not own_processes and not is_root
can_read_numa_maps = (
any((u.get("numa_maps") or {}).get("ok") for u in units)
if units else bool(preflight_info.get("numa_maps_self_ok"))
)
return {
"target_uids": target_uids,
"own_processes": own_processes,
"execution_mode": execution_mode,
"requires_privilege": requires_privilege,
"can_read_numa_maps": can_read_numa_maps,
}
def _collect_unit_deviations(units):
"""汇总 unit 偏差,并生成容量回退与证据提示。"""
deviations = []
notes = []
for u in units:
deviations.extend(u.get("deviations") or [])
if u.get("capacity_gap", 0) > 0:
notes.append("unit %s plan 核数 %d < 当前亲和 %d(差 %d);local-first 优先 NUMA 正确性" % (
u.get("unit_id"), len(u.get("cpuset_cpus") or []),
len(u.get("current_affinity_cpus") or []), u.get("capacity_gap")
))
for d in u.get("deviations") or []:
if d.get("kind") == "numa_maps_unavailable":
notes.append("unit %s numa_maps 不可读:远端页占比标 not_observable" % u.get("unit_id"))
if d.get("kind") == "nic_node_mismatch":
notes.append("unit %s:%s" % (u.get("unit_id"), d.get("evidence")))
return deviations, notes
def _hbm_invisible_notes(unknown):
"""对有 HBM 占用但无可见进程的 NPU 追加确认提示。"""
notes = []
for item in unknown:
notes.append("NPU%d 有 HBM 占用 %.0f MB 但无可见进程,可能存在其他容器/残留,绑核前需确认" % (
item["npu"], item["hbm_used_mb"]
))
return notes
def _environment_notes(preflight_info, form, locality_mode, requires_privilege, can_read_numa_maps,
fabric, die_groups):
"""按权限、可观测性与求解策略补充环境提示。"""
notes = []
if requires_privilege:
notes.append("目标进程属于其他用户/root;taskset/cgroup 变更需 sudo 或同用户执行,plan 不自动执行")
if form == "in_run" and not can_read_numa_maps:
notes.append("numa_maps 不可读;远端页占比 not_observable,已用 Cpus_allowed_list + 最近运行 CPU 作为弱证据")
if form == "in_run" and not preflight_info.get("has_taskset"):
notes.append("taskset 不可用;只能输出方案,由管理员选择 cgroup/启动包装执行")
if locality_mode == "global_slice":
notes.append(
"求解策略 global_slice:未观测到 NPU→CPU 亲和(%s 形态),按封装分组把 CPU 池均分到各 node,"
"保证 unit 间核集不重叠且切片对齐 NUMA 边界;NIC/存储/IRQ 仍按各自 node 就近"
% (fabric or "超节点矩阵")
)
if any(len(g) > 1 for g in (die_groups or [])):
notes.append(
"双 die 机型:同一封装的两个 die 上的 unit 已锁定同一 node(同封装 die 经 SIO 片内直连,"
"跨 node 拆分会让 die 间协同走远端)"
)
return notes
def _expected_effect(deviations, locality_mode):
"""评估收益上限等级,并给出证据依据。"""
level = "low"
if any(d.get("kind") in ("process_node_vs_npu_node", "affinity_outside_npu_range") for d in deviations):
level = "high"
elif any(d.get("kind") in ("remote_page_ratio", "thread_node_drift", "nic_node_mismatch") for d in deviations):
level = "medium"
basis = []
for d in deviations:
if d.get("kind") in ("process_node_vs_npu_node", "affinity_outside_npu_range",
"remote_page_ratio", "thread_node_drift"):
evidence = d.get("evidence")
if evidence:
basis.append(evidence)
if not basis:
if locality_mode == "global_slice":
basis.append("未观测到 NPU→CPU 亲和(超节点矩阵),按封装分组均分 CPU 池(global_slice)")
else:
basis.append("未发现跨 node 偏差;按拓扑生成预规划")
return level, basis
def _verify_spec():
"""验证方案:配置生效判据、运行时观测与收益指标。"""
return {
"config_effective": {
"commands": [
"grep -m1 '^Cpus_allowed_list:' /proc/<pid>/status",
"grep -m1 '^Cpus_allowed_list:' /proc/<pid>/task/<tid>/status",
],
},
"runtime": {
"consumer": "tracing --plan <plan.json>",
"check": "units[].roles[].cpus 落核/迁移/越界",
},
"effect": {
"metrics": ["sched_delay_p99_us", "remote_page_ratio", "migration_per_s"],
"threshold": "改善 ≥ 收益上限的 30%(§3.1.5)",
"window": "同参数复采",
},
}
def _preflight_block(preflight_info):
"""preflight 快照块。"""
return {
"npu_smi": preflight_info.get("npu_smi"),
"in_container": preflight_info.get("in_container"),
"cgroup_version": preflight_info.get("cgroup_version"),
"has_taskset": preflight_info.get("has_taskset"),
"user": preflight_info.get("user"),
"uid": preflight_info.get("uid"),
"is_root": preflight_info.get("is_root"),
"proc_hidden": preflight_info.get("proc_hidden"),
}
def _privilege_block(preflight_info, priv):
"""权限块:账号、目标 uid、可读性与执行模式。"""
return {
"current_user": preflight_info.get("user"),
"current_uid": preflight_info.get("uid"),
"is_root": preflight_info.get("is_root"),
"target_uids": priv["target_uids"],
"own_processes": priv["own_processes"],
"can_read_numa_maps": priv["can_read_numa_maps"],
# root 可跨用户执行;非 root 仅当目标进程全部属于当前用户时可直接绑核
"can_taskset": bool(preflight_info.get("has_taskset"))
and (priv["own_processes"] or bool(preflight_info.get("is_root"))),
"requires_privilege": priv["requires_privilege"],
"execution_mode": priv["execution_mode"],
"notes": [
n for n in [
"目标进程属于其他用户/root;需 sudo 或同用户执行" if priv["requires_privilege"] else None,
"numa_maps 不可读;使用 affinity + last_cpu 弱证据" if not priv["can_read_numa_maps"] else None,
"taskset 不可用;只能输出方案" if not preflight_info.get("has_taskset") else None,
] if n
],
}
def _apply_block(plan_path, bind_compute):
"""apply 动作契约块:bind_compute 与 unit 级 compute 绑定语义保持一致。"""
return {
"script": _script("apply_plan.py"),
"command": _apply_command(plan_path, " --roles comm,io"),
"rollback": _apply_command(plan_path, " --rollback"),
"roles": "comm,io",
"bind_compute": bool(bind_compute),
"dynamic_task_scan": True,
"cgroup_clip": True,
}
def _plan_flags(feasibility, preflight_info):
"""plan flags:feasibility 为权威来源,采集侧 view_limited 兜底覆盖。"""
flags = dict(feasibility.get("flags") or {})
flags["view_limited"] = bool(flags.get("view_limited") or preflight_info.get("view_limited"))
return flags
def _meta_value(topo, preflight_info, key):
"""拓扑 meta 优先,preflight_info 仅作 legacy 别名兜底。"""
value = (topo.get("meta") or {}).get(key)
if value not in (None, "", []):
return value
return preflight_info.get(key)
def _summary_block(ctx, flags):
"""summary 汇总块。"""
return {
"unit_count": len(ctx["units"]),
"deviation_count": len(ctx["deviations"]),
"finding_count": len(ctx["findings"]),
"unknown_hbm_npus": ctx["unknown"],
"view_limited": bool(flags.get("view_limited")),
"execution_mode": ctx["priv"]["execution_mode"],
"requires_privilege": ctx["priv"]["requires_privilege"],
}
def _assemble_plan(ctx):
"""装填 plan JSON 输出字段(字段顺序即对外契约)。"""
preflight_info = ctx["preflight_info"]
feasibility = ctx["feasibility"]
flags = ctx["flags"]
plan = {
"schema_version": "hostcpu-plan/1",
"plan_id": ctx["plan_id"],
"host": ctx["host"],
"ts_ns": int(time.time() * 1e9),
"mode": "container" if preflight_info.get("in_container") else "host",
"form": ctx["form"],
"fabric": ctx["fabric"],
"locality_mode": ctx["locality_mode"],
"affinity_granularity": ctx["affinity_granularity"],
"preflight": _preflight_block(preflight_info),
"privilege": _privilege_block(preflight_info, ctx["priv"]),
"topology_ref": {
"source": "self.topology",
"schema": "static-snapshot/topology",
"hash": _hash(ctx["topo"]),
},
"executable": ctx["priv"]["execution_mode"] == "self",
"plan_file": os.path.basename(ctx["plan_path"]),
"apply": _apply_block(ctx["plan_path"], ctx["bind_compute"]),
"before": {
"container_cpuset": ctx["container_cpuset"],
"source": "/proc/<pid>/cgroup",
},
"policy": ctx["zones_alloc"].get("policy"),
"zones": ctx["zones_alloc"].get("zones"),
"units": ctx["units"],
"baseline": {
"metrics": {
"sched_delay_p99_us": None,
"migration_per_s": None,
"remote_page_ratio": None,
"step_time_ms": None,
},
"source": "none",
"window": None,
},
"verify": ctx["verify"],
"expected_effect": {
"level": ctx["level"],
"basis": ctx["basis"][:5],
"upper_bound": "消除跨 node 亲和与越界核;容量回退见 notes",
},
"verify_script": os.path.basename(ctx["verify_path"]),
"verify_script_path": ctx["verify_path"],
"report": os.path.basename(ctx["report_path"]),
"unresolved": feasibility.get("unresolved") or [],
"warnings": feasibility.get("warnings") or [],
"findings": ctx["findings"],
"flags": flags,
"notes": ctx["notes"],
"summary": _summary_block(ctx, flags),
}
return plan
def build_plan(topo, units, zones_alloc, feasibility, preflight_info, form, out_dir):
ts = _ts()
# hostname 可能含空格/特殊字符:清洗后再进 plan_id 与文件名
host = re.sub(r"[^A-Za-z0-9_.-]", "_", socket.gethostname())
# pid 后缀避免同一秒内并发调用生成相同 plan_id
plan_id = "hostcpu-plan-%s-%s-%d" % (host, ts, os.getpid())
base = "hostcpu_plan_%s_%s" % (host, ts)
plan_path = os.path.join(out_dir, base + ".json")
verify_path = os.path.join(out_dir, "hostcpu_verify_%s_%s.sh" % (host, ts))
report_path = os.path.join(out_dir, "hostcpu_report_%s_%s.html" % (host, ts))
unit_out, all_before = _build_unit_output(units, plan_path, form)
container_cpuset = _common_container_cpuset(all_before)
priv = _privilege_profile(form, units, preflight_info)
# 求解策略与拓扑事实以 topo["meta"] 为唯一权威(与 model / feasibility 一致),
# preflight_info 仅保留 legacy 别名兜底,避免无亲和机器被误标 topo_affinity。
locality_mode = _meta_value(topo, preflight_info, "locality_mode") or "topo_affinity"
locality_mode = topology.LOCALITY_ALIAS.get(locality_mode, locality_mode)
fabric = _meta_value(topo, preflight_info, "fabric")
affinity_granularity = _meta_value(topo, preflight_info, "affinity_granularity")
die_groups = _meta_value(topo, preflight_info, "die_groups") or []
deviations, notes = _collect_unit_deviations(units)
unknown = (preflight_info.get("unknown_hbm_npus") or [])
notes.extend(_hbm_invisible_notes(unknown))
notes.extend(_environment_notes(preflight_info, form, locality_mode,
priv["requires_privilege"], priv["can_read_numa_maps"],
fabric, die_groups))
level, basis = _expected_effect(deviations, locality_mode)
flags = _plan_flags(feasibility, preflight_info)
findings = list(preflight_info.get("findings") or [])
# 存在 compute 核集且 apply=exec 时,执行块应声明 bind_compute=true
bind_compute = any(
b.get("role") == "compute" and b.get("apply") == "exec" and bool(b.get("cpuset"))
for u in units for b in (u.get("thread_binding") or [])
)
verify = _verify_spec()
ctx = {
"plan_id": plan_id,
"host": host,
"form": form,
"plan_path": plan_path,
"verify_path": verify_path,
"report_path": report_path,
"topo": topo,
"preflight_info": preflight_info,
"zones_alloc": zones_alloc,
"units": unit_out,
"container_cpuset": container_cpuset,
"priv": priv,
"verify": verify,
"locality_mode": locality_mode,
"fabric": fabric,
"affinity_granularity": affinity_granularity,
"level": level,
"basis": basis,
"deviations": deviations,
"unknown": unknown,
"notes": notes,
"feasibility": feasibility,
"flags": flags,
"findings": findings,
"bind_compute": bind_compute,
}
plan = _assemble_plan(ctx)
return plan, plan_path, verify_path, report_path
def write_plan(plan, path):
"""原子写入:同目录临时文件 + os.replace,避免并发/中断产生半截 plan。"""
directory = os.path.dirname(os.path.abspath(path))
fd, tmp = tempfile.mkstemp(prefix=".hostcpu-plan-", suffix=".tmp", dir=directory)
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(plan, f, ensure_ascii=False, indent=2)
os.replace(tmp, path)
except Exception:
try:
os.unlink(tmp)
except OSError:
pass
raise
def write_verify_script(plan, path):
verify_py = _script("verify_config.py")
plan_file = plan.get("plan_file") or ""
lines = [
"#!/bin/sh",
"# generated by msagent-perf-cpu-bind-tune",
"# plan: %s" % plan.get("plan_id"),
'DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)',
"PY=python3",
'command -v "$PY" >/dev/null 2>&1 || PY=/usr/bin/python3',
'exec "$PY" %s "$DIR"/%s --threads' % (shlex.quote(verify_py), shlex.quote(plan_file)),
]
with open(path, "w", encoding="utf-8") as f:
f.write("\n".join(lines) + "\n")
try:
os.chmod(path, 0o755)
except OSError as exc:
_LOG.warning("verify 脚本 chmod +x 失败(%s):请手动执行 chmod +x %s", exc, path)
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |