Skip to content

kali20gakki/msAgent / skills/quantizer/msmodelslim-model-adapt

msmodelslim-model-adapt

Verified

为 msModelSlim 创建基础 Transformers 模型适配器(Model Adapter)。 包含创建适配器、实现必需接口与注册安装流程。 适用:Decoder-only LLM、理解类 VLM(仅 LLM/text 部分)。 多模态生成(DiT/扩散,含双/多专家)见下方「多模态生成扩展」节,按需加载。 不适用:Encoder-only、非 Transformers 架构。

assets/vlm_model_adapter_template.py

View on GitHub
# -------------------------------------------------------------------------
# This file is part of the MindStudio project.
# Copyright (c) 2026 Huawei Technologies Co.,Ltd.
#
# MindStudio is licensed under Mulan PSL v2.
# You can use this software according to the terms and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#
#          http://license.coscl.org.cn/MulanPSL2
#
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.
# -------------------------------------------------------------------------

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
多模态理解模型(VLM)适配器模板。
"""

from pathlib import Path
from typing import Any, Generator, List

from torch import nn
from transformers import AutoProcessor

from msmodelslim.app.naive_quantization.model_info_interface import ModelInfoInterface
from msmodelslim.core.base.protocol import ProcessRequest
from msmodelslim.core.const import DeviceType
from msmodelslim.model.common.layer_wise_forward import generated_decoder_layer_visit_func, transformers_generated_forward_func
from msmodelslim.model.common.vlm_base import VLMBaseModelAdapter
from msmodelslim.model.interface_hub import ModelSlimPipelineInterfaceV1
from msmodelslim.utils.exception import InvalidModelError
from msmodelslim.utils.logging import logger_setter
from msmodelslim.utils.security import get_valid_read_path


@logger_setter()
class MyVLMModelAdapter(
    VLMBaseModelAdapter,
    ModelInfoInterface,
    ModelSlimPipelineInterfaceV1,
):
    """VLM 基础模板。"""

    def __init__(self, model_type: str, model_path: Path, trust_remote_code: bool = False):
        self._processor = None
        super().__init__(model_type, model_path, trust_remote_code)

    # ==================== ModelInfoInterface ====================
    def get_model_pedigree(self) -> str:
        return "my_vlm"

    def get_model_type(self) -> str:
        return self.model_type

    # ==================== ModelSlimPipelineInterfaceV1 ====================
    def handle_dataset(self, dataset: Any, device: DeviceType = DeviceType.NPU) -> List[Any]:
        """
        将图文样本转换为校准输入。
        样本建议格式:item.text + item.image(或 dict: {"text": ..., "image": ...})。
        """
        self._processor = AutoProcessor.from_pretrained(
            self.model_path,
            trust_remote_code=self.trust_remote_code,
            local_files_only=True,
        )
        processed = []
        for item in dataset:
            text = item.text if hasattr(item, "text") else item.get("text")
            image = item.image if hasattr(item, "image") else item.get("image")
            if text is None or image is None:
                raise InvalidModelError(
                    "VLM 校准样本需要同时包含 text 和 image。",
                    action="请提供 image+text 数据,避免纯文本样本。",
                )

            image = get_valid_read_path(str(image))
            messages = [{
                "role": "user",
                "content": [
                    {"type": "image", "image": str(image)},
                    {"type": "text", "text": str(text)},
                ],
            }]
            inputs = self._processor.apply_chat_template(
                messages,
                tokenize=True,
                add_generation_prompt=True,
                return_dict=True,
                return_tensors="pt",
            )
            processed.append(
                self._collect_inputs_to_device(
                    inputs,
                    device,
                    keys=[
                        "input_ids",
                        "attention_mask",
                        "position_ids",
                        "pixel_values",
                        "pixel_values_videos",
                        "image_grid_thw",
                        "video_grid_thw",
                        "cache_position",
                    ],
                    defaults={},
                )
            )
        return processed

    def init_model(self, device: DeviceType = DeviceType.NPU) -> nn.Module:
        return self._load_model(device)

    def generate_model_visit(self, model: nn.Module) -> Generator[ProcessRequest, Any, None]:
        yield from generated_decoder_layer_visit_func(model)

    def generate_model_forward(self, model: nn.Module, inputs: Any) -> Generator[ProcessRequest, Any, None]:
        yield from transformers_generated_forward_func(model, inputs)

    def enable_kv_cache(self, model: nn.Module, need_kv_cache: bool) -> None:
        return self._enable_kv_cache(model, need_kv_cache)
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall