Skip to content

vast-ai/vast-cli / vastai

vastai

Flagged

Vast.ai CLI to manage GPU instances, volumes, serverless endpoints, and billing.

FileKindSize
SKILL.mdskill md18.6 KB
__init__.pyscript1.8 KB
_base.pyscript1.6 KB
api/__init__.pyscript31 B
api/auth.pyscript15.3 KB
api/billing.pyscript10.4 KB
api/client.pyscript9.1 KB
api/clusters.pyscript4.5 KB
api/deployments.pyscript1.4 KB
api/endpoints.pyscript12.0 KB
api/instances.pyscript17.1 KB
api/keys.pyscript4.3 KB
api/machines.pyscript7.4 KB
api/metrics.pyscript2.6 KB
api/offers.pyscript21.2 KB
api/price_increase.pyscript4.0 KB
api/query.pyscript12.2 KB
api/storage.pyscript14.5 KB
api/teams.pyscript4.4 KB
async_/__init__.pyscript79 B
async_/client.pyscript9.6 KB
async_/results.pyscript1.3 KB
cli/__init__.pyscript25 B
cli/commands/__init__.pyscript757 B
cli/commands/auth.pyscript42.8 KB
cli/commands/benchmarks.pyscript42.0 KB
cli/commands/billing.pyscript30.4 KB
cli/commands/clusters.pyscript7.8 KB
cli/commands/deployments.pyscript7.8 KB
cli/commands/endpoints.pyscript17.3 KB
cli/commands/instances.pyscript55.4 KB
cli/commands/keys.pyscript7.9 KB
cli/commands/machines.pyscript78.4 KB
cli/commands/metrics.pyscript12.7 KB
cli/commands/misc.pyscript11.0 KB
cli/commands/offers.pyscript30.0 KB
cli/commands/price_increase.pyscript14.5 KB
cli/commands/repl.pyscript719 B
cli/commands/storage.pyscript38.9 KB
cli/commands/teams.pyscript6.4 KB
cli/commands/uninstall.pyscript2.4 KB
cli/commands/update.pyscript3.1 KB
cli/display.pyscript20.1 KB
cli/main.pyscript11.5 KB
cli/parser.pyscript18.1 KB
cli/repl/__init__.pyscript318 B
cli/repl/bridge.pyscript7.3 KB
cli/repl/catalog.pyscript6.3 KB
cli/repl/completion.pyscript8.2 KB
cli/repl/session.pyscript7.3 KB
cli/self_test/__init__.pyscript46 B
cli/self_test/machine_diagnostics.pyscript21.0 KB
cli/self_test/runtime_diagnostics.pyscript18.6 KB
cli/self_test/support_bundle.pyscript11.0 KB
cli/selfupdate.pyscript14.0 KB
cli/util.pyscript26.3 KB
cli/utils.pyscript577 B
data/__init__.pyscript303 B
data/deployment.pyscript3.5 KB
data/endpoint.pyscript2.1 KB
data/instance.pyscript3.6 KB
data/offer.pyscript4.4 KB
data/query.pyscript12.8 KB
data/workergroup.pyscript1.4 KB
pdf/__init__.pyscript0 B
pdf/vast.ai-logo-50pct.pngbinary7.4 KB
pdf/vast.ai-logo.pngbinary10.1 KB
pdf/vast_pdf.pyscript19.4 KB
pdf/watcher.shexecutablescript390 B
sdk.pyscript52.2 KB
serverless/__init__.pyscript193 B
serverless/client/__init__.pyscript346 B
serverless/client/client.pyscript36.7 KB
serverless/client/connection.pyscript11.5 KB
serverless/client/endpoint.pyscript6.9 KB
serverless/client/managed.pyscript7.3 KB
serverless/client/request_status.pyscript371 B
serverless/client/session.pyscript2.8 KB
serverless/client/worker.pyscript1.3 KB
serverless/remote/__init__.pyscript245 B
serverless/remote/base.pyscript6.3 KB
serverless/remote/deploy.pyscript12.6 KB
serverless/remote/serialization.pyscript3.7 KB
serverless/remote/serve.pyscript8.6 KB
serverless/remote/serve_deployment.pyscript6.3 KB
serverless/remote/utils.pyscript9.7 KB
serverless/server/__init__.pyscript0 B
serverless/server/lib/__init__.pyscript0 B
serverless/server/lib/backend.pyscript43.3 KB
serverless/server/lib/data_types.pyscript10.6 KB
serverless/server/lib/metrics.pyscript12.2 KB
serverless/server/lib/server.pyscript3.2 KB
serverless/server/worker.pyscript17.1 KB
sync/__init__.pyscript76 B
sync/client.pyscript4.9 KB
sync/results.pyscript1.2 KB
utils.pyscript12.2 KB

serverless/server/lib/metrics.py

View on GitHub
import os
import time
import logging
import json
from asyncio import sleep
from dataclasses import dataclass, asdict, field
from functools import cache
import asyncio
from aiohttp import ClientSession, ClientTimeout, TCPConnector, ClientResponseError

from .data_types import WorkerStatusData, SystemMetrics, ModelMetrics, RequestMetrics
from typing import Awaitable, NoReturn, List

METRICS_UPDATE_INTERVAL = 1
DELETE_REQUESTS_INTERVAL = 1

log = logging.getLogger(__file__)


@cache
def get_url() -> str:
    use_ssl = os.environ.get("USE_SSL", "false") == "true"
    worker_port = os.environ[f"VAST_TCP_PORT_{os.environ['WORKER_PORT']}"]
    public_ip = os.environ["PUBLIC_IPADDR"]
    return f"http{'s' if use_ssl else ''}://{public_ip}:{worker_port}"


@dataclass
class Metrics:
    version: str = "0"
    mtoken: str = ""
    last_metric_update: float = 0.0
    last_request_served: float = 0.0
    update_pending: bool = False
    id: int = field(default_factory=lambda: int(os.environ["CONTAINER_ID"]))
    report_addr: List[str] = field(
        default_factory=lambda: os.environ["REPORT_ADDR"].split(",")
    )
    url: str = field(default_factory=get_url)
    system_metrics: SystemMetrics = field(default_factory=SystemMetrics.empty)
    model_metrics: ModelMetrics = field(default_factory=ModelMetrics.empty)
    _session: ClientSession | None = field(default=None, init=False, repr=False)

    async def http(self) -> ClientSession:
        if self._session is None:
            self._session = ClientSession(
                timeout=ClientTimeout(total=10),
                connector=TCPConnector(
                    limit=8,
                    limit_per_host=4,
                    force_close=True,
                    enable_cleanup_closed=True,
                ),
            )
        return self._session
    
    async def aclose(self) -> None:
        if self._session is not None:
            await self._session.close()
            self._session = None

    def _request_id(self, request: RequestMetrics) -> str:
        """Return a formatted identifier for logging based on request type."""
        if request.is_session:
            return f"Session {request.request_idx}"
        elif request.session:
            return f"Request {request.session_reqnum} in Session {request.session.request_idx}"
        else:
            return f"Request {request.request_idx}"

    def _request_start(self, request: RequestMetrics) -> None:
        """
        this function is called prior to forwarding a request to a model API.
        """
        log.debug(f"Starting {self._request_id(request)}")
        request.status = "Started"
        self.model_metrics.workload_pending += request.workload
        self.model_metrics.workload_received += request.workload
        if not request.session:
            self.model_metrics.requests_recieved.add(request.reqnum)
            self.model_metrics.requests_working[request.reqnum] = request
        self.update_pending = True

    def _request_end(self, request: RequestMetrics) -> None:
        """
        this function is called after handling of a request ends, regardless of the outcome
        """
        log.debug(f"Ending {self._request_id(request)}")
        self.model_metrics.workload_pending -= request.workload
        if not request.session:
            self.model_metrics.requests_working.pop(request.reqnum, None)
            self.model_metrics.requests_deleting.append(request)
        self.last_request_served = time.time()

    def _request_success(self, request: RequestMetrics) -> None:
        """
        this function is called after a response from model API is received and forwarded.
        """
        log.debug(f"{self._request_id(request)} succeeded")
        request.work_completed_at = time.time()
        self.model_metrics.workload_served += request.workload
        request.status = "Success"
        request.success = True
        self.update_pending = True

    def _request_errored(self, request: RequestMetrics, message: str) -> None:
        """
        this function is called if model API returns an error
        """
        log.error(f"{self._request_id(request)} errored: {message}")
        request.work_completed_at = time.time()
        self.model_metrics.workload_errored += request.workload
        request.status = "Error"
        request.success = False
        self.update_pending = True

    def _request_canceled(self, request: RequestMetrics) -> None:
        """
        this function is called if client drops connection before model API has responded
        """
        log.debug(f"Canceling {self._request_id(request)}")
        self.model_metrics.workload_cancelled += request.workload
        request.success = True
        request.status = "Cancelled"
    
    def _request_reject(self, request: RequestMetrics) -> None:
        """
        this function is called if the current wait time for the model is above max_queue_time
        """
        log.debug(f"Rejecting {self._request_id(request)}")
        self.model_metrics.workload_rejected += request.workload
        if not request.session:
            self.model_metrics.requests_recieved.add(request.reqnum)
            self.model_metrics.requests_deleting.append(request)
        request.success = False
        request.status = "Rejected"
        self.update_pending = True

    async def _send_delete_requests_loop(self) -> Awaitable[NoReturn]:
        while True:
            await sleep(DELETE_REQUESTS_INTERVAL)
            if len(self.model_metrics.requests_deleting) > 0:
                await self.__send_delete_requests_and_reset()

    async def _send_metrics_loop(self) -> Awaitable[NoReturn]:
        while True:
            await sleep(METRICS_UPDATE_INTERVAL)
            elapsed = time.time() - self.last_metric_update
            if self.system_metrics.model_is_loaded is False and elapsed >= 10:
                await self.__send_metrics_and_reset()
            elif self.update_pending or elapsed > 10:
                await self.__send_metrics_and_reset()

    def _model_loaded(self, max_throughput: float) -> None:
        self.system_metrics.model_loading_time = (
            time.time() - self.system_metrics.model_loading_start
        )
        self.system_metrics.model_is_loaded = True
        self.model_metrics.max_throughput = max_throughput

    def _model_errored(self, error_msg: str) -> None:
        self.model_metrics.set_errored(error_msg)
        self.system_metrics.model_is_loaded = True

    def _set_version(self, version: str) -> None:
        self.version = version

    def _set_mtoken(self, mtoken: str) -> None:
        self.mtoken = mtoken

    #######################################Private#######################################

    async def __send_delete_requests_and_reset(self):
        async def post(report_addr: str, requests: list[dict]) -> bool:
            data = {
                "worker_id": self.id,
                "mtoken": self.mtoken,
                "requests": requests,
            }
            log.debug(
                f"Deleting requests: {[r['request_idx'] for r in requests]}"
            )
            full_path = report_addr.rstrip("/") + "/delete_requests/"
            for attempt in range(1, 4):
                try:
                    session = await self.http()
                    async with session.post(full_path, json=data) as res:
                        log.debug(f"delete_requests response: {res.status}")
                        res.raise_for_status()
                    return True
                except asyncio.TimeoutError:
                    log.debug("delete_requests timed out")
                except (ClientResponseError, Exception) as e:
                    log.debug(f"delete_requests failed with error: {e}")
                await asyncio.sleep(2)
                log.debug(f"Retrying delete_request, attempt: {attempt}")
            return False

        # Take a snapshot of what we plan to send this tick.
        # New arrivals after this snapshot will remain in the queue for the next tick.
        snapshot = list(self.model_metrics.requests_deleting)

        if not snapshot:
            return  # nothing to do

        requests_payload = [
            {
                "request_idx": r.request_idx,
                "success": r.success,
                "status": r.status,
                "entered_queue_at": r.entered_queue_at,
                "work_started_at": r.work_started_at,
                "work_completed_at": r.work_completed_at,
            }
            for r in snapshot
        ]

        for report_addr in self.report_addr:
            # TODO: Add a Redis subscriber queue for delete_requests
            if report_addr == "https://cloud.vast.ai/api/v0":
                # Patch: ignore the Redis API report_addr
                continue

            sent = await post(report_addr, requests_payload)

            if sent:
                # Remove only the items we actually sent from the live queue.
                sent_set = {r.request_idx for r in snapshot}
                self.model_metrics.requests_deleting[:] = [
                    r for r in self.model_metrics.requests_deleting
                    if r.request_idx not in sent_set
                ]
                break


    async def __send_metrics_and_reset(self):

        loadtime_snapshot = self.system_metrics.model_loading_time

        def get_worker_status_data() -> WorkerStatusData:
            return WorkerStatusData(
                id=self.id,
                mtoken=self.mtoken,
                version=self.version,
                loadtime=(loadtime_snapshot or 0.0), 
                new_load=self.model_metrics.workload_processing,
                cur_load=self.model_metrics.cur_load,
                rej_load=self.model_metrics.workload_rejected,
                max_perf=self.model_metrics.max_throughput,
                cur_perf=self.model_metrics.workload_served,
                error_msg=self.model_metrics.error_msg or "",
                num_requests_working=len(self.model_metrics.requests_working),
                num_requests_recieved=len(self.model_metrics.requests_recieved),
                additional_disk_usage=self.system_metrics.additional_disk_usage,
                disk_usage=self.system_metrics.disk_usage,
                working_request_idxs=self.model_metrics.working_request_idxs,
                cur_capacity=0,
                max_capacity=0,
                url=self.url,
            )

        async def send_data(report_addr: str) -> bool:
            data = get_worker_status_data()
            log_data = asdict(data)
            def obfuscate(secret: str) -> str:
                if secret is None:
                    return ""
                return secret[:7] + "..." if len(secret) > 12 else ("*" * len(secret))
            
            log_data["mtoken"] = obfuscate(log_data.get("mtoken"))
            log.debug(
                "\n".join(
                    [
                        "\n" + "#" * 20 + " Worker Status " + "#" * 20,
                        f"{json.dumps(log_data, indent=2)}",
                        "#" * 55,
                    ]
                )
            )

            full_path = report_addr.rstrip("/") + "/worker_status/"
            for attempt in range(1, 4):
                try:
                    session = await self.http()
                    async with session.post(full_path, json=asdict(data)) as res:
                        res.raise_for_status()
                    return True
                except asyncio.TimeoutError:
                    log.debug(f"Request to /worker_status/ timed out")
                except (ClientResponseError, Exception)  as e:
                    log.debug(f"Request to /worker_status/ failed with error: {e}")
                await asyncio.sleep(2)
                log.debug(f"Retrying /worker_status/ request, attempt: {attempt}")
            log.debug(f"Failed to send worker status through {report_addr}")
            return False

        ###########

        self.system_metrics.update_disk_usage()

        sent = False
        for report_addr in self.report_addr:
            if await send_data(report_addr):
                sent = True
                break

        if sent:
            self.update_pending = False
            self.model_metrics.reset()
            self.system_metrics.commit_disk_usage()
            self.last_metric_update = time.time()
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall