Skip to content

vast-ai/vast-cli / vastai

vastai

Flagged

Vast.ai CLI to manage GPU instances, volumes, serverless endpoints, and billing.

FileKindSize
SKILL.mdskill md18.6 KB
__init__.pyscript1.8 KB
_base.pyscript1.6 KB
api/__init__.pyscript31 B
api/auth.pyscript15.3 KB
api/billing.pyscript10.4 KB
api/client.pyscript9.1 KB
api/clusters.pyscript4.5 KB
api/deployments.pyscript1.4 KB
api/endpoints.pyscript12.0 KB
api/instances.pyscript17.1 KB
api/keys.pyscript4.3 KB
api/machines.pyscript7.4 KB
api/metrics.pyscript2.6 KB
api/offers.pyscript21.2 KB
api/price_increase.pyscript4.0 KB
api/query.pyscript12.2 KB
api/storage.pyscript14.5 KB
api/teams.pyscript4.4 KB
async_/__init__.pyscript79 B
async_/client.pyscript9.6 KB
async_/results.pyscript1.3 KB
cli/__init__.pyscript25 B
cli/commands/__init__.pyscript757 B
cli/commands/auth.pyscript42.8 KB
cli/commands/benchmarks.pyscript42.0 KB
cli/commands/billing.pyscript30.4 KB
cli/commands/clusters.pyscript7.8 KB
cli/commands/deployments.pyscript7.8 KB
cli/commands/endpoints.pyscript17.3 KB
cli/commands/instances.pyscript55.4 KB
cli/commands/keys.pyscript7.9 KB
cli/commands/machines.pyscript78.4 KB
cli/commands/metrics.pyscript12.7 KB
cli/commands/misc.pyscript11.0 KB
cli/commands/offers.pyscript30.0 KB
cli/commands/price_increase.pyscript14.5 KB
cli/commands/repl.pyscript719 B
cli/commands/storage.pyscript38.9 KB
cli/commands/teams.pyscript6.4 KB
cli/commands/uninstall.pyscript2.4 KB
cli/commands/update.pyscript3.1 KB
cli/display.pyscript20.1 KB
cli/main.pyscript11.5 KB
cli/parser.pyscript18.1 KB
cli/repl/__init__.pyscript318 B
cli/repl/bridge.pyscript7.3 KB
cli/repl/catalog.pyscript6.3 KB
cli/repl/completion.pyscript8.2 KB
cli/repl/session.pyscript7.3 KB
cli/self_test/__init__.pyscript46 B
cli/self_test/machine_diagnostics.pyscript21.0 KB
cli/self_test/runtime_diagnostics.pyscript18.6 KB
cli/self_test/support_bundle.pyscript11.0 KB
cli/selfupdate.pyscript14.0 KB
cli/util.pyscript26.3 KB
cli/utils.pyscript577 B
data/__init__.pyscript303 B
data/deployment.pyscript3.5 KB
data/endpoint.pyscript2.1 KB
data/instance.pyscript3.6 KB
data/offer.pyscript4.4 KB
data/query.pyscript12.8 KB
data/workergroup.pyscript1.4 KB
pdf/__init__.pyscript0 B
pdf/vast.ai-logo-50pct.pngbinary7.4 KB
pdf/vast.ai-logo.pngbinary10.1 KB
pdf/vast_pdf.pyscript19.4 KB
pdf/watcher.shexecutablescript390 B
sdk.pyscript52.2 KB
serverless/__init__.pyscript193 B
serverless/client/__init__.pyscript346 B
serverless/client/client.pyscript36.7 KB
serverless/client/connection.pyscript11.5 KB
serverless/client/endpoint.pyscript6.9 KB
serverless/client/managed.pyscript7.3 KB
serverless/client/request_status.pyscript371 B
serverless/client/session.pyscript2.8 KB
serverless/client/worker.pyscript1.3 KB
serverless/remote/__init__.pyscript245 B
serverless/remote/base.pyscript6.3 KB
serverless/remote/deploy.pyscript12.6 KB
serverless/remote/serialization.pyscript3.7 KB
serverless/remote/serve.pyscript8.6 KB
serverless/remote/serve_deployment.pyscript6.3 KB
serverless/remote/utils.pyscript9.7 KB
serverless/server/__init__.pyscript0 B
serverless/server/lib/__init__.pyscript0 B
serverless/server/lib/backend.pyscript43.3 KB
serverless/server/lib/data_types.pyscript10.6 KB
serverless/server/lib/metrics.pyscript12.2 KB
serverless/server/lib/server.pyscript3.2 KB
serverless/server/worker.pyscript17.1 KB
sync/__init__.pyscript76 B
sync/client.pyscript4.9 KB
sync/results.pyscript1.2 KB
utils.pyscript12.2 KB

cli/commands/benchmarks.py

View on GitHub
"""
A CLI command to benchmark a template against multiple GPUs in parallel.
The template's pyworker benchmark determines the measured perf (ex. tokens/$).

vastai run benchmarks
"""

import argparse
import atexit
import functools
import json
import math
import random
import re
import signal
import statistics
import sys
import threading
import time
import uuid
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait

from rich.console import Console
from rich.live import Live
from rich.table import Table

from vastai import VastAI
from vastai.cli.parser import argument
from vastai.cli.display import deindent
from vastai.cli.utils import get_parser as _get_parser, get_client  # noqa: F401  (used by test conftest)
from vastai.cli.util import _get_gpu_types
from vastai.data.query import OP_TO_STR
from vastai.data import query as _query_consts


parser = _get_parser()


# Default GPU sweep set + per-card VRAM fallback (MB). Catalog gpu_ram_mb
# overrides these values when populated; the keys stay the editorial sweep set.
DEFAULT_GPUS = {
    "RTX 5090":  32607,
    "RTX 4090":  24564,
    "RTX 3090":  24576,
    "RTX A6000": 49140,
}

# One worker per GPU, kept alive while idle so we can read its measured perf.
ENDPOINT_CONFIG = {"cold_workers": 1, "max_workers": 1, "min_load": 1.0}

RENTAL_TIMEOUT = 120                 
DEFAULT_BENCHMARK_TIMEOUT = 30 * 60  
WORKER_POLL_INTERVAL = 10.0          
PARALLEL_SUBMIT_DELAY = 4.0          # spacing between parallel endpoint submits
MAX_ABANDONS = 3                     # bail after this many workers vanish without one reaching idle

# states in which the autoscaler shouldnt retry the worker 
TERMINAL_STATES = {"stopped", "destroying", "unavail"} 
TERMINAL_GRACE_SECONDS = 30
TERMINAL_STATUSES = {"done", "skipped", "timeout", "failed", "no_worker"}


def format_time_elapsed(seconds):
    m, s = divmod(int(seconds), 60)
    return f"{m}:{s:02d}"


@functools.lru_cache(maxsize=1)
def _canonical_gpu_names():
    """Map of lower-cased GPU name -> canonical name, for resolving user input
    like "rtx_4090"/"RTX 4090" to the canonical "RTX 4090".

    Union of the hardcoded query.py constants (fallback floor) with the
    /api/v0/gpu_types/ catalog (picks up newly-onboarded SKUs). On a catalog
    fetch failure the constants alone are used, so resolution never gets worse.
    """
    canonical = {
        v.lower(): v for k, v in vars(_query_consts).items()
        if isinstance(v, str) and not k.startswith("_")
    }
    for gpu_type in (_get_gpu_types() or []):
        name = gpu_type.get("canonical_name")
        if name:
            canonical[name.lower()] = name
    return canonical


@functools.lru_cache(maxsize=1)
def _catalog_vram_map():
    """Map canonical_name -> gpu_ram_mb for entries the catalog has populated.
    Empty when the catalog is unreachable or the column is unbackfilled.
    """
    return {
        gpu_type["canonical_name"]: gpu_type["gpu_ram_mb"]
        for gpu_type in (_get_gpu_types() or [])
        if gpu_type.get("canonical_name") and gpu_type.get("gpu_ram_mb")
    }


def _per_card_vram_mb(gpu_name):
    """Per-card VRAM in MB: catalog gpu_ram_mb if populated, else the hardcoded
    DEFAULT_GPUS value. None when neither source knows the GPU.
    """
    return _catalog_vram_map().get(gpu_name) or DEFAULT_GPUS.get(gpu_name)


def parse_gpu_spec(token, default_num_gpus):
    """Parse a GPU token into (canonical_gpu_name, num_gpus).
       Example: 2x RTX_4090" -> ("RTX 4090", 2).  
    """

    # so "rtx_4090", "RTX 4090", "Rtx_4090" all map to "RTX 4090"
    canonical = _canonical_gpu_names()

    token = token.strip()
    # string to tuple. "2x RTX_4090" to ("RTX 4090", 2)
    m = re.match(r"^(\d+)[\s_]*x[\s_]*(.+)$", token, re.IGNORECASE)
    if m:
        raw_name, n = m.group(2).strip().replace("_", " "), int(m.group(1))
    else:
        raw_name, n = token.replace("_", " "), default_num_gpus
    return (canonical.get(raw_name.lower(), raw_name), n)


def format_filter_query(filters):
    """Render extra_filters dict into comparison string.
    e.g. {"gpu_ram": {"gte": 16000}} -> "gpu_ram>=16000"
    """
    parts = []
    for field, ops in (filters or {}).items():
        if not isinstance(ops, dict):
            continue
        for op, value in ops.items():
            sym = OP_TO_STR.get(op, op)
            parts.append(f"{field}{sym}{value}")
    return ", ".join(parts)


# ------------------------------------------------------------------------------------
# Pre-flight helpers: check whether each (template, gpu_name, num_gpus) is rentable
# before we benchmark it, so a run can't fail late due to 0 matching offers. On a
# miss they build a user-friendly explanation of which filter blocked it.
# ------------------------------------------------------------------------------------


def has_matching_offer(vast, *, gpu_name, num_gpus, extra_filters):
    """True if any verified+rentable offer matches the template's filters"""
    query = dict(extra_filters or {})
    query["gpu_name"] = {"eq": gpu_name}
    query["num_gpus"] = {"eq": num_gpus}
    return bool(vast.search_offers(query=query, limit=1))


def format_skip_message(vast, *, gpu_name, num_gpus, extra_filters):
    """Build a user-friendly explanation for why a GPU has 0 matching offers.
    Calls search_offers to find the blocking filter.
    """
    if not extra_filters:
        return (f"no offers for {gpu_name} (num_gpus={num_gpus})")

    diag = find_blockers(
        vast, gpu_name=gpu_name, num_gpus=num_gpus,
        extra_filters=extra_filters,
    )
    if diag["base_count"] == 0:
        return (f"no offers for {gpu_name} (num_gpus={num_gpus}), "
                f"independent of template filters")

    blockers = diag["single_blockers"]
    if blockers:
        sample = diag.get("sample_offer") or {}
        blocker_lines = []
        for key in blockers:
            ops = extra_filters.get(key)
            if not isinstance(ops, dict):
                blocker_lines.append(f"  {key}")
                continue
            gpu_value = sample.get(key)
            for op, threshold in ops.items():
                if check_template_filter(gpu_value, op, threshold) is True:
                    continue  # search-engine quirk, skip
                sym = OP_TO_STR.get(op, op)
                if gpu_value is None:
                    detail = "no sample available to check"
                else:
                    detail = f"{gpu_name} has {gpu_value}"
                blocker_lines.append(f"  {key}{sym}{threshold}: {detail}")

        if blocker_lines:
            plural = "s" if len(blocker_lines) > 1 else ""
            lines = [f"blocked by template filter{plural}:"] + blocker_lines
            # Only hint num_gpus when gpu_total_ram is the sole blocker; otherwise bumping it won't fix the run.
            if blockers == ["gpu_total_ram"]:
                ops = extra_filters.get("gpu_total_ram") or {}
                if ops:
                    op, value = next(iter(ops.items()))
                    raw = min_gpus_for_ram(
                        value, op, diag.get("per_card_gpu_ram"))
                    if raw and raw > num_gpus:
                        viable = get_gpu_chunk_size(raw)
                        lines.append(
                            f"  hint: try {viable}x {gpu_name} "
                            f"(host total then satisfies {value})")
            return "\n".join(lines)
        # All "blockers" turned out to be quirks; fall through to generic failure message

    return ("0 offers match. No single filter is the culprit; the combination "
            f"of template filters excludes {gpu_name} "
            f"({format_filter_query(extra_filters)})")


def find_blockers(vast, *, gpu_name, num_gpus, extra_filters):
    """Find which template filter blocks this GPU from having offers.

    Bails on the first real blocker. Returns a dict
    {base_count, single_blockers, per_card_gpu_ram, sample_offer}.
    """
    base_query = {"gpu_name": {"eq": gpu_name},
                  "num_gpus": {"eq": num_gpus}}
    base_offers = vast.search_offers(query=base_query, limit=1)
    base = len(base_offers)
    sample = base_offers[0] if base_offers else None
    per_card = sample.get("gpu_ram") if sample else None
    blockers = []
    if base > 0:
        for key, ops in (extra_filters or {}).items():
            # if GPU satisfies the given filter, its not a blocker so skip
            if isinstance(ops, dict) and sample is not None:
                gpu_value = sample.get(key)
                if gpu_value is not None and all(
                    check_template_filter(gpu_value, op, threshold) is True
                    for op, threshold in ops.items()
                ):
                    continue
            if not has_matching_offer(
                vast, gpu_name=gpu_name, num_gpus=num_gpus,
                extra_filters={key: ops},
            ):
                blockers.append(key)
                break
    return {"base_count": base, "single_blockers": blockers,
            "per_card_gpu_ram": per_card, "sample_offer": sample}


def check_template_filter(gpu_value, op, threshold):
    """Checks if gpu_value matches the template filters (e.g. gpu_total_ram>=16000)"""
    if gpu_value is None or threshold is None:
        return None
    try:
        a, t = float(gpu_value), float(threshold)
    except (TypeError, ValueError):
        return None
    if op == "gt":  return a > t
    if op == "gte": return a >= t
    if op == "lt":  return a < t
    if op == "lte": return a <= t
    if op == "eq":  return a == t
    if op == "neq": return a != t
    return None


def min_gpus_for_ram(threshold, op, per_card_gpu_ram):
    """Smallest ``num_gpus`` such that ``num_gpus * per_card_gpu_ram``
    satisfies the template's ``gpu_total_ram`` filter. Returns None if any
    input is missing or the operator isn't a comparison we can solve.
    """
    if not per_card_gpu_ram or not threshold:
        return None
    if op == "gt":
        return int(threshold // per_card_gpu_ram) + 1
    if op == "gte":
        return int(math.ceil(threshold / per_card_gpu_ram))
    return None


def get_gpu_chunk_size(n):
    """Round n up to the next common GPU chunk size (1, 2, 4, 8). Returns n
    unchanged for >8 since that's rare enough we'd rather be honest than guess.
    """
    for c in (1, 2, 4, 8):
        if n <= c:
            return c
    return n


def pick_num_gpus(gpu_name, extra_filters):
    """Smallest num_gpus where ``num_gpus * per_card_ram`` satisfies the
    template's ``gpu_total_ram`` filter, rounded up to a chassis size.
    Falls back to 1 when the template has no such filter or one card
    already satisfies it.
    """
    per_card = _per_card_vram_mb(gpu_name)
    if per_card is None:
        return 1
    ops = (extra_filters or {}).get("gpu_total_ram") or {}
    if not ops:
        return 1
    op, threshold = next(iter(ops.items()))
    raw_min = min_gpus_for_ram(threshold, op, per_card)
    if raw_min is None or raw_min <= 1:
        return 1
    return get_gpu_chunk_size(raw_min)


DEFAULT_CACHE_MAX_AGE_DAYS = 30
MAX_CACHE_ROWS = 100  # newest rows per template column; caps popular template+GPU combos


def lookup_cached_benchmark(vast, *, gpu_name, num_gpus, template_hash,
                             template_id, max_age_days):
    """Recent reported benchmarks for this exact spec (median + spread), or None.

    Queried once per template column rather than filtered client-side: a
    workergroup is created with either template_id or template_hash, so a row
    can carry either one. Both columns are indexed with last_update on the
    benchmarks table, so each query is index-backed instead of pulling every
    perf row for the GPU and discarding the non-matching ones.
    """
    base = {
        "type": {"eq": "perf"},
        "gpu_name": {"eq": gpu_name},
        "num_gpus": {"eq": num_gpus},
        "last_update": {"gte": time.time() - max_age_days * 86400},
    }
    rows = []
    seen_ids = set()
    for col, value in (("template_hash", template_hash), ("template_id", template_id)):
        if not value:
            continue
        found = vast.search_benchmarks(
            query=dict(base, **{col: {"eq": value}}),
            order=[{"col": "last_update", "dir": "desc"}],
            limit=MAX_CACHE_ROWS,
        )
        for r in found if isinstance(found, list) else []:
            if not isinstance(r, dict):
                continue
            # A row carrying both columns comes back from both queries; dedupe on
            # id, but keep rows without one rather than collapsing them together.
            row_id = r.get("id")
            if row_id is not None:
                if row_id in seen_ids:
                    continue
                seen_ids.add(row_id)
            rows.append(r)
    matched = [
        r for r in rows
        if isinstance(r.get("value"), (int, float)) and r["value"] > 0
    ]
    if not matched:
        return None
    values = [r["value"] for r in matched]
    newest = max((r.get("last_update") or 0) for r in matched)
    # skip null rows (from before we tracked dph_base in benchmarks)
    dphs = [
        r["dph_base"]
        for r in matched
        if isinstance(r.get("dph_base"), (int, float)) and r["dph_base"] > 0
    ]
    # use p5/p95 for the range
    if len(values) >= 2:
        qs = statistics.quantiles(values, n=20, method="inclusive")
        low, high = qs[0], qs[-1]
    else:
        low = high = values[0]
    return {
        "median": statistics.median(values),
        "low": low,
        "high": high,
        "n": len(matched),
        "age_days": max(0.0, (time.time() - newest) / 86400),
        "median_dph_base": statistics.median(dphs) if dphs else None,
    }


def update_worker_states(worker_states, current_workers, gpu_name):
    """Update worker_states with current poll, and print only on worker
    rotation (the live rich table already shows current status / elapsed).
    Tracks state_started per worker for the terminal-state grace check.
    Returns the count of workers abandoned during this poll.
    """
    now = time.monotonic()
    abandoned = 0
    waiting_key = "__waiting__"  # non-int sentinel: "no worker rented yet"

    if not current_workers:
        worker_states.setdefault(waiting_key, {"started": now})
        return abandoned

    worker_states.pop(waiting_key, None)

    seen = set()
    for w in current_workers:
        wid = w.get("id")
        if wid is None:
            continue
        seen.add(wid)
        status = str(w.get("status", "")).lower()
        prev = worker_states.get(wid)
        if prev is None:
            worker_states[wid] = {"status": status, "state_started": now}
            continue
        if prev["status"] != status:
            prev["status"] = status
            prev["state_started"] = now

    for wid in list(worker_states):
        if not isinstance(wid, int):
            continue  # skip the non-int waiting sentinel
        if wid not in seen:
            # Worker abandoned; live table only shows current worker, so rotation is otherwise invisible.
            prev = worker_states[wid]
            elapsed = format_time_elapsed(now - prev["state_started"])
            print(f"[{gpu_name}] worker {wid} abandoned "
                  f"(last state={prev['status']} for {elapsed})",
                  file=sys.stderr)
            del worker_states[wid]
            abandoned += 1
    return abandoned


def update_row(class_states, row_id, **fields):
    """Update one GPU's row in the live-table state dict. Sets
    ``run_started`` on first non-``queued`` status and ``run_ended`` on
    terminal status (so elapsed freezes); each thread only writes its own
    key so no lock needed.
    """
    if class_states is None:
        return
    cur = class_states.setdefault(row_id, {})
    new_status = fields.get("status")
    now_ts = time.monotonic()
    if new_status and new_status != cur.get("status"):
        if "run_started" not in cur and new_status not in ("queued", "cached"):
            cur["run_started"] = now_ts
        if new_status in TERMINAL_STATUSES:
            cur["run_ended"] = now_ts
        else:
            cur.pop("run_ended", None)  # recovered from a transient terminal status; unfreeze elapsed
    cur.update(fields)


def render_table(class_states):
    """Render the per-GPU progress as a rich.Table for the live display."""
    table = Table(show_header=True, header_style="bold cyan")
    table.add_column("GPU", no_wrap=True)
    table.add_column("Status", no_wrap=True)
    table.add_column("Endpoint", justify="right", no_wrap=True)
    table.add_column("Worker", justify="right", no_wrap=True)
    table.add_column("Elapsed", justify="right", no_wrap=True)
    table.add_column("Perf", justify="right", no_wrap=True)
    table.add_column("$/hr", justify="right", no_wrap=True)
    table.add_column("Perf/$/hr", justify="right", no_wrap=True)
    now = time.monotonic()
    for gpu in sorted(class_states):
        s = class_states[gpu]
        status = s.get("status") or "-"
        # Skipped rows are noise during the run; reason already printed at pre-flight, final summary still includes them.
        if status == "skipped":
            continue
        run_started = s.get("run_started")
        run_ended = s.get("run_ended")
        # Blank for not-yet-running, frozen for terminal, live otherwise.
        if run_started is None:
            elapsed_str = "-"
        elif run_ended is not None:
            elapsed_str = format_time_elapsed(run_ended - run_started)
        else:
            elapsed_str = format_time_elapsed(now - run_started)
        perf = s.get("perf")
        dph = s.get("dph")
        pps = (perf / dph) if (perf and dph) else None
        table.add_row(
            gpu,
            status,
            str(s.get("endpoint_id") or "-"),
            str(s.get("worker_id") or "-"),
            elapsed_str,
            f"{perf:.1f}" if perf else "-",
            f"${dph:.3f}" if dph else "-",
            f"{pps:.1f}" if pps else "-",
        )
    return table


def benchmark_gpu(vast, *, gpu_name, num_gpus, timeout,
                   workergroups, endpoints,
                   template_hash=None, template_id=None,
                   auto_instance=None, autoscaler_url=None,
                   class_states=None, stop_event=None):
    """Rent one instance for ``gpu_name``, poll until idle + measured_perf,
    tear down. Each call owns its own endpoint + workergroup so it can run
    in parallel safely. Returns ``(gpu_name, num_gpus, status, perf, err, dph_base)``.
    """
    endpoint_id = None
    workergroup_id = None
    # uuid suffix keeps the name unique when benchmarking the same GPU in parallel;
    # no parens/shell chars, the backend rejects them in endpoint_name.
    endpoint_name = f"benchmark {num_gpus}x {gpu_name} {uuid.uuid4().hex[:8]}"
    row_id = f"{num_gpus}x {gpu_name}" # id for the live table so 1x/2x/4x of the same GPU gets its own row
    start = time.monotonic()
    update_row(class_states, row_id, status="provisioning")
    search = f"gpu_name={gpu_name.replace(' ', '_')} num_gpus={num_gpus}"
    if autoscaler_url:
        status_vast = VastAI(api_key=vast.client.api_key, server_url=autoscaler_url, retry=vast.client.retry)
    else:
        status_vast = vast

    try:
        endpoint_kwargs = dict(ENDPOINT_CONFIG, endpoint_name=endpoint_name)
        if auto_instance is not None:
            endpoint_kwargs["auto_instance"] = auto_instance
        ep_resp = vast.create_endpoint(**endpoint_kwargs)
        # autoscaler returns the new id under either "result" or "id" depending on resource
        endpoint_id = ep_resp.get("result", ep_resp.get("id")) if isinstance(ep_resp, dict) else None
        if not isinstance(endpoint_id, int):
            return (gpu_name, num_gpus, "error", None,
                    f"create_endpoint returned no id: {ep_resp!r}", None)
        endpoints.add(endpoint_id)
        update_row(class_states, row_id, endpoint_id=endpoint_id)

        # cold_workers and min_load are set on ENDPOINT_CONFIG, not here; the autoscaler reads
        # both from the endpoint group, not the workergroup (verified via autoscaler.cpp).
        workergroup_kwargs = dict(
            endpoint_id=endpoint_id,
            endpoint_name=endpoint_name,
            search_params=search,
        )
        if template_id is not None:
            workergroup_kwargs["template_id"] = template_id
        elif template_hash is not None:
            workergroup_kwargs["template_hash"] = template_hash
        if auto_instance is not None:
            workergroup_kwargs["auto_instance"] = auto_instance
        resp = vast.create_workergroup(**workergroup_kwargs)
        workergroup_id = resp.get("result", resp.get("id")) if isinstance(resp, dict) else None
        if not isinstance(workergroup_id, int):
            return (gpu_name, num_gpus, "error", None,
                    f"create_workergroup returned no id: {resp!r}", None)
        workergroups.add(workergroup_id)
        update_row(class_states, row_id, status="waiting_for_worker")
        worker_states = {}
        dph_by_worker = {}  # cache so we don't re-fetch dph on every poll
        abandoned_count = 0  # bail when this hits MAX_ABANDONS; autoscaler is renting + giving up repeatedly, no progress

        while time.monotonic() - start < timeout:
            if stop_event is not None and stop_event.is_set():
                update_row(class_states, row_id, status="aborted")
                return (gpu_name, num_gpus, "aborted", None, "user aborted", None)
            # calls get endpoint workers which includes measured perf(get workergroup workers doesnt)
            resp = status_vast.get_endpoint_workers(endpoint_id)
            if isinstance(resp, list):
                workers = resp
            elif isinstance(resp, dict) and isinstance(resp.get("workers"), list):
                workers = resp["workers"]
            else:
                workers = []
            abandoned_count += update_worker_states(worker_states, workers, gpu_name)
            if abandoned_count >= MAX_ABANDONS:
                update_row(class_states, row_id, status="failed")
                return (gpu_name, num_gpus, "failed", None,
                        f"autoscaler abandoned {abandoned_count} workers without "
                        f"any reaching idle (likely template + GPU config not loadable "
                        f"on these hosts)", None)
            if workers:
                primary = workers[0]
                primary_id = primary.get("id")
                fields = {
                    "status": str(primary.get("status") or "?").lower(),
                    "worker_id": primary_id,
                }
                # Fetch dph once per new worker_id; surfaces $/hr in the live table as soon as the autoscaler rents (before model loads).
                if primary_id and primary_id not in dph_by_worker:
                    try:
                        inst = vast.show_instance(id=primary_id)
                        dph_by_worker[primary_id] = inst.get("dph_base")
                    except Exception as e:
                        # Cache None so we don't re-attempt every poll, but log so a real bug isn't silently swallowed.
                        dph_by_worker[primary_id] = None
                        print(f"[warn] failed to fetch dph for worker {primary_id}: {e}",
                              file=sys.stderr)
                if primary_id and dph_by_worker.get(primary_id) is not None:
                    fields["dph"] = dph_by_worker[primary_id]
                update_row(class_states, row_id, **fields)
            # measured_perf is only real once status==idle; before that it's a dlperf placeholder.
            ready = [w for w in workers
                     if str(w.get("status", "")).lower() == "idle"
                     and (w.get("measured_perf") or 0) > 0]
            if ready:
                # dph was already fetched and cached when this worker_id first appeared in polling.
                worker_id = ready[0].get("id")
                dph = dph_by_worker.get(worker_id) if worker_id else None
                update_row(class_states, row_id, status="done",
                                 perf=ready[0]["measured_perf"], dph=dph)
                return (gpu_name, num_gpus, "ok", ready[0]["measured_perf"], None, dph)
            # Fail fast: every worker stuck terminal past the grace period, none reached idle.
            now_ts = time.monotonic()
            terminal_workers_long = workers and all(
                str(w.get("status", "")).lower() in TERMINAL_STATES
                and isinstance(w.get("id"), int)
                and w["id"] in worker_states
                and now_ts - worker_states[w["id"]]["state_started"]
                    > TERMINAL_GRACE_SECONDS
                for w in workers
            )
            if terminal_workers_long:
                states = sorted({str(w.get("status", "")).lower()
                                 for w in workers})
                update_row(class_states, row_id, status="failed")
                return (gpu_name, num_gpus, "failed", None,
                        f"all workers terminal ({', '.join(states)}) for "
                        f">{TERMINAL_GRACE_SECONDS}s without reaching idle; "
                        f"autoscaler not rotating", None)
            # Fail fast if no real worker (int id) has ever appeared.
            if (not any(isinstance(k, int) for k in worker_states)
                    and time.monotonic() - start > RENTAL_TIMEOUT):
                update_row(class_states, row_id, status="no_worker")
                return (gpu_name, num_gpus, "no_worker", None,
                        f"autoscaler did not rent in {RENTAL_TIMEOUT}s "
                        f"(possible causes: insufficient credit, scoring issue, "
                        f"all candidates failed silently, or template+GPU mismatch "
                        f"missed by pre-flight)", None)
            # Jitter so N parallel threads don't all hit the autoscaler at the same instant.
            delay = WORKER_POLL_INTERVAL + random.uniform(0, 2.0)
            if stop_event is not None:
                if stop_event.wait(timeout=delay):
                    update_row(class_states, row_id, status="aborted")
                    return (gpu_name, num_gpus, "aborted", None, "user aborted", None)
            else:
                time.sleep(delay)

        update_row(class_states, row_id, status="timeout")
        return (gpu_name, num_gpus, "timeout", None,
                f"no measured_perf in {timeout}s", None)
    finally:
        # Workergroup first (stops new workers), then endpoint. atexit sweeps anything we miss.
        # 404s are silently absorbed since "already gone" is the desired end state.
        if workergroup_id is not None:
            try:
                vast.delete_workergroup(id=workergroup_id)
                workergroups.discard(workergroup_id)
            except Exception as e:
                if getattr(getattr(e, "response", None), "status_code", None) != 404:
                    print(f"[cleanup] failed to delete workergroup {workergroup_id}: {e}",
                          file=sys.stderr)
                else:
                    workergroups.discard(workergroup_id)
        if endpoint_id is not None:
            try:
                vast.delete_endpoint(id=endpoint_id)
                endpoints.discard(endpoint_id)
            except Exception as e:
                if getattr(getattr(e, "response", None), "status_code", None) != 404:
                    print(f"[cleanup] failed to delete endpoint {endpoint_id}: {e}",
                          file=sys.stderr)
                else:
                    endpoints.discard(endpoint_id)


@parser.command(
    argument("--template_hash", type=str, default=None,
             help="the template to benchmark; provide either --template_hash or --template_id"),
    argument("--template_id", type=int, default=None,
             help="the template to benchmark; provide either --template_hash or --template_id"),
    argument("--gpus", type=str,
             help="comma-separated GPU names to benchmark (e.g. RTX_4090,RTX_3090). Prefix a name with a count to set GPUs per instance, e.g. \"2x RTX_4090\" (overrides --num_gpus)"),
    argument("--num_gpus", type=int, default=None,
             help="GPUs per instance for tokens without an Nx prefix (default 1); overridden by inline Nx in --gpus"),
    argument("--timeout", type=int, default=DEFAULT_BENCHMARK_TIMEOUT,
             help=f"max seconds to wait for a benchmark before giving up (default {DEFAULT_BENCHMARK_TIMEOUT})"),
    argument("--no-cache", dest="no_cache", action="store_true",
             help="re-measure instead of reusing recent cached benchmark results"),
    argument("-y", "--yes", action="store_true",
             help="Skip confirmation prompt"),
    argument("--auto_instance", type=str, default=None, help=argparse.SUPPRESS),
    argument("--autoscaler_url", type=str, default=None, help=argparse.SUPPRESS),
    usage="vastai run benchmarks [OPTIONS]",
    help="Benchmark a template against one or more GPUs",
    epilog=deindent("""
        Rents one instance per GPU in parallel, measures perf, tears down.
        Each rental runs for up to --timeout seconds and costs real money.

        Specs benchmarked recently (same template, GPU, and count, by any
        user) are served from the benchmarks table as an estimated perf, and
        you're asked whether to reuse that or run a fresh benchmark. Pass
        --no-cache to always re-measure, or -y to always reuse the cache.

        Perf/$ uses the GPU $/hr, so cached and freshly measured rows are
        comparable. Cached rows recorded before that price was tracked show
        a perf with no price.

        Examples:
            # auto-sweep the default GPUs against TGI
            vastai run benchmarks --template_hash 79ebdd2ebfb9d42cedf7a221c42d37a5

            # specific GPUs against vLLM
            vastai run benchmarks --template_hash 393fa8572e6c73c927c8275fe4dffd53 --gpus RTX_4090,RTX_3090

            # multi-GPU configurations via inline Nx prefix against ComfyUI
            vastai run benchmarks --template_hash 40ef49becc953aa910ee05bd4653b9b3 --gpus "2x RTX_4090, 2x RTX_3090"

            # default count for tokens without an Nx prefix
            vastai run benchmarks --template_hash 79ebdd2ebfb9d42cedf7a221c42d37a5 --gpus RTX_4090,RTX_3090 --num_gpus 2

            # shorter timeout (10 min), skipping the cost prompt
            vastai run benchmarks --template_hash 393fa8572e6c73c927c8275fe4dffd53 --timeout 600 -y

            # ignore cached results and re-measure everything
            vastai run benchmarks --template_hash 79ebdd2ebfb9d42cedf7a221c42d37a5 --no-cache

            # raw JSON output for piping into another tool
            vastai run benchmarks --template_hash 40ef49becc953aa910ee05bd4653b9b3 --raw
    """),
)
def run__benchmarks(args):
    if args.template_id is None and args.template_hash is None:
        print("error: one of --template_id or --template_hash is required "
              "(run `vastai run benchmarks --help` for usage)",
              file=sys.stderr)
        return 1
    if args.template_id is not None:
        query = {"id": {"eq": args.template_id}}
        ident = f"id={args.template_id}"
    else:
        query = {"hash_id": {"eq": args.template_hash}}
        ident = f"hash={args.template_hash}"

    # bump retry budget so parallel polls don't exhaust VastClient's tiny default (3 attempts, ~0.7s) under autoscaler rate limits.
    retry = max(args.retry, 8)
    vast = VastAI(api_key=args.api_key, server_url=args.url, retry=retry,
                  explain=getattr(args, 'explain', False),
                  curl=getattr(args, 'curl', False))

    templates = vast.search_templates(query=query)
    if not templates:
        print(f"error: template not found ({ident})", file=sys.stderr)
        return 1
    template = templates[0]
    try:
        extra_filters = json.loads(template.get("extra_filters") or "{}")
    except (ValueError, TypeError):
        extra_filters = {}

    template_name = template.get("name") or template.get("title") or "?"
    filter_summary = format_filter_query(extra_filters) or "none"
    print(f"\nTemplate id={template.get('id')} {template_name}, filters: {filter_summary}")

    # if user provides specific GPU count like "2x RTX 4090", that takes precedence over --num_gpus
    if args.gpus:
        gpu_specs = [parse_gpu_spec(t, args.num_gpus or 1)
                     for t in args.gpus.split(",") if t.strip()]
    elif args.num_gpus is not None:
        gpu_specs = [(g, args.num_gpus) for g in DEFAULT_GPUS]
    else:
        gpu_specs = [
            (g, pick_num_gpus(g, extra_filters))
            for g in DEFAULT_GPUS
        ]

    console = Console(stderr=True)

    # deduplicates (gpu, num_gpus) so we dont benchmark the same GPU config twice
    seen = set()
    deduped = []
    for g, n in gpu_specs:
        if (g, n) in seen:
            console.print(f"[yellow]skipping duplicate spec {n}x {g}[/yellow]",
                          highlight=False)
            continue
        seen.add((g, n))
        deduped.append((g, n))
    gpu_specs = deduped

    # Pre-flight: for each spec, serve a recent cached benchmark if one exists
    # (unless --no-cache), else skip specs with 0 matching offers, so the user
    # sees cache hits and skip reasons before approving any rentals.
    compatible_specs = []
    skipped_results = []
    cached_results = []
    interactive = not (args.yes or args.raw)
    for g, n in gpu_specs:
        if not args.no_cache:
            hit = lookup_cached_benchmark(
                vast, gpu_name=g, num_gpus=n,
                template_hash=template.get("hash_id"),
                template_id=template.get("id"),
                max_age_days=DEFAULT_CACHE_MAX_AGE_DAYS,
            )
            if hit:
                rentable = has_matching_offer(
                    vast, gpu_name=g, num_gpus=n, extra_filters=extra_filters)
                age = (f"{hit['age_days']:.0f}d" if hit["age_days"] >= 1
                       else "<1d")
                note = None if rentable else "no offers available to rent right now"
                median_dph_base = hit.get("median_dph_base")
                pps = f", {hit['median'] / median_dph_base:.1f} perf/$" if median_dph_base else ""
                msg = (f"[green][{n}x {g}] cached:[/green] median perf "
                       f"{hit['median']:.1f}{pps} "
                       f"(n={hit['n']}, range {hit['low']:.1f}-{hit['high']:.1f}, "
                       f"newest {age} ago)")
                if note:
                    msg += f"; [yellow]{note}[/yellow]"
                console.print(msg, highlight=False)
                # Default is to reuse the cached result; only an interactive user
                # who opts in pays to re-measure (and only when offers exist).
                run_fresh = (
                    interactive and rentable
                    and input(
                        f"  [{n}x {g}] run a fresh benchmark anyway? this rents a "
                        f"real GPU and charges your account [y/N] "
                    ).strip().lower() in ("y", "yes")
                )
                if run_fresh:
                    compatible_specs.append((g, n))  # offers already confirmed
                else:
                    cached_results.append((g, n, "cached", hit["median"], None, hit.get("median_dph_base")))
                continue
        if not has_matching_offer(
            vast, gpu_name=g, num_gpus=n,
            extra_filters=extra_filters,
        ):
            msg = format_skip_message(
                vast, gpu_name=g, num_gpus=n,
                extra_filters=extra_filters,
            )
            console.print(f"[yellow][{n}x {g}] skipping:[/yellow] {msg}",
                          highlight=False)
            skipped_results.append((g, n, "skipped", None, msg, None))
        else:
            compatible_specs.append((g, n))

    # Live-table state. Pre-populate every GPU so the table is complete.
    class_states = {}
    for g, n in compatible_specs:
        update_row(class_states, f"{n}x {g}", status="queued")
    for sr in skipped_results:
        update_row(class_states, f"{sr[1]}x {sr[0]}", status="skipped")
    for cr in cached_results:
        update_row(class_states, f"{cr[1]}x {cr[0]}", status="cached",
                    perf=cr[3], dph=cr[5])

    timeout_minutes = args.timeout / 60.0
    n = len(compatible_specs)
    if n == 0:
        if cached_results:
            console.print()
            console.print(render_table(class_states))
        else:
            console.print(
                "\nNo compatible GPUs to benchmark for this template.",
                style="bold red")
        return print_results(args, skipped_results + cached_results)

    spec_strs = [f"{cnt}x {gpu}" for gpu, cnt in compatible_specs]
    spec_summary = ", ".join(spec_strs)
    if n == 1:
        print(f"\nWill rent 1 GPU configuration ({spec_summary}). "
              f"Each runs ~2 to {timeout_minutes:.0f} min.")
    else:
        print(f"\nWill rent {n} GPU configurations in parallel "
              f"({spec_summary}). Each runs ~2 to {timeout_minutes:.0f} min.")
    print("This rents real GPUs and charges your account for usage.")
    if not args.yes:
        if input("Continue? [y/N] ").strip().lower() not in ("y", "yes"):
            print("Aborted.", file=sys.stderr)
            sys.exit(130)
    print()

    # Backing sets for the atexit sweep; per-GPU teardown discards on success.
    workergroups = set()
    endpoints = set()
    stop_event = threading.Event()

    def _cleanup():
        # Mask SIGINT + catch BaseException so a second Ctrl+C can't leak records mid-iteration.
        prev_handler = None
        try:
            prev_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
        except (ValueError, OSError):
            pass
        try:
            for workergroup_id in list(workergroups):
                try:
                    vast.delete_workergroup(id=workergroup_id)
                    workergroups.discard(workergroup_id)
                except BaseException:
                    pass
            for ep_id in list(endpoints):
                try:
                    vast.delete_endpoint(id=ep_id)
                    endpoints.discard(ep_id)
                except BaseException:
                    pass
        finally:
            if prev_handler is not None:
                try:
                    signal.signal(signal.SIGINT, prev_handler)
                except (ValueError, OSError):
                    pass
    atexit.register(_cleanup)

    def _run_one_gpu(g, n):
        try:
            return benchmark_gpu(
                vast,
                gpu_name=g,
                num_gpus=n,
                timeout=args.timeout,
                workergroups=workergroups,
                endpoints=endpoints,
                template_hash=args.template_hash,
                template_id=args.template_id,
                auto_instance=args.auto_instance,
                autoscaler_url=args.autoscaler_url,
                class_states=class_states,
                stop_event=stop_event,
            )
        except Exception as e:
            update_row(class_states, f"{n}x {g}", status="error")
            # Surface response body for HTTPError so backend rate-limit / validation
            # messages aren't swallowed (default __str__ on HTTPError just shows the URL).
            body = ""
            resp = getattr(e, "response", None)
            if resp is not None:
                try:
                    body = f" | body: {resp.text[:500]}"
                except Exception:
                    pass
            return (g, n, "error", None, f"{type(e).__name__}: {e}{body}", None)

    run_results = []
    executor = ThreadPoolExecutor(max_workers=len(compatible_specs))
    try:
        # Stagger submits to keep cold-start POSTs from tripping rate limits.
        futures = []
        for i, (g, n_for_class) in enumerate(compatible_specs):
            futures.append(executor.submit(_run_one_gpu, g, n_for_class))
            if i < len(compatible_specs) - 1:
                time.sleep(PARALLEL_SUBMIT_DELAY)
        try:
            with Live(render_table(class_states), console=console,
                      refresh_per_second=2, transient=False) as live:
                pending = set(futures)
                while pending:
                    done, pending = wait(pending, timeout=0.5,
                                         return_when=FIRST_COMPLETED)
                    for fut in done:
                        result = fut.result()
                        run_results.append(result)
                        # Surface failures live above the table.
                        g, n, status, _perf, err, _price = result
                        if status not in ("ok", "skipped") and err:
                            console.print(
                                f"[red][{n}x {g}] {status}:[/red] {err}",
                                highlight=False)
                    live.update(render_table(class_states))
        except KeyboardInterrupt:
            # Set stop_event so threads exit their poll/sleep immediately, then cleanup synchronously.
            console.print(
                "\n[yellow]Deleting endpoints, please wait... "
                "If you exit now, you'll need to delete them manually with `vastai delete endpoint`.[/yellow]",
                highlight=False)
            stop_event.set()
            executor.shutdown(wait=False, cancel_futures=True)
            _cleanup()
            sys.exit(130)
    finally:
        executor.shutdown(wait=True)
        _cleanup()

    results = skipped_results + cached_results + run_results
    return print_results(args, results)


def print_results(args, results):
    rows = []
    for gpu, num_gpus, status, perf, err, price in results:
        pps = (perf / price) if (perf and price) else None
        rows.append({
            "gpu_name": gpu,
            "num_gpus": num_gpus,
            "rental_dph": price,
            "measured_perf": perf,
            "status": status,
            "perf_per_dollar": pps,
        })
    rows.sort(key=lambda r: (r["perf_per_dollar"] or -1), reverse=True)

    if args.raw:
        return rows

    n_ok = sum(1 for r in rows if r["status"] == "ok")
    n_cached = sum(1 for r in rows if r["status"] == "cached")
    line = f"\nBenchmark complete: {n_ok + n_cached}/{len(rows)} GPUs measured"
    print(line + (f" ({n_cached} from cache)." if n_cached else "."))
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall