Skip to content

vast-ai/vast-cli / vastai

vastai

Flagged

Vast.ai CLI to manage GPU instances, volumes, serverless endpoints, and billing.

FileKindSize
SKILL.mdskill md18.6 KB
__init__.pyscript1.8 KB
_base.pyscript1.6 KB
api/__init__.pyscript31 B
api/auth.pyscript15.3 KB
api/billing.pyscript10.4 KB
api/client.pyscript9.1 KB
api/clusters.pyscript4.5 KB
api/deployments.pyscript1.4 KB
api/endpoints.pyscript12.0 KB
api/instances.pyscript17.1 KB
api/keys.pyscript4.3 KB
api/machines.pyscript7.4 KB
api/metrics.pyscript2.6 KB
api/offers.pyscript21.2 KB
api/price_increase.pyscript4.0 KB
api/query.pyscript12.2 KB
api/storage.pyscript14.5 KB
api/teams.pyscript4.4 KB
async_/__init__.pyscript79 B
async_/client.pyscript9.6 KB
async_/results.pyscript1.3 KB
cli/__init__.pyscript25 B
cli/commands/__init__.pyscript757 B
cli/commands/auth.pyscript42.8 KB
cli/commands/benchmarks.pyscript42.0 KB
cli/commands/billing.pyscript30.4 KB
cli/commands/clusters.pyscript7.8 KB
cli/commands/deployments.pyscript7.8 KB
cli/commands/endpoints.pyscript17.3 KB
cli/commands/instances.pyscript55.4 KB
cli/commands/keys.pyscript7.9 KB
cli/commands/machines.pyscript78.4 KB
cli/commands/metrics.pyscript12.7 KB
cli/commands/misc.pyscript11.0 KB
cli/commands/offers.pyscript30.0 KB
cli/commands/price_increase.pyscript14.5 KB
cli/commands/repl.pyscript719 B
cli/commands/storage.pyscript38.9 KB
cli/commands/teams.pyscript6.4 KB
cli/commands/uninstall.pyscript2.4 KB
cli/commands/update.pyscript3.1 KB
cli/display.pyscript20.1 KB
cli/main.pyscript11.5 KB
cli/parser.pyscript18.1 KB
cli/repl/__init__.pyscript318 B
cli/repl/bridge.pyscript7.3 KB
cli/repl/catalog.pyscript6.3 KB
cli/repl/completion.pyscript8.2 KB
cli/repl/session.pyscript7.3 KB
cli/self_test/__init__.pyscript46 B
cli/self_test/machine_diagnostics.pyscript21.0 KB
cli/self_test/runtime_diagnostics.pyscript18.6 KB
cli/self_test/support_bundle.pyscript11.0 KB
cli/selfupdate.pyscript14.0 KB
cli/util.pyscript26.3 KB
cli/utils.pyscript577 B
data/__init__.pyscript303 B
data/deployment.pyscript3.5 KB
data/endpoint.pyscript2.1 KB
data/instance.pyscript3.6 KB
data/offer.pyscript4.4 KB
data/query.pyscript12.8 KB
data/workergroup.pyscript1.4 KB
pdf/__init__.pyscript0 B
pdf/vast.ai-logo-50pct.pngbinary7.4 KB
pdf/vast.ai-logo.pngbinary10.1 KB
pdf/vast_pdf.pyscript19.4 KB
pdf/watcher.shexecutablescript390 B
sdk.pyscript52.2 KB
serverless/__init__.pyscript193 B
serverless/client/__init__.pyscript346 B
serverless/client/client.pyscript36.7 KB
serverless/client/connection.pyscript11.5 KB
serverless/client/endpoint.pyscript6.9 KB
serverless/client/managed.pyscript7.3 KB
serverless/client/request_status.pyscript371 B
serverless/client/session.pyscript2.8 KB
serverless/client/worker.pyscript1.3 KB
serverless/remote/__init__.pyscript245 B
serverless/remote/base.pyscript6.3 KB
serverless/remote/deploy.pyscript12.6 KB
serverless/remote/serialization.pyscript3.7 KB
serverless/remote/serve.pyscript8.6 KB
serverless/remote/serve_deployment.pyscript6.3 KB
serverless/remote/utils.pyscript9.7 KB
serverless/server/__init__.pyscript0 B
serverless/server/lib/__init__.pyscript0 B
serverless/server/lib/backend.pyscript43.3 KB
serverless/server/lib/data_types.pyscript10.6 KB
serverless/server/lib/metrics.pyscript12.2 KB
serverless/server/lib/server.pyscript3.2 KB
serverless/server/worker.pyscript17.1 KB
sync/__init__.pyscript76 B
sync/client.pyscript4.9 KB
sync/results.pyscript1.2 KB
utils.pyscript12.2 KB

serverless/client/client.py

View on GitHub
# client.py
from .connection import _make_request, _require_public_worker_url
from .endpoint import Endpoint, Endpoint_
from .managed import ManagedEndpoint, ManagedDeployment
from .worker import Worker
from .session import Session
from .request_status import RequestStatus
from vastai.data.endpoint import EndpointConfig, EndpointData
from vastai.data.deployment import (
    DeploymentConfig,
    DeploymentData,
    DeploymentPutResponse,
)
from vastai.data.workergroup import WorkergroupConfig
import asyncio
import aiohttp
import ssl
import os
import tempfile
import logging
import random
import time
import collections
from abc import abstractmethod
from typing import (
    Any,
    Awaitable,
    Callable,
    Coroutine,
    Deque,
    Dict,
    Generic,
    Optional,
    TypeVar,
    Union,
    List,
)

logger = logging.getLogger("vastai")

R = TypeVar("R", bound=Awaitable)


class ServerlessRequest(asyncio.Future):
    """A fire-and-observe request to a Serverless endpoint.

    Composes a RequestStatus tracker for status/timestamp tracking.
    Properties delegate to the tracker for backward compatibility.
    """

    def __init__(self, tracker: Optional[RequestStatus] = None):
        super().__init__()
        self.tracker = tracker if tracker is not None else RequestStatus()

    @property
    def status(self):
        return self.tracker.status

    @status.setter
    def status(self, value):
        self.tracker.status = value

    @property
    def create_time(self):
        return self.tracker.create_time

    @create_time.setter
    def create_time(self, value):
        self.tracker.create_time = value

    @property
    def start_time(self):
        return self.tracker.start_time

    @start_time.setter
    def start_time(self, value):
        self.tracker.start_time = value

    @property
    def complete_time(self):
        return self.tracker.complete_time

    @complete_time.setter
    def complete_time(self, value):
        self.tracker.complete_time = value

    @property
    def req_idx(self):
        return self.tracker.req_idx

    @req_idx.setter
    def req_idx(self, value):
        self.tracker.req_idx = value

    def then(self, callback) -> "ServerlessRequest":
        def _done(fut):
            if fut.exception() is not None:
                print(fut.exception())
                return
            callback(fut.result())

        self.add_done_callback(_done)
        return self


class _ServerlessBase(Generic[R]):
    SSL_CERT_URL = "https://console.vast.ai/static/jvastai_root.cer"
    VAST_WEB_URL = "https://console.vast.ai"
    VAST_SERVERLESS_URL = "https://run.vast.ai"

    def __init__(
        self,
        api_key: Optional[str] = os.environ.get("VAST_API_KEY", None),
        *,
        debug: bool = False,
        instance: str = "prod",
        autoscaler_url: Optional[str] = None,
        webserver_url: str = VAST_WEB_URL,
        connection_limit: int = 500,
        default_request_timeout: float = 600.0,
        max_poll_interval: float = 5.0,
    ):
        if api_key is None or api_key == "":
            raise AttributeError(
                "API key missing. Please set VAST_API_KEY in your environment variables."
            )
        self.api_key = api_key
        self.vast_web_url = webserver_url
        if autoscaler_url:
            self.autoscaler_url = autoscaler_url
        else:
            match instance:
                case "prod":
                    self.autoscaler_url = "https://run.vast.ai"
                case "alpha":
                    self.autoscaler_url = "https://run-alpha.vast.ai"
                case "candidate":
                    self.autoscaler_url = "https://run-candidate.vast.ai"
                case "local":
                    self.autoscaler_url = "http://localhost:8080"
                case _:
                    self.autoscaler_url = "https://run.vast.ai"

        self.latencies = collections.deque(maxlen=50)
        self.debug = debug
        self.default_request_timeout = float(default_request_timeout)
        self.max_poll_interval = float(max_poll_interval)
        self.logger = logging.getLogger(self.__class__.__name__)

        if not self.logger.handlers:
            self.logger.addHandler(logging.NullHandler())

        if debug:
            handler = logging.StreamHandler()
            formatter = logging.Formatter(
                "[%(asctime)s] %(name)s - %(levelname)s - %(message)s"
            )
            handler.setFormatter(formatter)
            handler.setLevel(logging.DEBUG)

            self.logger.addHandler(handler)
            self.logger.setLevel(logging.DEBUG)

            # If we attach our own handler, avoid double-logging via root handlers.
            self.logger.propagate = False
        else:
            # Let the application decide if/where logs go.
            self.logger.propagate = True

        self.connection_limit = connection_limit
        self._session: aiohttp.ClientSession | None = None
        self._ssl_context: ssl.SSLContext | None = None
        self._transport_owner = (
            None  # if set, delegate _get_session/get_ssl_context and skip close
        )

    async def __aenter__(self):
        await self._get_session()
        return self

    async def __aexit__(self, exc_type, exc, tb):
        await self.close()

    async def _get_session(self) -> aiohttp.ClientSession:
        if self._transport_owner is not None:
            return await self._transport_owner._get_session()
        if self._session is None or self._session.closed:
            self.logger.info("Started aiohttp ClientSession")
            connector = aiohttp.TCPConnector(
                limit=self.connection_limit, ssl=await self.get_ssl_context()
            )
            self._session = aiohttp.ClientSession(connector=connector)
        return self._session

    def is_open(self):
        if self._transport_owner is not None:
            return self._transport_owner.is_open()
        return self._session is not None and not self._session.closed

    async def close(self):
        if self._transport_owner is not None:
            return  # transport owned by another client
        if self._session and not self._session.closed:
            await self._session.close()
            self.logger.info("Closed aiohttp ClientSession")

    async def get_ssl_context(self) -> ssl.SSLContext:
        """Download Vast.ai root cert and build SSL context (cached)."""
        if self._transport_owner is not None:
            return await self._transport_owner.get_ssl_context()
        if self._ssl_context is None:
            async with aiohttp.ClientSession() as s:
                async with s.get(self.SSL_CERT_URL) as resp:
                    if resp.status != 200:
                        raise Exception(f"Failed to fetch SSL cert: {resp.status}")
                    cert_bytes = await resp.read()

            tmpfile = tempfile.NamedTemporaryFile(delete=False, suffix=".cer")
            tmpfile.write(cert_bytes)
            tmpfile.close()

            ctx = ssl.create_default_context()
            ctx.load_verify_locations(cafile=tmpfile.name)
            # The Vast.ai root CA cert has pathlen set without keyCertSign in
            # Key Usage.  OpenSSL 3.x (Python ≥3.10) rejects this by default.
            # Clearing VERIFY_X509_STRICT relaxes that single check while
            # keeping full chain-of-trust and signature verification intact.
            ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT
            self.logger.info("Loaded Vast.ai SSL certificate")

            self._ssl_context = ctx
            os.unlink(tmpfile.name)

        return self._ssl_context

    async def get_endpoint(self, name="") -> Endpoint_[R]:
        endpoints = await self.get_endpoints()
        for e in endpoints:
            if e.name == name:
                return e
        raise Exception(f"Endpoint {name} could not be found")

    async def get_endpoints(self) -> list[Endpoint_[R]]:
        try:
            result = await _make_request(
                client=self,
                url=self.vast_web_url,
                route="/api/v0/endptjobs/",
                api_key=self.api_key,
                params={"client_id": "me"},
            )
        except Exception as ex:
            raise Exception(f"Failed to get endpoints:\nReason={ex}")

        if not result.get("ok"):
            raise Exception(
                f"Failed to get endpoints: HTTP {result.get('status')} - {result.get('text', '')[:512]}"
            )

        response = result.get("json") or {}
        endpoints = []
        for e in response.get("results", []):
            data = EndpointData.from_dict(e)
            endpoints.append(Endpoint(client=self, data=data))
        self.logger.info(f"Found {len(endpoints)} endpoints")
        return endpoints

    async def fetch_endpoint(self, endpoint_id: int) -> EndpointData:
        """Fetch full endpoint data by ID. Used internally by ManagedEndpoint.get()."""
        try:
            result = await _make_request(
                client=self,
                url=self.vast_web_url,
                route=f"/api/v0/endptjobs/{endpoint_id}/",
                api_key=self.api_key,
            )
        except Exception as ex:
            raise Exception(f"Failed to fetch endpoint {endpoint_id}:\nReason={ex}")

        if not result.get("ok"):
            raise Exception(
                f"Failed to fetch endpoint {endpoint_id}: HTTP {result.get('status')} - {result.get('text', '')[:512]}"
            )

        return EndpointData.from_dict(result["json"]["result"])

    async def fetch_deployment(self, deployment_id: int) -> DeploymentData:
        """Fetch full deployment data by ID. Used internally by ManagedDeployment.get()."""
        try:
            result = await _make_request(
                client=self,
                url=self.vast_web_url,
                route=f"/api/v0/deployment/{deployment_id}/",
                api_key=self.api_key,
            )
        except Exception as ex:
            raise Exception(f"Failed to fetch deployment {deployment_id}:\nReason={ex}")

        if not result.get("ok"):
            raise Exception(
                f"Failed to fetch deployment {deployment_id}: HTTP {result.get('status')} - {result.get('text', '')[:512]}"
            )

        return DeploymentData.from_dict(result["json"]["deployment"])

    async def create_endpoint(self, config: EndpointConfig) -> ManagedEndpoint[R]:
        """Create a new endpoint job. Returns a ManagedEndpoint (data fetched lazily)."""
        try:
            result = await _make_request(
                client=self,
                url=self.vast_web_url,
                route="/api/v0/endptjobs/",
                api_key=self.api_key,
                body=config.to_dict(),
                method="POST",
            )
        except Exception as ex:
            raise Exception(f"Failed to create endpoint:\nReason={ex}")

        if not result.get("ok"):
            raise Exception(
                f"Failed to create endpoint: HTTP {result.get('status')} - {result.get('text', '')[:512]}"
            )

        endpoint_id = result["json"]["result"]
        self.logger.info(f"Created endpoint {endpoint_id}")
        return ManagedEndpoint(id=endpoint_id, client=self)

    async def put_deployment(self, config: DeploymentConfig) -> ManagedDeployment[R]:
        """Create or update a deployment. Returns a ManagedDeployment (data fetched lazily)."""
        logger.debug(f"putting deployment: \n {config.to_dict()}")
        try:
            result = await _make_request(
                client=self,
                url=self.vast_web_url,
                route="/api/v0/deployments/",
                api_key=self.api_key,
                body=config.to_dict(),
                method="PUT",
            )
        except Exception as ex:
            raise Exception(f"Failed to put deployment:\nReason={ex}")

        if not result.get("ok"):
            raise Exception(
                f"Failed to put deployment: HTTP {result.get('status')} - {result.get('text', '')[:512]}"
            )

        resp = DeploymentPutResponse.from_dict(result["json"])
        self.logger.info(
            f"Deployment {resp.deployment_id} ({resp.action}), endpoint {resp.endpoint_id}"
        )
        return ManagedDeployment(
            id=resp.deployment_id,
            endpoint_id=resp.endpoint_id,
            client=self,
            put_response=resp,
        )

    async def create_workergroup(self, config: WorkergroupConfig) -> int:
        """Create a workergroup (autoscale job). Returns the workergroup ID."""
        try:
            result = await _make_request(
                client=self,
                url=self.vast_web_url,
                route="/api/v0/workergroups/",
                api_key=self.api_key,
                body=config.to_dict(),
                method="POST",
            )
        except Exception as ex:
            raise Exception(f"Failed to create workergroup:\nReason={ex}")

        if not result.get("ok"):
            raise Exception(
                f"Failed to create workergroup: HTTP {result.get('status')} - {result.get('text', '')[:512]}"
            )

        wg_id = result["json"]["id"]
        self.logger.info(f"Created workergroup {wg_id}")
        return wg_id

    async def delete_workergroup(self, workergroup_id: int) -> None:
        """Delete a workergroup (autoscale job) by ID."""
        try:
            result = await _make_request(
                client=self,
                url=self.vast_web_url,
                route=f"/api/v0/workergroups/{workergroup_id}/",
                api_key=self.api_key,
                method="DELETE",
            )
        except Exception as ex:
            raise Exception(
                f"Failed to delete workergroup {workergroup_id}:\nReason={ex}"
            )

        if not result.get("ok"):
            raise Exception(
                f"Failed to delete workergroup {workergroup_id}: HTTP {result.get('status')} - {result.get('text', '')[:512]}"
            )

        self.logger.info(f"Deleted workergroup {workergroup_id}")

    async def update_workers(self, workergroup_id: int) -> dict:
        """Trigger a rolling update for all workers in a workergroup via the autoscaler."""
        try:
            result = await _make_request(
                client=self,
                url=self.autoscaler_url,
                route="/update_workers/",
                api_key=self.api_key,
                body={"workergroup_id": workergroup_id, "api_key": self.api_key},
                method="POST",
            )
        except Exception as ex:
            raise Exception(
                f"Failed to trigger worker update for workergroup {workergroup_id}:\nReason={ex}"
            )

        if not result.get("ok"):
            raise Exception(
                f"Failed to trigger worker update for workergroup {workergroup_id}: HTTP {result.get('status')} - {result.get('text', '')[:512]}"
            )

        self.logger.info(f"Triggered rolling update for workergroup {workergroup_id}")
        return result.get("json", {})

    async def find_workergroup_for_endpoint(self, endpoint_id: int) -> Optional[int]:
        """Fetch all workergroups and return the first one matching endpoint_id, or None."""
        try:
            result = await _make_request(
                client=self,
                url=self.vast_web_url,
                route="/api/v0/workergroups/",
                api_key=self.api_key,
                method="GET",
            )
        except Exception:
            return None
        if not result.get("ok"):
            return None
        for wg in result.get("json", {}).get("results", []):
            if wg.get("endpoint_id") == endpoint_id:
                return wg["id"]
        return None

    async def delete_endpoint(self, endpoint_id: int) -> None:
        """Delete an endpoint job by ID."""
        try:
            result = await _make_request(
                client=self,
                url=self.vast_web_url,
                route=f"/api/v0/endptjobs/{endpoint_id}/",
                api_key=self.api_key,
                method="DELETE",
            )
        except Exception as ex:
            raise Exception(f"Failed to delete endpoint {endpoint_id}:\nReason={ex}")

        if not result.get("ok"):
            raise Exception(
                f"Failed to delete endpoint {endpoint_id}: HTTP {result.get('status')} - {result.get('text', '')[:512]}"
            )

        self.logger.info(f"Deleted endpoint {endpoint_id}")

    async def get_endpoint_workers(self, endpoint: Endpoint_[R]) -> List[Worker]:
        if not isinstance(endpoint, Endpoint_):
            raise ValueError("endpoint must be an Endpoint")

        url = f"{self.autoscaler_url}/get_endpoint_workers/"
        payload = {"id": endpoint.id, "api_key": self.api_key}

        async with self._session.post(
            url, json=payload, timeout=aiohttp.ClientTimeout(total=30)
        ) as resp:
            if resp.status != 200:
                text = await resp.text()
                raise RuntimeError(
                    f"get_endpoint_workers failed: HTTP {resp.status} - {text}"
                )

            data = await resp.json(content_type=None)

            # If error message from authenticate_endpoint_apikey_by_id occurs, there is a possibility that
            # the endpoint's worker instances are not ready to be queried. If an error message occurs,
            # return an empty list and print the error message to the user. The endpoint get_endpoint_workers
            # should normally return a list of dictionaries containing worker instance information.
            if isinstance(data, dict):
                if "error_msg" in data.keys():
                    self.logger.warning(
                        f"Received the following error from get_endpoint_workers:{data['error_msg']}.\nEndpoint may not be ready for query. Check credentials or wait a few minutes and try again."
                    )
                    return []

            if not isinstance(data, list):
                raise RuntimeError(
                    f"Unexpected response type (wanted list): {type(data)}"
                )

            return [Worker.from_dict(item) for item in data]

    async def get_endpoint_session(
        self, endpoint, session_id: int, session_auth: str, timeout: float = 10.0
    ):
        try:
            _require_public_worker_url(session_auth.get("url"))
            result = await _make_request(
                client=self,
                url=session_auth.get("url"),
                api_key="",
                route="/session/get",
                body={"session_id": session_id, "session_auth": session_auth},
                method="POST",
                retries=1,
                timeout=timeout,
                stream=False,
                allow_redirects=False,
            )

            if not result.get("ok"):
                error_msg = f"Error on /session/get: {result.get('json', {}).get('error', result.get('text', 'Unknown error'))}"
                raise Exception(error_msg)

            worker_response = result.get("json") or {}

            auth_data = worker_response.get("auth_data")
            if auth_data is None:
                raise Exception("Missing auth_data in response")

            session = Session(
                endpoint=endpoint,
                session_id=session_id,
                lifetime=worker_response.get("lifetime"),
                expiration=worker_response.get("expiration"),
                auth_data=auth_data,
                url=auth_data.get("url"),
            )
            return session

        except asyncio.TimeoutError:
            raise
        except Exception as ex:
            error_msg = f"Failed to get session {session_id}: {ex}"
            self.logger.error(error_msg)
            raise Exception(error_msg)

    async def end_endpoint_session(self, session: Session, timeout: float = 10.0):
        try:
            self.logger.debug(
                f"Attempting to end session {session.session_id} at {session.url}"
            )
            _require_public_worker_url(session.url)
            result = await _make_request(
                client=self,
                url=session.url,
                api_key="",
                route="/session/end",
                body={
                    "session_id": session.session_id,
                    "session_auth": session.auth_data,
                },
                method="POST",
                retries=1,
                timeout=timeout,
                stream=False,
                allow_redirects=False,
            )

            if not result.get("ok"):
                error_msg = f"Error on /session/end: {result.get('json', {}).get('error', result.get('text', 'Unknown error'))}"
                raise Exception(error_msg)

            self.logger.debug(f"Successfully ended session {session.session_id}")
            return

        except asyncio.TimeoutError:
            raise
        except Exception as ex:
            error_msg = f"Failed to end session {session.session_id}: {ex}"
            self.logger.error(error_msg)
            raise Exception(error_msg)

    async def start_endpoint_session(
        self,
        endpoint: Endpoint_[R],
        cost: int = 100,
        lifetime: float = 60,
        on_close_route: str = None,
        on_close_payload: dict = None,
        timeout: float = None,
    ) -> Session:
        try:
            session_start_response = await self.queue_endpoint_request(
                endpoint=endpoint,
                worker_route="/session/create",
                worker_payload={
                    "lifetime": lifetime,
                    "on_close_route": on_close_route,
                    "on_close_payload": on_close_payload,
                },
                cost=cost,
                timeout=timeout,
                worker_timeout=10.0,
            )
            if not session_start_response.get("ok"):
                error_msg = f"Error on /session/create: {session_start_response.get('json', {}).get('error', session_start_response.get('text', 'Unknown error'))}"
                raise Exception(error_msg)
            response = session_start_response.get("response")
            if response is None:
                raise Exception("No response from /session/create")
            if not isinstance(response, dict):
                raise Exception(
                    "Invalid response from /session/create: expected mapping"
                )
            session_id = response.get("session_id")
            if session_id is None:
                raise Exception("Missing session id")
            expiration = response.get("expiration")
            url = session_start_response.get("url")
            if url is None:
                raise Exception("Missing URL")
            auth_data = session_start_response.get("auth_data")
            if auth_data is None:
                raise Exception("Missing auth data")
            return Session(
                endpoint=endpoint,
                session_id=session_id,
                lifetime=lifetime,
                expiration=expiration,
                url=url,
                auth_data=auth_data,
            )
        except asyncio.TimeoutError:
            raise
        except Exception as ex:
            error_msg = f"Failed to create session: {ex}"
            self.logger.error(error_msg)
            raise Exception(error_msg)

    async def _do_request(
        self,
        endpoint: Endpoint_[R],
        worker_route: str,
        worker_payload: dict,
        session: Optional[Session] = None,
        tracker: Optional[RequestStatus] = None,
        cost: int = 100,
        timeout: Optional[float] = None,
        worker_timeout: Optional[float] = 600,
        retry: bool = True,
        max_retries: Optional[int] = None,
        stream: bool = False,
    ) -> dict:
        """Core request logic: route to a worker, execute, retry on failure. Returns the response dict."""
        if tracker is None:
            tracker = RequestStatus()

        request_idx: int = 0
        total_attempts = 0
        start_time = time.time()
        try:
            while True:
                total_attempts += 1
                tracker.status = "Queued"
                worker_url = ""
                auth_data = {}
                session_id = None

                # Check total elapsed time
                if timeout is not None and (time.time() - start_time) >= timeout:
                    raise asyncio.TimeoutError(
                        f"Timed out after {time.time() - start_time:.1f}s waiting for worker"
                    )

                if session is None:
                    if request_idx == 0:
                        self.logger.debug(
                            f"Sending initial route call for request_idx {request_idx}"
                        )
                    else:
                        self.logger.debug(
                            f"Sending retry route call for request_idx {request_idx}"
                        )

                    route = await endpoint._route(
                        cost=cost, req_idx=request_idx, timeout=60.0
                    )

                    request_idx = route.request_idx
                    if request_idx:
                        self.logger.debug(f"Got request index {request_idx}")
                    else:
                        self.logger.info(  # usually non-fatal due to get_serverless_groups polling delay; we should fix that and make this an error again.
                            "Did not get request_idx from initial route"
                        )

                    poll_interval = 1
                    poll_elapsed = 0
                    attempt = 0
                    while route.status != "READY":
                        tracker.status = "Polling"

                        # Check total elapsed time
                        if (
                            timeout is not None
                            and (time.time() - start_time) >= timeout
                        ):
                            raise asyncio.TimeoutError(
                                f"Timed out after {time.time() - start_time:.1f}s waiting for worker to become ready"
                            )

                        await asyncio.sleep(poll_interval)
                        poll_elapsed += poll_interval

                        route = await endpoint._route(
                            cost=cost, req_idx=request_idx, timeout=60.0
                        )
                        request_idx = route.request_idx or request_idx

                        attempt += 1
                        poll_interval = random.uniform(
                            0.1,
                            min(
                                2 ** min(attempt, 20) + random.uniform(0, 1),
                                self.max_poll_interval,
                            ),
                        )
                        self.logger.debug(f"Polling route, attempt {attempt}")

                    worker_url = route.get_url()
                    auth_data = route.body
                else:
                    if session.url is not None:
                        worker_url = session.url
                        auth_data = session.auth_data
                        session_id = session.session_id

                payload = worker_payload
                worker_request_body = {
                    "auth_data": auth_data,
                    "session_id": session_id,
                    "payload": payload,
                }

                self.logger.debug("Found worker machine, starting work")
                if tracker.status != "Retrying":
                    tracker.status = "In Progress"
                    tracker.start_time = time.time()

                _require_public_worker_url(worker_url)
                # Transport/JSON failures may raise; HTTP errors return ok=False.
                try:
                    result = await _make_request(
                        client=self,
                        url=worker_url,
                        route=worker_route,
                        api_key=endpoint.api_key,
                        body=worker_request_body,
                        method="POST",
                        retries=1,  # avoid stacking retries with the outer loop
                        timeout=worker_timeout,
                        stream=stream,
                        allow_non_json=True,
                        allow_redirects=False,
                    )
                except (
                    aiohttp.ClientConnectorError,
                    aiohttp.ServerDisconnectedError,
                ) as ex:
                    # Worker is gone
                    if session is not None:
                        # Session is bound to this worker - can't re-route
                        self.logger.error(
                            f"Session worker unavailable ({type(ex).__name__})"
                        )
                        session.open = False
                        raise ConnectionError(
                            f"Session worker unavailable: {ex}"
                        ) from ex
                    # No session - reset request_idx to force a fresh route
                    self.logger.warning(
                        f"Worker unavailable ({type(ex).__name__}), re-routing to new worker"
                    )
                    tracker.status = "Retrying"
                    continue
                except Exception as ex:
                    self.logger.error(f"Worker request failed: {ex}")
                    tracker.status = "Retrying"
                    continue

                if not result.get("ok"):
                    if (
                        retry
                        and result.get("retryable")
                        and (max_retries is None or total_attempts < max_retries)
                    ):
                        # Check if we have time left before retrying
                        if (
                            timeout is not None
                            and (time.time() - start_time) >= timeout
                        ):
                            raise asyncio.TimeoutError(
                                f"Request timed out after {time.time() - start_time:.1f}s"
                            )

                        tracker.status = "Retrying"
                        await asyncio.sleep(
                            min(
                                (2 ** min(total_attempts, 20)) + random.uniform(0, 1),
                                self.max_poll_interval,
                            )
                        )
                        continue

                    # Return the raw HTTP result to the caller
                    tracker.status = "Complete"
                    tracker.complete_time = time.time()
                    if tracker.start_time is not None:
                        self.latencies.append(
                            tracker.complete_time - tracker.start_time
                        )

                    return {
                        "response": result.get("json")
                        if result.get("json") is not None
                        else {"error": result.get("text", "")},
                        "ok": result.get("ok"),
                        "status": result.get("status"),
                        "text": result.get("text"),
                        "latency": (tracker.complete_time - tracker.start_time)
                        if tracker.start_time
                        else None,
                        "url": worker_url,
                        "request_idx": request_idx,
                        "auth_data": auth_data,
                    }

                # Success
                if stream:
                    worker_response = result.get("stream")
                elif result.get("content") is not None:
                    worker_response = result.get("content")
                else:
                    worker_response = result.get("json")

                tracker.status = "Complete"
                tracker.complete_time = time.time()
                self.latencies.append(tracker.complete_time - tracker.start_time)
                self.logger.info("Endpoint request task completed")

                return {
                    "response": worker_response,
                    "ok": result.get("ok"),
                    "status": result.get("status"),
                    "text": result.get("text"),
                    "content_type": result.get("content_type"),
                    "latency": tracker.complete_time - tracker.start_time,
                    "url": worker_url,
                    "request_idx": request_idx,
                    "auth_data": auth_data,
                }

        except asyncio.CancelledError:
            tracker.status = "Cancelled"
            raise
        except Exception as ex:
            tracker.status = "Errored"
            self.logger.error(f"Request errored: {ex}")
            raise

    @abstractmethod
    def queue_endpoint_request(
        self,
        endpoint: Endpoint_[R],
        worker_route: str,
        worker_payload: dict,
        session: Optional[Session] = None,
        serverless_request: Optional[Union[ServerlessRequest, RequestStatus]] = None,
        cost: int = 100,
        timeout: Optional[float] = None,
        worker_timeout: Optional[float] = 600,
        retry: bool = True,
        max_retries: Optional[int] = None,
        stream: bool = False,
    ) -> R:
        """Dispatch a request to a serverless endpoint. Return type depends on subclass."""
        ...


class Serverless(_ServerlessBase[ServerlessRequest]):
    """Fire-and-observe client: queue_endpoint_request returns a ServerlessRequest (Future)."""

    def queue_endpoint_request(
        self,
        endpoint: Endpoint,
        worker_route: str,
        worker_payload: dict,
        session: Optional[Session] = None,
        serverless_request: Optional[Union[ServerlessRequest, RequestStatus]] = None,
        cost: int = 100,
        timeout: Optional[float] = None,
        worker_timeout: Optional[float] = 600,
        retry: bool = True,
        max_retries: Optional[int] = None,
        stream: bool = False,
    ) -> ServerlessRequest:
        """Return a Future that will resolve once the request completes."""
        if serverless_request is None:
            serverless_request = ServerlessRequest()
        elif isinstance(serverless_request, RequestStatus):
            serverless_request = ServerlessRequest(tracker=serverless_request)

        async def task(request: ServerlessRequest):
            try:
                result = await self._do_request(
                    endpoint=endpoint,
                    worker_route=worker_route,
                    worker_payload=worker_payload,
                    session=session,
                    tracker=request.tracker,
                    cost=cost,
                    timeout=timeout,
                    worker_timeout=worker_timeout,
                    retry=retry,
                    max_retries=max_retries,
                    stream=stream,
                )
                request.set_result(result)
            except asyncio.CancelledError:
                pass  # tracker already set by _do_request
            except Exception as ex:
                request.set_exception(ex)

        bg_task = asyncio.create_task(task(serverless_request))

        def _propagate_cancel(fut: ServerlessRequest):
            if fut.cancelled():
                bg_task.cancel()

        serverless_request.add_done_callback(_propagate_cancel)
        self.logger.info("Queued endpoint request")
        return serverless_request


class CoroutineServerless(_ServerlessBase[Coroutine[Any, Any, dict]]):
    """Async/await client: queue_endpoint_request returns a coroutine."""

    def queue_endpoint_request(
        self,
        endpoint: Endpoint,
        worker_route: str,
        worker_payload: dict,
        session: Optional[Session] = None,
        serverless_request: Optional[Union[ServerlessRequest, RequestStatus]] = None,
        cost: int = 100,
        timeout: Optional[float] = None,
        worker_timeout: Optional[float] = 600,
        retry: bool = True,
        max_retries: Optional[int] = None,
        stream: bool = False,
    ) -> Coroutine[Any, Any, dict]:
        """Return a coroutine that resolves to the response dict when awaited."""
        if isinstance(serverless_request, ServerlessRequest):
            tracker = serverless_request.tracker
        elif isinstance(serverless_request, RequestStatus):
            tracker = serverless_request
        else:
            tracker = None
        return self._do_request(
            endpoint=endpoint,
            worker_route=worker_route,
            worker_payload=worker_payload,
            session=session,
            tracker=tracker,
            cost=cost,
            timeout=timeout,
            worker_timeout=worker_timeout,
            retry=retry,
            max_retries=max_retries,
            stream=stream,
        )
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall