Skip to content

datarobot-oss/datarobot-agent-skills / skills/datarobot-external-agent-monitoring

datarobot-external-agent-monitoring

Verified

Instrument any external or existing AI agent with OpenTelemetry to send traces, logs, and metrics to DataRobot for monitoring, observability, and governance. Use when the user says "add tracing/observability/monitoring to my agent", wants to instrument an existing agent project in their IDE, wants to send agent traces, logs, or metrics to DataRobot, or wants to record and replay agent trajectories (Tensile) from DataRobot traces.

FileKindSize
SKILL.mdskill md19.1 KB
frameworks/crewai.mdmarkdown2.9 KB
frameworks/generic-python.mdmarkdown6.7 KB
frameworks/google-adk.mdmarkdown14.7 KB
frameworks/langchain-langgraph.mdmarkdown5.0 KB
frameworks/llamaindex.mdmarkdown3.3 KB
frameworks/pydantic-ai.mdmarkdown8.0 KB
reference/dr_otel_config.mdmarkdown8.5 KB
scripts/create_shell_deployment.pyexecutablescript6.3 KB
scripts/create_use_case.pyexecutablescript4.0 KB
scripts/verify_otel_connection.pyexecutablescript5.9 KB

scripts/create_shell_deployment.py

View on GitHub
#!/usr/bin/env python3
# Copyright (c) 2026 DataRobot, Inc. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Create a shell deployment in DataRobot for receiving external agent OTel telemetry.

Uses RegisteredModelVersion.create_for_external (DataRobot Python SDK 3.x).

Usage:
    python create_shell_deployment.py --name "My Agent" --description "OTel sink"

Env vars:
    DATAROBOT_API_TOKEN  - DataRobot API token
    DATAROBOT_ENDPOINT   - DataRobot API endpoint (e.g. https://app.datarobot.com/api/v2)
"""

from __future__ import annotations

import argparse
import json
import os
import sys

import datarobot as dr

# Raised when a tenant does not support an optional target type or deployment
# setting: the API rejects the call (ClientError) or the async settings update
# never resolves. Each call site below degrades gracefully instead of failing.
_UNSUPPORTED_FEATURE_ERRORS = (
    dr.errors.ClientError,
    dr.errors.AsyncTimeoutError,
    dr.errors.AsyncFailureError,
    dr.errors.AsyncProcessUnsuccessfulError,
)


def _otel_endpoint(api_endpoint: str) -> str:
    """Derive OTel endpoint from the API endpoint.

    API endpoint:  https://app.datarobot.com/api/v2
    OTel endpoint: https://app.datarobot.com/otel
    """
    base = api_endpoint.rstrip("/")
    base = base.removesuffix("/api/v2")
    return f"{base}/otel"


def _find_or_create_prediction_environment() -> dr.PredictionEnvironment:
    """Find or create a prediction environment for external agent monitoring.

    Not all tenants support platform="external". Try common platforms in order
    of preference: gcp, other, aws, azure. Reuse an existing environment when
    possible to avoid clutter.
    """
    preferred_platforms = ("gcp", "other", "aws", "azure")
    envs = dr.PredictionEnvironment.list()

    for platform in preferred_platforms:
        for env in envs:
            if getattr(env, "platform", None) == platform:
                return env

    # No compatible environment found — create one
    return dr.PredictionEnvironment.create(
        name="External Agent OTel Environment",
        platform="other",
        description="Prediction environment for external agent OTel monitoring",
    )


def create_shell_deployment(name: str, description: str) -> dict:
    """Create a shell deployment in DataRobot for external agent monitoring.

    Creates an external registered model version (target type: AgenticWorkflow)
    and deploys it with prediction row storage and automatic association ID
    generation enabled. The deployment ID is used to route OTel telemetry to
    the correct DataRobot monitoring dashboard.

    Args:
        name: Display name for the deployment
        description: Description of what agent this monitors

    Returns:
        Dict with deployment_id, entity_id, and otel_endpoint
    """
    token = os.getenv("DATAROBOT_API_TOKEN")
    endpoint = os.getenv("DATAROBOT_ENDPOINT", "https://app.datarobot.com/api/v2")

    if not token:
        print("Error: DATAROBOT_API_TOKEN env var is required", file=sys.stderr)
        sys.exit(1)

    dr.Client(token=token, endpoint=endpoint)

    # --- Prediction environment ---
    pred_env = _find_or_create_prediction_environment()

    # --- Registered model version (external shell) ---
    # Prefer AgenticWorkflow (enables full monitoring dashboards);
    # fall back to TextGeneration if the tenant doesn't support it.
    try:
        model_version = dr.RegisteredModelVersion.create_for_external(
            name="external-agent-shell-v1",
            target={"name": "agent_output", "type": "AgenticWorkflow"},
            registered_model_name=name,
            registered_model_description=description,
        )
    except _UNSUPPORTED_FEATURE_ERRORS as e:
        print(
            f"Warning: AgenticWorkflow target type failed ({e}). "
            "Retrying with TextGeneration.",
            file=sys.stderr,
        )
        model_version = dr.RegisteredModelVersion.create_for_external(
            name="external-agent-shell-v1",
            target={"name": "prediction", "type": "TextGeneration"},
            registered_model_name=name,
            registered_model_description=description,
        )

    # --- Deploy ---
    deployment = dr.Deployment.create_from_registered_model_version(
        model_package_id=model_version.id,
        label=name,
        description=description,
        prediction_environment_id=pred_env.id,
    )

    # --- Enable monitoring settings ---
    # Prediction row storage: stores prediction inputs/outputs for monitoring
    try:
        deployment.update_predictions_data_collection_settings(enabled=True)
    except _UNSUPPORTED_FEATURE_ERRORS as e:
        print(
            f"Warning: Could not enable prediction row storage: {e}",
            file=sys.stderr,
        )

    # Automatic association ID generation: assigns unique IDs to prediction rows.
    # The API requires a column name alongside autoGenerateId.
    try:
        client = dr.client.get_client()
        resp = client.patch(
            f"deployments/{deployment.id}/settings/",
            json={
                "associationId": {
                    "columnNames": ["association_id"],
                    "autoGenerateId": True,
                    "requiredInPredictionRequests": False,
                },
            },
        )
        from datarobot.utils.waiters import wait_for_async_resolution

        wait_for_async_resolution(client, resp.headers["Location"])
    except (*_UNSUPPORTED_FEATURE_ERRORS, KeyError) as e:
        print(
            f"Warning: Could not enable automatic association ID: {e}",
            file=sys.stderr,
        )

    return {
        "deployment_id": deployment.id,
        "entity_id": f"deployment-{deployment.id}",
        "otel_endpoint": _otel_endpoint(endpoint),
    }


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Create a DataRobot shell deployment for external agent monitoring"
    )
    parser.add_argument(
        "--name",
        required=True,
        help="Display name for the deployment",
    )
    parser.add_argument(
        "--description",
        default="External agent OTel telemetry sink",
        help="Description of what agent this monitors",
    )
    args = parser.parse_args()

    result = create_shell_deployment(args.name, args.description)
    print(json.dumps(result, indent=2))
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 323kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 323kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 323kUpdated Claude CodeCodexVerifiedInstall