datarobot-oss/datarobot-agent-skills / skills/datarobot-external-agent-monitoring
datarobot-external-agent-monitoring
VerifiedInstrument any external or existing AI agent with OpenTelemetry to send traces, logs, and metrics to DataRobot for monitoring, observability, and governance. Use when the user says "add tracing/observability/monitoring to my agent", wants to instrument an existing agent project in their IDE, wants to send agent traces, logs, or metrics to DataRobot, or wants to record and replay agent trajectories (Tensile) from DataRobot traces.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 19.1 KB |
| frameworks/crewai.md | markdown | 2.9 KB |
| frameworks/generic-python.md | markdown | 6.7 KB |
| frameworks/google-adk.md | markdown | 14.7 KB |
| frameworks/langchain-langgraph.md | markdown | 5.0 KB |
| frameworks/llamaindex.md | markdown | 3.3 KB |
| frameworks/pydantic-ai.md | markdown | 8.0 KB |
| reference/dr_otel_config.md | markdown | 8.5 KB |
| scripts/create_shell_deployment.pyexecutable | script | 6.3 KB |
| scripts/create_use_case.pyexecutable | script | 4.0 KB |
| scripts/verify_otel_connection.pyexecutable | script | 5.9 KB |
scripts/create_shell_deployment.py
View on GitHub#!/usr/bin/env python3
# Copyright (c) 2026 DataRobot, Inc. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Create a shell deployment in DataRobot for receiving external agent OTel telemetry.
Uses RegisteredModelVersion.create_for_external (DataRobot Python SDK 3.x).
Usage:
python create_shell_deployment.py --name "My Agent" --description "OTel sink"
Env vars:
DATAROBOT_API_TOKEN - DataRobot API token
DATAROBOT_ENDPOINT - DataRobot API endpoint (e.g. https://app.datarobot.com/api/v2)
"""
from __future__ import annotations
import argparse
import json
import os
import sys
import datarobot as dr
# Raised when a tenant does not support an optional target type or deployment
# setting: the API rejects the call (ClientError) or the async settings update
# never resolves. Each call site below degrades gracefully instead of failing.
_UNSUPPORTED_FEATURE_ERRORS = (
dr.errors.ClientError,
dr.errors.AsyncTimeoutError,
dr.errors.AsyncFailureError,
dr.errors.AsyncProcessUnsuccessfulError,
)
def _otel_endpoint(api_endpoint: str) -> str:
"""Derive OTel endpoint from the API endpoint.
API endpoint: https://app.datarobot.com/api/v2
OTel endpoint: https://app.datarobot.com/otel
"""
base = api_endpoint.rstrip("/")
base = base.removesuffix("/api/v2")
return f"{base}/otel"
def _find_or_create_prediction_environment() -> dr.PredictionEnvironment:
"""Find or create a prediction environment for external agent monitoring.
Not all tenants support platform="external". Try common platforms in order
of preference: gcp, other, aws, azure. Reuse an existing environment when
possible to avoid clutter.
"""
preferred_platforms = ("gcp", "other", "aws", "azure")
envs = dr.PredictionEnvironment.list()
for platform in preferred_platforms:
for env in envs:
if getattr(env, "platform", None) == platform:
return env
# No compatible environment found — create one
return dr.PredictionEnvironment.create(
name="External Agent OTel Environment",
platform="other",
description="Prediction environment for external agent OTel monitoring",
)
def create_shell_deployment(name: str, description: str) -> dict:
"""Create a shell deployment in DataRobot for external agent monitoring.
Creates an external registered model version (target type: AgenticWorkflow)
and deploys it with prediction row storage and automatic association ID
generation enabled. The deployment ID is used to route OTel telemetry to
the correct DataRobot monitoring dashboard.
Args:
name: Display name for the deployment
description: Description of what agent this monitors
Returns:
Dict with deployment_id, entity_id, and otel_endpoint
"""
token = os.getenv("DATAROBOT_API_TOKEN")
endpoint = os.getenv("DATAROBOT_ENDPOINT", "https://app.datarobot.com/api/v2")
if not token:
print("Error: DATAROBOT_API_TOKEN env var is required", file=sys.stderr)
sys.exit(1)
dr.Client(token=token, endpoint=endpoint)
# --- Prediction environment ---
pred_env = _find_or_create_prediction_environment()
# --- Registered model version (external shell) ---
# Prefer AgenticWorkflow (enables full monitoring dashboards);
# fall back to TextGeneration if the tenant doesn't support it.
try:
model_version = dr.RegisteredModelVersion.create_for_external(
name="external-agent-shell-v1",
target={"name": "agent_output", "type": "AgenticWorkflow"},
registered_model_name=name,
registered_model_description=description,
)
except _UNSUPPORTED_FEATURE_ERRORS as e:
print(
f"Warning: AgenticWorkflow target type failed ({e}). "
"Retrying with TextGeneration.",
file=sys.stderr,
)
model_version = dr.RegisteredModelVersion.create_for_external(
name="external-agent-shell-v1",
target={"name": "prediction", "type": "TextGeneration"},
registered_model_name=name,
registered_model_description=description,
)
# --- Deploy ---
deployment = dr.Deployment.create_from_registered_model_version(
model_package_id=model_version.id,
label=name,
description=description,
prediction_environment_id=pred_env.id,
)
# --- Enable monitoring settings ---
# Prediction row storage: stores prediction inputs/outputs for monitoring
try:
deployment.update_predictions_data_collection_settings(enabled=True)
except _UNSUPPORTED_FEATURE_ERRORS as e:
print(
f"Warning: Could not enable prediction row storage: {e}",
file=sys.stderr,
)
# Automatic association ID generation: assigns unique IDs to prediction rows.
# The API requires a column name alongside autoGenerateId.
try:
client = dr.client.get_client()
resp = client.patch(
f"deployments/{deployment.id}/settings/",
json={
"associationId": {
"columnNames": ["association_id"],
"autoGenerateId": True,
"requiredInPredictionRequests": False,
},
},
)
from datarobot.utils.waiters import wait_for_async_resolution
wait_for_async_resolution(client, resp.headers["Location"])
except (*_UNSUPPORTED_FEATURE_ERRORS, KeyError) as e:
print(
f"Warning: Could not enable automatic association ID: {e}",
file=sys.stderr,
)
return {
"deployment_id": deployment.id,
"entity_id": f"deployment-{deployment.id}",
"otel_endpoint": _otel_endpoint(endpoint),
}
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Create a DataRobot shell deployment for external agent monitoring"
)
parser.add_argument(
"--name",
required=True,
help="Display name for the deployment",
)
parser.add_argument(
"--description",
default="External agent OTel telemetry sink",
help="Description of what agent this monitors",
)
args = parser.parse_args()
result = create_shell_deployment(args.name, args.description)
print(json.dumps(result, indent=2))
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |