Skip to content

datarobot-oss/datarobot-agent-skills / skills/datarobot-external-agent-monitoring

datarobot-external-agent-monitoring

Verified

Instrument any external or existing AI agent with OpenTelemetry to send traces, logs, and metrics to DataRobot for monitoring, observability, and governance. Use when the user says "add tracing/observability/monitoring to my agent", wants to instrument an existing agent project in their IDE, wants to send agent traces, logs, or metrics to DataRobot, or wants to record and replay agent trajectories (Tensile) from DataRobot traces.

FileKindSize
SKILL.mdskill md19.1 KB
frameworks/crewai.mdmarkdown2.9 KB
frameworks/generic-python.mdmarkdown6.7 KB
frameworks/google-adk.mdmarkdown14.7 KB
frameworks/langchain-langgraph.mdmarkdown5.0 KB
frameworks/llamaindex.mdmarkdown3.3 KB
frameworks/pydantic-ai.mdmarkdown8.0 KB
reference/dr_otel_config.mdmarkdown8.5 KB
scripts/create_shell_deployment.pyexecutablescript6.3 KB
scripts/create_use_case.pyexecutablescript4.0 KB
scripts/verify_otel_connection.pyexecutablescript5.9 KB

scripts/verify_otel_connection.py

View on GitHub
#!/usr/bin/env python3
# Copyright (c) 2026 DataRobot, Inc. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""
Verify OTel connection to DataRobot by sending test telemetry.

Sends a test trace span, log record, and metric to DataRobot's OTel endpoint
to confirm the pipeline is working before deploying the instrumented agent.

Usage:
    python verify_otel_connection.py

Env vars:
    DATAROBOT_API_TOKEN      - DataRobot API token
    DATAROBOT_ENTITY_ID      - experiment_container-<use_case_id> (or deployment-<deployment_id>)
    DATAROBOT_OTEL_ENDPOINT  - https://<instance>.datarobot.com/otel
"""

import json
import logging
import os
import sys

from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import SimpleLogRecordProcessor
from opentelemetry.sdk.metrics import (
    Counter,
    Histogram,
    MeterProvider,
    ObservableCounter,
)
from opentelemetry.sdk.metrics.export import (
    AggregationTemporality,
    PeriodicExportingMetricReader,
)
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor


def verify_connection() -> dict:
    """Send test telemetry to DataRobot and report results.

    Returns:
        Dict with status and per-signal results
    """
    api_key = os.environ.get("DATAROBOT_API_TOKEN", "")
    entity_id = os.environ.get("DATAROBOT_ENTITY_ID", "")
    endpoint = os.environ.get("DATAROBOT_OTEL_ENDPOINT", "")

    errors = []
    if not api_key:
        errors.append("DATAROBOT_API_TOKEN env var is required")
    if not entity_id:
        errors.append("DATAROBOT_ENTITY_ID env var is required")
    if not endpoint:
        errors.append("DATAROBOT_OTEL_ENDPOINT env var is required")
    if entity_id and not entity_id.startswith(("experiment_container-", "deployment-")):
        errors.append(
            "DATAROBOT_ENTITY_ID must start with 'experiment_container-' (Use Case) "
            f"or 'deployment-' (deployment), got: {entity_id}"
        )

    if errors:
        return {"status": "error", "errors": errors}

    headers = {
        "X-DataRobot-Entity-Id": entity_id,
        "X-DataRobot-Api-Key": api_key,
    }
    resource = Resource.create()
    results = {
        "status": "success",
        "traces": "pending",
        "logs": "pending",
        "metrics": "pending",
    }

    # --- Traces ---
    try:
        trace_provider = TracerProvider(resource=resource)
        trace_provider.add_span_processor(
            SimpleSpanProcessor(
                OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces", headers=headers)
            )
        )
        tracer = trace_provider.get_tracer("datarobot-otel-verification")
        with tracer.start_as_current_span("datarobot.otel.verification") as span:
            span.set_attribute("verification", True)
            span.set_attribute("source", "datarobot-external-agent-monitoring")
        trace_provider.shutdown()
        results["traces"] = "sent"
    # Broad catch is intentional: this is a connectivity diagnostic, and any
    # exporter/transport failure must be reported per-signal rather than abort
    # the remaining checks.
    except Exception as e:  # noqa: BLE001
        results["traces"] = f"error: {e}"
        results["status"] = "partial"

    # --- Logs ---
    try:
        logger_provider = LoggerProvider(resource=resource)
        logger_provider.add_log_record_processor(
            SimpleLogRecordProcessor(
                OTLPLogExporter(endpoint=f"{endpoint}/v1/logs", headers=headers)
            )
        )
        handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
        test_logger = logging.getLogger("datarobot-otel-verification")
        test_logger.addHandler(handler)
        test_logger.setLevel(logging.INFO)
        test_logger.info("DataRobot OTel verification — test log record")
        test_logger.removeHandler(handler)
        logger_provider.shutdown()
        results["logs"] = "sent"
    # Broad catch is intentional: this is a connectivity diagnostic, and any
    # exporter/transport failure must be reported per-signal rather than abort
    # the remaining checks.
    except Exception as e:  # noqa: BLE001
        results["logs"] = f"error: {e}"
        results["status"] = "partial"

    # --- Metrics ---
    try:
        preferred_temporality = {
            Counter: AggregationTemporality.DELTA,
            Histogram: AggregationTemporality.DELTA,
            ObservableCounter: AggregationTemporality.DELTA,
        }
        meter_provider = MeterProvider(
            metric_readers=[
                PeriodicExportingMetricReader(
                    OTLPMetricExporter(
                        endpoint=f"{endpoint}/v1/metrics",
                        headers=headers,
                        preferred_temporality=preferred_temporality,
                    )
                )
            ],
            resource=resource,
        )
        meter = meter_provider.get_meter("datarobot-otel-verification")
        counter = meter.create_counter("datarobot.otel.verification", unit="1")
        counter.add(1, {"source": "verification"})
        meter_provider.shutdown()
        results["metrics"] = "sent"
    # Broad catch is intentional: this is a connectivity diagnostic, and any
    # exporter/transport failure must be reported per-signal rather than abort
    # the remaining checks.
    except Exception as e:  # noqa: BLE001
        results["metrics"] = f"error: {e}"
        results["status"] = "partial"

    return results


if __name__ == "__main__":
    result = verify_connection()
    print(json.dumps(result, indent=2))
    if result["status"] != "success":
        sys.exit(1)
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 323kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 323kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 323kUpdated Claude CodeCodexVerifiedInstall