Skip to content

dominodatalab/domino-ai-plugin / skills/domino-trace-setup

domino-trace-setup

Verified

Set up GenAI tracing for an LLM or agent application in Domino by checking the Domino SDK and MLflow requirements (the DSE needs nothing installed), then adding a tracing_setup.py helper, span evaluators, a config.yaml, and @add_tracing / DominoAgentContext wiring. Use when the user asks to add tracing, observability, or evaluation to an agent or LLM app running on Domino.

FileKindSize
SKILL.mdskill md6.9 KB
assets/config.yamlconfig1.7 KB
assets/tracing_setup.pyscript7.4 KB

assets/tracing_setup.py

View on GitHub
"""
Domino GenAI tracing helpers.

Copy this file next to the agent's entry point. It wraps the Domino agents SDK so an
agent can be traced with @add_tracing, scored by evaluators, and grouped into an
evaluation run with DominoAgentContext (or DominoRun).

Requirements: see the domino-trace-setup skill (SKILL.md, step 1).

Usage:
    from tracing_setup import (setup_tracing, create_evaluator,
                               create_llm_judge_evaluator, get_aggregation_metrics)
    from domino.agents.tracing import add_tracing
    from domino.agents.logging import DominoAgentContext

    setup_tracing("openai")

    @add_tracing(name="my_agent", span_type="AGENT", evaluator=create_evaluator())
    def my_agent(query: str) -> dict:
        ...
        return {"response": text}

    with DominoAgentContext(agent_config_path="config.yaml",
                            custom_summary_metrics=get_aggregation_metrics()) as run:
        my_agent("Hello")
        print(run.info.run_id)       # run is the mlflow ActiveRun the context manager returns
"""

import os
from contextlib import contextmanager

import mlflow

FRAMEWORKS = ("openai", "anthropic", "langchain")
DEFAULT_JUDGE_MODEL = "gpt-6.1-sol"   # keep in sync with models.judge in config.yaml
_openai_autolog_enabled = False       # set by setup_tracing("openai"); read by the judge helper


def setup_tracing(framework: str = "openai") -> None:
    """Enable MLflow autologging for one LLM framework before any model call is made."""
    global _openai_autolog_enabled
    if framework not in FRAMEWORKS:
        raise ValueError(f"Unknown framework: {framework!r}. Use one of {FRAMEWORKS}")
    getattr(mlflow, framework).autolog()
    if framework == "openai":
        _openai_autolog_enabled = True
    print(f"Enabled {framework} auto-tracing")


def output_text(outputs) -> str:
    """Best-effort text of a traced function's return value (span.outputs)."""
    if outputs is None:
        return ""
    if isinstance(outputs, dict):
        for key in ("response", "content", "answer", "text"):
            if key in outputs:
                return str(outputs[key])
    return str(outputs)


def input_text(inputs) -> str:
    """Best-effort text of a traced function's arguments (span.inputs)."""
    if isinstance(inputs, dict):
        for key in ("query", "question", "prompt", "message", "messages"):
            if key in inputs:
                return str(inputs[key])
    return str(inputs)


def create_evaluator(metrics=None):
    """
    Return a span evaluator for @add_tracing(evaluator=...).

    Supported metric names: "response_length" (characters in the output text) and
    "confidence" (reported only when the output dict carries a numeric "confidence").
    Add your own metrics to the dict; every value must be a number.
    """
    metrics = list(metrics or ["response_length"])

    def evaluator(span):
        outputs = span.outputs
        scores = {}
        if "response_length" in metrics:
            scores["response_length"] = len(output_text(outputs))
        if "confidence" in metrics and isinstance(outputs, dict):
            confidence = outputs.get("confidence")
            if isinstance(confidence, (int, float)) and not isinstance(confidence, bool):
                scores["confidence"] = float(confidence)
        return scores

    return evaluator


@contextmanager
def _openai_autolog_paused():
    """
    Keep the judge's own OpenAI call out of the agent's traces.

    Autologging is process-wide: while the judge runs, OpenAI calls in other threads are not
    traced either. Run evaluation loops single-threaded, or score traces post-hoc in a separate
    Job (see the domino-genai-tracing skill, EVALUATORS.md). No-op unless setup_tracing("openai")
    enabled autologging.
    """
    if not _openai_autolog_enabled:
        yield
        return
    mlflow.openai.autolog(disable=True)
    try:
        yield
    finally:
        mlflow.openai.autolog()


def create_llm_judge_evaluator(judge_model: str = None, client=None):
    """
    Return an LLM-as-judge span evaluator that scores the output 0-1 with an OpenAI-style model.

    client is the OpenAI-compatible client the agent already uses (a provider SDK, a Domino-hosted
    endpoint or LLM Gateway 2.0, where judge_model is a gateway alias); defaults to OpenAI().

    judge_model defaults to the JUDGE_MODEL environment variable, then DEFAULT_JUDGE_MODEL;
    pass the value from config.yaml (models.judge) so the run's parameters and the judge agree.
    A non-numeric judge reply or an API error raises; the SDK logs it and records no score.
    """
    judge_model = judge_model or os.environ.get("JUDGE_MODEL", DEFAULT_JUDGE_MODEL)

    def llm_judge(span):
        from openai import OpenAI

        judge_client = client or OpenAI()
        query = input_text(span.inputs)
        response = output_text(span.outputs)
        prompt = (
            "Rate this response on a scale of 0-10 for relevance, completeness and clarity.\n\n"
            f"Question: {query}\nResponse: {response}\n\n"
            "Reply with only a number from 0 to 10."
        )
        with _openai_autolog_paused():
            completion = judge_client.chat.completions.create(
                model=judge_model,
                messages=[{"role": "user", "content": prompt}],
                max_tokens=10,
                temperature=0,
            )
        text = completion.choices[0].message.content.strip()
        try:
            score = float(text)
        except ValueError:
            raise ValueError(f"LLM judge returned a non-numeric reply: {text!r}")
        return {
            "llm_judge_score": max(0.0, min(10.0, score)) / 10.0,
            "judge_model": judge_model,     # a string: stored as a label, not aggregated
        }

    return llm_judge


def get_aggregation_metrics(metric_names=None):
    """
    Build the custom_summary_metrics list for DominoAgentContext / DominoRun.

    Each (metric, statistic) pair becomes a run metric named "<statistic>_<metric>", for
    example mean_response_length. Without custom_summary_metrics the SDK logs the mean of
    every metric it finds.
    """
    metric_names = list(metric_names or ["response_length", "llm_judge_score"])
    aggregations = []
    for name in metric_names:
        aggregations.append((name, "mean"))
        if name in ("response_length", "latency"):
            aggregations.append((name, "max"))
        if name in ("llm_judge_score", "confidence"):
            aggregations.append((name, "min"))
    return aggregations


if __name__ == "__main__":
    # Minimal end-to-end example; needs OPENAI_API_KEY as a Domino environment variable.
    from domino.agents.logging import DominoAgentContext
    from domino.agents.tracing import add_tracing
    from openai import OpenAI

    setup_tracing("openai")
    client = OpenAI()
    model = "gpt-6.1-sol"

    @add_tracing(name="example_agent", span_type="AGENT", evaluator=create_evaluator())
    def example_agent(query: str) -> dict:
        response = client.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": query}],
        )
        return {"response": response.choices[0].message.content, "model": model}

    with DominoAgentContext(
        agent_config_path="config.yaml",
        custom_summary_metrics=get_aggregation_metrics(["response_length"]),
    ) as run:
        result = example_agent("What is machine learning?")
        print(f"Response: {result['response'][:100]}...")
        print(f"Run ID: {run.info.run_id}")
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall