dominodatalab/domino-ai-plugin / skills/domino-trace-setup
domino-trace-setup
VerifiedSet up GenAI tracing for an LLM or agent application in Domino by checking the Domino SDK and MLflow requirements (the DSE needs nothing installed), then adding a tracing_setup.py helper, span evaluators, a config.yaml, and @add_tracing / DominoAgentContext wiring. Use when the user asks to add tracing, observability, or evaluation to an agent or LLM app running on Domino.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 6.9 KB |
| assets/config.yaml | config | 1.7 KB |
| assets/tracing_setup.py | script | 7.4 KB |
assets/tracing_setup.py
View on GitHub"""
Domino GenAI tracing helpers.
Copy this file next to the agent's entry point. It wraps the Domino agents SDK so an
agent can be traced with @add_tracing, scored by evaluators, and grouped into an
evaluation run with DominoAgentContext (or DominoRun).
Requirements: see the domino-trace-setup skill (SKILL.md, step 1).
Usage:
from tracing_setup import (setup_tracing, create_evaluator,
create_llm_judge_evaluator, get_aggregation_metrics)
from domino.agents.tracing import add_tracing
from domino.agents.logging import DominoAgentContext
setup_tracing("openai")
@add_tracing(name="my_agent", span_type="AGENT", evaluator=create_evaluator())
def my_agent(query: str) -> dict:
...
return {"response": text}
with DominoAgentContext(agent_config_path="config.yaml",
custom_summary_metrics=get_aggregation_metrics()) as run:
my_agent("Hello")
print(run.info.run_id) # run is the mlflow ActiveRun the context manager returns
"""
import os
from contextlib import contextmanager
import mlflow
FRAMEWORKS = ("openai", "anthropic", "langchain")
DEFAULT_JUDGE_MODEL = "gpt-6.1-sol" # keep in sync with models.judge in config.yaml
_openai_autolog_enabled = False # set by setup_tracing("openai"); read by the judge helper
def setup_tracing(framework: str = "openai") -> None:
"""Enable MLflow autologging for one LLM framework before any model call is made."""
global _openai_autolog_enabled
if framework not in FRAMEWORKS:
raise ValueError(f"Unknown framework: {framework!r}. Use one of {FRAMEWORKS}")
getattr(mlflow, framework).autolog()
if framework == "openai":
_openai_autolog_enabled = True
print(f"Enabled {framework} auto-tracing")
def output_text(outputs) -> str:
"""Best-effort text of a traced function's return value (span.outputs)."""
if outputs is None:
return ""
if isinstance(outputs, dict):
for key in ("response", "content", "answer", "text"):
if key in outputs:
return str(outputs[key])
return str(outputs)
def input_text(inputs) -> str:
"""Best-effort text of a traced function's arguments (span.inputs)."""
if isinstance(inputs, dict):
for key in ("query", "question", "prompt", "message", "messages"):
if key in inputs:
return str(inputs[key])
return str(inputs)
def create_evaluator(metrics=None):
"""
Return a span evaluator for @add_tracing(evaluator=...).
Supported metric names: "response_length" (characters in the output text) and
"confidence" (reported only when the output dict carries a numeric "confidence").
Add your own metrics to the dict; every value must be a number.
"""
metrics = list(metrics or ["response_length"])
def evaluator(span):
outputs = span.outputs
scores = {}
if "response_length" in metrics:
scores["response_length"] = len(output_text(outputs))
if "confidence" in metrics and isinstance(outputs, dict):
confidence = outputs.get("confidence")
if isinstance(confidence, (int, float)) and not isinstance(confidence, bool):
scores["confidence"] = float(confidence)
return scores
return evaluator
@contextmanager
def _openai_autolog_paused():
"""
Keep the judge's own OpenAI call out of the agent's traces.
Autologging is process-wide: while the judge runs, OpenAI calls in other threads are not
traced either. Run evaluation loops single-threaded, or score traces post-hoc in a separate
Job (see the domino-genai-tracing skill, EVALUATORS.md). No-op unless setup_tracing("openai")
enabled autologging.
"""
if not _openai_autolog_enabled:
yield
return
mlflow.openai.autolog(disable=True)
try:
yield
finally:
mlflow.openai.autolog()
def create_llm_judge_evaluator(judge_model: str = None, client=None):
"""
Return an LLM-as-judge span evaluator that scores the output 0-1 with an OpenAI-style model.
client is the OpenAI-compatible client the agent already uses (a provider SDK, a Domino-hosted
endpoint or LLM Gateway 2.0, where judge_model is a gateway alias); defaults to OpenAI().
judge_model defaults to the JUDGE_MODEL environment variable, then DEFAULT_JUDGE_MODEL;
pass the value from config.yaml (models.judge) so the run's parameters and the judge agree.
A non-numeric judge reply or an API error raises; the SDK logs it and records no score.
"""
judge_model = judge_model or os.environ.get("JUDGE_MODEL", DEFAULT_JUDGE_MODEL)
def llm_judge(span):
from openai import OpenAI
judge_client = client or OpenAI()
query = input_text(span.inputs)
response = output_text(span.outputs)
prompt = (
"Rate this response on a scale of 0-10 for relevance, completeness and clarity.\n\n"
f"Question: {query}\nResponse: {response}\n\n"
"Reply with only a number from 0 to 10."
)
with _openai_autolog_paused():
completion = judge_client.chat.completions.create(
model=judge_model,
messages=[{"role": "user", "content": prompt}],
max_tokens=10,
temperature=0,
)
text = completion.choices[0].message.content.strip()
try:
score = float(text)
except ValueError:
raise ValueError(f"LLM judge returned a non-numeric reply: {text!r}")
return {
"llm_judge_score": max(0.0, min(10.0, score)) / 10.0,
"judge_model": judge_model, # a string: stored as a label, not aggregated
}
return llm_judge
def get_aggregation_metrics(metric_names=None):
"""
Build the custom_summary_metrics list for DominoAgentContext / DominoRun.
Each (metric, statistic) pair becomes a run metric named "<statistic>_<metric>", for
example mean_response_length. Without custom_summary_metrics the SDK logs the mean of
every metric it finds.
"""
metric_names = list(metric_names or ["response_length", "llm_judge_score"])
aggregations = []
for name in metric_names:
aggregations.append((name, "mean"))
if name in ("response_length", "latency"):
aggregations.append((name, "max"))
if name in ("llm_judge_score", "confidence"):
aggregations.append((name, "min"))
return aggregations
if __name__ == "__main__":
# Minimal end-to-end example; needs OPENAI_API_KEY as a Domino environment variable.
from domino.agents.logging import DominoAgentContext
from domino.agents.tracing import add_tracing
from openai import OpenAI
setup_tracing("openai")
client = OpenAI()
model = "gpt-6.1-sol"
@add_tracing(name="example_agent", span_type="AGENT", evaluator=create_evaluator())
def example_agent(query: str) -> dict:
response = client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": query}],
)
return {"response": response.choices[0].message.content, "model": model}
with DominoAgentContext(
agent_config_path="config.yaml",
custom_summary_metrics=get_aggregation_metrics(["response_length"]),
) as run:
result = example_agent("What is machine learning?")
print(f"Response: {result['response'][:100]}...")
print(f"Run ID: {run.info.run_id}")
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |