datarobot-oss/datarobot-agent-skills / skills/datarobot-agent-assist/agent-assist-simulate
datarobot-agent-assist-simulate
FlaggedUse when the user wants to adversarially test, evaluate, or harden an implemented AI agent before deployment; mentions swarm simulation, attack testing, persistence testing, evaluation criteria, or eval_report.md.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 17.9 KB |
| prompts/evaluate-result.md | markdown | 3.8 KB |
| prompts/generate-attack.md | markdown | 4.0 KB |
| prompts/generate-behavior.md | markdown | 3.9 KB |
| prompts/generate-persistence.md | markdown | 4.2 KB |
| prompts/generate-tool-return.md | markdown | 4.7 KB |
| prompts/industry.md | markdown | 13.1 KB |
| prompts/run-scenario.md | markdown | 2.8 KB |
| scripts/_bootstrap.py | script | 2.7 KB |
| scripts/artifacts.pyexecutable | script | 8.1 KB |
| scripts/gateway_worker.pyexecutable | script | 9.6 KB |
| scripts/native_convergence.pyexecutable | script | 12.7 KB |
| scripts/native_execution.pyexecutable | script | 15.7 KB |
| scripts/native_scenarios.pyexecutable | script | 10.8 KB |
| scripts/native_swarm.pyexecutable | script | 25.5 KB |
| scripts/prompt_inputs.pyexecutable | script | 3.9 KB |
| scripts/swarm_contracts.pyexecutable | script | 6.8 KB |
| scripts/tool_executor.pyexecutable | script | 5.6 KB |
| scripts/write_report.pyexecutable | script | 6.4 KB |
scripts/artifacts.py
View on GitHub#!/usr/bin/env python3
# Copyright (c) 2026 DataRobot, Inc. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Deterministic spec, criteria, and configuration artifact helpers."""
import json
import os
import re
from pathlib import Path
from typing import Any
import yaml
from pydantic import ValidationError
from swarm_contracts import AgentSpec, Scenario, SimulationConfig
class CriteriaError(ValueError):
"""Raised when confirmed evaluation criteria cannot be loaded safely."""
class ConfigError(ValueError):
"""Raised when simulation configuration cannot be loaded safely."""
def resolve_swarm_dir() -> Path:
"""Return the swarm working directory, persisting the chosen path for cross-process consistency."""
swarm_dir_env = os.environ.get("SWARM_DIR")
if swarm_dir_env:
return Path(swarm_dir_env)
swarm_temp = Path(".datarobot/swarm/swarm_temp")
if swarm_temp.exists():
return Path(swarm_temp.read_text(encoding="utf-8").strip())
resolved = Path(".datarobot/swarm").resolve()
swarm_temp.parent.mkdir(parents=True, exist_ok=True)
swarm_temp.write_text(str(resolved), encoding="utf-8")
return resolved
def write_json(path: Path, data: object) -> None:
"""Write an internal JSON artifact, creating its parent directory."""
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
)
def load_json(path: Path) -> Any:
"""Load an internal JSON artifact."""
with path.open(encoding="utf-8") as artifact_file:
return json.load(artifact_file)
def read_generated_code(directory: Path | None = None) -> str | None:
"""Read a bounded implementation-code summary for scenario generation."""
priority = ["tools.py", "agent.py", "myagent.py", "app.py"]
candidates: list[Path] = []
root = directory or Path.cwd()
for name in priority:
path = root / name
if path.is_file():
candidates.append(path)
if not candidates:
return None
parts: list[str] = []
for path in candidates[:3]:
try:
lines = path.read_text(encoding="utf-8").splitlines()[:200]
parts.append(f"# File: {path.name}\n" + "\n".join(lines) + "\n")
except OSError:
continue
return "\n".join(parts) if parts else None
def resolve_spec_path(path: Path) -> Path:
"""Prefer the canonical YAML spec, falling back to the legacy Markdown name."""
if path.name not in {"agent_spec.yaml", "agent_spec.md"}:
return path
yaml_path = path.with_name("agent_spec.yaml")
markdown_path = path.with_name("agent_spec.md")
if yaml_path.is_file():
return yaml_path
if markdown_path.is_file():
return markdown_path
return path
def load_spec(path: Path) -> AgentSpec:
"""Load and validate an agent specification."""
path = resolve_spec_path(path)
with path.open(encoding="utf-8") as spec_file:
data = yaml.safe_load(spec_file)
return AgentSpec.model_validate(data)
def write_criteria(scenarios: list[Scenario], path: Path) -> None:
"""Persist generated or confirmed evaluation criteria."""
data = [scenario.model_dump() for scenario in scenarios]
path.write_text(
yaml.dump(data, default_flow_style=False, allow_unicode=True), encoding="utf-8"
)
def load_criteria(path: Path) -> list[Scenario]:
"""Load a non-empty, validated confirmed scenario list."""
try:
text = path.read_text(encoding="utf-8")
except OSError as exc:
raise CriteriaError(f"could not read {path}: {exc}") from exc
try:
data = yaml.safe_load(text)
except yaml.YAMLError as exc:
raise CriteriaError(f"{path} contains invalid YAML: {exc}") from exc
if not isinstance(data, list) or not data:
raise CriteriaError(f"{path} must contain a non-empty list of scenarios")
try:
return [Scenario.model_validate(scenario) for scenario in data]
except ValidationError as exc:
raise CriteriaError(f"{path} contains an invalid scenario: {exc}") from exc
def load_native_config(path: Path) -> tuple[SimulationConfig, list[str]]:
"""Load native configuration, migrating legacy Gateway fields in memory."""
try:
data = yaml.safe_load(path.read_text(encoding="utf-8"))
except OSError as exc:
raise ConfigError(f"could not read {path}: {exc}") from exc
except yaml.YAMLError as exc:
raise ConfigError(f"{path} contains invalid YAML: {exc}") from exc
if not isinstance(data, dict):
raise ConfigError(f"{path} must contain a configuration mapping")
try:
if data.get("schema_version") == 1:
return SimulationConfig.model_validate(data), []
if "user_type" not in data:
raise ConfigError(
f"{path} is neither schema_version 1 nor a recognized legacy config"
)
migrated = SimulationConfig.model_validate(
{
"schema_version": 1,
"persona": {"description": data["user_type"]},
"grounding": {"context_path": None},
"evaluation": {
"mode": data.get("judge_mode", "standard"),
"fail_on": ["high", "critical"],
},
"convergence": {
"max_iterations": data.get("max_convergence_iterations", 3)
},
"turn_limits": {"attack": 6, "behavior": 3, "persistence": 6},
"execution": {
"mode": "simulated",
"requested_scope": {"tools": [], "resources": []},
},
}
)
except ConfigError:
raise
except ValidationError as exc:
raise ConfigError(f"{path} contains invalid configuration: {exc}") from exc
warnings = ["Migrated legacy Gateway configuration in memory for native execution."]
if data.get("llm_judge_model"):
warnings.append(
"Ignored legacy llm_judge_model; native execution uses the active harness model."
)
return migrated, warnings
def save_native_config(config: SimulationConfig, path: Path) -> None:
"""Persist the versioned native simulation configuration."""
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
yaml.dump(
config.model_dump(mode="json"),
default_flow_style=False,
sort_keys=False,
allow_unicode=True,
),
encoding="utf-8",
)
def one_line(exc: Exception) -> str:
return re.sub(r"\s+", " ", str(exc)).strip()
def scenario_id(scenario: Scenario) -> str:
if not scenario.scenario_id:
raise ValueError(f"confirmed scenario is missing scenario_id: {scenario.name}")
return scenario.scenario_id
def resolve_under_root(project_root: Path, path: Path, label: str) -> Path:
candidate = path if path.is_absolute() else project_root / path
resolved = candidate.resolve()
if not resolved.is_relative_to(project_root):
raise ValueError(f"{label} escapes project root: {path}")
return resolved
def resolve_project_file(project_root: Path, path: Path, label: str) -> Path:
resolved = resolve_under_root(project_root, path, label)
if not resolved.is_file():
raise ValueError(f"{label} does not exist or is not a file: {resolved}")
return resolved
def merge_metrics(metrics_dir: Path) -> None:
"""Merge per-worker metrics shards into metrics.jsonl, then remove the shards."""
shards = sorted(metrics_dir.glob("metrics-*.jsonl"))
if not shards:
return
lines: list[str] = []
for shard in shards:
try:
lines.append(shard.read_text(encoding="utf-8"))
except OSError:
continue
try:
metrics_path = metrics_dir / "metrics.jsonl"
metrics_path.parent.mkdir(parents=True, exist_ok=True)
with metrics_path.open("a", encoding="utf-8") as f:
for line in lines:
f.write(line)
except OSError:
return
for shard in shards:
try:
shard.unlink()
except OSError:
pass
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |