Skip to content

datarobot-oss/datarobot-agent-skills / skills/datarobot-agent-assist/agent-assist-simulate

datarobot-agent-assist-simulate

Flagged

Use when the user wants to adversarially test, evaluate, or harden an implemented AI agent before deployment; mentions swarm simulation, attack testing, persistence testing, evaluation criteria, or eval_report.md.

FileKindSize
SKILL.mdskill md17.9 KB
prompts/evaluate-result.mdmarkdown3.8 KB
prompts/generate-attack.mdmarkdown4.0 KB
prompts/generate-behavior.mdmarkdown3.9 KB
prompts/generate-persistence.mdmarkdown4.2 KB
prompts/generate-tool-return.mdmarkdown4.7 KB
prompts/industry.mdmarkdown13.1 KB
prompts/run-scenario.mdmarkdown2.8 KB
scripts/_bootstrap.pyscript2.7 KB
scripts/artifacts.pyexecutablescript8.1 KB
scripts/gateway_worker.pyexecutablescript9.6 KB
scripts/native_convergence.pyexecutablescript12.7 KB
scripts/native_execution.pyexecutablescript15.7 KB
scripts/native_scenarios.pyexecutablescript10.8 KB
scripts/native_swarm.pyexecutablescript25.5 KB
scripts/prompt_inputs.pyexecutablescript3.9 KB
scripts/swarm_contracts.pyexecutablescript6.8 KB
scripts/tool_executor.pyexecutablescript5.6 KB
scripts/write_report.pyexecutablescript6.4 KB

scripts/artifacts.py

View on GitHub
#!/usr/bin/env python3
# Copyright (c) 2026 DataRobot, Inc. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Deterministic spec, criteria, and configuration artifact helpers."""

import json
import os
import re
from pathlib import Path
from typing import Any

import yaml
from pydantic import ValidationError
from swarm_contracts import AgentSpec, Scenario, SimulationConfig


class CriteriaError(ValueError):
    """Raised when confirmed evaluation criteria cannot be loaded safely."""


class ConfigError(ValueError):
    """Raised when simulation configuration cannot be loaded safely."""


def resolve_swarm_dir() -> Path:
    """Return the swarm working directory, persisting the chosen path for cross-process consistency."""
    swarm_dir_env = os.environ.get("SWARM_DIR")
    if swarm_dir_env:
        return Path(swarm_dir_env)
    swarm_temp = Path(".datarobot/swarm/swarm_temp")
    if swarm_temp.exists():
        return Path(swarm_temp.read_text(encoding="utf-8").strip())
    resolved = Path(".datarobot/swarm").resolve()
    swarm_temp.parent.mkdir(parents=True, exist_ok=True)
    swarm_temp.write_text(str(resolved), encoding="utf-8")
    return resolved


def write_json(path: Path, data: object) -> None:
    """Write an internal JSON artifact, creating its parent directory."""
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(
        json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
    )


def load_json(path: Path) -> Any:
    """Load an internal JSON artifact."""
    with path.open(encoding="utf-8") as artifact_file:
        return json.load(artifact_file)


def read_generated_code(directory: Path | None = None) -> str | None:
    """Read a bounded implementation-code summary for scenario generation."""
    priority = ["tools.py", "agent.py", "myagent.py", "app.py"]
    candidates: list[Path] = []
    root = directory or Path.cwd()
    for name in priority:
        path = root / name
        if path.is_file():
            candidates.append(path)
    if not candidates:
        return None

    parts: list[str] = []
    for path in candidates[:3]:
        try:
            lines = path.read_text(encoding="utf-8").splitlines()[:200]
            parts.append(f"# File: {path.name}\n" + "\n".join(lines) + "\n")
        except OSError:
            continue
    return "\n".join(parts) if parts else None


def resolve_spec_path(path: Path) -> Path:
    """Prefer the canonical YAML spec, falling back to the legacy Markdown name."""
    if path.name not in {"agent_spec.yaml", "agent_spec.md"}:
        return path
    yaml_path = path.with_name("agent_spec.yaml")
    markdown_path = path.with_name("agent_spec.md")
    if yaml_path.is_file():
        return yaml_path
    if markdown_path.is_file():
        return markdown_path
    return path


def load_spec(path: Path) -> AgentSpec:
    """Load and validate an agent specification."""
    path = resolve_spec_path(path)
    with path.open(encoding="utf-8") as spec_file:
        data = yaml.safe_load(spec_file)
    return AgentSpec.model_validate(data)


def write_criteria(scenarios: list[Scenario], path: Path) -> None:
    """Persist generated or confirmed evaluation criteria."""
    data = [scenario.model_dump() for scenario in scenarios]
    path.write_text(
        yaml.dump(data, default_flow_style=False, allow_unicode=True), encoding="utf-8"
    )


def load_criteria(path: Path) -> list[Scenario]:
    """Load a non-empty, validated confirmed scenario list."""
    try:
        text = path.read_text(encoding="utf-8")
    except OSError as exc:
        raise CriteriaError(f"could not read {path}: {exc}") from exc

    try:
        data = yaml.safe_load(text)
    except yaml.YAMLError as exc:
        raise CriteriaError(f"{path} contains invalid YAML: {exc}") from exc

    if not isinstance(data, list) or not data:
        raise CriteriaError(f"{path} must contain a non-empty list of scenarios")

    try:
        return [Scenario.model_validate(scenario) for scenario in data]
    except ValidationError as exc:
        raise CriteriaError(f"{path} contains an invalid scenario: {exc}") from exc


def load_native_config(path: Path) -> tuple[SimulationConfig, list[str]]:
    """Load native configuration, migrating legacy Gateway fields in memory."""
    try:
        data = yaml.safe_load(path.read_text(encoding="utf-8"))
    except OSError as exc:
        raise ConfigError(f"could not read {path}: {exc}") from exc
    except yaml.YAMLError as exc:
        raise ConfigError(f"{path} contains invalid YAML: {exc}") from exc

    if not isinstance(data, dict):
        raise ConfigError(f"{path} must contain a configuration mapping")
    try:
        if data.get("schema_version") == 1:
            return SimulationConfig.model_validate(data), []
        if "user_type" not in data:
            raise ConfigError(
                f"{path} is neither schema_version 1 nor a recognized legacy config"
            )
        migrated = SimulationConfig.model_validate(
            {
                "schema_version": 1,
                "persona": {"description": data["user_type"]},
                "grounding": {"context_path": None},
                "evaluation": {
                    "mode": data.get("judge_mode", "standard"),
                    "fail_on": ["high", "critical"],
                },
                "convergence": {
                    "max_iterations": data.get("max_convergence_iterations", 3)
                },
                "turn_limits": {"attack": 6, "behavior": 3, "persistence": 6},
                "execution": {
                    "mode": "simulated",
                    "requested_scope": {"tools": [], "resources": []},
                },
            }
        )
    except ConfigError:
        raise
    except ValidationError as exc:
        raise ConfigError(f"{path} contains invalid configuration: {exc}") from exc

    warnings = ["Migrated legacy Gateway configuration in memory for native execution."]
    if data.get("llm_judge_model"):
        warnings.append(
            "Ignored legacy llm_judge_model; native execution uses the active harness model."
        )
    return migrated, warnings


def save_native_config(config: SimulationConfig, path: Path) -> None:
    """Persist the versioned native simulation configuration."""
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(
        yaml.dump(
            config.model_dump(mode="json"),
            default_flow_style=False,
            sort_keys=False,
            allow_unicode=True,
        ),
        encoding="utf-8",
    )


def one_line(exc: Exception) -> str:
    return re.sub(r"\s+", " ", str(exc)).strip()


def scenario_id(scenario: Scenario) -> str:
    if not scenario.scenario_id:
        raise ValueError(f"confirmed scenario is missing scenario_id: {scenario.name}")
    return scenario.scenario_id


def resolve_under_root(project_root: Path, path: Path, label: str) -> Path:
    candidate = path if path.is_absolute() else project_root / path
    resolved = candidate.resolve()
    if not resolved.is_relative_to(project_root):
        raise ValueError(f"{label} escapes project root: {path}")
    return resolved


def resolve_project_file(project_root: Path, path: Path, label: str) -> Path:
    resolved = resolve_under_root(project_root, path, label)
    if not resolved.is_file():
        raise ValueError(f"{label} does not exist or is not a file: {resolved}")
    return resolved


def merge_metrics(metrics_dir: Path) -> None:
    """Merge per-worker metrics shards into metrics.jsonl, then remove the shards."""
    shards = sorted(metrics_dir.glob("metrics-*.jsonl"))
    if not shards:
        return
    lines: list[str] = []
    for shard in shards:
        try:
            lines.append(shard.read_text(encoding="utf-8"))
        except OSError:
            continue
    try:
        metrics_path = metrics_dir / "metrics.jsonl"
        metrics_path.parent.mkdir(parents=True, exist_ok=True)
        with metrics_path.open("a", encoding="utf-8") as f:
            for line in lines:
                f.write(line)
    except OSError:
        return
    for shard in shards:
        try:
            shard.unlink()
        except OSError:
            pass
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall