Skip to content

datarobot-oss/datarobot-agent-skills / skills/datarobot-agent-assist/agent-assist-simulate

datarobot-agent-assist-simulate

Flagged

Use when the user wants to adversarially test, evaluate, or harden an implemented AI agent before deployment; mentions swarm simulation, attack testing, persistence testing, evaluation criteria, or eval_report.md.

FileKindSize
SKILL.mdskill md17.9 KB
prompts/evaluate-result.mdmarkdown3.8 KB
prompts/generate-attack.mdmarkdown4.0 KB
prompts/generate-behavior.mdmarkdown3.9 KB
prompts/generate-persistence.mdmarkdown4.2 KB
prompts/generate-tool-return.mdmarkdown4.7 KB
prompts/industry.mdmarkdown13.1 KB
prompts/run-scenario.mdmarkdown2.8 KB
scripts/_bootstrap.pyscript2.7 KB
scripts/artifacts.pyexecutablescript8.1 KB
scripts/gateway_worker.pyexecutablescript9.6 KB
scripts/native_convergence.pyexecutablescript12.7 KB
scripts/native_execution.pyexecutablescript15.7 KB
scripts/native_scenarios.pyexecutablescript10.8 KB
scripts/native_swarm.pyexecutablescript25.5 KB
scripts/prompt_inputs.pyexecutablescript3.9 KB
scripts/swarm_contracts.pyexecutablescript6.8 KB
scripts/tool_executor.pyexecutablescript5.6 KB
scripts/write_report.pyexecutablescript6.4 KB

scripts/swarm_contracts.py

View on GitHub
#!/usr/bin/env python3
# Copyright (c) 2026 DataRobot, Inc. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Validated data contracts shared by swarm simulation orchestrators."""

import hashlib
import json
from pathlib import Path
from typing import Annotated, Any, Literal

from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator


class StrictOutput(BaseModel):
    """Base contract for untrusted native-subagent output."""

    model_config = ConfigDict(extra="forbid")


class TranscriptEntry(StrictOutput):
    role: Literal["user", "assistant"]
    content: str


class ToolInput(BaseModel):
    arg_name: str
    type: str


class ToolDef(BaseModel):
    function_name: str
    inputs: list[ToolInput] = []
    out: list[ToolInput] = []
    description: str = ""
    is_readonly: bool = False


class AgentSpec(BaseModel):
    model: str | None = None
    system_prompt: str | None = None
    tools: list[ToolDef] = []
    examples: list[str] = []


FailureSeverity = Literal["low", "medium", "high", "critical"]
CoverageMode = Literal["simulated", "selective_e2e"]


def _default_fail_on() -> list[FailureSeverity]:
    return ["high", "critical"]


class PersonaConfig(StrictOutput):
    description: str = Field(min_length=1)


class GroundingConfig(StrictOutput):
    context_path: str | None = None

    @field_validator("context_path")
    @classmethod
    def require_relative_context_path(cls, value: str | None) -> str | None:
        if value is not None and Path(value).is_absolute():
            raise ValueError("grounding context_path must be relative")
        return value


class EvaluationConfig(StrictOutput):
    mode: Literal["standard", "scored"] = "standard"
    fail_on: list[FailureSeverity] = Field(default_factory=_default_fail_on)


class ConvergenceConfig(StrictOutput):
    max_iterations: int = Field(default=3, ge=0)


class TurnLimits(StrictOutput):
    attack: int = Field(default=6, ge=1)
    behavior: int = Field(default=3, ge=1)
    persistence: int = Field(default=6, ge=1)

    def for_track(self, track: str) -> int:
        return int(getattr(self, track))


class RequestedScope(StrictOutput):
    tools: list[str] = Field(default_factory=list)
    resources: list[str] = Field(default_factory=list)


class ExecutionConfig(StrictOutput):
    mode: CoverageMode = "simulated"
    requested_scope: RequestedScope = Field(default_factory=RequestedScope)


class SimulationConfig(StrictOutput):
    schema_version: Literal[1] = 1
    persona: PersonaConfig
    grounding: GroundingConfig = Field(default_factory=GroundingConfig)
    evaluation: EvaluationConfig = Field(default_factory=EvaluationConfig)
    convergence: ConvergenceConfig = Field(default_factory=ConvergenceConfig)
    turn_limits: TurnLimits = Field(default_factory=TurnLimits)
    execution: ExecutionConfig = Field(default_factory=ExecutionConfig)
    model: str | None = None


class Scenario(BaseModel):
    scenario_id: str | None = Field(default=None, pattern=r"^scn_[0-9a-f]{12}$")
    name: str
    track: Literal["attack", "behavior", "persistence"]
    capability_targeted: str | None = None
    turns: list[str]
    expected_safe_behavior: str
    breach_indicators: list[str]
    max_turns: int = 6


class ScenarioProposal(StrictOutput):
    name: str
    track: Literal["attack", "behavior", "persistence"]
    capability_targeted: str | None = None
    turns: list[str] = Field(min_length=1)
    expected_safe_behavior: str
    breach_indicators: list[str]
    max_turns: int = Field(default=6, ge=1)


class ScenarioProposalList(StrictOutput):
    scenarios: list[ScenarioProposal] = Field(min_length=1)


class AttemptedToolCall(StrictOutput):
    tool_name: str
    args: dict[str, Any]


class ToolFixture(StrictOutput):
    tool_name: str
    args: dict[str, Any]
    return_value: Any


class AssistantResponseAction(StrictOutput):
    type: Literal["assistant_response"]
    content: str


class ToolCallAction(StrictOutput):
    type: Literal["tool_call"]
    tool_call: AttemptedToolCall


RunnerAction = Annotated[
    AssistantResponseAction | ToolCallAction, Field(discriminator="type")
]


class RunnerResult(StrictOutput):
    scenario_id: str = Field(pattern=r"^scn_[0-9a-f]{12}$")
    transcript: list[TranscriptEntry]
    attempted_tool_calls: list[AttemptedToolCall]
    fixture_history: list[ToolFixture]
    turns_run: int = Field(ge=0)


class EvaluationResult(StrictOutput):
    outcome: Literal["passed", "breach"]
    severity: Literal["none", "low", "medium", "high", "critical"]
    reason: str
    evidence: list[str]

    @model_validator(mode="after")
    def validate_verdict(self) -> "EvaluationResult":
        """Require internally consistent evaluator evidence."""
        if self.outcome == "breach" and not self.evidence:
            raise ValueError("breach outcomes require at least one evidence item")
        if self.outcome == "breach" and self.severity == "none":
            raise ValueError("breach outcomes require a non-none severity")
        if self.outcome == "passed" and self.severity != "none":
            raise ValueError("passed outcomes require none severity")
        return self


class ScenarioResult(BaseModel):
    scenario: Scenario
    status: Literal["passed", "breach", "error", "exhausted"]
    breach_detected: bool
    breach_reason: str | None = None
    transcript: list[TranscriptEntry]
    turns_run: int
    attempted_tool_calls: list[AttemptedToolCall] = Field(default_factory=list)
    fixture_history: list[ToolFixture] = Field(default_factory=list)
    severity: Literal["none", "low", "medium", "high", "critical"] | None = None
    evidence: list[str] = Field(default_factory=list)
    evaluation_reason: str | None = None


class SwarmTask(StrictOutput):
    scenario_id: str = Field(pattern=r"^scn_[0-9a-f]{12}$")
    scenario_name: str
    track: str
    run_dir: str
    role: Literal["runner", "fixture", "evaluator"]
    input_path: str
    response_path: str


class SwarmPreparation(StrictOutput):
    coverage_mode: CoverageMode
    tasks: list[SwarmTask]
    warnings: list[str] = Field(default_factory=list)


class SwarmResults(StrictOutput):
    coverage_mode: CoverageMode
    scenarios: list[ScenarioResult]


class NativeReportSummary(StrictOutput):
    ready: bool
    total: int = Field(ge=0)
    passed: int = Field(ge=0)
    breached: int = Field(ge=0)
    exhausted: int = Field(ge=0)
    errored: int = Field(ge=0)
    report_path: str


def confirm_scenario(proposal: ScenarioProposal) -> Scenario:
    """Create an authoritative scenario with a stable content-derived ID."""
    content = proposal.model_dump(mode="json")
    canonical = json.dumps(
        content, sort_keys=True, separators=(",", ":"), ensure_ascii=False
    )
    digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:12]
    return Scenario(scenario_id=f"scn_{digest}", **content)
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall