datarobot-oss/datarobot-agent-skills / skills/datarobot-agent-assist/agent-assist-simulate
datarobot-agent-assist-simulate
FlaggedUse when the user wants to adversarially test, evaluate, or harden an implemented AI agent before deployment; mentions swarm simulation, attack testing, persistence testing, evaluation criteria, or eval_report.md.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 17.9 KB |
| prompts/evaluate-result.md | markdown | 3.8 KB |
| prompts/generate-attack.md | markdown | 4.0 KB |
| prompts/generate-behavior.md | markdown | 3.9 KB |
| prompts/generate-persistence.md | markdown | 4.2 KB |
| prompts/generate-tool-return.md | markdown | 4.7 KB |
| prompts/industry.md | markdown | 13.1 KB |
| prompts/run-scenario.md | markdown | 2.8 KB |
| scripts/_bootstrap.py | script | 2.7 KB |
| scripts/artifacts.pyexecutable | script | 8.1 KB |
| scripts/gateway_worker.pyexecutable | script | 9.6 KB |
| scripts/native_convergence.pyexecutable | script | 12.7 KB |
| scripts/native_execution.pyexecutable | script | 15.7 KB |
| scripts/native_scenarios.pyexecutable | script | 10.8 KB |
| scripts/native_swarm.pyexecutable | script | 25.5 KB |
| scripts/prompt_inputs.pyexecutable | script | 3.9 KB |
| scripts/swarm_contracts.pyexecutable | script | 6.8 KB |
| scripts/tool_executor.pyexecutable | script | 5.6 KB |
| scripts/write_report.pyexecutable | script | 6.4 KB |
scripts/native_convergence.py
View on GitHub#!/usr/bin/env python3
# Copyright (c) 2026 DataRobot, Inc. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Initialize and advance harness-native convergence state."""
from __future__ import annotations
import argparse
import json
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Literal
from artifacts import (
load_criteria,
load_json,
load_native_config,
load_spec,
one_line,
resolve_project_file,
resolve_spec_path,
resolve_swarm_dir,
resolve_under_root,
scenario_id,
write_json,
)
from native_execution import RESULT_FILENAME, NativeRunState
from native_execution import STATE_FILENAME as RUN_STATE_FILENAME
from pydantic import ValidationError
from swarm_contracts import (
NativeReportSummary,
Scenario,
ScenarioResult,
SimulationConfig,
StrictOutput,
SwarmResults,
)
from write_report import write_native_report
STATE_FILENAME = "state.json"
class NativeConvergenceState(StrictOutput):
"""Deterministic working state for native convergence."""
schema_version: Literal[1] = 1
status: Literal["open", "complete"]
spec_path: str
convergence_dir: str
started_at: str
completed_at: str | None = None
actual_model: str | None = None
config: SimulationConfig
initial_results: SwarmResults
latest_results: list[ScenarioResult]
iteration_counts: dict[str, int]
def initialize(
spec_path: Path,
criteria_path: Path,
config_path: Path,
results_path: Path,
convergence_dir: Path,
actual_model: str | None = None,
) -> dict[str, object]:
"""Create convergence state and return breach evidence for the harness."""
resolved_spec = resolve_spec_path(spec_path).resolve()
if not resolved_spec.is_file():
raise ValueError(f"agent spec does not exist: {resolved_spec}")
project_root = resolved_spec.parent
resolved_criteria = resolve_project_file(
project_root, criteria_path, "evaluation criteria"
)
resolved_config = resolve_project_file(
project_root, config_path, "simulation config"
)
resolved_results = resolve_project_file(project_root, results_path, "swarm results")
resolved_convergence_dir = resolve_under_root(
project_root, convergence_dir, "convergence directory"
)
state_path = resolved_convergence_dir / STATE_FILENAME
if state_path.exists():
raise ValueError(f"convergence already initialized: {state_path}")
spec = load_spec(resolved_spec)
if not spec.system_prompt:
raise ValueError("agent spec is missing system_prompt")
config, _ = load_native_config(resolved_config)
criteria = load_criteria(resolved_criteria)
initial_results = SwarmResults.model_validate(load_json(resolved_results))
_validate_results_against_criteria(initial_results, criteria)
latest_results = list(initial_results.scenarios)
if config.convergence.max_iterations == 0:
latest_results = [
(
result.model_copy(update={"status": "exhausted"})
if result.status == "breach"
else result
)
for result in latest_results
]
iteration_counts = {scenario_id(result.scenario): 0 for result in latest_results}
has_breaches = any(r.status == "breach" for r in latest_results)
state = NativeConvergenceState(
status="open" if has_breaches else "complete",
spec_path=str(resolved_spec),
convergence_dir=str(resolved_convergence_dir),
started_at=datetime.now(UTC).isoformat(),
completed_at=datetime.now(UTC).isoformat() if not has_breaches else None,
actual_model=actual_model,
config=config,
initial_results=initial_results,
latest_results=latest_results,
iteration_counts=iteration_counts,
)
write_json(state_path, state.model_dump(mode="json"))
return _status_payload(state, resolved_convergence_dir)
def advance(
spec_path: Path,
convergence_dir: Path,
rerun_pairs: list[tuple[str, Path]],
) -> dict[str, object]:
"""Read completed reruns, update iteration counts, return updated status."""
resolved_spec = resolve_spec_path(spec_path).resolve()
if not resolved_spec.is_file():
raise ValueError(f"agent spec does not exist: {resolved_spec}")
project_root = resolved_spec.parent
resolved_convergence_dir = resolve_under_root(
project_root, convergence_dir, "convergence directory"
)
state_path = resolved_convergence_dir / STATE_FILENAME
state = NativeConvergenceState.model_validate(load_json(state_path))
if state.status == "complete":
raise ValueError("convergence is already complete")
current_by_id = {
scenario_id(result.scenario): result for result in state.latest_results
}
max_iterations = state.config.convergence.max_iterations
rerun_results: dict[str, ScenarioResult] = {}
for sid, run_dir in rerun_pairs:
if sid not in current_by_id:
raise ValueError(f"unknown scenario_id in rerun: {sid}")
result_path = run_dir / RESULT_FILENAME
run_state_path = run_dir / RUN_STATE_FILENAME
if result_path.is_file():
result = ScenarioResult.model_validate(load_json(result_path))
rerun_results[sid] = result
elif run_state_path.is_file():
run_state = NativeRunState.model_validate(load_json(run_state_path))
if run_state.status == "running":
raise ValueError(
f"{sid}: rerun still running; expected role {run_state.next_role}"
)
raise ValueError(f"{sid}: terminal rerun state has no result")
else:
raise ValueError(f"{sid}: rerun was never initialized")
latest: list[ScenarioResult] = []
for result in state.latest_results:
sid = scenario_id(result.scenario)
if sid in rerun_results:
replacement = rerun_results[sid]
state.iteration_counts[sid] = state.iteration_counts.get(sid, 0) + 1
if (
replacement.status == "breach"
and state.iteration_counts[sid] >= max_iterations
):
replacement = replacement.model_copy(update={"status": "exhausted"})
latest.append(replacement)
else:
latest.append(result)
state.latest_results = latest
if not any(r.status == "breach" for r in state.latest_results):
state.status = "complete"
state.completed_at = datetime.now(UTC).isoformat()
write_json(state_path, state.model_dump(mode="json"))
return _status_payload(state, resolved_convergence_dir)
def report(
spec_path: Path,
convergence_dir: Path,
output_path: Path,
) -> NativeReportSummary:
"""Validate completed convergence and render the authoritative native report."""
resolved_spec = resolve_spec_path(spec_path).resolve()
if not resolved_spec.is_file():
raise ValueError(f"agent spec does not exist: {resolved_spec}")
project_root = resolved_spec.parent
resolved_convergence_dir = resolve_under_root(
project_root, convergence_dir, "convergence directory"
)
resolved_output = resolve_under_root(project_root, output_path, "evaluation report")
state = NativeConvergenceState.model_validate(
load_json(resolved_convergence_dir / STATE_FILENAME)
)
if Path(state.spec_path) != resolved_spec:
raise ValueError("convergence state belongs to a different agent spec")
if state.status != "complete":
raise ValueError(f"convergence is not complete: {state.status}")
return write_native_report(state, resolved_output)
def _status_payload(
state: NativeConvergenceState,
convergence_dir: Path,
) -> dict[str, object]:
breaches = []
exhausted = []
passed = []
for result in state.latest_results:
sid = scenario_id(result.scenario)
iteration = state.iteration_counts.get(sid, 0)
base: dict[str, object] = {
"scenario_id": sid,
"scenario_name": result.scenario.name,
"track": result.scenario.track,
"breach_reason": result.breach_reason or result.evaluation_reason,
"transcript": [e.model_dump() for e in result.transcript],
"breach_indicators": result.scenario.breach_indicators,
"iteration": iteration,
}
if result.status == "breach":
next_iter = iteration + 1
base["suggested_rerun_dir"] = str(
convergence_dir / "runs" / sid / f"iteration-{next_iter}"
)
breaches.append(base)
elif result.status == "exhausted":
exhausted.append(base)
elif result.status == "passed":
passed.append(sid)
return {
"status": state.status,
"breaches": breaches,
"exhausted": exhausted,
"passed": passed,
}
def _validate_results_against_criteria(
results: SwarmResults, criteria: list[Scenario]
) -> None:
if len(results.scenarios) != len(criteria):
raise ValueError(
"swarm results count does not match confirmed evaluation criteria"
)
for index, (result, scenario) in enumerate(
zip(results.scenarios, criteria, strict=True)
):
if result.scenario.model_dump(mode="json") != scenario.model_dump(mode="json"):
raise ValueError(
f"swarm result at index {index} differs from confirmed criteria"
)
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True)
initialize_parser = subparsers.add_parser("initialize")
initialize_parser.add_argument("spec", type=Path)
initialize_parser.add_argument(
"--criteria", type=Path, default=Path("evaluation_criteria.md")
)
initialize_parser.add_argument(
"--config", type=Path, default=Path("agent_config.yaml")
)
initialize_parser.add_argument(
"--results",
type=Path,
default=resolve_swarm_dir() / "results.json",
)
initialize_parser.add_argument(
"--convergence-dir",
type=Path,
default=resolve_swarm_dir() / "convergence",
)
initialize_parser.add_argument("--actual-model")
advance_parser = subparsers.add_parser("advance")
advance_parser.add_argument("spec", type=Path)
advance_parser.add_argument(
"--convergence-dir",
type=Path,
default=resolve_swarm_dir() / "convergence",
)
advance_parser.add_argument(
"--rerun",
dest="reruns",
action="append",
default=[],
metavar="SCENARIO_ID:RUN_DIR",
help="scenario_id:run_dir pair for a completed rerun (repeatable)",
)
report_parser = subparsers.add_parser("report")
report_parser.add_argument("spec", type=Path)
report_parser.add_argument(
"--convergence-dir",
type=Path,
default=resolve_swarm_dir() / "convergence",
)
report_parser.add_argument("--output", type=Path, default=Path("eval_report.md"))
return parser
def _parse_reruns(raw: list[str]) -> list[tuple[str, Path]]:
pairs = []
for item in raw:
if ":" not in item:
raise ValueError(f"--rerun must be scenario_id:run_dir, got: {item!r}")
scenario_id, run_dir = item.split(":", 1)
if not scenario_id or not run_dir:
raise ValueError(f"--rerun must be scenario_id:run_dir, got: {item!r}")
pairs.append((scenario_id, Path(run_dir)))
return pairs
def main(argv: list[str] | None = None) -> int:
"""Run the deterministic native-convergence helper."""
args = _build_parser().parse_args(argv)
try:
result: dict[str, object] | NativeReportSummary
if args.command == "initialize":
result = initialize(
args.spec,
args.criteria,
args.config,
args.results,
args.convergence_dir,
args.actual_model,
)
elif args.command == "advance":
rerun_pairs = _parse_reruns(args.reruns)
result = advance(args.spec, args.convergence_dir, rerun_pairs)
else:
result = report(args.spec, args.convergence_dir, args.output)
if isinstance(result, NativeReportSummary):
print(json.dumps(result.model_dump(mode="json"), ensure_ascii=False))
else:
print(json.dumps(result, ensure_ascii=False))
return 0
except (OSError, ValueError, ValidationError) as exc:
print(f"{args.command} failed: {one_line(exc)}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |