MiaoDX/roboclaws / skills/eval-harness
eval-harness
VerifiedSelect and run Roboclaws validation, product, eval-suite, and live-agent eval rows from a plan, diff, or explicit capability request.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 7.4 KB |
| catalog/rows.json | config | 36.1 KB |
| skill.json | config | 1.5 KB |
catalog/rows.json
View on GitHub{
"schema": "roboclaws_eval_harness_row_catalog_v1",
"execution_defaults": {
"deterministic": {
"execution_requirements": [
"cpu",
"python-env",
"artifact-storage"
],
"timeout_s": 900,
"concurrency_group": ""
},
"local-sim": {
"execution_requirements": [
"cpu",
"python-env",
"artifact-storage",
"simulator:mujoco"
],
"timeout_s": 1800,
"concurrency_group": "visual-backend"
},
"dino": {
"execution_requirements": [
"gpu",
"python-env",
"artifact-storage",
"simulator:mujoco",
"detector:grounding-dino"
],
"timeout_s": 1800,
"concurrency_group": "visual-backend"
},
"live-agent": {
"execution_requirements": [
"cpu",
"python-env",
"artifact-storage",
"simulator:mujoco",
"openai-agents-sdk"
],
"timeout_s": 3600,
"concurrency_group": "visual-backend"
}
},
"provider_execution_policies": {
"mimo-responses": {
"provider_network_scope": "internal",
"allowed_execution_targets": ["local"],
"execution_requirements": [
"network:configured-endpoint"
]
},
"mimo-tp-openai-chat": {
"provider_network_scope": "external",
"allowed_execution_targets": ["local"],
"execution_requirements": ["network:external-egress"]
},
"kimi-openai-chat": {
"provider_network_scope": "external",
"allowed_execution_targets": [
"local"
],
"execution_requirements": [
"network:external-egress"
]
},
"minimax-responses": {
"provider_network_scope": "external",
"allowed_execution_targets": [
"local"
],
"execution_requirements": [
"network:external-egress"
]
}
},
"rows": [
{
"row_id": "map-build-consumer-openai-agents-sdk-mimo-tp-openai-chat",
"row_kind": "live_agent_eval",
"command": [".venv/bin/python", "-m", "roboclaws.evals.cli", "suite=map_consumer_fixed_prior", "runtime_map_prior={runtime_map_prior}", "budget=focused", "output_dir={eval_output_root}", "stamp=map-build-consumer-openai-agents-sdk-mimo-tp-openai-chat", "agent_engine=openai-agents-sdk", "provider_profile=mimo-tp-openai-chat", "live_execution=run"],
"axes": {"agent_engine": "openai-agents-sdk", "provider_profile": "mimo-tp-openai-chat", "intent": "map-build-consumer", "suite": "map_consumer_fixed_prior", "evidence_lane": "world-public-labels", "backend": "{default_backend}", "world": "{default_world}", "provider_cell_count": "{provider_cell_count}", "parallel_group_id": "{map_build_consumer_parallel_group}", "default_local_concurrency_width": "1", "concurrency_policy": "{concurrency_policy}"},
"reason_selected": "MapBuild consumer proof keeps MiMo TP Chat as an explicit fixed-prior provider cell.",
"expense": "live-agent",
"selection_rule_ids": ["map_build"],
"requires": ["python_env", "openai_agents_package", "provider_profile", "runtime_map_prior"],
"profiles": ["baseline-refresh"]
},
{
"row_id": "route-trace-contract-tests",
"row_kind": "deterministic_gate",
"command": [
"./scripts/dev/run_pytest_standalone.sh",
"-q",
"tests/contract/dev_tools/test_task_agent_just_recipes_backends.py",
"tests/contract/dev_tools/test_task_agent_just_recipes_facades.py",
"tests/contract/dev_tools/test_task_agent_just_recipes_launch.py",
"tests/contract/dev_tools/test_task_agent_just_recipes_misc.py",
"tests/contract/dev_tools/test_task_agent_just_recipes_prompts.py"
],
"axes": {
"intent": "route-trace"
},
"reason_selected": "Launch catalog, provider profile, or command-surface changes need route trace coverage.",
"expense": "deterministic",
"selection_rule_ids": [
"launch_catalog"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "eval-unit-tests",
"row_kind": "deterministic_gate",
"command": [
"./scripts/dev/run_pytest_standalone.sh",
"-q",
"tests/unit/evals"
],
"axes": {
"intent": "eval-harness"
},
"reason_selected": "Eval harness, suite, manifest, or regression-promotion changes need eval tests.",
"expense": "deterministic",
"selection_rule_ids": [
"eval_harness"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "cleanup-contract-tests",
"row_kind": "deterministic_gate",
"command": [
"./scripts/dev/run_pytest_standalone.sh",
"-q",
"tests/unit/molmo_cleanup/test_molmo_cleanup_semantic_acceptability.py",
"tests/unit/molmo_cleanup/test_molmo_semantic_cleanup_loop.py"
],
"axes": {
"intent": "cleanup",
"agent_engine": "direct-runner"
},
"reason_selected": "Cleanup, MCP, checker, and report changes need deterministic cleanup contract checks.",
"expense": "deterministic",
"selection_rule_ids": [
"cleanup_skill",
"mcp_checker",
"agent_view_module"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "agent-view-contract-tests",
"row_kind": "deterministic_gate",
"command": [
"./scripts/dev/run_pytest_standalone.sh",
"-q",
"tests/contract/molmo_cleanup/test_household_runtime_contract_tools.py::test_realworld_agent_view_payload_keeps_private_evaluation_out",
"tests/contract/molmo_cleanup/test_household_runtime_contract_raw_fpv_misc.py::test_realworld_raw_fpv_mode_suppresses_structured_detections",
"tests/contract/molmo_cleanup/test_household_runtime_contract_policy.py::test_realworld_camera_model_policy_registers_model_labelled_candidates",
"tests/contract/molmo_cleanup/test_household_runtime_contract_policy.py::test_realworld_camera_model_policy_records_sim_pipeline_provenance",
"tests/contract/molmo_cleanup/test_household_runtime_contract_camera_labels.py::test_realworld_camera_labels_http_failure_is_visible_without_sim_fallback",
"tests/contract/molmo_cleanup/test_household_runtime_contract_camera_labels.py::test_realworld_camera_labels_missing_raw_image_fails_before_sidecar",
"tests/contract/molmo_cleanup/test_household_runtime_contract_camera_labels.py::test_realworld_camera_labels_http_success_uses_destination_resolver",
"tests/contract/molmo_cleanup/test_household_mcp_server_tools.py::test_realworld_mcp_registered_tools_match_profile_public_surface",
"tests/contract/molmo_cleanup/test_household_mcp_server_misc.py::test_agent_sdk_camera_grounded_composite_flag_cannot_expand_entitlement",
"tests/contract/molmo_cleanup/test_household_mcp_server_tools.py::test_realworld_mcp_rejects_removed_static_fixture_projection_tool",
"tests/contract/mcp/test_semantic_profiles.py",
"tests/unit/molmo_cleanup/test_visual_grounding.py::test_visual_grounding_request_rejects_static_fixture_projection_field",
"tests/unit/molmo_cleanup/test_visual_grounding.py::test_visual_grounding_request_rejects_private_public_map_hints",
"tests/unit/molmo_cleanup/test_visual_grounding.py::test_visual_grounding_request_rejects_private_hint_keys",
"tests/contract/molmo_cleanup/test_physical_agibot_pilot.py::test_agibot_map_build_camera_labels_call_external_grounding",
"tests/contract/agibot/test_agibot_map_context_scripts_context.py::test_vendor_sdk_runner_exports_base_metric_context_generated_candidates"
],
"axes": {
"intent": "agent-view",
"agent_engine": "direct-runner"
},
"reason_selected": "Agent View module changes need artifact schema, privacy guard, MCP response, profile capability, active-perception, sidecar public-input, and Agibot export coverage.",
"expense": "deterministic",
"selection_rule_ids": [
"agent_view_module"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "household-direct-world-public-product",
"row_kind": "product_run",
"command": [
".venv/bin/python",
"-m",
"roboclaws.cli.main",
"run",
"surface",
"surface=household-world",
"world={default_world}",
"backend={default_backend}",
"preset=cleanup",
"agent_engine=direct-runner",
"evidence_lane=world-public-labels",
"seed={default_seed}",
"output_dir={row_dir}/household-direct-world-public-product/run"
],
"axes": {
"agent_engine": "direct-runner",
"intent": "cleanup",
"preset": "cleanup",
"evidence_lane": "world-public-labels",
"backend": "{default_backend}",
"world": "{default_world}"
},
"reason_selected": "MCP/server/checker changes need at least one public cleanup product route.",
"expense": "local-sim",
"selection_rule_ids": [
"mcp_checker",
"agent_view_module"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "open-ended-household-contract-tests",
"row_kind": "deterministic_gate",
"command": [
"./scripts/dev/run_pytest_standalone.sh",
"-q",
"tests/contract/dev_tools/test_task_agent_just_recipes_launch.py::test_surface_open_ended_supports_mcp_smoke_for_local_gate",
"tests/contract/molmo_cleanup/test_household_mcp_server_misc.py::test_realworld_mcp_open_ended_intent_is_recorded_in_run_result",
"tests/contract/checkers/test_cleanup_validation_maps.py::test_checker_allows_open_ended_agent_view_with_no_visible_objects"
],
"axes": {
"intent": "open-ended"
},
"reason_selected": "Open-ended, goal-contract, or completion-claim changes need a first-class open-ended launch and artifact contract gate.",
"expense": "deterministic",
"selection_rule_ids": [
"open_ended"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "smoke-regression-eval-suite",
"row_kind": "eval_suite",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=smoke_regression",
"budget=smoke",
"output_dir={eval_output_root}",
"stamp=smoke-regression-eval-suite",
"agent_engine=direct-runner"
],
"axes": {
"intent": "eval-suite",
"suite": "smoke_regression"
},
"reason_selected": "Eval harness or command-surface changes need the smoke regression suite.",
"expense": "deterministic",
"selection_rule_ids": [
"eval_harness",
"launch_catalog"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "map-build-consumer-eval-suite",
"row_kind": "eval_suite",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=map_build_quality",
"budget=smoke",
"output_dir={eval_output_root}",
"stamp=map-build-quality-eval-suite",
"agent_engine=direct-runner"
],
"axes": {
"intent": "eval-suite",
"suite": "map_build_quality"
},
"reason_selected": "Map-build and actionability changes need the map consumer suite and runtime-map-prior flow.",
"expense": "deterministic",
"selection_rule_ids": [
"map_build"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "map-build-consumer-openai-agents-sdk-mimo-responses",
"row_kind": "live_agent_eval",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=map_consumer_fixed_prior",
"runtime_map_prior={runtime_map_prior}",
"budget=focused",
"output_dir={eval_output_root}",
"stamp=map-build-consumer-openai-agents-sdk-mimo-responses",
"agent_engine=openai-agents-sdk",
"provider_profile=mimo-responses",
"live_execution=run"
],
"axes": {
"agent_engine": "openai-agents-sdk",
"provider_profile": "mimo-responses",
"intent": "map-build-consumer",
"suite": "map_consumer_fixed_prior",
"evidence_lane": "world-public-labels",
"backend": "{default_backend}",
"world": "{default_world}",
"parallel_group_id": "{map_build_consumer_parallel_group}",
"provider_cell_count": "{provider_cell_count}",
"default_local_concurrency_width": "1",
"concurrency_policy": "{concurrency_policy}"
},
"reason_selected": "MapBuild consumer proof keeps MiMo Responses as one explicit provider cell.",
"expense": "live-agent",
"selection_rule_ids": [
"map_build"
],
"requires": [
"python_env",
"openai_agents_package",
"provider_profile",
"runtime_map_prior"
],
"profiles": [
"baseline-refresh"
]
},
{
"row_id": "map-build-consumer-openai-agents-sdk-kimi-openai-chat",
"row_kind": "live_agent_eval",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=map_consumer_fixed_prior",
"runtime_map_prior={runtime_map_prior}",
"budget=focused",
"output_dir={eval_output_root}",
"stamp=map-build-consumer-openai-agents-sdk-kimi-openai-chat",
"agent_engine=openai-agents-sdk",
"provider_profile=kimi-openai-chat",
"live_execution=run"
],
"axes": {
"agent_engine": "openai-agents-sdk",
"provider_profile": "kimi-openai-chat",
"intent": "map-build-consumer",
"suite": "map_consumer_fixed_prior",
"evidence_lane": "world-public-labels",
"backend": "{default_backend}",
"world": "{default_world}",
"parallel_group_id": "{map_build_consumer_parallel_group}",
"provider_cell_count": "{provider_cell_count}",
"default_local_concurrency_width": "1",
"concurrency_policy": "{concurrency_policy}"
},
"reason_selected": "MapBuild consumer plan/model-matrix proof needs this provider profile as a first-class availability-or-behavior cell. Run provider cells serially unless ROBOCLAWS_MOLMO_MAX_VISUAL_BACKENDS is explicitly raised and each live row has an isolated stamp/output root.",
"expense": "live-agent",
"selection_rule_ids": [
"map_build"
],
"requires": [
"python_env",
"openai_agents_package",
"provider_profile",
"runtime_map_prior"
],
"profiles": [
"baseline-refresh"
]
},
{
"row_id": "map-build-consumer-openai-agents-sdk-minimax-responses",
"row_kind": "live_agent_eval",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=map_consumer_fixed_prior",
"runtime_map_prior={runtime_map_prior}",
"budget=focused",
"output_dir={eval_output_root}",
"stamp=map-build-consumer-openai-agents-sdk-minimax-responses",
"agent_engine=openai-agents-sdk",
"provider_profile=minimax-responses",
"live_execution=run"
],
"axes": {
"agent_engine": "openai-agents-sdk",
"provider_profile": "minimax-responses",
"intent": "map-build-consumer",
"suite": "map_consumer_fixed_prior",
"evidence_lane": "world-public-labels",
"backend": "{default_backend}",
"world": "{default_world}",
"parallel_group_id": "{map_build_consumer_parallel_group}",
"provider_cell_count": "{provider_cell_count}",
"default_local_concurrency_width": "1",
"concurrency_policy": "{concurrency_policy}"
},
"reason_selected": "MapBuild consumer plan/model-matrix proof needs this provider profile as a first-class availability-or-behavior cell. Run provider cells serially unless ROBOCLAWS_MOLMO_MAX_VISUAL_BACKENDS is explicitly raised and each live row has an isolated stamp/output root.",
"expense": "live-agent",
"selection_rule_ids": [
"map_build"
],
"requires": [
"python_env",
"openai_agents_package",
"provider_profile",
"runtime_map_prior"
],
"profiles": [
"baseline-refresh"
]
},
{
"row_id": "open-ended-goals-eval-suite",
"row_kind": "eval_suite",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=open_ended_goals",
"runtime_map_prior={runtime_map_prior}",
"runtime_map_prior={runtime_map_prior}",
"budget=smoke",
"output_dir={eval_output_root}",
"stamp=open-ended-goals-eval-suite",
"agent_engine=direct-runner"
],
"axes": {
"intent": "eval-suite",
"suite": "open_ended_goals"
},
"reason_selected": "Open-ended household goal changes need the dedicated no-preset open-task capability suite.",
"expense": "deterministic",
"selection_rule_ids": [
"open_ended"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "scene-sampler-stress-eval-suite",
"row_kind": "eval_suite",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=scene_sampler_stress",
"budget=smoke",
"output_dir={eval_output_root}",
"stamp=scene-sampler-stress-eval-suite",
"agent_engine=direct-runner"
],
"axes": {
"intent": "eval-suite",
"suite": "scene_sampler_stress"
},
"reason_selected": "MolmoSpaces scene-source sampling changes need the static sampler stress projection with partial/blocked source metadata.",
"expense": "deterministic",
"selection_rule_ids": [
"scene_sampler",
"launch_catalog",
"eval_harness"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "cleanup-capability-eval-suite",
"row_kind": "eval_suite",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=cleanup_capability",
"budget=smoke",
"output_dir={eval_output_root}",
"stamp=cleanup-capability-eval-suite",
"agent_engine=direct-runner"
],
"axes": {
"intent": "eval-suite",
"suite": "cleanup_capability"
},
"reason_selected": "Cleanup skill or MCP changes need the repeated cleanup capability suite.",
"expense": "deterministic",
"selection_rule_ids": [
"cleanup_skill",
"mcp_checker"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "long-horizon-tasks-eval-suite",
"row_kind": "eval_suite",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=long_horizon_tasks",
"budget=smoke",
"output_dir={eval_output_root}",
"stamp=long-horizon-tasks-eval-suite",
"agent_engine=direct-runner"
],
"axes": {
"intent": "eval-suite",
"suite": "long_horizon_tasks"
},
"reason_selected": "Long-horizon household task changes need the real MolmoSpaces multi-room manipulation suite.",
"expense": "local-sim",
"selection_rule_ids": [
"long_horizon"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "openai-agents-sdk-open-task-live-eval",
"row_kind": "live_agent_eval",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=open_ended_goals",
"runtime_map_prior={runtime_map_prior}",
"budget=smoke",
"output_dir={eval_output_root}",
"stamp=openai-agents-sdk-open-task-live-eval",
"agent_engine=openai-agents-sdk",
"provider_profile={agent_sdk_provider}",
"live_execution=run"
],
"axes": {
"agent_engine": "openai-agents-sdk",
"provider_profile": "{agent_sdk_provider}",
"intent": "open-ended",
"preset": "",
"evidence_lane": "world-public-labels",
"backend": "{default_backend}",
"world": "{default_world}"
},
"reason_selected": "Agent SDK or open-task launch changes need a representative live-agent capability eval row.",
"expense": "live-agent",
"selection_rule_ids": [
"agent_sdk",
"open_ended"
],
"requires": [
"python_env",
"openai_agents_package",
"provider_profile",
"runtime_map_prior"
],
"profiles": [
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "openai-agents-sdk-session-live-eval",
"row_kind": "live_agent_eval",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"session-live",
"budget=smoke",
"output_dir={eval_output_root}",
"stamp=openai-agents-sdk-session-live-eval",
"agent_engine=openai-agents-sdk",
"provider_profile={agent_sdk_provider}",
"live_execution=run"
],
"axes": {
"agent_engine": "openai-agents-sdk",
"provider_profile": "{agent_sdk_provider}",
"intent": "operator-session",
"preset": "",
"evidence_lane": "world-public-labels",
"backend": "{default_backend}",
"world": "{default_world}"
},
"reason_selected": "Operator Session chaining changes need live SDK evidence that Steer is consumed and terminal Next Goal starts a linked child run with sanitized follow-up context.",
"expense": "live-agent",
"selection_rule_ids": [
"agent_sdk",
"open_ended",
"launch_catalog",
"eval_harness"
],
"requires": [
"python_env",
"openai_agents_package",
"provider_profile"
],
"profiles": [
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "openai-agents-sdk-cleanup-live-eval",
"skill_delivery_cell": "static-full",
"row_kind": "live_agent_eval",
"command": [
".venv/bin/python",
"-m",
"roboclaws.evals.cli",
"suite=cleanup_capability",
"runtime_map_prior={runtime_map_prior}",
"budget=smoke",
"output_dir={eval_output_root}",
"stamp=openai-agents-sdk-cleanup-live-eval",
"agent_engine=openai-agents-sdk",
"provider_profile={agent_sdk_provider}",
"skill_delivery_cell=static-full",
"live_execution=run"
],
"axes": {
"agent_engine": "openai-agents-sdk",
"provider_profile": "{agent_sdk_provider}",
"intent": "cleanup",
"preset": "cleanup",
"evidence_lane": "world-public-labels",
"backend": "{default_backend}",
"world": "{default_world}"
},
"reason_selected": "Cleanup skill or MCP changes need a representative SDK live-agent cleanup capability eval row with enough stall budget for long reasoning calls.",
"expense": "live-agent",
"selection_rule_ids": [
"cleanup_skill",
"mcp_checker"
],
"requires": [
"python_env",
"openai_agents_package",
"provider_profile",
"runtime_map_prior"
],
"profiles": [
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "openai-agents-sdk-cleanup-no-skill-eval",
"skill_delivery_cell": "no-skill",
"row_kind": "live_agent_eval",
"command": [".venv/bin/python", "-m", "roboclaws.evals.cli", "suite=cleanup_capability", "budget=smoke", "output_dir={eval_output_root}", "stamp=openai-agents-sdk-cleanup-no-skill-eval", "agent_engine=openai-agents-sdk", "provider_profile={agent_sdk_provider}", "skill_delivery_cell=no-skill", "live_execution=run"],
"axes": {"agent_engine": "openai-agents-sdk", "provider_profile": "{agent_sdk_provider}", "intent": "cleanup", "preset": "cleanup", "evidence_lane": "world-public-labels", "backend": "{default_backend}", "world": "{default_world}"},
"reason_selected": "Skill-delivery evaluation no-skill control.",
"expense": "live-agent",
"selection_rule_ids": ["cleanup_skill", "mcp_checker"],
"requires": ["python_env", "openai_agents_package", "provider_profile"],
"profiles": ["baseline-refresh"]
},
{
"row_id": "openai-agents-sdk-cleanup-dynamic-full-eval",
"skill_delivery_cell": "dynamic-full",
"row_kind": "live_agent_eval",
"command": [".venv/bin/python", "-m", "roboclaws.evals.cli", "suite=cleanup_capability", "budget=smoke", "output_dir={eval_output_root}", "stamp=openai-agents-sdk-cleanup-dynamic-full-eval", "agent_engine=openai-agents-sdk", "provider_profile={agent_sdk_provider}", "skill_delivery_cell=dynamic-full", "live_execution=run"],
"axes": {"agent_engine": "openai-agents-sdk", "provider_profile": "{agent_sdk_provider}", "intent": "cleanup", "preset": "cleanup", "evidence_lane": "world-public-labels", "backend": "{default_backend}", "world": "{default_world}"},
"reason_selected": "Skill-delivery evaluation byte-identical dynamic callback control.",
"expense": "live-agent",
"selection_rule_ids": ["cleanup_skill", "mcp_checker"],
"requires": ["python_env", "openai_agents_package", "provider_profile"],
"profiles": ["baseline-refresh"]
},
{
"row_id": "openai-agents-sdk-cleanup-dynamic-routed-eval",
"skill_delivery_cell": "dynamic-routed",
"row_kind": "live_agent_eval",
"command": [".venv/bin/python", "-m", "roboclaws.evals.cli", "suite=cleanup_capability", "budget=smoke", "output_dir={eval_output_root}", "stamp=openai-agents-sdk-cleanup-dynamic-routed-eval", "agent_engine=openai-agents-sdk", "provider_profile={agent_sdk_provider}", "skill_delivery_cell=dynamic-routed", "live_execution=run"],
"axes": {"agent_engine": "openai-agents-sdk", "provider_profile": "{agent_sdk_provider}", "intent": "cleanup", "preset": "cleanup", "evidence_lane": "world-public-labels", "backend": "{default_backend}", "world": "{default_world}"},
"reason_selected": "Skill-delivery evaluation frozen shared, intent, and evidence guidance.",
"expense": "live-agent",
"selection_rule_ids": ["cleanup_skill", "mcp_checker"],
"requires": ["python_env", "openai_agents_package", "provider_profile"],
"profiles": ["baseline-refresh"]
},
{
"row_id": "openai-agents-sdk-cleanup-sandbox-skills-eval",
"skill_delivery_cell": "sandbox-skills",
"row_kind": "live_agent_eval",
"command": [".venv/bin/python", "-m", "roboclaws.evals.cli", "suite=cleanup_capability", "budget=smoke", "output_dir={eval_output_root}", "stamp=openai-agents-sdk-cleanup-sandbox-skills-eval", "agent_engine=openai-agents-sdk", "provider_profile={agent_sdk_provider}", "skill_delivery_cell=sandbox-skills", "live_execution=run"],
"axes": {"agent_engine": "openai-agents-sdk", "provider_profile": "{agent_sdk_provider}", "intent": "cleanup", "preset": "cleanup", "evidence_lane": "world-public-labels", "backend": "{default_backend}", "world": "{default_world}"},
"reason_selected": "Exploratory official SandboxAgent plus Skills readiness and isolation row.",
"expense": "live-agent",
"selection_rule_ids": ["cleanup_skill", "mcp_checker"],
"requires": ["python_env", "openai_agents_package", "provider_profile", "sandbox_skills"],
"profiles": ["baseline-refresh"]
},
{
"row_id": "planner-proof-dry-run-product",
"row_kind": "product_run",
"command": [
".venv/bin/python",
"-m",
"roboclaws.cli.main",
"run",
"surface",
"surface=planner-proof",
"world=planner-proof/default",
"backend={default_backend}",
"intent=planner-proof",
"agent_engine=direct-runner",
"mode=dry-run",
"output_dir={row_dir}/planner-proof-dry-run-product/run"
],
"axes": {
"agent_engine": "direct-runner",
"intent": "planner-proof",
"backend": "{default_backend}",
"world": "planner-proof/default"
},
"reason_selected": "Planner-proof changes need the current public planner-proof dry-run proof row; lower-level harness recipes remain private mechanics.",
"expense": "local-sim",
"selection_rule_ids": [
"planner_proof"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "direct-camera-grounded-grounding-dino",
"row_kind": "product_run",
"command": [
".venv/bin/python",
"-m",
"roboclaws.cli.main",
"run",
"surface",
"surface=household-world",
"world={default_world}",
"backend={default_backend}",
"preset=cleanup",
"agent_engine=direct-runner",
"evidence_lane=camera-grounded-labels",
"seed={default_seed}",
"scenario_setup=relocate-cleanup-related-objects",
"relocation_count=5",
"output_dir={row_dir}/direct-camera-grounded-grounding-dino/run",
"camera_labeler=grounding-dino"
],
"axes": {
"agent_engine": "direct-runner",
"intent": "cleanup",
"preset": "cleanup",
"evidence_lane": "camera-grounded-labels",
"camera_labeler": "grounding-dino",
"backend": "{default_backend}",
"world": "{default_world}"
},
"reason_selected": "Visual-grounding changes need a real Grounding DINO camera-labeler gate.",
"expense": "dino",
"selection_rule_ids": [
"visual_grounding"
],
"requires": [
"python_env",
"dino_sidecar"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "direct-map-build-grounding-dino",
"row_kind": "product_run",
"scene_scope": "selected",
"command": [
".venv/bin/python",
"-m",
"roboclaws.cli.main",
"run",
"surface",
"surface=household-world",
"world={default_world}",
"backend={default_backend}",
"preset=map-build",
"agent_engine=direct-runner",
"evidence_lane=camera-grounded-labels",
"camera_labeler=grounding-dino",
"seed={default_seed}",
"scenario_setup=baseline",
"output_dir={row_dir}/direct-map-build-grounding-dino/run"
],
"axes": {
"agent_engine": "direct-runner",
"intent": "map-build",
"preset": "map-build",
"evidence_lane": "camera-grounded-labels",
"camera_labeler": "grounding-dino",
"backend": "{default_backend}",
"world": "{default_world}"
},
"reason_selected": "Map-build product proof should exercise the deployable Grounding-DINO camera-labeler route.",
"expense": "dino",
"selection_rule_ids": [
"map_build",
"visual_grounding"
],
"requires": [
"python_env",
"dino_sidecar"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "direct-camera-raw-fpv",
"row_kind": "product_run",
"command": [
".venv/bin/python",
"-m",
"roboclaws.cli.main",
"run",
"surface",
"surface=household-world",
"world={default_world}",
"backend={default_backend}",
"preset=cleanup",
"agent_engine=direct-runner",
"evidence_lane=camera-raw-fpv",
"seed={default_seed}",
"scenario_setup=relocate-cleanup-related-objects",
"relocation_count=5",
"output_dir={row_dir}/direct-camera-raw-fpv/run"
],
"axes": {
"agent_engine": "direct-runner",
"intent": "cleanup",
"preset": "cleanup",
"evidence_lane": "camera-raw-fpv",
"backend": "{default_backend}",
"world": "{default_world}"
},
"reason_selected": "RAW-FPV changes need a direct RAW-FPV product gate.",
"expense": "local-sim",
"selection_rule_ids": [
"raw_fpv"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "direct-map-build-world-public",
"row_kind": "product_run",
"scene_scope": "selected",
"command": [
".venv/bin/python",
"-m",
"roboclaws.cli.main",
"run",
"surface",
"surface=household-world",
"world={default_world}",
"backend={default_backend}",
"preset=map-build",
"agent_engine=direct-runner",
"evidence_lane=world-public-labels",
"seed={default_seed}",
"scenario_setup=baseline",
"output_dir={row_dir}/direct-map-build-world-public/run"
],
"axes": {
"agent_engine": "direct-runner",
"intent": "map-build",
"preset": "map-build",
"evidence_lane": "world-public-labels",
"backend": "{default_backend}",
"world": "{default_world}"
},
"reason_selected": "Map-build and actionability changes need a map-build lane gate.",
"expense": "local-sim",
"selection_rule_ids": [
"map_build"
],
"requires": [
"python_env"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
},
{
"row_id": "direct-cleanup-runtime-prior-consumer",
"row_kind": "product_run",
"command": [
".venv/bin/python",
"-m",
"roboclaws.cli.main",
"run",
"surface",
"surface=household-world",
"world={default_world}",
"backend={default_backend}",
"preset=cleanup",
"agent_engine=direct-runner",
"evidence_lane=world-public-labels",
"seed={default_seed}",
"scenario_setup=relocate-cleanup-related-objects",
"relocation_count=5",
"output_dir={row_dir}/direct-cleanup-runtime-prior-consumer/run",
"runtime_map_prior=${direct-map-build-world-public:runtime_metric_map.json}"
],
"axes": {
"agent_engine": "direct-runner",
"intent": "cleanup",
"preset": "cleanup",
"evidence_lane": "world-public-labels",
"runtime_map_prior": "required",
"backend": "{default_backend}",
"world": "{default_world}"
},
"reason_selected": "Map-build changes need a cleanup consumer gate with the runtime-map prior.",
"expense": "local-sim",
"selection_rule_ids": [
"map_build"
],
"requires": [
"python_env",
"runtime_map_prior"
],
"depends_on": [
"direct-map-build-world-public"
],
"profiles": [
"baseline-core",
"baseline-live-default",
"baseline-refresh"
]
}
]
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |