Skip to content

MiaoDX/roboclaws / skills/eval-harness

eval-harness

Verified

Select and run Roboclaws validation, product, eval-suite, and live-agent eval rows from a plan, diff, or explicit capability request.

FileKindSize
SKILL.mdskill md7.4 KB
catalog/rows.jsonconfig36.1 KB
skill.jsonconfig1.5 KB

catalog/rows.json

View on GitHub
{
  "schema": "roboclaws_eval_harness_row_catalog_v1",
  "execution_defaults": {
    "deterministic": {
      "execution_requirements": [
        "cpu",
        "python-env",
        "artifact-storage"
      ],
      "timeout_s": 900,
      "concurrency_group": ""
    },
    "local-sim": {
      "execution_requirements": [
        "cpu",
        "python-env",
        "artifact-storage",
        "simulator:mujoco"
      ],
      "timeout_s": 1800,
      "concurrency_group": "visual-backend"
    },
    "dino": {
      "execution_requirements": [
        "gpu",
        "python-env",
        "artifact-storage",
        "simulator:mujoco",
        "detector:grounding-dino"
      ],
      "timeout_s": 1800,
      "concurrency_group": "visual-backend"
    },
    "live-agent": {
      "execution_requirements": [
        "cpu",
        "python-env",
        "artifact-storage",
        "simulator:mujoco",
        "openai-agents-sdk"
      ],
      "timeout_s": 3600,
      "concurrency_group": "visual-backend"
    }
  },
  "provider_execution_policies": {
    "mimo-responses": {
      "provider_network_scope": "internal",
      "allowed_execution_targets": ["local"],
      "execution_requirements": [
        "network:configured-endpoint"
      ]
    },
    "mimo-tp-openai-chat": {
      "provider_network_scope": "external",
      "allowed_execution_targets": ["local"],
      "execution_requirements": ["network:external-egress"]
    },
    "kimi-openai-chat": {
      "provider_network_scope": "external",
      "allowed_execution_targets": [
        "local"
      ],
      "execution_requirements": [
        "network:external-egress"
      ]
    },
    "minimax-responses": {
      "provider_network_scope": "external",
      "allowed_execution_targets": [
        "local"
      ],
      "execution_requirements": [
        "network:external-egress"
      ]
    }
  },
  "rows": [
    {
      "row_id": "map-build-consumer-openai-agents-sdk-mimo-tp-openai-chat",
      "row_kind": "live_agent_eval",
      "command": [".venv/bin/python", "-m", "roboclaws.evals.cli", "suite=map_consumer_fixed_prior", "runtime_map_prior={runtime_map_prior}", "budget=focused", "output_dir={eval_output_root}", "stamp=map-build-consumer-openai-agents-sdk-mimo-tp-openai-chat", "agent_engine=openai-agents-sdk", "provider_profile=mimo-tp-openai-chat", "live_execution=run"],
      "axes": {"agent_engine": "openai-agents-sdk", "provider_profile": "mimo-tp-openai-chat", "intent": "map-build-consumer", "suite": "map_consumer_fixed_prior", "evidence_lane": "world-public-labels", "backend": "{default_backend}", "world": "{default_world}", "provider_cell_count": "{provider_cell_count}", "parallel_group_id": "{map_build_consumer_parallel_group}", "default_local_concurrency_width": "1", "concurrency_policy": "{concurrency_policy}"},
      "reason_selected": "MapBuild consumer proof keeps MiMo TP Chat as an explicit fixed-prior provider cell.",
      "expense": "live-agent",
      "selection_rule_ids": ["map_build"],
      "requires": ["python_env", "openai_agents_package", "provider_profile", "runtime_map_prior"],
      "profiles": ["baseline-refresh"]
    },
    {
      "row_id": "route-trace-contract-tests",
      "row_kind": "deterministic_gate",
      "command": [
        "./scripts/dev/run_pytest_standalone.sh",
        "-q",
        "tests/contract/dev_tools/test_task_agent_just_recipes_backends.py",
        "tests/contract/dev_tools/test_task_agent_just_recipes_facades.py",
        "tests/contract/dev_tools/test_task_agent_just_recipes_launch.py",
        "tests/contract/dev_tools/test_task_agent_just_recipes_misc.py",
        "tests/contract/dev_tools/test_task_agent_just_recipes_prompts.py"
      ],
      "axes": {
        "intent": "route-trace"
      },
      "reason_selected": "Launch catalog, provider profile, or command-surface changes need route trace coverage.",
      "expense": "deterministic",
      "selection_rule_ids": [
        "launch_catalog"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "eval-unit-tests",
      "row_kind": "deterministic_gate",
      "command": [
        "./scripts/dev/run_pytest_standalone.sh",
        "-q",
        "tests/unit/evals"
      ],
      "axes": {
        "intent": "eval-harness"
      },
      "reason_selected": "Eval harness, suite, manifest, or regression-promotion changes need eval tests.",
      "expense": "deterministic",
      "selection_rule_ids": [
        "eval_harness"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "cleanup-contract-tests",
      "row_kind": "deterministic_gate",
      "command": [
        "./scripts/dev/run_pytest_standalone.sh",
        "-q",
        "tests/unit/molmo_cleanup/test_molmo_cleanup_semantic_acceptability.py",
        "tests/unit/molmo_cleanup/test_molmo_semantic_cleanup_loop.py"
      ],
      "axes": {
        "intent": "cleanup",
        "agent_engine": "direct-runner"
      },
      "reason_selected": "Cleanup, MCP, checker, and report changes need deterministic cleanup contract checks.",
      "expense": "deterministic",
      "selection_rule_ids": [
        "cleanup_skill",
        "mcp_checker",
        "agent_view_module"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "agent-view-contract-tests",
      "row_kind": "deterministic_gate",
      "command": [
        "./scripts/dev/run_pytest_standalone.sh",
        "-q",
        "tests/contract/molmo_cleanup/test_household_runtime_contract_tools.py::test_realworld_agent_view_payload_keeps_private_evaluation_out",
        "tests/contract/molmo_cleanup/test_household_runtime_contract_raw_fpv_misc.py::test_realworld_raw_fpv_mode_suppresses_structured_detections",
        "tests/contract/molmo_cleanup/test_household_runtime_contract_policy.py::test_realworld_camera_model_policy_registers_model_labelled_candidates",
        "tests/contract/molmo_cleanup/test_household_runtime_contract_policy.py::test_realworld_camera_model_policy_records_sim_pipeline_provenance",
        "tests/contract/molmo_cleanup/test_household_runtime_contract_camera_labels.py::test_realworld_camera_labels_http_failure_is_visible_without_sim_fallback",
        "tests/contract/molmo_cleanup/test_household_runtime_contract_camera_labels.py::test_realworld_camera_labels_missing_raw_image_fails_before_sidecar",
        "tests/contract/molmo_cleanup/test_household_runtime_contract_camera_labels.py::test_realworld_camera_labels_http_success_uses_destination_resolver",
        "tests/contract/molmo_cleanup/test_household_mcp_server_tools.py::test_realworld_mcp_registered_tools_match_profile_public_surface",
        "tests/contract/molmo_cleanup/test_household_mcp_server_misc.py::test_agent_sdk_camera_grounded_composite_flag_cannot_expand_entitlement",
        "tests/contract/molmo_cleanup/test_household_mcp_server_tools.py::test_realworld_mcp_rejects_removed_static_fixture_projection_tool",
        "tests/contract/mcp/test_semantic_profiles.py",
        "tests/unit/molmo_cleanup/test_visual_grounding.py::test_visual_grounding_request_rejects_static_fixture_projection_field",
        "tests/unit/molmo_cleanup/test_visual_grounding.py::test_visual_grounding_request_rejects_private_public_map_hints",
        "tests/unit/molmo_cleanup/test_visual_grounding.py::test_visual_grounding_request_rejects_private_hint_keys",
        "tests/contract/molmo_cleanup/test_physical_agibot_pilot.py::test_agibot_map_build_camera_labels_call_external_grounding",
        "tests/contract/agibot/test_agibot_map_context_scripts_context.py::test_vendor_sdk_runner_exports_base_metric_context_generated_candidates"
      ],
      "axes": {
        "intent": "agent-view",
        "agent_engine": "direct-runner"
      },
      "reason_selected": "Agent View module changes need artifact schema, privacy guard, MCP response, profile capability, active-perception, sidecar public-input, and Agibot export coverage.",
      "expense": "deterministic",
      "selection_rule_ids": [
        "agent_view_module"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "household-direct-world-public-product",
      "row_kind": "product_run",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.cli.main",
        "run",
        "surface",
        "surface=household-world",
        "world={default_world}",
        "backend={default_backend}",
        "preset=cleanup",
        "agent_engine=direct-runner",
        "evidence_lane=world-public-labels",
        "seed={default_seed}",
        "output_dir={row_dir}/household-direct-world-public-product/run"
      ],
      "axes": {
        "agent_engine": "direct-runner",
        "intent": "cleanup",
        "preset": "cleanup",
        "evidence_lane": "world-public-labels",
        "backend": "{default_backend}",
        "world": "{default_world}"
      },
      "reason_selected": "MCP/server/checker changes need at least one public cleanup product route.",
      "expense": "local-sim",
      "selection_rule_ids": [
        "mcp_checker",
        "agent_view_module"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "open-ended-household-contract-tests",
      "row_kind": "deterministic_gate",
      "command": [
        "./scripts/dev/run_pytest_standalone.sh",
        "-q",
        "tests/contract/dev_tools/test_task_agent_just_recipes_launch.py::test_surface_open_ended_supports_mcp_smoke_for_local_gate",
        "tests/contract/molmo_cleanup/test_household_mcp_server_misc.py::test_realworld_mcp_open_ended_intent_is_recorded_in_run_result",
        "tests/contract/checkers/test_cleanup_validation_maps.py::test_checker_allows_open_ended_agent_view_with_no_visible_objects"
      ],
      "axes": {
        "intent": "open-ended"
      },
      "reason_selected": "Open-ended, goal-contract, or completion-claim changes need a first-class open-ended launch and artifact contract gate.",
      "expense": "deterministic",
      "selection_rule_ids": [
        "open_ended"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "smoke-regression-eval-suite",
      "row_kind": "eval_suite",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=smoke_regression",
        "budget=smoke",
        "output_dir={eval_output_root}",
        "stamp=smoke-regression-eval-suite",
        "agent_engine=direct-runner"
      ],
      "axes": {
        "intent": "eval-suite",
        "suite": "smoke_regression"
      },
      "reason_selected": "Eval harness or command-surface changes need the smoke regression suite.",
      "expense": "deterministic",
      "selection_rule_ids": [
        "eval_harness",
        "launch_catalog"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "map-build-consumer-eval-suite",
      "row_kind": "eval_suite",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=map_build_quality",
        "budget=smoke",
        "output_dir={eval_output_root}",
        "stamp=map-build-quality-eval-suite",
        "agent_engine=direct-runner"
      ],
      "axes": {
        "intent": "eval-suite",
        "suite": "map_build_quality"
      },
      "reason_selected": "Map-build and actionability changes need the map consumer suite and runtime-map-prior flow.",
      "expense": "deterministic",
      "selection_rule_ids": [
        "map_build"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "map-build-consumer-openai-agents-sdk-mimo-responses",
      "row_kind": "live_agent_eval",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=map_consumer_fixed_prior",
        "runtime_map_prior={runtime_map_prior}",
        "budget=focused",
        "output_dir={eval_output_root}",
        "stamp=map-build-consumer-openai-agents-sdk-mimo-responses",
        "agent_engine=openai-agents-sdk",
        "provider_profile=mimo-responses",
        "live_execution=run"
      ],
      "axes": {
        "agent_engine": "openai-agents-sdk",
        "provider_profile": "mimo-responses",
        "intent": "map-build-consumer",
        "suite": "map_consumer_fixed_prior",
        "evidence_lane": "world-public-labels",
        "backend": "{default_backend}",
        "world": "{default_world}",
        "parallel_group_id": "{map_build_consumer_parallel_group}",
        "provider_cell_count": "{provider_cell_count}",
        "default_local_concurrency_width": "1",
        "concurrency_policy": "{concurrency_policy}"
      },
      "reason_selected": "MapBuild consumer proof keeps MiMo Responses as one explicit provider cell.",
      "expense": "live-agent",
      "selection_rule_ids": [
        "map_build"
      ],
      "requires": [
        "python_env",
        "openai_agents_package",
        "provider_profile",
        "runtime_map_prior"
      ],
      "profiles": [
        "baseline-refresh"
      ]
    },
    {
      "row_id": "map-build-consumer-openai-agents-sdk-kimi-openai-chat",
      "row_kind": "live_agent_eval",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=map_consumer_fixed_prior",
        "runtime_map_prior={runtime_map_prior}",
        "budget=focused",
        "output_dir={eval_output_root}",
        "stamp=map-build-consumer-openai-agents-sdk-kimi-openai-chat",
        "agent_engine=openai-agents-sdk",
        "provider_profile=kimi-openai-chat",
        "live_execution=run"
      ],
      "axes": {
        "agent_engine": "openai-agents-sdk",
        "provider_profile": "kimi-openai-chat",
        "intent": "map-build-consumer",
        "suite": "map_consumer_fixed_prior",
        "evidence_lane": "world-public-labels",
        "backend": "{default_backend}",
        "world": "{default_world}",
        "parallel_group_id": "{map_build_consumer_parallel_group}",
        "provider_cell_count": "{provider_cell_count}",
        "default_local_concurrency_width": "1",
        "concurrency_policy": "{concurrency_policy}"
      },
      "reason_selected": "MapBuild consumer plan/model-matrix proof needs this provider profile as a first-class availability-or-behavior cell. Run provider cells serially unless ROBOCLAWS_MOLMO_MAX_VISUAL_BACKENDS is explicitly raised and each live row has an isolated stamp/output root.",
      "expense": "live-agent",
      "selection_rule_ids": [
        "map_build"
      ],
      "requires": [
        "python_env",
        "openai_agents_package",
        "provider_profile",
        "runtime_map_prior"
      ],
      "profiles": [
        "baseline-refresh"
      ]
    },
    {
      "row_id": "map-build-consumer-openai-agents-sdk-minimax-responses",
      "row_kind": "live_agent_eval",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=map_consumer_fixed_prior",
        "runtime_map_prior={runtime_map_prior}",
        "budget=focused",
        "output_dir={eval_output_root}",
        "stamp=map-build-consumer-openai-agents-sdk-minimax-responses",
        "agent_engine=openai-agents-sdk",
        "provider_profile=minimax-responses",
        "live_execution=run"
      ],
      "axes": {
        "agent_engine": "openai-agents-sdk",
        "provider_profile": "minimax-responses",
        "intent": "map-build-consumer",
        "suite": "map_consumer_fixed_prior",
        "evidence_lane": "world-public-labels",
        "backend": "{default_backend}",
        "world": "{default_world}",
        "parallel_group_id": "{map_build_consumer_parallel_group}",
        "provider_cell_count": "{provider_cell_count}",
        "default_local_concurrency_width": "1",
        "concurrency_policy": "{concurrency_policy}"
      },
      "reason_selected": "MapBuild consumer plan/model-matrix proof needs this provider profile as a first-class availability-or-behavior cell. Run provider cells serially unless ROBOCLAWS_MOLMO_MAX_VISUAL_BACKENDS is explicitly raised and each live row has an isolated stamp/output root.",
      "expense": "live-agent",
      "selection_rule_ids": [
        "map_build"
      ],
      "requires": [
        "python_env",
        "openai_agents_package",
        "provider_profile",
        "runtime_map_prior"
      ],
      "profiles": [
        "baseline-refresh"
      ]
    },
    {
      "row_id": "open-ended-goals-eval-suite",
      "row_kind": "eval_suite",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=open_ended_goals",
        "runtime_map_prior={runtime_map_prior}",
        "runtime_map_prior={runtime_map_prior}",
        "budget=smoke",
        "output_dir={eval_output_root}",
        "stamp=open-ended-goals-eval-suite",
        "agent_engine=direct-runner"
      ],
      "axes": {
        "intent": "eval-suite",
        "suite": "open_ended_goals"
      },
      "reason_selected": "Open-ended household goal changes need the dedicated no-preset open-task capability suite.",
      "expense": "deterministic",
      "selection_rule_ids": [
        "open_ended"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "scene-sampler-stress-eval-suite",
      "row_kind": "eval_suite",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=scene_sampler_stress",
        "budget=smoke",
        "output_dir={eval_output_root}",
        "stamp=scene-sampler-stress-eval-suite",
        "agent_engine=direct-runner"
      ],
      "axes": {
        "intent": "eval-suite",
        "suite": "scene_sampler_stress"
      },
      "reason_selected": "MolmoSpaces scene-source sampling changes need the static sampler stress projection with partial/blocked source metadata.",
      "expense": "deterministic",
      "selection_rule_ids": [
        "scene_sampler",
        "launch_catalog",
        "eval_harness"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "cleanup-capability-eval-suite",
      "row_kind": "eval_suite",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=cleanup_capability",
        "budget=smoke",
        "output_dir={eval_output_root}",
        "stamp=cleanup-capability-eval-suite",
        "agent_engine=direct-runner"
      ],
      "axes": {
        "intent": "eval-suite",
        "suite": "cleanup_capability"
      },
      "reason_selected": "Cleanup skill or MCP changes need the repeated cleanup capability suite.",
      "expense": "deterministic",
      "selection_rule_ids": [
        "cleanup_skill",
        "mcp_checker"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "long-horizon-tasks-eval-suite",
      "row_kind": "eval_suite",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=long_horizon_tasks",
        "budget=smoke",
        "output_dir={eval_output_root}",
        "stamp=long-horizon-tasks-eval-suite",
        "agent_engine=direct-runner"
      ],
      "axes": {
        "intent": "eval-suite",
        "suite": "long_horizon_tasks"
      },
      "reason_selected": "Long-horizon household task changes need the real MolmoSpaces multi-room manipulation suite.",
      "expense": "local-sim",
      "selection_rule_ids": [
        "long_horizon"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "openai-agents-sdk-open-task-live-eval",
      "row_kind": "live_agent_eval",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=open_ended_goals",
        "runtime_map_prior={runtime_map_prior}",
        "budget=smoke",
        "output_dir={eval_output_root}",
        "stamp=openai-agents-sdk-open-task-live-eval",
        "agent_engine=openai-agents-sdk",
        "provider_profile={agent_sdk_provider}",
        "live_execution=run"
      ],
      "axes": {
        "agent_engine": "openai-agents-sdk",
        "provider_profile": "{agent_sdk_provider}",
        "intent": "open-ended",
        "preset": "",
        "evidence_lane": "world-public-labels",
        "backend": "{default_backend}",
        "world": "{default_world}"
      },
      "reason_selected": "Agent SDK or open-task launch changes need a representative live-agent capability eval row.",
      "expense": "live-agent",
      "selection_rule_ids": [
        "agent_sdk",
        "open_ended"
      ],
      "requires": [
        "python_env",
        "openai_agents_package",
        "provider_profile",
        "runtime_map_prior"
      ],
      "profiles": [
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "openai-agents-sdk-session-live-eval",
      "row_kind": "live_agent_eval",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "session-live",
        "budget=smoke",
        "output_dir={eval_output_root}",
        "stamp=openai-agents-sdk-session-live-eval",
        "agent_engine=openai-agents-sdk",
        "provider_profile={agent_sdk_provider}",
        "live_execution=run"
      ],
      "axes": {
        "agent_engine": "openai-agents-sdk",
        "provider_profile": "{agent_sdk_provider}",
        "intent": "operator-session",
        "preset": "",
        "evidence_lane": "world-public-labels",
        "backend": "{default_backend}",
        "world": "{default_world}"
      },
      "reason_selected": "Operator Session chaining changes need live SDK evidence that Steer is consumed and terminal Next Goal starts a linked child run with sanitized follow-up context.",
      "expense": "live-agent",
      "selection_rule_ids": [
        "agent_sdk",
        "open_ended",
        "launch_catalog",
        "eval_harness"
      ],
      "requires": [
        "python_env",
        "openai_agents_package",
        "provider_profile"
      ],
      "profiles": [
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "openai-agents-sdk-cleanup-live-eval",
      "skill_delivery_cell": "static-full",
      "row_kind": "live_agent_eval",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.evals.cli",
        "suite=cleanup_capability",
        "runtime_map_prior={runtime_map_prior}",
        "budget=smoke",
        "output_dir={eval_output_root}",
        "stamp=openai-agents-sdk-cleanup-live-eval",
        "agent_engine=openai-agents-sdk",
        "provider_profile={agent_sdk_provider}",
        "skill_delivery_cell=static-full",
        "live_execution=run"
      ],
      "axes": {
        "agent_engine": "openai-agents-sdk",
        "provider_profile": "{agent_sdk_provider}",
        "intent": "cleanup",
        "preset": "cleanup",
        "evidence_lane": "world-public-labels",
        "backend": "{default_backend}",
        "world": "{default_world}"
      },
      "reason_selected": "Cleanup skill or MCP changes need a representative SDK live-agent cleanup capability eval row with enough stall budget for long reasoning calls.",
      "expense": "live-agent",
      "selection_rule_ids": [
        "cleanup_skill",
        "mcp_checker"
      ],
      "requires": [
        "python_env",
        "openai_agents_package",
        "provider_profile",
        "runtime_map_prior"
      ],
      "profiles": [
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "openai-agents-sdk-cleanup-no-skill-eval",
      "skill_delivery_cell": "no-skill",
      "row_kind": "live_agent_eval",
      "command": [".venv/bin/python", "-m", "roboclaws.evals.cli", "suite=cleanup_capability", "budget=smoke", "output_dir={eval_output_root}", "stamp=openai-agents-sdk-cleanup-no-skill-eval", "agent_engine=openai-agents-sdk", "provider_profile={agent_sdk_provider}", "skill_delivery_cell=no-skill", "live_execution=run"],
      "axes": {"agent_engine": "openai-agents-sdk", "provider_profile": "{agent_sdk_provider}", "intent": "cleanup", "preset": "cleanup", "evidence_lane": "world-public-labels", "backend": "{default_backend}", "world": "{default_world}"},
      "reason_selected": "Skill-delivery evaluation no-skill control.",
      "expense": "live-agent",
      "selection_rule_ids": ["cleanup_skill", "mcp_checker"],
      "requires": ["python_env", "openai_agents_package", "provider_profile"],
      "profiles": ["baseline-refresh"]
    },
    {
      "row_id": "openai-agents-sdk-cleanup-dynamic-full-eval",
      "skill_delivery_cell": "dynamic-full",
      "row_kind": "live_agent_eval",
      "command": [".venv/bin/python", "-m", "roboclaws.evals.cli", "suite=cleanup_capability", "budget=smoke", "output_dir={eval_output_root}", "stamp=openai-agents-sdk-cleanup-dynamic-full-eval", "agent_engine=openai-agents-sdk", "provider_profile={agent_sdk_provider}", "skill_delivery_cell=dynamic-full", "live_execution=run"],
      "axes": {"agent_engine": "openai-agents-sdk", "provider_profile": "{agent_sdk_provider}", "intent": "cleanup", "preset": "cleanup", "evidence_lane": "world-public-labels", "backend": "{default_backend}", "world": "{default_world}"},
      "reason_selected": "Skill-delivery evaluation byte-identical dynamic callback control.",
      "expense": "live-agent",
      "selection_rule_ids": ["cleanup_skill", "mcp_checker"],
      "requires": ["python_env", "openai_agents_package", "provider_profile"],
      "profiles": ["baseline-refresh"]
    },
    {
      "row_id": "openai-agents-sdk-cleanup-dynamic-routed-eval",
      "skill_delivery_cell": "dynamic-routed",
      "row_kind": "live_agent_eval",
      "command": [".venv/bin/python", "-m", "roboclaws.evals.cli", "suite=cleanup_capability", "budget=smoke", "output_dir={eval_output_root}", "stamp=openai-agents-sdk-cleanup-dynamic-routed-eval", "agent_engine=openai-agents-sdk", "provider_profile={agent_sdk_provider}", "skill_delivery_cell=dynamic-routed", "live_execution=run"],
      "axes": {"agent_engine": "openai-agents-sdk", "provider_profile": "{agent_sdk_provider}", "intent": "cleanup", "preset": "cleanup", "evidence_lane": "world-public-labels", "backend": "{default_backend}", "world": "{default_world}"},
      "reason_selected": "Skill-delivery evaluation frozen shared, intent, and evidence guidance.",
      "expense": "live-agent",
      "selection_rule_ids": ["cleanup_skill", "mcp_checker"],
      "requires": ["python_env", "openai_agents_package", "provider_profile"],
      "profiles": ["baseline-refresh"]
    },
    {
      "row_id": "openai-agents-sdk-cleanup-sandbox-skills-eval",
      "skill_delivery_cell": "sandbox-skills",
      "row_kind": "live_agent_eval",
      "command": [".venv/bin/python", "-m", "roboclaws.evals.cli", "suite=cleanup_capability", "budget=smoke", "output_dir={eval_output_root}", "stamp=openai-agents-sdk-cleanup-sandbox-skills-eval", "agent_engine=openai-agents-sdk", "provider_profile={agent_sdk_provider}", "skill_delivery_cell=sandbox-skills", "live_execution=run"],
      "axes": {"agent_engine": "openai-agents-sdk", "provider_profile": "{agent_sdk_provider}", "intent": "cleanup", "preset": "cleanup", "evidence_lane": "world-public-labels", "backend": "{default_backend}", "world": "{default_world}"},
      "reason_selected": "Exploratory official SandboxAgent plus Skills readiness and isolation row.",
      "expense": "live-agent",
      "selection_rule_ids": ["cleanup_skill", "mcp_checker"],
      "requires": ["python_env", "openai_agents_package", "provider_profile", "sandbox_skills"],
      "profiles": ["baseline-refresh"]
    },
    {
      "row_id": "planner-proof-dry-run-product",
      "row_kind": "product_run",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.cli.main",
        "run",
        "surface",
        "surface=planner-proof",
        "world=planner-proof/default",
        "backend={default_backend}",
        "intent=planner-proof",
        "agent_engine=direct-runner",
        "mode=dry-run",
        "output_dir={row_dir}/planner-proof-dry-run-product/run"
      ],
      "axes": {
        "agent_engine": "direct-runner",
        "intent": "planner-proof",
        "backend": "{default_backend}",
        "world": "planner-proof/default"
      },
      "reason_selected": "Planner-proof changes need the current public planner-proof dry-run proof row; lower-level harness recipes remain private mechanics.",
      "expense": "local-sim",
      "selection_rule_ids": [
        "planner_proof"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "direct-camera-grounded-grounding-dino",
      "row_kind": "product_run",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.cli.main",
        "run",
        "surface",
        "surface=household-world",
        "world={default_world}",
        "backend={default_backend}",
        "preset=cleanup",
        "agent_engine=direct-runner",
        "evidence_lane=camera-grounded-labels",
        "seed={default_seed}",
        "scenario_setup=relocate-cleanup-related-objects",
        "relocation_count=5",
        "output_dir={row_dir}/direct-camera-grounded-grounding-dino/run",
        "camera_labeler=grounding-dino"
      ],
      "axes": {
        "agent_engine": "direct-runner",
        "intent": "cleanup",
        "preset": "cleanup",
        "evidence_lane": "camera-grounded-labels",
        "camera_labeler": "grounding-dino",
        "backend": "{default_backend}",
        "world": "{default_world}"
      },
      "reason_selected": "Visual-grounding changes need a real Grounding DINO camera-labeler gate.",
      "expense": "dino",
      "selection_rule_ids": [
        "visual_grounding"
      ],
      "requires": [
        "python_env",
        "dino_sidecar"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "direct-map-build-grounding-dino",
      "row_kind": "product_run",
      "scene_scope": "selected",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.cli.main",
        "run",
        "surface",
        "surface=household-world",
        "world={default_world}",
        "backend={default_backend}",
        "preset=map-build",
        "agent_engine=direct-runner",
        "evidence_lane=camera-grounded-labels",
        "camera_labeler=grounding-dino",
        "seed={default_seed}",
        "scenario_setup=baseline",
        "output_dir={row_dir}/direct-map-build-grounding-dino/run"
      ],
      "axes": {
        "agent_engine": "direct-runner",
        "intent": "map-build",
        "preset": "map-build",
        "evidence_lane": "camera-grounded-labels",
        "camera_labeler": "grounding-dino",
        "backend": "{default_backend}",
        "world": "{default_world}"
      },
      "reason_selected": "Map-build product proof should exercise the deployable Grounding-DINO camera-labeler route.",
      "expense": "dino",
      "selection_rule_ids": [
        "map_build",
        "visual_grounding"
      ],
      "requires": [
        "python_env",
        "dino_sidecar"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "direct-camera-raw-fpv",
      "row_kind": "product_run",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.cli.main",
        "run",
        "surface",
        "surface=household-world",
        "world={default_world}",
        "backend={default_backend}",
        "preset=cleanup",
        "agent_engine=direct-runner",
        "evidence_lane=camera-raw-fpv",
        "seed={default_seed}",
        "scenario_setup=relocate-cleanup-related-objects",
        "relocation_count=5",
        "output_dir={row_dir}/direct-camera-raw-fpv/run"
      ],
      "axes": {
        "agent_engine": "direct-runner",
        "intent": "cleanup",
        "preset": "cleanup",
        "evidence_lane": "camera-raw-fpv",
        "backend": "{default_backend}",
        "world": "{default_world}"
      },
      "reason_selected": "RAW-FPV changes need a direct RAW-FPV product gate.",
      "expense": "local-sim",
      "selection_rule_ids": [
        "raw_fpv"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "direct-map-build-world-public",
      "row_kind": "product_run",
      "scene_scope": "selected",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.cli.main",
        "run",
        "surface",
        "surface=household-world",
        "world={default_world}",
        "backend={default_backend}",
        "preset=map-build",
        "agent_engine=direct-runner",
        "evidence_lane=world-public-labels",
        "seed={default_seed}",
        "scenario_setup=baseline",
        "output_dir={row_dir}/direct-map-build-world-public/run"
      ],
      "axes": {
        "agent_engine": "direct-runner",
        "intent": "map-build",
        "preset": "map-build",
        "evidence_lane": "world-public-labels",
        "backend": "{default_backend}",
        "world": "{default_world}"
      },
      "reason_selected": "Map-build and actionability changes need a map-build lane gate.",
      "expense": "local-sim",
      "selection_rule_ids": [
        "map_build"
      ],
      "requires": [
        "python_env"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    },
    {
      "row_id": "direct-cleanup-runtime-prior-consumer",
      "row_kind": "product_run",
      "command": [
        ".venv/bin/python",
        "-m",
        "roboclaws.cli.main",
        "run",
        "surface",
        "surface=household-world",
        "world={default_world}",
        "backend={default_backend}",
        "preset=cleanup",
        "agent_engine=direct-runner",
        "evidence_lane=world-public-labels",
        "seed={default_seed}",
        "scenario_setup=relocate-cleanup-related-objects",
        "relocation_count=5",
        "output_dir={row_dir}/direct-cleanup-runtime-prior-consumer/run",
        "runtime_map_prior=${direct-map-build-world-public:runtime_metric_map.json}"
      ],
      "axes": {
        "agent_engine": "direct-runner",
        "intent": "cleanup",
        "preset": "cleanup",
        "evidence_lane": "world-public-labels",
        "runtime_map_prior": "required",
        "backend": "{default_backend}",
        "world": "{default_world}"
      },
      "reason_selected": "Map-build changes need a cleanup consumer gate with the runtime-map prior.",
      "expense": "local-sim",
      "selection_rule_ids": [
        "map_build"
      ],
      "requires": [
        "python_env",
        "runtime_map_prior"
      ],
      "depends_on": [
        "direct-map-build-world-public"
      ],
      "profiles": [
        "baseline-core",
        "baseline-live-default",
        "baseline-refresh"
      ]
    }
  ]
}
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 322kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 322kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 322kUpdated Claude CodeCodexVerifiedInstall