Skip to content

amazon-science/ammo / ai_cli_session/.codex/skills/ammo

Profile and optimize GPU kernels for vLLM inference on NVIDIA GPUs. Use when targeting a specific model, hardware, dtype, parallelism, and workload to improve latency.

FileKindSize
RUNTIME.mdmarkdown4.9 KB
SKILL.mdskill md18.5 KB
agents/ammo-auditor.mdmarkdown9.1 KB
agents/ammo-champion.mdmarkdown9.6 KB
agents/ammo-delegate.mdmarkdown3.2 KB
agents/ammo-implementer.mdmarkdown13.8 KB
agents/ammo-investigator.mdmarkdown4.0 KB
agents/ammo-report.mdmarkdown2.9 KB
agents/ammo-researcher.mdmarkdown10.3 KB
agents/ammo-resolver.mdmarkdown4.4 KB
agents/ammo-transcript-monitor.mdmarkdown7.8 KB
agents/openai.yamlexecutablescript313 B
data/ATTRIBUTION.mdmarkdown1.4 KB
data/gsm8k_full.jsonexecutablescript735.5 KB
data/gsm8k_subset.jsonexecutablescript116.8 KB
eval/EVAL-SKILL.mdexecutablescript21.8 KB
eval/agents/transcript_grader.mdexecutablescript18.0 KB
eval/causal/__init__.pyexecutablescript137 B
eval/causal/agents/causal_analyzer.mdexecutablescript10.5 KB
eval/causal/build_dag.pyexecutablescript21.6 KB
eval/causal/diff_versions.pyexecutablescript38.5 KB
eval/causal/extract_events.pyexecutablescript36.3 KB
eval/causal/generate_postmortem.pyexecutablescript17.0 KB
eval/causal/score_nodes.pyexecutablescript21.5 KB
eval/causal/templates/narrative_template.mdexecutablescript472 B
eval/causal/templates/viz_template.htmlexecutablescript9.5 KB
eval/causal/tests/__init__.pyexecutablescript151 B
eval/causal/tests/conftest.pyexecutablescript448 B
eval/causal/tests/fixtures/sample_session.jsonlexecutablescript10.4 KB
eval/causal/tests/fixtures/sample_session_data.jsonexecutablescript3.4 KB
eval/causal/tests/fixtures/sample_snapshot.jsonexecutablescript2.5 KB
eval/causal/tests/test_build_dag.pyexecutablescript2.7 KB
eval/causal/tests/test_diff_versions.pyexecutablescript13.5 KB
eval/causal/tests/test_extract_events.pyexecutablescript3.7 KB
eval/causal/tests/test_generate_postmortem.pyexecutablescript3.1 KB
eval/causal/tests/test_score_nodes.pyexecutablescript3.3 KB
eval/scripts/aggregate_versions.pyexecutablescript15.2 KB
eval/scripts/archive_run.pyexecutablescript9.1 KB
eval/scripts/codex_session.pyexecutablescript10.6 KB
eval/scripts/generate_dashboard.pyexecutablescript5.3 KB
eval/scripts/parse_artifacts.pyexecutablescript48.1 KB
eval/scripts/parse_session_logs.pyexecutablescript51.3 KB
eval/scripts/rescore_archived.pyexecutablescript5.5 KB
eval/scripts/score_campaign.pyexecutablescript43.8 KB
eval/scripts/snapshot_changes.pyexecutablescript11.1 KB
eval/scripts/verify_citations.pyexecutablescript8.8 KB
eval/tests/test_dashboard_bugs.pyexecutablescript8.8 KB
eval/tests/test_parse_session_logs.pyexecutablescript3.9 KB
eval/tests/test_parse_session_logs_round_inference.pyexecutablescript11.0 KB
eval/tests/test_score_campaign_diluted.pyexecutablescript9.4 KB
eval/viewer/dashboard_template.htmlexecutablescript15.1 KB
instrumentation/sitecustomize.pyexecutablescript3.6 KB
orchestration/audit-protocol.mdmarkdown9.3 KB
orchestration/debate-protocol.mdmarkdown11.2 KB
orchestration/integration-logic.mdmarkdown10.3 KB
orchestration/parallel-tracks.mdmarkdown8.9 KB
references/artifact-layout.mdexecutablescript22.4 KB
references/audit-invariants.mdmarkdown10.6 KB
references/champion-common-patterns.mdmarkdown3.4 KB
references/crossover-probing.mdexecutablescript4.7 KB
references/cudagraph-safety.mdmarkdown2.7 KB
references/debate-rules.mdmarkdown4.7 KB
references/debate-scoring-rubric.mdexecutablescript16.7 KB
references/e2e-delta-math.mdexecutablescript17.9 KB
references/e2e-latency-guide.mdexecutablescript9.6 KB
references/fusion-feasibility-heuristics.mdmarkdown5.6 KB
references/gpu-pool.mdexecutablescript5.4 KB
references/impl-track-rules.mdexecutablescript13.8 KB
references/nsys-profiling-guide.mdmarkdown9.8 KB
references/optimization-categories.mdexecutablescript11.0 KB
references/shutdown-protocol.mdexecutablescript3.1 KB
references/technology-selection.mdmarkdown5.0 KB
references/torch-compile-contract.mdmarkdown13.8 KB
references/validation-defaults.mdexecutablescript34.2 KB
references/workload-invocation.mdexecutablescript4.9 KB
references/writing-style.mdexecutablescript6.9 KB
scripts/ammo_state.pyexecutablescript142.0 KB
scripts/artifact_layout.jsonconfig16.3 KB
scripts/check_projection_accuracy.pyexecutablescript11.7 KB
scripts/create_evidence_template.pyexecutablescript9.7 KB
scripts/create_worktree_with_build.shexecutablescript10.2 KB
scripts/generate_validation_report.pyexecutablescript35.7 KB
scripts/gpu_reservation.pyexecutablescript27.1 KB
scripts/hook_cmd_classify.pyscript13.3 KB
scripts/mine_trace.pyscript29.4 KB
scripts/monitor_queue_ack.pyexecutablescript6.1 KB
scripts/ncu_sanity.shexecutablescript5.2 KB
scripts/ncu_sanity_driver.pyexecutablescript8.5 KB
scripts/new_target.pyexecutablescript22.0 KB
scripts/preflight_vllm_api.pyexecutablescript12.3 KB
scripts/reconcile_track_state.pyexecutablescript15.8 KB
scripts/remove_worktree_cleanup.shexecutablescript1.1 KB
scripts/resolve_codex_transcript.pyexecutablescript3.6 KB
scripts/run_vllm_bench_latency_sweep.pyexecutablescript249.6 KB
scripts/transcript_filter.pyexecutablescript38.4 KB
scripts/transitions.jsonexecutablescript1.5 KB
scripts/verify_validation_gates.pyexecutablescript88.8 KB
tests/agents/test-champion.mdexecutablescript20.4 KB
tests/agents/test-impl-champion.mdexecutablescript14.3 KB
tests/agents/test-implementer.mdexecutablescript15.8 KB
tests/agents/test-orchestrator.mdexecutablescript33.8 KB
tests/agents/test-researcher.mdexecutablescript21.0 KB
tests/agents/test-transcript-monitor.mdexecutablescript13.4 KB
tests/fixtures/diluted_pass/legacy_no_phase.golden.mdmarkdown1.1 KB
tests/fixtures/diluted_pass/legacy_no_phase.golden.summary.jsonconfig582 B
tests/fixtures/diluted_pass/legacy_no_phase.jsonconfig400 B
tests/fixtures/diluted_pass/legacy_scalar_phase.golden.mdmarkdown2.0 KB
tests/fixtures/diluted_pass/legacy_scalar_phase.golden.summary.jsonconfig1.0 KB
tests/fixtures/diluted_pass/legacy_scalar_phase.jsonconfig626 B
tests/fixtures/mining/real_campaign_bottleneck_analysis.mdmarkdown20.7 KB
tests/run_all.shexecutablescript1.6 KB
tests/scripts/test_transcript_filter.pyexecutablescript41.1 KB
tests/test_agent_runtime_contracts.pyexecutablescript1.8 KB
tests/test_ammo_state.pyexecutablescript71.7 KB
tests/test_artifact_layout_manifest.pyscript11.6 KB
tests/test_audit_started.pyscript18.2 KB
tests/test_correctness_comparator.pyscript12.6 KB
tests/test_diluted_pass.pyscript24.4 KB
tests/test_drift_compat.pyscript5.1 KB
tests/test_evidence_freshness.pyexecutablescript5.6 KB
tests/test_fail_closed.pyexecutablescript14.7 KB
tests/test_gpu_reservation.pyexecutablescript24.8 KB
tests/test_gpu_reservation_session_id.pyexecutablescript11.2 KB
tests/test_guidance_semantic_contract.pyscript24.9 KB
tests/test_hook_cmd_classify_parity.pyscript11.7 KB
tests/test_hook_semantics.pyexecutablescript95.3 KB
tests/test_mine_trace.pyscript24.2 KB
tests/test_mining_invalidated_schema.pyexecutablescript9.4 KB
tests/test_new_target_dp_ep.pyexecutablescript7.4 KB
tests/test_new_target_v2_scaffold.pyexecutablescript13.1 KB
tests/test_parse_artifacts.pyexecutablescript3.0 KB
tests/test_policy_convergence.pyscript797 B
tests/test_preflight_vllm_api.pyscript13.3 KB
tests/test_projection_accuracy_v2.pyexecutablescript4.1 KB
tests/test_reconcile_track_state.pyexecutablescript18.2 KB
tests/test_resolve_codex_transcript.pyexecutablescript3.1 KB
tests/test_state_schema_semantics.pyexecutablescript7.6 KB
tests/test_status_set_sync.pyexecutablescript9.1 KB
tests/test_sweep_baseline_from_arm.pyscript11.0 KB
tests/test_sweep_decode_capture.pyexecutablescript31.8 KB
tests/test_sweep_dp.pyexecutablescript45.4 KB
tests/test_sweep_fresh_cache.pyscript5.5 KB
tests/test_sweep_profiler_guard.pyscript22.6 KB
tests/test_sweep_real_prompts.pyscript20.1 KB
tests/test_sweep_refactor.pyexecutablescript37.6 KB
tests/test_sweep_round_slot.pyexecutablescript7.4 KB
tests/test_validation_report_phase_triage.pyexecutablescript10.0 KB
tests/test_validation_report_v2.pyexecutablescript6.2 KB
tests/test_verify_validation_gates.pyexecutablescript52.9 KB
tests/test_workload_matrix.pyexecutablescript19.3 KB

tests/test_sweep_profiler_guard.py

View on GitHub
#!/usr/bin/env python3
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: CC-BY-NC-4.0
"""Tests for the profiler-baseline contamination guard in run_vllm_bench_latency_sweep.py.

Verifies:
  - --slot baseline + --nsys-profile = hard SystemExit
  - --slot baseline + --torch-profile = hard SystemExit
  - --slot baseline + both profiling flags = hard SystemExit (nsys takes priority in msg)
  - --slot profiling + --nsys-profile = allowed
  - --slot profiling + --torch-profile = allowed
  - --slot opt/{op_id} + --nsys-profile = allowed (not baseline)
  - --slot integration + --torch-profile = allowed (not baseline)
  - No --slot + --nsys-profile = allowed (legacy mode or child)
  - Error message includes guidance for two-invocation pattern
  - e2e_is_reduced = True when nsys_profile or torch_profile is set
  - e2e_is_reduced = False when neither profiling flag is set
  - runner.json status dict includes nsys_profile and torch_profile fields
  - profiling slot resolves to rounds/{N}/sweeps/profiling/
  - _v2_profiling_dir routes traces independent of slot value
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path
from unittest import mock

import pytest

_SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "scripts"
if str(_SCRIPTS_DIR) not in sys.path:
    sys.path.insert(0, str(_SCRIPTS_DIR))


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------

def _make_v2_artifact(tmp_path: Path, current_round: int = 1) -> Path:
    """Create a v2-layout artifact dir with rounds/{current_round}/ scaffold + state.json."""
    artifact = tmp_path / "artifact"
    (artifact / "rounds" / str(current_round) / "sweeps").mkdir(parents=True)
    state = {
        "schema_version": "4.1",
        "campaign": {"current_round": current_round, "rounds": []},
    }
    (artifact / "state.json").write_text(json.dumps(state), encoding="utf-8")
    target = {
        "artifact_dir": str(artifact),
        "target": {"model_id": "x", "dtype": "fp16", "tp": 1, "ep": 1, "max_model_len": 4096},
        "workload": {"input_len": 64, "output_len": 512, "batch_sizes": [1], "num_iters": 1},
        "bench": {
            "runner": "vllm_bench_latency",
            "vllm_cmd": "vllm",
            "extra_args": [],
            "baseline_extra_args": [],
            "opt_extra_args": [],
            "baseline_env": {},
            "opt_env": {},
            "baseline_label": "baseline",
            "opt_label": "opt",
        },
    }
    (artifact / "target.json").write_text(json.dumps(target), encoding="utf-8")
    return artifact


def _make_args(**kwargs) -> argparse.Namespace:
    """Create a minimal argparse.Namespace mimicking sweep script args."""
    defaults = {
        "slot": None,
        "round": None,
        "nsys_profile": False,
        "torch_profile": False,
        "labels": "baseline",
        "verify_correctness": False,
        "_out_root": None,
        "out_name": "e2e_latency",
    }
    defaults.update(kwargs)
    return argparse.Namespace(**defaults)


# ---------------------------------------------------------------------------
# Profiler-baseline guard tests
# ---------------------------------------------------------------------------

class TestProfilerBaselineGuard:
    """--slot baseline + any profiling flag = hard SystemExit."""

    def _run_guard(self, args: argparse.Namespace):
        """Execute only the guard logic extracted from the sweep script."""
        slot = getattr(args, "slot", None)
        if slot == "baseline" and (args.nsys_profile or args.torch_profile):
            profiler_flag = "--nsys-profile" if args.nsys_profile else "--torch-profile"
            raise SystemExit(
                f"ERROR: {profiler_flag} cannot be combined with --slot baseline. "
                "Profiling adds overhead that contaminates E2E timing (nsys: 10-30% "
                "process-wide, torch: CUPTI subscription). The baseline slot is the "
                "authoritative timing source for speedup calculations.\n\n"
                "Correct Stage 1 workflow (two invocations):\n"
                "  1. Clean E2E:  --round N --slot baseline --labels baseline "
                "--capture-golden-refs\n"
                "  2. Profiling:  --round N --slot profiling --nsys-profile "
                "(or --torch-profile)\n"
                "     (traces route to rounds/{N}/profiling/ regardless of slot)\n\n"
                "See references/e2e-latency-guide.md for details."
            )

    def test_baseline_nsys_profile_raises(self):
        args = _make_args(slot="baseline", nsys_profile=True)
        with pytest.raises(SystemExit, match="--nsys-profile cannot be combined with --slot baseline"):
            self._run_guard(args)

    def test_baseline_torch_profile_raises(self):
        args = _make_args(slot="baseline", torch_profile=True)
        with pytest.raises(SystemExit, match="--torch-profile cannot be combined with --slot baseline"):
            self._run_guard(args)

    def test_baseline_both_profilers_raises_nsys_priority(self):
        """When both flags set, error message names nsys (checked first)."""
        args = _make_args(slot="baseline", nsys_profile=True, torch_profile=True)
        with pytest.raises(SystemExit, match="--nsys-profile"):
            self._run_guard(args)

    def test_profiling_slot_nsys_allowed(self):
        args = _make_args(slot="profiling", nsys_profile=True)
        self._run_guard(args)  # Should not raise

    def test_profiling_slot_torch_allowed(self):
        args = _make_args(slot="profiling", torch_profile=True)
        self._run_guard(args)  # Should not raise

    def test_opt_slot_nsys_allowed(self):
        args = _make_args(slot="opt/op001", nsys_profile=True)
        self._run_guard(args)  # Should not raise

    def test_integration_slot_torch_allowed(self):
        args = _make_args(slot="integration", torch_profile=True)
        self._run_guard(args)  # Should not raise

    def test_no_slot_nsys_allowed(self):
        """Legacy/child mode: no --slot, profiling is fine."""
        args = _make_args(slot=None, nsys_profile=True)
        self._run_guard(args)  # Should not raise

    def test_baseline_no_profiling_allowed(self):
        """--slot baseline without profiling flags is fine."""
        args = _make_args(slot="baseline", nsys_profile=False, torch_profile=False)
        self._run_guard(args)  # Should not raise

    def test_error_message_contains_two_invocation_guidance(self):
        args = _make_args(slot="baseline", nsys_profile=True)
        with pytest.raises(SystemExit) as exc_info:
            self._run_guard(args)
        msg = str(exc_info.value)
        assert "two invocations" in msg.lower() or "Clean E2E" in msg
        assert "--slot profiling" in msg
        assert "--slot baseline" in msg
        assert "e2e-latency-guide.md" in msg

    def test_golden_capture_slot_nsys_allowed(self):
        args = _make_args(slot="golden_capture", nsys_profile=True)
        self._run_guard(args)  # Should not raise


# ---------------------------------------------------------------------------
# Guard integration: verify the actual script enforces it via parse_args path
# ---------------------------------------------------------------------------

class TestProfilerGuardIntegration:
    """Run the sweep script as subprocess with forbidden flag combos, verify exit code."""

    @pytest.fixture
    def artifact_dir(self, tmp_path):
        return _make_v2_artifact(tmp_path)

    def _run_sweep(self, artifact_dir: Path, extra_args: list[str]) -> tuple[int, str]:
        """Run the sweep script and return (exit_code, stderr)."""
        import subprocess
        cmd = [
            sys.executable, str(_SCRIPTS_DIR / "run_vllm_bench_latency_sweep.py"),
            "--artifact-dir", str(artifact_dir),
            "--round", "1",
            *extra_args,
        ]
        result = subprocess.run(
            cmd, capture_output=True, text=True, timeout=30,
            env={**__import__("os").environ, "CUDA_VISIBLE_DEVICES": ""},
        )
        return result.returncode, result.stderr + result.stdout

    def test_subprocess_baseline_nsys_exits_nonzero(self, artifact_dir):
        code, output = self._run_sweep(artifact_dir, [
            "--slot", "baseline", "--nsys-profile", "--labels", "baseline",
        ])
        assert code != 0, f"Expected nonzero exit, got {code}. Output:\n{output}"
        assert "--nsys-profile cannot be combined with --slot baseline" in output

    def test_subprocess_baseline_torch_exits_nonzero(self, artifact_dir):
        code, output = self._run_sweep(artifact_dir, [
            "--slot", "baseline", "--torch-profile", "--labels", "baseline",
        ])
        assert code != 0, f"Expected nonzero exit, got {code}. Output:\n{output}"
        assert "--torch-profile cannot be combined with --slot baseline" in output

    def test_subprocess_profiling_slot_nsys_passes_guard(self, artifact_dir):
        """--slot profiling + --nsys-profile passes the guard (may fail later for missing nsys binary, that's OK)."""
        code, output = self._run_sweep(artifact_dir, [
            "--slot", "profiling", "--nsys-profile", "--labels", "baseline",
        ])
        # Should NOT fail with the baseline guard error
        assert "cannot be combined with --slot baseline" not in output
        # May fail with "nsys not found" — that's expected in test env without nsys
        # The point is it passed the baseline guard check


# ---------------------------------------------------------------------------
# e2e_is_reduced logic
# ---------------------------------------------------------------------------

class TestE2eIsReduced:
    """e2e_is_reduced should be True when ANY profiling flag is active."""

    def test_nsys_only_is_reduced(self):
        args = _make_args(nsys_profile=True, torch_profile=False)
        e2e_is_reduced = bool(args.nsys_profile or args.torch_profile)
        assert e2e_is_reduced is True

    def test_torch_only_is_reduced(self):
        args = _make_args(nsys_profile=False, torch_profile=True)
        e2e_is_reduced = bool(args.nsys_profile or args.torch_profile)
        assert e2e_is_reduced is True

    def test_both_is_reduced(self):
        args = _make_args(nsys_profile=True, torch_profile=True)
        e2e_is_reduced = bool(args.nsys_profile or args.torch_profile)
        assert e2e_is_reduced is True

    def test_neither_is_not_reduced(self):
        args = _make_args(nsys_profile=False, torch_profile=False)
        e2e_is_reduced = bool(args.nsys_profile or args.torch_profile)
        assert e2e_is_reduced is False

    def test_full_benchmark_label_false_when_reduced(self):
        """Sidecar label full_benchmark should be 'false' when profiling active."""
        args = _make_args(nsys_profile=True)
        e2e_is_reduced = bool(args.nsys_profile or args.torch_profile)
        label_value = "false" if e2e_is_reduced else "true"
        assert label_value == "false"

    def test_full_benchmark_label_true_when_clean(self):
        """Sidecar label full_benchmark should be 'true' when no profiling."""
        args = _make_args(nsys_profile=False, torch_profile=False)
        e2e_is_reduced = bool(args.nsys_profile or args.torch_profile)
        label_value = "false" if e2e_is_reduced else "true"
        assert label_value == "true"


# ---------------------------------------------------------------------------
# Runner.json status dict fields
# ---------------------------------------------------------------------------

class TestRunnerJsonProfilingFields:
    """runner.json status dict must include nsys_profile and torch_profile booleans."""

    def _build_status(self, args: argparse.Namespace) -> dict:
        """Simulate status dict construction from sweep script L2154-2163."""
        from datetime import datetime, timezone
        start = datetime.now(timezone.utc)
        return {
            "ok": False,
            "label": "baseline",
            "batch_size": 1,
            "input_len": 64,
            "output_len": 512,
            "start_time": start.isoformat(),
            "nsys_profile": bool(args.nsys_profile),
            "torch_profile": bool(args.torch_profile),
        }

    def test_nsys_true_in_status(self):
        args = _make_args(nsys_profile=True, torch_profile=False)
        status = self._build_status(args)
        assert status["nsys_profile"] is True
        assert status["torch_profile"] is False

    def test_torch_true_in_status(self):
        args = _make_args(nsys_profile=False, torch_profile=True)
        status = self._build_status(args)
        assert status["nsys_profile"] is False
        assert status["torch_profile"] is True

    def test_both_false_in_status(self):
        args = _make_args(nsys_profile=False, torch_profile=False)
        status = self._build_status(args)
        assert status["nsys_profile"] is False
        assert status["torch_profile"] is False

    def test_both_true_in_status(self):
        args = _make_args(nsys_profile=True, torch_profile=True)
        status = self._build_status(args)
        assert status["nsys_profile"] is True
        assert status["torch_profile"] is True

    def test_fields_are_booleans_not_truthy(self):
        """Ensure we write proper booleans, not truthy ints or strings."""
        args = _make_args(nsys_profile=True, torch_profile=False)
        status = self._build_status(args)
        assert isinstance(status["nsys_profile"], bool)
        assert isinstance(status["torch_profile"], bool)

    def test_audit_invariant_14_detects_contamination(self):
        """Simulate auditor sub-check: baseline runner.json must not have nsys_profile: true."""
        args_clean = _make_args(nsys_profile=False, torch_profile=False)
        args_contaminated = _make_args(nsys_profile=True, torch_profile=False)

        status_clean = self._build_status(args_clean)
        status_contaminated = self._build_status(args_contaminated)

        # Auditor check: fail if any runner.json in baseline has profiling = true
        assert not (status_clean["nsys_profile"] or status_clean["torch_profile"]), \
            "Clean baseline should pass audit"
        assert (status_contaminated["nsys_profile"] or status_contaminated["torch_profile"]), \
            "Contaminated baseline should fail audit"


# ---------------------------------------------------------------------------
# Slot resolution: profiling slot
# ---------------------------------------------------------------------------

class TestProfilingSlotResolution:
    """The 'profiling' slot resolves like any other named slot."""

    def test_profiling_slot_resolves_to_sweeps_profiling(self, tmp_path):
        from run_vllm_bench_latency_sweep import _resolve_sweep_out_name
        artifact = _make_v2_artifact(tmp_path, current_round=1)
        ns = argparse.Namespace(round=1, slot="profiling", out_name="e2e_latency", _out_root=None)
        result = _resolve_sweep_out_name(ns, artifact)
        assert result == "rounds/1/sweeps/profiling"

    def test_profiling_slot_round2_resolves(self, tmp_path):
        from run_vllm_bench_latency_sweep import _resolve_sweep_out_name
        artifact = _make_v2_artifact(tmp_path, current_round=2)
        (artifact / "rounds" / "2" / "sweeps").mkdir(parents=True, exist_ok=True)
        ns = argparse.Namespace(round=2, slot="profiling", out_name="e2e_latency", _out_root=None)
        result = _resolve_sweep_out_name(ns, artifact)
        assert result == "rounds/2/sweeps/profiling"

    def test_profiling_slot_state_json_fallback(self, tmp_path):
        """No --round given, --slot profiling: reads state.json current_round."""
        from run_vllm_bench_latency_sweep import _resolve_sweep_out_name
        artifact = _make_v2_artifact(tmp_path, current_round=3)
        ns = argparse.Namespace(round=None, slot="profiling", out_name="e2e_latency", _out_root=None)
        result = _resolve_sweep_out_name(ns, artifact)
        assert result == "rounds/3/sweeps/profiling"


# ---------------------------------------------------------------------------
# _v2_profiling_dir: slot-independent trace routing
# ---------------------------------------------------------------------------

class TestV2ProfilingDir:
    """_v2_profiling_dir routes traces to rounds/{N}/profiling/{kind}/ regardless of slot."""

    def test_baseline_slot_routes_to_profiling_nsys(self, tmp_path):
        from run_vllm_bench_latency_sweep import _v2_profiling_dir
        artifact = _make_v2_artifact(tmp_path, current_round=1)
        out_root = artifact / "rounds" / "1" / "sweeps" / "baseline"
        out_root.mkdir(parents=True, exist_ok=True)
        result = _v2_profiling_dir(out_root, "nsys")
        assert result == artifact / "rounds" / "1" / "profiling" / "nsys"

    def test_profiling_slot_routes_to_profiling_nsys(self, tmp_path):
        from run_vllm_bench_latency_sweep import _v2_profiling_dir
        artifact = _make_v2_artifact(tmp_path, current_round=1)
        out_root = artifact / "rounds" / "1" / "sweeps" / "profiling"
        out_root.mkdir(parents=True, exist_ok=True)
        result = _v2_profiling_dir(out_root, "nsys")
        assert result == artifact / "rounds" / "1" / "profiling" / "nsys"

    def test_opt_slot_routes_to_profiling_torch(self, tmp_path):
        from run_vllm_bench_latency_sweep import _v2_profiling_dir
        artifact = _make_v2_artifact(tmp_path, current_round=2)
        out_root = artifact / "rounds" / "2" / "sweeps" / "opt" / "op001"
        out_root.mkdir(parents=True, exist_ok=True)
        result = _v2_profiling_dir(out_root, "torch_profile")
        assert result == artifact / "rounds" / "2" / "profiling" / "torch_profile"

    def test_all_slots_converge_same_profiling_dir(self, tmp_path):
        """All slot values produce the same profiling sibling for a given round."""
        from run_vllm_bench_latency_sweep import _v2_profiling_dir
        artifact = _make_v2_artifact(tmp_path, current_round=1)

        results = []
        for slot_path in ["baseline", "profiling", "opt/op001", "integration", "golden_capture"]:
            out_root = artifact / "rounds" / "1" / "sweeps" / slot_path
            out_root.mkdir(parents=True, exist_ok=True)
            results.append(_v2_profiling_dir(out_root, "nsys"))

        # All should resolve to the same path
        expected = artifact / "rounds" / "1" / "profiling" / "nsys"
        for r in results:
            assert r == expected, f"Expected {expected}, got {r}"


# ---------------------------------------------------------------------------
# Archive behavior with profiling slot
# ---------------------------------------------------------------------------

class TestArchiveProfilingSlot:
    """Archive semantics work correctly for the two-invocation pattern."""

    def test_second_invocation_archives_first(self, tmp_path):
        """Profiling run populates slot, then clean run archives it."""
        from run_vllm_bench_latency_sweep import _prepare_out_root
        artifact = _make_v2_artifact(tmp_path, current_round=1)

        # Simulate Invocation 1 (profiling) writing to opt/op001
        opt_slot = artifact / "rounds" / "1" / "sweeps" / "opt" / "op001"
        opt_slot.mkdir(parents=True, exist_ok=True)
        contaminated = opt_slot / "e2e_latency_results.json"
        contaminated.write_text('{"contaminated": true}', encoding="utf-8")

        # Simulate Invocation 2 (clean): _prepare_out_root archives existing
        out_root = _prepare_out_root(
            artifact_dir=artifact,
            out_name="rounds/1/sweeps/opt/op001",
            overwrite=False,
        )

        # Contaminated file should be archived
        assert not contaminated.exists()
        assert out_root == opt_slot
        assert out_root.is_dir() and not any(out_root.iterdir())

        # Should be in _archive
        archive_dir = artifact / "rounds" / "1" / "_archive"
        assert archive_dir.is_dir()
        archived_dirs = list(archive_dir.glob("opt_op001_*"))
        assert len(archived_dirs) == 1
        assert (archived_dirs[0] / "e2e_latency_results.json").exists()

    def test_profiling_traces_survive_archive(self, tmp_path):
        """Traces in rounds/{N}/profiling/nsys/ are NOT affected by sweep slot archiving."""
        from run_vllm_bench_latency_sweep import _prepare_out_root
        artifact = _make_v2_artifact(tmp_path, current_round=1)

        # Simulate profiling traces written by Invocation 1
        profiling_dir = artifact / "rounds" / "1" / "profiling" / "nsys"
        profiling_dir.mkdir(parents=True, exist_ok=True)
        trace = profiling_dir / "opt_bs1.nsys-rep"
        trace.write_text("fake trace data", encoding="utf-8")

        # Simulate sweep slot populated by Invocation 1
        opt_slot = artifact / "rounds" / "1" / "sweeps" / "opt" / "op001"
        opt_slot.mkdir(parents=True, exist_ok=True)
        (opt_slot / "e2e_latency_results.json").write_text("{}", encoding="utf-8")

        # Invocation 2 archives the sweep slot
        _prepare_out_root(
            artifact_dir=artifact,
            out_name="rounds/1/sweeps/opt/op001",
            overwrite=False,
        )

        # Traces must still exist (they're siblings, not children of the sweep slot)
        assert trace.exists(), "nsys trace should survive sweep slot archiving"
        assert trace.read_text() == "fake trace data"


# ---------------------------------------------------------------------------
# Slot enumeration in error messages
# ---------------------------------------------------------------------------

class TestSlotEnumeration:
    """Error messages for missing --slot include 'profiling' in the valid list."""

    def test_round_only_error_mentions_profiling(self, tmp_path):
        from run_vllm_bench_latency_sweep import _resolve_sweep_out_name
        artifact = _make_v2_artifact(tmp_path)
        ns = argparse.Namespace(round=1, slot=None, out_name="e2e_latency", _out_root=None)
        with pytest.raises(SystemExit) as exc_info:
            _resolve_sweep_out_name(ns, artifact)
        msg = str(exc_info.value)
        assert "profiling" in msg, f"Error should mention 'profiling' as valid slot: {msg}"
        assert "baseline" in msg
        assert "opt/" in msg or "opt/{" in msg

    def test_out_name_error_mentions_profiling(self, tmp_path):
        from run_vllm_bench_latency_sweep import _resolve_sweep_out_name
        artifact = _make_v2_artifact(tmp_path)
        ns = argparse.Namespace(round=None, slot=None, out_name="custom_name", _out_root=None)
        with pytest.raises(SystemExit) as exc_info:
            _resolve_sweep_out_name(ns, artifact)
        msg = str(exc_info.value)
        assert "profiling" in msg, f"Error should mention 'profiling' as valid slot: {msg}"


if __name__ == "__main__":
    pytest.main([__file__, "-v"])
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall