Skip to content

amazon-science/ammo / ai_cli_session/.codex/skills/ammo

Profile and optimize GPU kernels for vLLM inference on NVIDIA GPUs. Use when targeting a specific model, hardware, dtype, parallelism, and workload to improve latency.

FileKindSize
RUNTIME.mdmarkdown4.9 KB
SKILL.mdskill md18.5 KB
agents/ammo-auditor.mdmarkdown9.1 KB
agents/ammo-champion.mdmarkdown9.6 KB
agents/ammo-delegate.mdmarkdown3.2 KB
agents/ammo-implementer.mdmarkdown13.8 KB
agents/ammo-investigator.mdmarkdown4.0 KB
agents/ammo-report.mdmarkdown2.9 KB
agents/ammo-researcher.mdmarkdown10.3 KB
agents/ammo-resolver.mdmarkdown4.4 KB
agents/ammo-transcript-monitor.mdmarkdown7.8 KB
agents/openai.yamlexecutablescript313 B
data/ATTRIBUTION.mdmarkdown1.4 KB
data/gsm8k_full.jsonexecutablescript735.5 KB
data/gsm8k_subset.jsonexecutablescript116.8 KB
eval/EVAL-SKILL.mdexecutablescript21.8 KB
eval/agents/transcript_grader.mdexecutablescript18.0 KB
eval/causal/__init__.pyexecutablescript137 B
eval/causal/agents/causal_analyzer.mdexecutablescript10.5 KB
eval/causal/build_dag.pyexecutablescript21.6 KB
eval/causal/diff_versions.pyexecutablescript38.5 KB
eval/causal/extract_events.pyexecutablescript36.3 KB
eval/causal/generate_postmortem.pyexecutablescript17.0 KB
eval/causal/score_nodes.pyexecutablescript21.5 KB
eval/causal/templates/narrative_template.mdexecutablescript472 B
eval/causal/templates/viz_template.htmlexecutablescript9.5 KB
eval/causal/tests/__init__.pyexecutablescript151 B
eval/causal/tests/conftest.pyexecutablescript448 B
eval/causal/tests/fixtures/sample_session.jsonlexecutablescript10.4 KB
eval/causal/tests/fixtures/sample_session_data.jsonexecutablescript3.4 KB
eval/causal/tests/fixtures/sample_snapshot.jsonexecutablescript2.5 KB
eval/causal/tests/test_build_dag.pyexecutablescript2.7 KB
eval/causal/tests/test_diff_versions.pyexecutablescript13.5 KB
eval/causal/tests/test_extract_events.pyexecutablescript3.7 KB
eval/causal/tests/test_generate_postmortem.pyexecutablescript3.1 KB
eval/causal/tests/test_score_nodes.pyexecutablescript3.3 KB
eval/scripts/aggregate_versions.pyexecutablescript15.2 KB
eval/scripts/archive_run.pyexecutablescript9.1 KB
eval/scripts/codex_session.pyexecutablescript10.6 KB
eval/scripts/generate_dashboard.pyexecutablescript5.3 KB
eval/scripts/parse_artifacts.pyexecutablescript48.1 KB
eval/scripts/parse_session_logs.pyexecutablescript51.3 KB
eval/scripts/rescore_archived.pyexecutablescript5.5 KB
eval/scripts/score_campaign.pyexecutablescript43.8 KB
eval/scripts/snapshot_changes.pyexecutablescript11.1 KB
eval/scripts/verify_citations.pyexecutablescript8.8 KB
eval/tests/test_dashboard_bugs.pyexecutablescript8.8 KB
eval/tests/test_parse_session_logs.pyexecutablescript3.9 KB
eval/tests/test_parse_session_logs_round_inference.pyexecutablescript11.0 KB
eval/tests/test_score_campaign_diluted.pyexecutablescript9.4 KB
eval/viewer/dashboard_template.htmlexecutablescript15.1 KB
instrumentation/sitecustomize.pyexecutablescript3.6 KB
orchestration/audit-protocol.mdmarkdown9.3 KB
orchestration/debate-protocol.mdmarkdown11.2 KB
orchestration/integration-logic.mdmarkdown10.3 KB
orchestration/parallel-tracks.mdmarkdown8.9 KB
references/artifact-layout.mdexecutablescript22.4 KB
references/audit-invariants.mdmarkdown10.6 KB
references/champion-common-patterns.mdmarkdown3.4 KB
references/crossover-probing.mdexecutablescript4.7 KB
references/cudagraph-safety.mdmarkdown2.7 KB
references/debate-rules.mdmarkdown4.7 KB
references/debate-scoring-rubric.mdexecutablescript16.7 KB
references/e2e-delta-math.mdexecutablescript17.9 KB
references/e2e-latency-guide.mdexecutablescript9.6 KB
references/fusion-feasibility-heuristics.mdmarkdown5.6 KB
references/gpu-pool.mdexecutablescript5.4 KB
references/impl-track-rules.mdexecutablescript13.8 KB
references/nsys-profiling-guide.mdmarkdown9.8 KB
references/optimization-categories.mdexecutablescript11.0 KB
references/shutdown-protocol.mdexecutablescript3.1 KB
references/technology-selection.mdmarkdown5.0 KB
references/torch-compile-contract.mdmarkdown13.8 KB
references/validation-defaults.mdexecutablescript34.2 KB
references/workload-invocation.mdexecutablescript4.9 KB
references/writing-style.mdexecutablescript6.9 KB
scripts/ammo_state.pyexecutablescript142.0 KB
scripts/artifact_layout.jsonconfig16.3 KB
scripts/check_projection_accuracy.pyexecutablescript11.7 KB
scripts/create_evidence_template.pyexecutablescript9.7 KB
scripts/create_worktree_with_build.shexecutablescript10.2 KB
scripts/generate_validation_report.pyexecutablescript35.7 KB
scripts/gpu_reservation.pyexecutablescript27.1 KB
scripts/hook_cmd_classify.pyscript13.3 KB
scripts/mine_trace.pyscript29.4 KB
scripts/monitor_queue_ack.pyexecutablescript6.1 KB
scripts/ncu_sanity.shexecutablescript5.2 KB
scripts/ncu_sanity_driver.pyexecutablescript8.5 KB
scripts/new_target.pyexecutablescript22.0 KB
scripts/preflight_vllm_api.pyexecutablescript12.3 KB
scripts/reconcile_track_state.pyexecutablescript15.8 KB
scripts/remove_worktree_cleanup.shexecutablescript1.1 KB
scripts/resolve_codex_transcript.pyexecutablescript3.6 KB
scripts/run_vllm_bench_latency_sweep.pyexecutablescript249.6 KB
scripts/transcript_filter.pyexecutablescript38.4 KB
scripts/transitions.jsonexecutablescript1.5 KB
scripts/verify_validation_gates.pyexecutablescript88.8 KB
tests/agents/test-champion.mdexecutablescript20.4 KB
tests/agents/test-impl-champion.mdexecutablescript14.3 KB
tests/agents/test-implementer.mdexecutablescript15.8 KB
tests/agents/test-orchestrator.mdexecutablescript33.8 KB
tests/agents/test-researcher.mdexecutablescript21.0 KB
tests/agents/test-transcript-monitor.mdexecutablescript13.4 KB
tests/fixtures/diluted_pass/legacy_no_phase.golden.mdmarkdown1.1 KB
tests/fixtures/diluted_pass/legacy_no_phase.golden.summary.jsonconfig582 B
tests/fixtures/diluted_pass/legacy_no_phase.jsonconfig400 B
tests/fixtures/diluted_pass/legacy_scalar_phase.golden.mdmarkdown2.0 KB
tests/fixtures/diluted_pass/legacy_scalar_phase.golden.summary.jsonconfig1.0 KB
tests/fixtures/diluted_pass/legacy_scalar_phase.jsonconfig626 B
tests/fixtures/mining/real_campaign_bottleneck_analysis.mdmarkdown20.7 KB
tests/run_all.shexecutablescript1.6 KB
tests/scripts/test_transcript_filter.pyexecutablescript41.1 KB
tests/test_agent_runtime_contracts.pyexecutablescript1.8 KB
tests/test_ammo_state.pyexecutablescript71.7 KB
tests/test_artifact_layout_manifest.pyscript11.6 KB
tests/test_audit_started.pyscript18.2 KB
tests/test_correctness_comparator.pyscript12.6 KB
tests/test_diluted_pass.pyscript24.4 KB
tests/test_drift_compat.pyscript5.1 KB
tests/test_evidence_freshness.pyexecutablescript5.6 KB
tests/test_fail_closed.pyexecutablescript14.7 KB
tests/test_gpu_reservation.pyexecutablescript24.8 KB
tests/test_gpu_reservation_session_id.pyexecutablescript11.2 KB
tests/test_guidance_semantic_contract.pyscript24.9 KB
tests/test_hook_cmd_classify_parity.pyscript11.7 KB
tests/test_hook_semantics.pyexecutablescript95.3 KB
tests/test_mine_trace.pyscript24.2 KB
tests/test_mining_invalidated_schema.pyexecutablescript9.4 KB
tests/test_new_target_dp_ep.pyexecutablescript7.4 KB
tests/test_new_target_v2_scaffold.pyexecutablescript13.1 KB
tests/test_parse_artifacts.pyexecutablescript3.0 KB
tests/test_policy_convergence.pyscript797 B
tests/test_preflight_vllm_api.pyscript13.3 KB
tests/test_projection_accuracy_v2.pyexecutablescript4.1 KB
tests/test_reconcile_track_state.pyexecutablescript18.2 KB
tests/test_resolve_codex_transcript.pyexecutablescript3.1 KB
tests/test_state_schema_semantics.pyexecutablescript7.6 KB
tests/test_status_set_sync.pyexecutablescript9.1 KB
tests/test_sweep_baseline_from_arm.pyscript11.0 KB
tests/test_sweep_decode_capture.pyexecutablescript31.8 KB
tests/test_sweep_dp.pyexecutablescript45.4 KB
tests/test_sweep_fresh_cache.pyscript5.5 KB
tests/test_sweep_profiler_guard.pyscript22.6 KB
tests/test_sweep_real_prompts.pyscript20.1 KB
tests/test_sweep_refactor.pyexecutablescript37.6 KB
tests/test_sweep_round_slot.pyexecutablescript7.4 KB
tests/test_validation_report_phase_triage.pyexecutablescript10.0 KB
tests/test_validation_report_v2.pyexecutablescript6.2 KB
tests/test_verify_validation_gates.pyexecutablescript52.9 KB
tests/test_workload_matrix.pyexecutablescript19.3 KB

tests/test_validation_report_phase_triage.py

View on GitHub
#!/usr/bin/env python3
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: CC-BY-NC-4.0
"""Tests for the phase-decomposition triage layer in generate_validation_report.py.

The layer consumes ONLY fields the sweep already harvests (prefill_avg_s /
decode_avg_s / decode_share_of_e2e / tpot_s / otps + row-level
tpot_improvement_pct / otps_gain_pct) — zero extra benchmark runs. It is
informational: flags must NEVER change a per-BS or track verdict.

Pinned behaviors:
  1. DILUTED-WIN: decode win >= floor, e2e below PASS, Amdahl-consistent
     -> diluted_decode_win=true, verdict unchanged (still NOISE).
  2. PHASE-REGRESSION: e2e PASS but a phase regressed beyond noise tolerance
     -> phase_warnings emitted, verdict unchanged (still PASS).
  3. INCONSISTENT: actual e2e delta disagrees with the phase-decomposition
     expectation -> amdahl_consistent=false, no diluted_decode_win.
  4. Legacy rows without phase fields -> no phase section, no phase keys
     (byte-compatible report path).
  5. Clean PASS with consistent phases -> no flags, table still rendered.
"""

from __future__ import annotations

import json
import subprocess
import sys
from pathlib import Path

import pytest

_SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "scripts"
_GEN = _SCRIPTS_DIR / "generate_validation_report.py"


def _payload(rows: list[dict]) -> str:
    return json.dumps({
        "model_id": "test/model",
        "tp": 1,
        "max_model_len": 4096,
        "workload": {"input_len": 1024, "output_len": 128, "num_iters": 10},
        "results": rows,
        "bench": {"baseline_label": "baseline", "opt_label": "opt"},
    })


def _row(bs: int, speedup: float, *, b_prefill: float | None = None,
         b_decode: float | None = None, o_prefill: float | None = None,
         o_decode: float | None = None, significant: bool | None = None,
         phase_significance: dict | None = None) -> dict:
    b_avg = 1.0
    o_avg = b_avg / speedup
    b: dict = {"avg_s": b_avg}
    o: dict = {"avg_s": o_avg}
    if b_prefill is not None and b_decode is not None:
        b["prefill_avg_s"] = b_prefill
        b["decode_avg_s"] = b_decode
        b["decode_share_of_e2e"] = b_decode / (b_prefill + b_decode)
    if o_prefill is not None and o_decode is not None:
        o["prefill_avg_s"] = o_prefill
        o["decode_avg_s"] = o_decode
        o["decode_share_of_e2e"] = o_decode / (o_prefill + o_decode)
    row = {
        "batch_size": bs,
        "speedup": speedup,
        "improvement_pct": (1.0 - o_avg / b_avg) * 100.0,
        "baseline": b,
        "opt": o,
    }
    if significant is not None:
        row["significance"] = {"significant": significant}
    if phase_significance is not None:
        row["phase_significance"] = phase_significance
    return row


def _run(tmp_path: Path, rows: list[dict]) -> tuple[str, dict]:
    artifact = tmp_path / "artifact"
    e2e_dir = artifact / "e2e_latency"
    e2e_dir.mkdir(parents=True)
    (e2e_dir / "e2e_latency_results.json").write_text(_payload(rows))
    result = subprocess.run(
        [sys.executable, str(_GEN), "--artifact-dir", str(artifact)],
        capture_output=True, text=True,
    )
    assert result.returncode == 0, f"report generation failed: {result.stderr}"
    md = (artifact / "validation_results.md").read_text()
    summary = json.loads((artifact / "validation_summary.json").read_text())
    return md, summary


class TestDilutedWin:
    """A real decode win diluted below the e2e floor by prefill share."""

    def test_flags_diluted_decode_win_without_promoting_verdict(self, tmp_path):
        # decode improves 4% but decode share is only 0.10:
        # expected e2e ~ 0.10*4% = 0.4% < 0.5% floor -> NOISE.
        # speedup consistent with the phase expectation.
        b_pf, b_dc = 0.90, 0.10
        o_dc = b_dc * (1 - 0.04)          # 4% decode win
        o_pf = b_pf                        # prefill unchanged
        o_avg_ratio = (o_pf + o_dc) / (b_pf + b_dc)
        speedup = 1.0 / o_avg_ratio        # ~1.004
        md, summary = _run(tmp_path, [_row(
            1, speedup, b_prefill=b_pf, b_decode=b_dc,
            o_prefill=o_pf, o_decode=o_dc)])

        verdicts = summary["e2e_gate"]["per_bs_verdicts"]
        assert verdicts[0]["verdict"] == "NOISE", \
            "phase triage must not promote the verdict"
        assert verdicts[0]["phase"]["diluted_decode_win"] is True
        assert summary["e2e_gate"]["phase_triage"]["diluted_decode_win_bs"] == [1]
        assert "DILUTED-WIN" in md
        assert "Amdahl-consistent" in md

    def test_track_verdict_unchanged_by_dilution_flag(self, tmp_path):
        b_pf, b_dc = 0.90, 0.10
        o_dc = b_dc * (1 - 0.04)
        o_avg_ratio = (b_pf + o_dc) / (b_pf + b_dc)
        md, summary = _run(tmp_path, [_row(
            1, 1.0 / o_avg_ratio, b_prefill=b_pf, b_decode=b_dc,
            o_prefill=b_pf, o_decode=o_dc)])
        # All-NOISE rows -> FAIL candidate; the flag must not rescue it.
        assert summary["e2e_gate"]["track_verdict"] == "FAIL"


class TestPhaseRegression:
    """A net e2e PASS masking a phase-level regression."""

    def test_pass_with_prefill_regression_warns(self, tmp_path):
        # decode improves 6%, prefill regresses 2%; decode share 0.8
        # -> e2e ~ +4.4% PASS, but prefill regressed beyond 0.5% noise.
        b_pf, b_dc = 0.20, 0.80
        o_pf = b_pf * 1.02
        o_dc = b_dc * (1 - 0.06)
        o_avg_ratio = (o_pf + o_dc) / (b_pf + b_dc)
        md, summary = _run(tmp_path, [_row(
            8, 1.0 / o_avg_ratio, b_prefill=b_pf, b_decode=b_dc,
            o_prefill=o_pf, o_decode=o_dc, significant=True)])

        verdicts = summary["e2e_gate"]["per_bs_verdicts"]
        assert verdicts[0]["verdict"] == "PASS", \
            "phase warning must not demote a PASS"
        warns = verdicts[0]["phase"]["phase_warnings"]
        assert [w["phase"] for w in warns] == ["prefill"]
        assert summary["e2e_gate"]["phase_triage"]["phase_regression_bs"] == [8]
        assert "PHASE-REGRESSION" in md

    def test_clean_pass_no_flags(self, tmp_path):
        # Uniform 3% win in both phases: PASS, no flags, table rendered.
        b_pf, b_dc = 0.30, 0.70
        md, summary = _run(tmp_path, [_row(
            1, 1.0 / 0.97, b_prefill=b_pf, b_decode=b_dc,
            o_prefill=b_pf * 0.97, o_decode=b_dc * 0.97, significant=True)])
        phase = summary["e2e_gate"]["per_bs_verdicts"][0]["phase"]
        assert "phase_warnings" not in phase
        assert phase.get("diluted_decode_win") is not True
        triage = summary["e2e_gate"]["phase_triage"]
        assert triage["diluted_decode_win_bs"] == []
        assert triage["phase_regression_bs"] == []
        assert "Phase decomposition" in md
        assert "DILUTED-WIN" not in md
        assert "PHASE-REGRESSION" not in md


class TestAmdahlConsistency:
    def test_inconsistent_phase_math_flagged_not_diluted(self, tmp_path):
        # Phases claim a 4% decode win at share 0.9 (expected e2e ~3.6%)
        # but wall-clock speedup says 0.1% -> INCONSISTENT, not DILUTED-WIN.
        b_pf, b_dc = 0.10, 0.90
        o_dc = b_dc * (1 - 0.04)
        md, summary = _run(tmp_path, [_row(
            1, 1.001, b_prefill=b_pf, b_decode=b_dc,
            o_prefill=b_pf, o_decode=o_dc)])
        phase = summary["e2e_gate"]["per_bs_verdicts"][0]["phase"]
        assert phase["amdahl_consistent"] is False
        assert phase.get("diluted_decode_win") is not True
        assert summary["e2e_gate"]["phase_triage"]["amdahl_inconsistent_bs"] == [1]
        assert "INCONSISTENT" in md


class TestLegacyPassthrough:
    def test_rows_without_phase_fields_render_original_report(self, tmp_path):
        md, summary = _run(tmp_path, [_row(1, 1.05)])
        assert "Phase decomposition" not in md
        assert "phase" not in summary["e2e_gate"]["per_bs_verdicts"][0]
        assert "phase_triage" not in summary["e2e_gate"]


class TestDilutedPassIntegration:
    """Two cases wired into the existing triage suite (design §5 line item).

    Uses the file's default gating (no target.json, so min_e2e floor and
    noise are both 0.5%). The DILUTED_PASS window opens by having a
    non-significant prefill regression pull e2e into the NOISE band while a
    Welch-significant decode win clears the e2e-equivalent decode floor
    (decode_improvement_pct * decode_share >= 2*noise = 1.0%)."""

    def _diluted_row(self, bs: int) -> dict:
        # b: prefill 0.2 / decode 0.8. opt: prefill regresses 10% (0.22),
        # decode wins 3% (0.776). e2e improvement ~0.4% (NOISE), speedup
        # 1.004 (>=1.0). decode e2e-equiv = 3% * 0.8 = 2.4% (>=1.0 floor).
        return _row(
            bs, 1.0 / 0.996, b_prefill=0.2, b_decode=0.8,
            o_prefill=0.22, o_decode=0.776,
            phase_significance={
                "decode": {"significant": True, "n_baseline": 30, "n_opt": 30},
                "prefill": {"significant": False, "n_baseline": 30, "n_opt": 30},
            },
        )

    def test_legacy_all_noise_row_stays_fail_no_diluted_pass(self, tmp_path):
        md, summary = _run(tmp_path, [_row(1, 1.004), _row(8, 1.003)])
        assert summary["e2e_gate"]["track_verdict"] == "FAIL"
        assert "diluted_pass" not in summary["e2e_gate"]
        assert "Ships as PASS (diluted)" not in md

    def test_fully_qualifying_row_ships_pass_diluted(self, tmp_path):
        rows = [self._diluted_row(1), self._diluted_row(8)]
        md, summary = _run(tmp_path, rows)
        assert summary["e2e_gate"]["track_verdict"] == "PASS"
        assert summary["e2e_gate"]["diluted_pass"]["eligible"] is True
        assert "Ships as PASS (diluted)" in md


class TestSignificanceFloor:
    def test_1_0001x_non_significant_result_is_not_pass(self, tmp_path):
        _, summary = _run(tmp_path, [_row(1, 1.0001, significant=False)])
        verdict = summary["e2e_gate"]["per_bs_verdicts"][0]
        assert verdict["verdict"] == "NOISE"
        assert verdict["significant"] is False
        assert summary["e2e_gate"]["track_verdict"] == "FAIL"


if __name__ == "__main__":
    pytest.main([__file__, "-v"])

slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall