Skip to content

culiops/culiops-skills / skills/sre-investigate

sre-investigate

Unverified

FileKindSize
SKILL.mdskill md71.5 KB
access/standalone.mdmarkdown19.0 KB
agents/evidence.mdmarkdown8.1 KB
agents/helper.mdmarkdown3.9 KB
analytics/bin/logs-fetchexecutablescript229 B
analytics/bin/logs-inventoryexecutablescript209 B
analytics/bin/logs-statusexecutablescript192 B
analytics/bin/sqlqexecutablescript212 B
analytics/lib/s3logs/__init__.pyscript62 B
analytics/lib/s3logs/awscli.pyscript10.0 KB
analytics/lib/s3logs/budget.pyscript9.6 KB
analytics/lib/s3logs/cli_fetch.pyscript34.0 KB
analytics/lib/s3logs/cli_inventory.pyscript7.0 KB
analytics/lib/s3logs/cli_sqlq.pyscript18.4 KB
analytics/lib/s3logs/cli_status.pyscript3.0 KB
analytics/lib/s3logs/config.pyscript7.4 KB
analytics/lib/s3logs/convert.pyscript23.6 KB
analytics/lib/s3logs/duck.pyscript11.4 KB
analytics/lib/s3logs/layout.pyscript8.8 KB
analytics/lib/s3logs/paths.pyscript4.9 KB
analytics/lib/s3logs/redact.pyscript22.1 KB
analytics/lib/s3logs/s5.pyscript9.1 KB
analytics/lib/s3logs/sqlguard.pyscript11.8 KB
analytics/lib/s3logs/validate.pyscript4.4 KB
analytics/logs.env.exampleother4.2 KB
analytics/sql/alb/by_minute.sqlother918 B
analytics/sql/alb/lb_vs_target_status.sqlother629 B
analytics/sql/alb/status_by_target.sqlother887 B
analytics/sql/alb/summary.sqlother1.2 KB
analytics/sql/alb/target_latency_pct.sqlother1.4 KB
analytics/sql/alb/top_clients.sqlother429 B
analytics/sql/alb/top_paths_5xx.sqlother673 B
analytics/sql/cloudfront/aborted.sqlother1.1 KB
analytics/sql/cloudfront/by_day.sqlother499 B
analytics/sql/cloudfront/by_hour.sqlother489 B
analytics/sql/cloudfront/by_object.sqlother544 B
analytics/sql/cloudfront/bytes_per_viewer.sqlother1.1 KB
analytics/sql/cloudfront/cache_results.sqlother823 B
analytics/sql/cloudfront/client_concentration.sqlother707 B
analytics/sql/cloudfront/edges.sqlother434 B
analytics/sql/cloudfront/object_by_day.sqlother599 B
analytics/sql/cloudfront/range_requests.sqlother528 B
analytics/sql/cloudfront/referer_pages.sqlother393 B
analytics/sql/cloudfront/status_codes.sqlother342 B
analytics/sql/cloudfront/summary.sqlother719 B
analytics/sql/cloudfront/top_clients.sqlother353 B
analytics/sql/cloudfront/top_objects.sqlother349 B
analytics/sql/cloudfront/top_referers.sqlother492 B
analytics/sql/cloudfront/user_agents.sqlother533 B
analytics/sql/waf/asns.sqlother378 B
analytics/sql/waf/by_day.sqlother453 B
analytics/sql/waf/by_minute.sqlother314 B
analytics/sql/waf/client_keys.sqlother443 B
analytics/sql/waf/clients.sqlother521 B
analytics/sql/waf/compare_days.sqlother924 B
analytics/sql/waf/concentration.sqlother775 B
analytics/sql/waf/count_rules.sqlother556 B
analytics/sql/waf/countries.sqlother373 B
analytics/sql/waf/fingerprints.sqlother424 B
analytics/sql/waf/header_shape.sqlother425 B
analytics/sql/waf/labels.sqlother347 B
analytics/sql/waf/matched_data.sqlother530 B
analytics/sql/waf/rules_by_minute.sqlother497 B
analytics/sql/waf/summary.sqlother735 B
analytics/sql/waf/top_rules.sqlother361 B
analytics/sql/waf/uris.sqlother402 B
analytics/sql/waf/user_agents.sqlother557 B
analytics/tests/conftest.pyscript4.7 KB
analytics/tests/gen_fixtures.pyscript19.7 KB
analytics/tests/stubs/awsexecutablescript4.9 KB
analytics/tests/stubs/s5cmdexecutablescript4.5 KB
analytics/tests/test_awscli.pyscript16.0 KB
analytics/tests/test_budget.pyscript13.3 KB
analytics/tests/test_config.pyscript9.1 KB
analytics/tests/test_convert.pyscript25.9 KB
analytics/tests/test_duck.pyscript10.6 KB
analytics/tests/test_fetch.pyscript42.8 KB
analytics/tests/test_gen_fixtures.pyscript1.9 KB
analytics/tests/test_inventory.pyscript12.9 KB
analytics/tests/test_layout.pyscript9.8 KB
analytics/tests/test_named_queries.pyscript26.7 KB
analytics/tests/test_paths.pyscript3.2 KB
analytics/tests/test_redact.pyscript32.0 KB
analytics/tests/test_s5.pyscript17.8 KB
analytics/tests/test_sqlguard.pyscript10.8 KB
analytics/tests/test_sqlq.pyscript37.2 KB
analytics/tests/test_status.pyscript5.8 KB
analytics/tests/test_validate.pyscript2.9 KB
environment.template.mdmarkdown7.1 KB
playbooks/changes.mdmarkdown37.2 KB
playbooks/errors.mdmarkdown10.4 KB
playbooks/external.mdmarkdown15.4 KB
playbooks/infra-state.mdmarkdown13.7 KB
playbooks/logs-s3.mdmarkdown29.5 KB
playbooks/logs.mdmarkdown21.3 KB
playbooks/metrics.mdmarkdown39.0 KB
playbooks/other-backends.mdmarkdown7.2 KB
playbooks/traces.mdmarkdown16.2 KB
playbooks/waf.mdmarkdown17.2 KB
templates/incident.mdmarkdown3.9 KB
templates/ledger.mdmarkdown9.7 KB
templates/lesson.mdmarkdown2.7 KB
templates/profile.mdmarkdown4.5 KB
templates/report.mdmarkdown6.5 KB

analytics/tests/test_inventory.py

View on GitHub
from __future__ import annotations

import datetime as dt
import json
import os
import subprocess

import pytest

import gen_fixtures as gf
from conftest import BIN

BUCKET = "example-logs-bucket"
ALB_TEMPLATE = "AWSLogs/111122223333/elasticloadbalancing/us-east-1/{yyyy}/{MM}/{dd}/"
CF_TEMPLATE = "cf/E2EXAMPLE.{yyyy}-{MM}-{dd}-{HH}"
DAY = dt.date(2026, 3, 9)
NEXT = dt.date(2026, 3, 10)
GAP = "0 files — logging may be off or delivery late; this is a gap, not zero traffic"


def write_config(home, extra="", profile="sre-read"):
    d = home / ".config/culiops/sre-investigate"
    d.mkdir(parents=True, exist_ok=True)
    lines = []
    if profile:
        lines.append("AWS_PROFILE=%s" % profile)
    lines += [
        "DATASET_cf=cloudfront:s3://%s/%s" % (BUCKET, CF_TEMPLATE),
        "DATASET_alb=alb:s3://%s/%s" % (BUCKET, ALB_TEMPLATE),
        "DATASET_waf=waf:s3://%s/AWSLogs/111122223333/WAFLogs/us-east-1/example-acl/{yyyy}/{MM}/{dd}/{HH}/" % BUCKET,
    ]
    (d / "logs.env").write_text("\n".join(lines) + "\n" + extra)


def run(*args):
    return subprocess.run([str(BIN / "logs-inventory")] + list(args), capture_output=True, text=True,
                          env=os.environ.copy(), timeout=60)


def manifest_path(home, ds, frm, to):
    return home / ".local/share/culiops/sre-investigate/manifests" / ds / ("%s_%s.json" % (frm, to))


@pytest.fixture
def env(home, fake_s3):
    write_config(home)
    fake_s3.bucket(BUCKET)
    return fake_s3


def rows(out):
    return [l.split() for l in out.splitlines()]


def test_inventory_lists_day_and_writes_manifest(home, env):
    gf.make(env.root, "alb", DAY, hours=[0, 1], minutes=(0, 30))      # 4 files
    r = run("alb", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    lines = r.stdout.splitlines()
    assert lines[0].split()[:4] == ["day", "files", "MB", "est_cost"]
    day_row = [l.split() for l in lines if l.startswith("2026-03-09")][0]
    assert day_row[1] == "4"
    total = [l.split() for l in lines if l.startswith("total")][0]
    assert total[1] == "4"
    mpath = manifest_path(home, "alb", "2026-03-09", "2026-03-09")
    assert lines[-1] == "manifest: %s" % mpath
    m = json.loads(mpath.read_text())
    assert m["dataset"] == "alb" and m["type"] == "alb" and m["bucket"] == BUCKET
    assert (m["from"], m["to"]) == ("2026-03-09", "2026-03-09")
    assert m["skipped_unsafe"] == 0 and m["skipped_unparsed"] == 0 and m["unsupported_format"] is False
    assert m["listed_at"].endswith("Z")
    assert len(m["files"]) == 4
    f0 = m["files"][0]
    assert set(f0) == {"key", "size", "etag", "start", "end"}
    assert f0["start"] == "2026-03-09T23:55:00Z" or f0["start"].endswith(":00Z")
    assert f0["end"] > f0["start"]
    assert oct(mpath.stat().st_mode & 0o777) == "0o600"


def test_inventory_hourly_template_lists_once_per_day(home, env):
    gf.make(env.root, "cloudfront", DAY, hours=[3, 4, 5])
    r = run("cf", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    assert len(env.calls()) == 1                        # one LIST for the whole day, not 24
    r = run("cf", "2026-03-09", "2026-03-11")
    assert r.returncode == 0, r.stderr
    assert len(env.calls()) == 1 + 3                    # one per day
    r = run("waf", "2026-03-09", "2026-03-10")
    assert r.returncode == 0, r.stderr
    assert len(env.calls()) == 1 + 3 + 2
    assert all(c[:2] == ["s3api", "list-objects-v2"] for c in env.calls())
    m = json.loads(manifest_path(home, "cf", "2026-03-09", "2026-03-09").read_text())
    assert [f["start"] for f in m["files"]] == ["2026-03-09T03:00:00Z", "2026-03-09T04:00:00Z",
                                                  "2026-03-09T05:00:00Z"]


def test_inventory_hourly_template_does_not_count_strays_under_the_day_prefix(home, env, tmp_path):
    gf.make(env.root, "cloudfront", DAY, hours=[3])
    b = env.bucket(BUCKET)
    (b / "cf/E2EXAMPLE.2026-03-09-7.AAA.gz").write_bytes(b"x")      # hour part does not match
    (b / "cf/E2EXAMPLE.2026-03-09-99.AAA.gz").write_bytes(b"x")     # not an hour
    (b / "cf/E2EXAMPLE.2026-03-09-.AAA.gz").write_bytes(b"x")
    (b / "cf/E2EXAMPLE.2026-03-10-03.AAA.gz").write_bytes(b"x")     # next day: outside the day prefix
    r = run("cf", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    m = json.loads(manifest_path(home, "cf", "2026-03-09", "2026-03-09").read_text())
    assert len(m["files"]) == 1 and m["files"][0]["start"] == "2026-03-09T03:00:00Z"
    assert m["skipped_unparsed"] == 3
    assert m["days"]["2026-03-09"]["files"] == 1


def test_inventory_waf_stray_key_directly_under_the_day_dir_is_not_counted(home, env):
    gf.make(env.root, "waf", DAY, hours=[3], minutes=(0,))
    b = env.bucket(BUCKET)
    stray = b / "AWSLogs/111122223333/WAFLogs/us-east-1/example-acl/2026/03/09/loose_20260309T0500Z_a.log.gz"
    stray.write_bytes(b"x")
    r = run("waf", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    m = json.loads(manifest_path(home, "waf", "2026-03-09", "2026-03-09").read_text())
    assert all("loose_" not in f["key"] for f in m["files"])
    assert m["skipped_unparsed"] == 1


def test_manifest_files_are_sorted_and_ranges_span_days(home, env):
    gf.make(env.root, "alb", DAY, hours=[5], minutes=(0,))
    gf.make(env.root, "alb", NEXT, hours=[5], minutes=(0,))
    r = run("alb", "2026-03-09", "2026-03-10")
    assert r.returncode == 0, r.stderr
    day_rows = [l for l in rows(r.stdout) if l and l[0].startswith("2026-03-")]
    assert [x[0] for x in day_rows] == ["2026-03-09", "2026-03-10"]
    m = json.loads(manifest_path(home, "alb", "2026-03-09", "2026-03-10").read_text())
    assert [f["start"] for f in m["files"]] == sorted(f["start"] for f in m["files"])
    assert len(m["files"]) == 2


def test_empty_day_reported_as_gap(home, env):
    gf.make(env.root, "alb", DAY, hours=[1], minutes=(0,))
    r = run("alb", "2026-03-09", "2026-03-10")
    assert r.returncode == 0, r.stderr
    assert "2026-03-10: " + GAP in r.stdout
    assert "2026-03-09: " + GAP not in r.stdout
    assert "zero traffic" in r.stdout and "no traffic" not in r.stdout


def test_range_over_92_days_refused(home, env):
    r = run("alb", "2026-01-01", "2026-04-03")           # 93 days inclusive
    assert r.returncode == 2
    assert "92" in r.stderr
    assert env.calls() == []
    ok = run("alb", "2026-01-01", "2026-04-02")          # exactly 92 days is allowed
    assert ok.returncode == 0, ok.stderr


@pytest.mark.parametrize("args", [
    ("alb", "2026-03-10", "2026-03-09"),
    ("alb", "2026-3-9", "2026-03-09"),
    ("nope", "2026-03-09", "2026-03-09"),
    ("ALB", "2026-03-09", "2026-03-09"),
    ("alb", "2026-03-09"),
])
def test_bad_arguments_exit_2(home, env, args):
    r = run(*args)
    assert r.returncode == 2
    assert env.calls() == []


def test_not_configured_exit_2(home, fake_s3):
    r = run("alb", "2026-03-09", "2026-03-09")
    assert r.returncode == 2
    assert "S3 log module: off (not configured)" in r.stderr


def test_expired_session_exit_3_with_hint(home, env, monkeypatch):
    monkeypatch.setenv("FAKE_AWS_ERROR", "An error occurred (ExpiredToken) when calling the ListObjectsV2 "
                                          "operation: The provided token has expired.")
    r = run("alb", "2026-03-09", "2026-03-09")
    assert r.returncode == 3
    assert "! aws sso login --profile sre-read" in r.stderr
    assert not manifest_path(home, "alb", "2026-03-09", "2026-03-09").exists()


def test_unsafe_keys_skipped_in_manifest(home, env, tmp_path, monkeypatch):
    base = "AWSLogs/111122223333/elasticloadbalancing/us-east-1/2026/03/09/"
    gf.make(env.root, "alb", DAY, hours=[1], minutes=(0,))
    extra = tmp_path / "extra.json"
    extra.write_text(json.dumps([
        {"key": base + "../x.gz", "size": 10},
        {"key": base + "a b.gz", "size": 10},
        {"key": base + "a\nb.gz", "size": 10},
    ]))
    monkeypatch.setenv("FAKE_EXTRA_KEYS_JSON", str(extra))
    r = run("alb", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    mpath = manifest_path(home, "alb", "2026-03-09", "2026-03-09")
    text = mpath.read_text()
    m = json.loads(text)
    assert m["skipped_unsafe"] == 3
    assert len(m["files"]) == 1
    for needle in ("x.gz", "a b", "a\\nb", "a\nb", "../"):
        assert needle not in text
        assert needle not in r.stdout and needle not in r.stderr
    assert "3" in r.stdout and "unsafe" in r.stdout


def test_v2_json_marked_unsupported(home, env):
    gf.make(env.root, "cloudfront", DAY, hours=[1], fmt="json")
    gf.make(env.root, "cloudfront", DAY, hours=[2])
    r = run("cf", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    m = json.loads(manifest_path(home, "cf", "2026-03-09", "2026-03-09").read_text())
    assert m["unsupported_format"] is True
    assert [f["start"] for f in m["files"]] == ["2026-03-09T02:00:00Z"]      # the json.gz is not offered to fetch
    assert "format unsupported (v2 JSON/Parquet)" in r.stdout


def test_unparsed_files_counted(home, env):
    gf.make(env.root, "alb", DAY, hours=[1], minutes=(0,))
    (env.root / BUCKET / "AWSLogs/111122223333/elasticloadbalancing/us-east-1/2026/03/09/readme.txt").write_text("x")
    r = run("alb", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    m = json.loads(manifest_path(home, "alb", "2026-03-09", "2026-03-09").read_text())
    assert m["skipped_unparsed"] == 1 and len(m["files"]) == 1
    assert "file time" in r.stdout


def thousand_files(tmp_path, monkeypatch):
    prefix = "AWSLogs/111122223333/elasticloadbalancing/us-east-1/2026/03/09/"
    extra = tmp_path / "thousand.json"
    extra.write_text(json.dumps([
        {"key": "%s111122223333_elasticloadbalancing_us-east-1_app.example-alb.1234567890abcdef_20260309T%02d%02dZ_192.0.2.1_%04d.log.gz"
                % (prefix, i // 60 % 24, i % 60, i), "size": 1048576}
        for i in range(1000)]))
    monkeypatch.setenv("FAKE_EXTRA_KEYS_JSON", str(extra))


def test_cost_na_without_prices(home, env, tmp_path, monkeypatch):
    thousand_files(tmp_path, monkeypatch)
    r = run("alb", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    day_row = [l.split() for l in r.stdout.splitlines() if l.startswith("2026-03-09")][0]
    assert day_row[1] == "1000" and day_row[2] == "1000.00" and day_row[3] == "n/a"
    assert [l.split() for l in r.stdout.splitlines() if l.startswith("total")][0][3] == "n/a"


def test_cost_na_with_only_one_price(home, env, tmp_path, monkeypatch):
    write_config(home, "PRICE_PER_GB=0.09\n")
    thousand_files(tmp_path, monkeypatch)
    r = run("alb", "2026-03-09", "2026-03-09")
    assert "n/a" in r.stdout


def test_cost_estimate(home, env, tmp_path, monkeypatch):
    write_config(home, "PRICE_PER_GB=0.09\nPRICE_PER_1K_GET=0.0004\n")
    thousand_files(tmp_path, monkeypatch)
    r = run("alb", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    # GB = 1000 * 1 MiB / 1024^3 = 0.9765625 ; 0.9765625*0.09 + 1000/1000*0.0004 = 0.0882906...
    day_row = [l.split() for l in r.stdout.splitlines() if l.startswith("2026-03-09")][0]
    assert day_row[3] == "$0.0883"
    assert [l.split() for l in r.stdout.splitlines() if l.startswith("total")][0][3] == "$0.0883"


def test_estimate_cost_unit():
    from decimal import Decimal
    from s3logs import cli_inventory as ci
    assert ci.estimate_cost(Decimal("0.09"), Decimal("0.0004"), 1000 * 1048576, 1000) == "$0.0883"
    assert ci.estimate_cost(None, Decimal("0.0004"), 10, 1) == "n/a"
    assert ci.estimate_cost(Decimal("0.09"), None, 10, 1) == "n/a"
    assert ci.estimate_cost(Decimal("0.09"), Decimal("0.0004"), 0, 0) == "$0.0000"


def test_out_of_range_stamp_counted_unparsed(home, env):
    base = "AWSLogs/111122223333/elasticloadbalancing/us-east-1/2026/03/09/"
    gf.make(env.root, "alb", DAY, hours=[1], minutes=(0,))
    (env.root / BUCKET / (base + "planted_00010101T0000Z_y.log.gz")).write_bytes(b"x")
    r = run("alb", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    mpath = manifest_path(home, "alb", "2026-03-09", "2026-03-09")
    text = mpath.read_text()
    m = json.loads(text)
    assert m["skipped_unparsed"] >= 1 and len(m["files"]) == 1
    for out in (text, r.stdout, r.stderr):
        assert "planted" not in out and "00010101" not in out
    assert "Traceback" not in r.stderr


def test_day_with_only_a_folder_placeholder_is_still_a_gap(home, env, tmp_path, monkeypatch):
    """A console-created folder marker (zero bytes, key ends in /) is listed but is no log file."""
    extra = tmp_path / "extra.json"
    extra.write_text(json.dumps([{"key": "AWSLogs/111122223333/WAFLogs/us-east-1/example-acl/2026/03/09/",
                                  "size": 0}]))
    monkeypatch.setenv("FAKE_EXTRA_KEYS_JSON", str(extra))
    r = run("waf", "2026-03-09", "2026-03-09")
    assert r.returncode == 0, r.stderr
    assert "2026-03-09: " + GAP in r.stdout
    m = json.loads(manifest_path(home, "waf", "2026-03-09", "2026-03-09").read_text())
    assert m["files"] == [] and m["days"]["2026-03-09"]["listed"] == 1 and m["days"]["2026-03-09"]["files"] == 0
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall