Skip to content

culiops/culiops-skills / skills/sre-investigate

sre-investigate

Unverified

FileKindSize
SKILL.mdskill md71.5 KB
access/standalone.mdmarkdown19.0 KB
agents/evidence.mdmarkdown8.1 KB
agents/helper.mdmarkdown3.9 KB
analytics/bin/logs-fetchexecutablescript229 B
analytics/bin/logs-inventoryexecutablescript209 B
analytics/bin/logs-statusexecutablescript192 B
analytics/bin/sqlqexecutablescript212 B
analytics/lib/s3logs/__init__.pyscript62 B
analytics/lib/s3logs/awscli.pyscript10.0 KB
analytics/lib/s3logs/budget.pyscript9.6 KB
analytics/lib/s3logs/cli_fetch.pyscript34.0 KB
analytics/lib/s3logs/cli_inventory.pyscript7.0 KB
analytics/lib/s3logs/cli_sqlq.pyscript18.4 KB
analytics/lib/s3logs/cli_status.pyscript3.0 KB
analytics/lib/s3logs/config.pyscript7.4 KB
analytics/lib/s3logs/convert.pyscript23.6 KB
analytics/lib/s3logs/duck.pyscript11.4 KB
analytics/lib/s3logs/layout.pyscript8.8 KB
analytics/lib/s3logs/paths.pyscript4.9 KB
analytics/lib/s3logs/redact.pyscript22.1 KB
analytics/lib/s3logs/s5.pyscript9.1 KB
analytics/lib/s3logs/sqlguard.pyscript11.8 KB
analytics/lib/s3logs/validate.pyscript4.4 KB
analytics/logs.env.exampleother4.2 KB
analytics/sql/alb/by_minute.sqlother918 B
analytics/sql/alb/lb_vs_target_status.sqlother629 B
analytics/sql/alb/status_by_target.sqlother887 B
analytics/sql/alb/summary.sqlother1.2 KB
analytics/sql/alb/target_latency_pct.sqlother1.4 KB
analytics/sql/alb/top_clients.sqlother429 B
analytics/sql/alb/top_paths_5xx.sqlother673 B
analytics/sql/cloudfront/aborted.sqlother1.1 KB
analytics/sql/cloudfront/by_day.sqlother499 B
analytics/sql/cloudfront/by_hour.sqlother489 B
analytics/sql/cloudfront/by_object.sqlother544 B
analytics/sql/cloudfront/bytes_per_viewer.sqlother1.1 KB
analytics/sql/cloudfront/cache_results.sqlother823 B
analytics/sql/cloudfront/client_concentration.sqlother707 B
analytics/sql/cloudfront/edges.sqlother434 B
analytics/sql/cloudfront/object_by_day.sqlother599 B
analytics/sql/cloudfront/range_requests.sqlother528 B
analytics/sql/cloudfront/referer_pages.sqlother393 B
analytics/sql/cloudfront/status_codes.sqlother342 B
analytics/sql/cloudfront/summary.sqlother719 B
analytics/sql/cloudfront/top_clients.sqlother353 B
analytics/sql/cloudfront/top_objects.sqlother349 B
analytics/sql/cloudfront/top_referers.sqlother492 B
analytics/sql/cloudfront/user_agents.sqlother533 B
analytics/sql/waf/asns.sqlother378 B
analytics/sql/waf/by_day.sqlother453 B
analytics/sql/waf/by_minute.sqlother314 B
analytics/sql/waf/client_keys.sqlother443 B
analytics/sql/waf/clients.sqlother521 B
analytics/sql/waf/compare_days.sqlother924 B
analytics/sql/waf/concentration.sqlother775 B
analytics/sql/waf/count_rules.sqlother556 B
analytics/sql/waf/countries.sqlother373 B
analytics/sql/waf/fingerprints.sqlother424 B
analytics/sql/waf/header_shape.sqlother425 B
analytics/sql/waf/labels.sqlother347 B
analytics/sql/waf/matched_data.sqlother530 B
analytics/sql/waf/rules_by_minute.sqlother497 B
analytics/sql/waf/summary.sqlother735 B
analytics/sql/waf/top_rules.sqlother361 B
analytics/sql/waf/uris.sqlother402 B
analytics/sql/waf/user_agents.sqlother557 B
analytics/tests/conftest.pyscript4.7 KB
analytics/tests/gen_fixtures.pyscript19.7 KB
analytics/tests/stubs/awsexecutablescript4.9 KB
analytics/tests/stubs/s5cmdexecutablescript4.5 KB
analytics/tests/test_awscli.pyscript16.0 KB
analytics/tests/test_budget.pyscript13.3 KB
analytics/tests/test_config.pyscript9.1 KB
analytics/tests/test_convert.pyscript25.9 KB
analytics/tests/test_duck.pyscript10.6 KB
analytics/tests/test_fetch.pyscript42.8 KB
analytics/tests/test_gen_fixtures.pyscript1.9 KB
analytics/tests/test_inventory.pyscript12.9 KB
analytics/tests/test_layout.pyscript9.8 KB
analytics/tests/test_named_queries.pyscript26.7 KB
analytics/tests/test_paths.pyscript3.2 KB
analytics/tests/test_redact.pyscript32.0 KB
analytics/tests/test_s5.pyscript17.8 KB
analytics/tests/test_sqlguard.pyscript10.8 KB
analytics/tests/test_sqlq.pyscript37.2 KB
analytics/tests/test_status.pyscript5.8 KB
analytics/tests/test_validate.pyscript2.9 KB
environment.template.mdmarkdown7.1 KB
playbooks/changes.mdmarkdown37.2 KB
playbooks/errors.mdmarkdown10.4 KB
playbooks/external.mdmarkdown15.4 KB
playbooks/infra-state.mdmarkdown13.7 KB
playbooks/logs-s3.mdmarkdown29.5 KB
playbooks/logs.mdmarkdown21.3 KB
playbooks/metrics.mdmarkdown39.0 KB
playbooks/other-backends.mdmarkdown7.2 KB
playbooks/traces.mdmarkdown16.2 KB
playbooks/waf.mdmarkdown17.2 KB
templates/incident.mdmarkdown3.9 KB
templates/ledger.mdmarkdown9.7 KB
templates/lesson.mdmarkdown2.7 KB
templates/profile.mdmarkdown4.5 KB
templates/report.mdmarkdown6.5 KB

analytics/tests/test_layout.py

View on GitHub
from __future__ import annotations

import datetime as dt

import pytest

import gen_fixtures as gf
from s3logs import layout
from s3logs.awscli import Entry
from s3logs.config import Dataset
from s3logs.validate import Refused

UTC = dt.timezone.utc

CF = Dataset("cf", "cloudfront", "example-logs-bucket", "cf/E2EXAMPLE.{yyyy}-{MM}-{dd}-{HH}")
CF_V2 = Dataset("cfv2", "cloudfront", "example-logs-bucket",
                "AWSLogs/111122223333/CloudFront/E2EXAMPLE/{yyyy}/{MM}/{dd}/{HH}/")
ALB = Dataset("alb", "alb", "example-logs-bucket",
              "AWSLogs/111122223333/elasticloadbalancing/us-east-1/{yyyy}/{MM}/{dd}/")
WAF = Dataset("waf", "waf", "example-logs-bucket",
              "AWSLogs/111122223333/WAFLogs/us-east-1/example-acl/{yyyy}/{MM}/{dd}/{HH}/")
DAY = dt.date(2026, 3, 9)
NEXT = dt.date(2026, 3, 10)


def entry(key):
    return Entry(key, 100, "abc")


def alb_entry(delivery_day, hh, mm, day_offset=0):
    end = dt.datetime(delivery_day.year, delivery_day.month, delivery_day.day) + dt.timedelta(
        days=day_offset, hours=hh, minutes=mm)
    return entry(gf.alb_key(delivery_day, end))


def cf_entry(day, hh):
    return entry(gf.cf_key(day, hh))


def keys(entries):
    return [e.key for e in entries]


def test_cloudfront_legacy_prefixes_one_per_day():
    assert layout.list_prefixes(CF, DAY) == ["cf/E2EXAMPLE.2026-03-09-"]


def test_cloudfront_v2_and_waf_one_prefix_per_day():
    assert layout.list_prefixes(CF_V2, DAY) == ["AWSLogs/111122223333/CloudFront/E2EXAMPLE/2026/03/09/"]
    assert layout.list_prefixes(WAF, DAY) == ["AWSLogs/111122223333/WAFLogs/us-east-1/example-acl/2026/03/09/"]


def test_hour_template_with_a_day_token_after_the_hour_keeps_per_hour_prefixes():
    odd = Dataset("odd", "cloudfront", "example-logs-bucket", "cf/{yyyy}-{MM}/h{HH}/{dd}/")
    got = layout.list_prefixes(odd, DAY)
    assert got == ["cf/2026-03/h%02d/09/" % h for h in range(24)]


def test_hour_template_missing_no_day_token_before_hour_keeps_per_hour_prefixes():
    odd = Dataset("odd", "cloudfront", "example-logs-bucket", "cf/{HH}/{yyyy}/{MM}/{dd}/")
    assert layout.list_prefixes(odd, DAY) == ["cf/%02d/2026/03/09/" % h for h in range(24)]


def test_day_prefix_is_cut_at_the_first_hour_token_only():
    t = Dataset("t", "cloudfront", "example-logs-bucket", "cf/{yyyy}/{MM}/{dd}/{HH}/E.{yyyy}-{MM}-{dd}-{HH}")
    assert layout.list_prefixes(t, DAY) == ["cf/2026/03/09/"]


def test_in_day_accepts_only_keys_of_that_day_and_template():
    assert layout.in_day(CF, gf.cf_key(DAY, 7), DAY)
    assert not layout.in_day(CF, gf.cf_key(NEXT, 7), DAY)                 # another day
    assert not layout.in_day(CF, "cf/E2EXAMPLE.2026-03-09-7.AAA.gz", DAY)  # hour part does not match
    assert not layout.in_day(CF, "cf/E2EXAMPLE.2026-03-09.AAA.gz", DAY)
    assert layout.in_day(CF, "cf/E2EXAMPLE.2026-03-09-99.AAA.gz", DAY)   # shape ok; file_time rejects it
    assert layout.file_time(CF, "cf/E2EXAMPLE.2026-03-09-99.AAA.gz") is None
    assert not layout.in_day(WAF, "AWSLogs/111122223333/WAFLogs/us-east-1/example-acl/2026/03/09/loose_20260309T0500Z_a.log.gz", DAY)
    assert layout.in_day(ALB, "anything/at/all", DAY)                     # no {HH}: nothing to check


def test_alb_prefix_is_day_dir():
    assert layout.list_prefixes(ALB, DAY) == [
        "AWSLogs/111122223333/elasticloadbalancing/us-east-1/2026/03/09/"]


def test_cloudfront_file_time_hour():
    ft = layout.file_time(CF, gf.cf_key(DAY, 7))
    assert ft.start == dt.datetime(2026, 3, 9, 7, 0, tzinfo=UTC)
    assert ft.end == dt.datetime(2026, 3, 9, 8, 0, tzinfo=UTC)
    assert ft.start.utcoffset() == dt.timedelta(0)


def test_cloudfront_v2_file_time_from_path_hour():
    ft = layout.file_time(CF_V2, "AWSLogs/111122223333/CloudFront/E2EXAMPLE/2026/03/09/23/x_1.gz")
    assert (ft.start.hour, ft.end) == (23, dt.datetime(2026, 3, 10, 0, 0, tzinfo=UTC))


def test_alb_file_time_from_end_time():
    end = dt.datetime(2026, 3, 9, 23, 40)
    ft = layout.file_time(ALB, gf.alb_key(DAY, end))
    assert ft.start == dt.datetime(2026, 3, 9, 23, 35, tzinfo=UTC)
    assert ft.end == dt.datetime(2026, 3, 9, 23, 40, tzinfo=UTC)


def test_waf_file_time_minute():
    ft = layout.file_time(WAF, gf.waf_key(dt.datetime(2026, 3, 9, 19, 50)))
    assert ft.start == dt.datetime(2026, 3, 9, 19, 45, tzinfo=UTC)
    assert ft.end == dt.datetime(2026, 3, 9, 19, 50, tzinfo=UTC)


@pytest.mark.parametrize("ds,key", [
    (CF, "cf/E2EXAMPLE.not-a-date.abc.gz"),
    (CF, "cf/E2EXAMPLE.2026-13-09-00.abc.gz"),
    (CF, "other/E2EXAMPLE.2026-03-09-00.abc.gz"),
    (ALB, "AWSLogs/111122223333/elasticloadbalancing/us-east-1/2026/03/09/readme.txt"),
    (ALB, "AWSLogs/x/y_20261340T9999Z_z.log.gz"),
])
def test_unparsable_file_time_is_none(ds, key):
    assert layout.file_time(ds, key) is None


def test_alb_slice_widens_into_next_day_dir():
    # Requests at 23:57 land in the file ending 00:00, which lives in the NEXT day's directory.
    listing = [
        alb_entry(DAY, 23, 50), alb_entry(DAY, 23, 55),
        alb_entry(NEXT, 0, 0), alb_entry(NEXT, 0, 5), alb_entry(NEXT, 0, 10),
    ]
    got = layout.select(ALB, listing, DAY, hours=(23, 23))
    names = keys(got)
    assert listing[2].key in names                 # ..._20260310T0000Z_...: 23:55-00:00
    assert listing[3].key in names                 # widen by one 5-minute unit
    assert listing[4].key not in names             # but nothing later
    assert listing[0].key in names and listing[1].key in names


def test_whole_day_widens_to_next_day_first_hour():
    listing = [cf_entry(DAY, h) for h in (0, 12, 23)] + [cf_entry(NEXT, h) for h in (0, 1)]
    got = layout.select(CF, listing, DAY)
    assert keys(got) == [e.key for e in listing[:4]]       # day + next day's hour 00
    assert listing[4].key not in keys(got)                  # hour 01 of the next day stays out


def test_whole_day_widens_alb_to_next_days_first_file():
    listing = [alb_entry(DAY, 0, 0), alb_entry(DAY, 0, 5), alb_entry(DAY, 12, 0),
               alb_entry(NEXT, 0, 0), alb_entry(NEXT, 0, 5), alb_entry(NEXT, 0, 10)]
    got = keys(layout.select(ALB, listing, DAY))
    assert listing[0].key not in got        # ends 00:00 on the day: covers the previous day
    assert listing[1].key in got and listing[2].key in got
    assert listing[3].key in got and listing[4].key in got
    assert listing[5].key not in got


def test_hour_range_inclusive():
    listing = [cf_entry(DAY, h) for h in range(8, 16)]
    got = keys(layout.select(CF, listing, DAY, hours=(10, 12)))
    # 10, 11, 12 inclusive, plus the widened hour 13; 9 and 14 stay out
    assert got == [cf_entry(DAY, h).key for h in (10, 11, 12, 13)]


def test_minute_range_start_inclusive_end_exclusive():
    listing = [alb_entry(DAY, 10, m) for m in (0, 5, 10, 15, 20)]
    got = keys(layout.select(ALB, listing, DAY, minutes=(10 * 60, 10 * 60 + 10)))
    # window [10:00, 10:10): files ending 10:05 and 10:10; widen takes 10:15; 10:00 ends at window start
    assert got == [listing[i].key for i in (1, 2, 3)]


def test_waf_minute_slice():
    listing = [entry(gf.waf_key(dt.datetime(2026, 3, 9, 19, m))) for m in (40, 45, 50, 55)]
    got = layout.select(WAF, listing, DAY, minutes=(19 * 60 + 45, 19 * 60 + 50))
    assert keys(got) == [listing[i].key for i in (2, 3)]    # 19:50 covers [19:45,19:50), 19:55 is the widen


def test_minute_range_to_end_of_day():
    listing = [alb_entry(DAY, 23, 55), alb_entry(NEXT, 0, 0), alb_entry(NEXT, 0, 5)]
    got = layout.select(ALB, listing, DAY, minutes=(23 * 60 + 55, 1440))
    # 23:55 ends where the window starts, so it is out; 00:00 closes the last minutes of the
    # day (next day's directory); 00:05 is the widen.
    assert keys(got) == [listing[1].key, listing[2].key]


def test_cloudfront_minute_slice_refused():
    with pytest.raises(Refused):
        layout.select(CF, [cf_entry(DAY, 1)], DAY, minutes=(60, 120))


def test_hours_and_minutes_together_refused():
    with pytest.raises(Refused):
        layout.select(ALB, [], DAY, hours=(1, 2), minutes=(60, 120))


def test_select_drops_unparsable_and_sorts_by_time():
    listing = [cf_entry(DAY, 5), entry("cf/junk.gz"), cf_entry(DAY, 3)]
    got = layout.select(CF, listing, DAY, hours=(0, 23))
    assert keys(got) == [cf_entry(DAY, 3).key, cf_entry(DAY, 5).key]


@pytest.mark.parametrize("key,expected", [
    ("cf/E2EXAMPLE.2026-03-09-00.abc.gz", False),
    ("AWSLogs/x/CloudFront/E2EXAMPLE/2026/03/09/00/part.json.gz", True),
    ("AWSLogs/x/CloudFront/E2EXAMPLE/2026/03/09/00/part.parquet", True),
    ("x/y.JSON", True),
])
def test_unsupported_format_cloudfront(key, expected):
    assert layout.is_unsupported_format(CF, key) is expected


def test_unsupported_format_only_for_cloudfront():
    assert layout.is_unsupported_format(WAF, "x/y.json.gz") is False
    assert layout.is_unsupported_format(ALB, "x/y.parquet") is False


@pytest.mark.parametrize("ds,key", [
    (ALB, "AWSLogs/x/y_00010101T0000Z_z.log.gz"),            # end - 5 min underflows year 1
    (WAF, "AWSLogs/x/y_00010101T0004Z_z.log.gz"),
    (CF, "cf/E2EXAMPLE.9999-12-31-23.abc.gz"),               # start + 1 h overflows year 9999
])
def test_overflowing_stamp_is_unparsable_not_a_crash(ds, key):
    assert layout.file_time(ds, key) is None


def test_valid_extreme_stamps_still_parse():
    ft = layout.file_time(ALB, "AWSLogs/x/y_99991231T2359Z_z.log.gz")
    assert ft.end == dt.datetime(9999, 12, 31, 23, 59, tzinfo=UTC)
    ft = layout.file_time(ALB, "AWSLogs/x/y_00010101T0005Z_z.log.gz")
    assert ft.start == dt.datetime(1, 1, 1, 0, 0, tzinfo=UTC)


def test_select_on_last_calendar_day_refuses_instead_of_crashing():
    listing = [entry("AWSLogs/x/y_99991231T2359Z_z.log.gz")]
    with pytest.raises(Refused):
        layout.select(ALB, listing, dt.date(9999, 12, 31))                     # day+1 does not exist
    got = layout.select(ALB, listing, dt.date(9999, 12, 31), minutes=(23 * 60, 23 * 60 + 59))
    assert keys(got) == [listing[0].key]                                       # widen overflow is tolerated
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall