Skip to content

culiops/culiops-skills / skills/sre-investigate

sre-investigate

Unverified

FileKindSize
SKILL.mdskill md71.5 KB
access/standalone.mdmarkdown19.0 KB
agents/evidence.mdmarkdown8.1 KB
agents/helper.mdmarkdown3.9 KB
analytics/bin/logs-fetchexecutablescript229 B
analytics/bin/logs-inventoryexecutablescript209 B
analytics/bin/logs-statusexecutablescript192 B
analytics/bin/sqlqexecutablescript212 B
analytics/lib/s3logs/__init__.pyscript62 B
analytics/lib/s3logs/awscli.pyscript10.0 KB
analytics/lib/s3logs/budget.pyscript9.6 KB
analytics/lib/s3logs/cli_fetch.pyscript34.0 KB
analytics/lib/s3logs/cli_inventory.pyscript7.0 KB
analytics/lib/s3logs/cli_sqlq.pyscript18.4 KB
analytics/lib/s3logs/cli_status.pyscript3.0 KB
analytics/lib/s3logs/config.pyscript7.4 KB
analytics/lib/s3logs/convert.pyscript23.6 KB
analytics/lib/s3logs/duck.pyscript11.4 KB
analytics/lib/s3logs/layout.pyscript8.8 KB
analytics/lib/s3logs/paths.pyscript4.9 KB
analytics/lib/s3logs/redact.pyscript22.1 KB
analytics/lib/s3logs/s5.pyscript9.1 KB
analytics/lib/s3logs/sqlguard.pyscript11.8 KB
analytics/lib/s3logs/validate.pyscript4.4 KB
analytics/logs.env.exampleother4.2 KB
analytics/sql/alb/by_minute.sqlother918 B
analytics/sql/alb/lb_vs_target_status.sqlother629 B
analytics/sql/alb/status_by_target.sqlother887 B
analytics/sql/alb/summary.sqlother1.2 KB
analytics/sql/alb/target_latency_pct.sqlother1.4 KB
analytics/sql/alb/top_clients.sqlother429 B
analytics/sql/alb/top_paths_5xx.sqlother673 B
analytics/sql/cloudfront/aborted.sqlother1.1 KB
analytics/sql/cloudfront/by_day.sqlother499 B
analytics/sql/cloudfront/by_hour.sqlother489 B
analytics/sql/cloudfront/by_object.sqlother544 B
analytics/sql/cloudfront/bytes_per_viewer.sqlother1.1 KB
analytics/sql/cloudfront/cache_results.sqlother823 B
analytics/sql/cloudfront/client_concentration.sqlother707 B
analytics/sql/cloudfront/edges.sqlother434 B
analytics/sql/cloudfront/object_by_day.sqlother599 B
analytics/sql/cloudfront/range_requests.sqlother528 B
analytics/sql/cloudfront/referer_pages.sqlother393 B
analytics/sql/cloudfront/status_codes.sqlother342 B
analytics/sql/cloudfront/summary.sqlother719 B
analytics/sql/cloudfront/top_clients.sqlother353 B
analytics/sql/cloudfront/top_objects.sqlother349 B
analytics/sql/cloudfront/top_referers.sqlother492 B
analytics/sql/cloudfront/user_agents.sqlother533 B
analytics/sql/waf/asns.sqlother378 B
analytics/sql/waf/by_day.sqlother453 B
analytics/sql/waf/by_minute.sqlother314 B
analytics/sql/waf/client_keys.sqlother443 B
analytics/sql/waf/clients.sqlother521 B
analytics/sql/waf/compare_days.sqlother924 B
analytics/sql/waf/concentration.sqlother775 B
analytics/sql/waf/count_rules.sqlother556 B
analytics/sql/waf/countries.sqlother373 B
analytics/sql/waf/fingerprints.sqlother424 B
analytics/sql/waf/header_shape.sqlother425 B
analytics/sql/waf/labels.sqlother347 B
analytics/sql/waf/matched_data.sqlother530 B
analytics/sql/waf/rules_by_minute.sqlother497 B
analytics/sql/waf/summary.sqlother735 B
analytics/sql/waf/top_rules.sqlother361 B
analytics/sql/waf/uris.sqlother402 B
analytics/sql/waf/user_agents.sqlother557 B
analytics/tests/conftest.pyscript4.7 KB
analytics/tests/gen_fixtures.pyscript19.7 KB
analytics/tests/stubs/awsexecutablescript4.9 KB
analytics/tests/stubs/s5cmdexecutablescript4.5 KB
analytics/tests/test_awscli.pyscript16.0 KB
analytics/tests/test_budget.pyscript13.3 KB
analytics/tests/test_config.pyscript9.1 KB
analytics/tests/test_convert.pyscript25.9 KB
analytics/tests/test_duck.pyscript10.6 KB
analytics/tests/test_fetch.pyscript42.8 KB
analytics/tests/test_gen_fixtures.pyscript1.9 KB
analytics/tests/test_inventory.pyscript12.9 KB
analytics/tests/test_layout.pyscript9.8 KB
analytics/tests/test_named_queries.pyscript26.7 KB
analytics/tests/test_paths.pyscript3.2 KB
analytics/tests/test_redact.pyscript32.0 KB
analytics/tests/test_s5.pyscript17.8 KB
analytics/tests/test_sqlguard.pyscript10.8 KB
analytics/tests/test_sqlq.pyscript37.2 KB
analytics/tests/test_status.pyscript5.8 KB
analytics/tests/test_validate.pyscript2.9 KB
environment.template.mdmarkdown7.1 KB
playbooks/changes.mdmarkdown37.2 KB
playbooks/errors.mdmarkdown10.4 KB
playbooks/external.mdmarkdown15.4 KB
playbooks/infra-state.mdmarkdown13.7 KB
playbooks/logs-s3.mdmarkdown29.5 KB
playbooks/logs.mdmarkdown21.3 KB
playbooks/metrics.mdmarkdown39.0 KB
playbooks/other-backends.mdmarkdown7.2 KB
playbooks/traces.mdmarkdown16.2 KB
playbooks/waf.mdmarkdown17.2 KB
templates/incident.mdmarkdown3.9 KB
templates/ledger.mdmarkdown9.7 KB
templates/lesson.mdmarkdown2.7 KB
templates/profile.mdmarkdown4.5 KB
templates/report.mdmarkdown6.5 KB

analytics/lib/s3logs/cli_inventory.py

View on GitHub
"""logs-inventory <dataset> <from-day> <to-day>: list S3, count, estimate, write a manifest.

Exit codes: 0 ok, 2 refused (bad input, module off), 3 AWS/S3 error.
"""
from __future__ import annotations

import argparse
import datetime as dt
import json
import os
import sys
from decimal import ROUND_HALF_UP, Decimal
from pathlib import Path
from typing import Dict, List, Optional

if __name__ == "__main__":
    # python3 -I drops the script dir from sys.path; add the lib dir that holds this package.
    sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from s3logs import awscli, config, layout, paths, validate  # noqa: E402
from s3logs.validate import Refused  # noqa: E402

MAX_DAYS = 92
GAP_SENTENCE = "0 files — logging may be off or delivery late; this is a gap, not zero traffic"
_MIB = 1024 * 1024
_GIB = Decimal(1024 ** 3)


def estimate_cost(price_per_gb: Optional[Decimal], price_per_1k_get: Optional[Decimal],
                  nbytes: int, nfiles: int) -> str:
    """GB * price_per_gb + files / 1000 * price_per_1k_get, GB = bytes / 1024^3, 4 decimals."""
    if price_per_gb is None or price_per_1k_get is None:
        return "n/a"
    cost = Decimal(nbytes) / _GIB * price_per_gb + Decimal(nfiles) / 1000 * price_per_1k_get
    return "$" + str(cost.quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP))


def _iso(t: dt.datetime) -> str:
    return t.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")


def _write_manifest(path: Path, doc: dict) -> None:
    """Atomic write, mode 0600, into a private directory."""
    paths.ensure_private_dirs()
    paths.private_subdir(paths.state_root() / "manifests", path.parent.name)
    tmp = path.with_name(".%s.%d.tmp" % (path.name, os.getpid()))
    fd = os.open(str(tmp), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
    try:
        with os.fdopen(fd, "w") as f:
            json.dump(doc, f, indent=2, sort_keys=True)
            f.write("\n")
        os.replace(str(tmp), str(path))
    except BaseException:
        try:
            os.unlink(str(tmp))
        except OSError:
            pass
        raise


def _load_config() -> config.Config:
    cfg_path = paths.config_path()
    if not (cfg_path.exists() or cfg_path.is_symlink()):
        raise Refused("S3 log module: off (not configured)")
    try:
        return config.load(cfg_path)
    except config.ConfigError as e:
        raise Refused("S3 log module: off (config error: %s)" % e)


def run(dataset: str, day_from: dt.date, day_to: dt.date) -> int:
    if day_to < day_from:
        raise Refused("to-day is before from-day")
    ndays = (day_to - day_from).days + 1
    if ndays > MAX_DAYS:
        raise Refused("range of %d days is over the %d-day limit; split it" % (ndays, MAX_DAYS))
    cfg = _load_config()
    ds = cfg.datasets.get(dataset)
    if ds is None:
        raise Refused("unknown dataset %r (configured: %s)" % (dataset, ", ".join(cfg.datasets) or "none"))

    days = [day_from + dt.timedelta(days=i) for i in range(ndays)]
    files: List[dict] = []
    per_day: Dict[dt.date, Dict[str, int]] = {}
    seen = set()
    unsafe = unparsed = unsupported = 0
    for d in days:
        tally = per_day.setdefault(d, {"listed": 0, "files": 0, "bytes": 0})
        for prefix in layout.list_prefixes(ds, d):
            for e in awscli.list_objects(ds.bucket, prefix, cfg.aws_profile, cfg.aws_region):
                tally["listed"] += 1
                try:
                    validate.safe_key(e.key)
                except Refused:
                    unsafe += 1          # counted only: hostile key text is never kept or printed
                    continue
                if e.key in seen:
                    continue
                seen.add(e.key)
                if layout.is_unsupported_format(ds, e.key):
                    unsupported += 1
                    continue
                ft = layout.file_time(ds, e.key) if layout.in_day(ds, e.key, d) else None
                if ft is None:
                    unparsed += 1
                    continue
                tally["files"] += 1
                tally["bytes"] += e.size
                files.append({"key": e.key, "size": e.size, "etag": e.etag,
                              "start": _iso(ft.start), "end": _iso(ft.end)})
    files.sort(key=lambda f: (f["start"], f["key"]))

    now = dt.datetime.now(dt.timezone.utc)
    mpath = paths.manifest_path(ds.name, day_from.isoformat(), day_to.isoformat())
    _write_manifest(mpath, {
        "dataset": ds.name, "type": ds.type, "bucket": ds.bucket,
        "from": day_from.isoformat(), "to": day_to.isoformat(), "listed_at": _iso(now),
        "files": files, "skipped_unsafe": unsafe, "skipped_unparsed": unparsed,
        "skipped_unsupported": unsupported, "unsupported_format": unsupported > 0,
        "days": {d.isoformat(): {"listed": t["listed"], "files": t["files"], "bytes": t["bytes"]}
                 for d, t in per_day.items()},
    })

    out = ["%-10s  %8s  %10s  %9s" % ("day", "files", "MB", "est_cost")]
    tot_files = tot_bytes = 0
    for d in days:
        t = per_day[d]
        tot_files += t["files"]
        tot_bytes += t["bytes"]
        out.append("%-10s  %8d  %10.2f  %9s" % (
            d.isoformat(), t["files"], t["bytes"] / _MIB,
            estimate_cost(cfg.price_per_gb, cfg.price_per_1k_get, t["bytes"], t["files"])))
    out.append("%-10s  %8d  %10.2f  %9s" % (
        "total", tot_files, tot_bytes / _MIB,
        estimate_cost(cfg.price_per_gb, cfg.price_per_1k_get, tot_bytes, tot_files)))
    for d in days:
        if per_day[d]["files"] == 0:
            out.append("%s: %s" % (d.isoformat(), GAP_SENTENCE))
    if unsupported:
        out.append("format unsupported (v2 JSON/Parquet): %d objects left out; this is a gap, "
                   "those logs cannot be queried" % unsupported)
    if unsafe:
        out.append("skipped %d objects with unsafe key names (not listed, never fetched)" % unsafe)
    if unparsed:
        out.append("skipped %d objects whose file time could not be read from the key" % unparsed)
    out.append("manifest: %s" % mpath)
    sys.stdout.write("\n".join(out) + "\n")
    return 0


class _Parser(argparse.ArgumentParser):
    def error(self, message):
        raise Refused(message)


def main(argv: Optional[List[str]] = None) -> int:
    p = _Parser(prog="logs-inventory", description="List an S3 log dataset for a day range and write a manifest.")
    p.add_argument("dataset")
    p.add_argument("from_day")
    p.add_argument("to_day")
    try:
        args = p.parse_args(argv)
        dataset = validate.dataset_name(args.dataset)
        return run(dataset, validate.day(args.from_day), validate.day(args.to_day))
    except Refused as e:
        sys.stderr.write("refused: %s\n" % e)
        return 2
    except awscli.AwsError as e:
        sys.stderr.write("aws error: %s\n" % e.msg)
        if e.hint:
            sys.stderr.write("%s\n" % e.hint)
        return 3
    except OSError as e:
        sys.stderr.write("refused: cannot write the manifest: %s\n" % (e.strerror or e))
        return 2


if __name__ == "__main__":
    sys.exit(main())
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall