Skip to content

culiops/culiops-skills / skills/sre-investigate

sre-investigate

Unverified

FileKindSize
SKILL.mdskill md71.5 KB
access/standalone.mdmarkdown19.0 KB
agents/evidence.mdmarkdown8.1 KB
agents/helper.mdmarkdown3.9 KB
analytics/bin/logs-fetchexecutablescript229 B
analytics/bin/logs-inventoryexecutablescript209 B
analytics/bin/logs-statusexecutablescript192 B
analytics/bin/sqlqexecutablescript212 B
analytics/lib/s3logs/__init__.pyscript62 B
analytics/lib/s3logs/awscli.pyscript10.0 KB
analytics/lib/s3logs/budget.pyscript9.6 KB
analytics/lib/s3logs/cli_fetch.pyscript34.0 KB
analytics/lib/s3logs/cli_inventory.pyscript7.0 KB
analytics/lib/s3logs/cli_sqlq.pyscript18.4 KB
analytics/lib/s3logs/cli_status.pyscript3.0 KB
analytics/lib/s3logs/config.pyscript7.4 KB
analytics/lib/s3logs/convert.pyscript23.6 KB
analytics/lib/s3logs/duck.pyscript11.4 KB
analytics/lib/s3logs/layout.pyscript8.8 KB
analytics/lib/s3logs/paths.pyscript4.9 KB
analytics/lib/s3logs/redact.pyscript22.1 KB
analytics/lib/s3logs/s5.pyscript9.1 KB
analytics/lib/s3logs/sqlguard.pyscript11.8 KB
analytics/lib/s3logs/validate.pyscript4.4 KB
analytics/logs.env.exampleother4.2 KB
analytics/sql/alb/by_minute.sqlother918 B
analytics/sql/alb/lb_vs_target_status.sqlother629 B
analytics/sql/alb/status_by_target.sqlother887 B
analytics/sql/alb/summary.sqlother1.2 KB
analytics/sql/alb/target_latency_pct.sqlother1.4 KB
analytics/sql/alb/top_clients.sqlother429 B
analytics/sql/alb/top_paths_5xx.sqlother673 B
analytics/sql/cloudfront/aborted.sqlother1.1 KB
analytics/sql/cloudfront/by_day.sqlother499 B
analytics/sql/cloudfront/by_hour.sqlother489 B
analytics/sql/cloudfront/by_object.sqlother544 B
analytics/sql/cloudfront/bytes_per_viewer.sqlother1.1 KB
analytics/sql/cloudfront/cache_results.sqlother823 B
analytics/sql/cloudfront/client_concentration.sqlother707 B
analytics/sql/cloudfront/edges.sqlother434 B
analytics/sql/cloudfront/object_by_day.sqlother599 B
analytics/sql/cloudfront/range_requests.sqlother528 B
analytics/sql/cloudfront/referer_pages.sqlother393 B
analytics/sql/cloudfront/status_codes.sqlother342 B
analytics/sql/cloudfront/summary.sqlother719 B
analytics/sql/cloudfront/top_clients.sqlother353 B
analytics/sql/cloudfront/top_objects.sqlother349 B
analytics/sql/cloudfront/top_referers.sqlother492 B
analytics/sql/cloudfront/user_agents.sqlother533 B
analytics/sql/waf/asns.sqlother378 B
analytics/sql/waf/by_day.sqlother453 B
analytics/sql/waf/by_minute.sqlother314 B
analytics/sql/waf/client_keys.sqlother443 B
analytics/sql/waf/clients.sqlother521 B
analytics/sql/waf/compare_days.sqlother924 B
analytics/sql/waf/concentration.sqlother775 B
analytics/sql/waf/count_rules.sqlother556 B
analytics/sql/waf/countries.sqlother373 B
analytics/sql/waf/fingerprints.sqlother424 B
analytics/sql/waf/header_shape.sqlother425 B
analytics/sql/waf/labels.sqlother347 B
analytics/sql/waf/matched_data.sqlother530 B
analytics/sql/waf/rules_by_minute.sqlother497 B
analytics/sql/waf/summary.sqlother735 B
analytics/sql/waf/top_rules.sqlother361 B
analytics/sql/waf/uris.sqlother402 B
analytics/sql/waf/user_agents.sqlother557 B
analytics/tests/conftest.pyscript4.7 KB
analytics/tests/gen_fixtures.pyscript19.7 KB
analytics/tests/stubs/awsexecutablescript4.9 KB
analytics/tests/stubs/s5cmdexecutablescript4.5 KB
analytics/tests/test_awscli.pyscript16.0 KB
analytics/tests/test_budget.pyscript13.3 KB
analytics/tests/test_config.pyscript9.1 KB
analytics/tests/test_convert.pyscript25.9 KB
analytics/tests/test_duck.pyscript10.6 KB
analytics/tests/test_fetch.pyscript42.8 KB
analytics/tests/test_gen_fixtures.pyscript1.9 KB
analytics/tests/test_inventory.pyscript12.9 KB
analytics/tests/test_layout.pyscript9.8 KB
analytics/tests/test_named_queries.pyscript26.7 KB
analytics/tests/test_paths.pyscript3.2 KB
analytics/tests/test_redact.pyscript32.0 KB
analytics/tests/test_s5.pyscript17.8 KB
analytics/tests/test_sqlguard.pyscript10.8 KB
analytics/tests/test_sqlq.pyscript37.2 KB
analytics/tests/test_status.pyscript5.8 KB
analytics/tests/test_validate.pyscript2.9 KB
environment.template.mdmarkdown7.1 KB
playbooks/changes.mdmarkdown37.2 KB
playbooks/errors.mdmarkdown10.4 KB
playbooks/external.mdmarkdown15.4 KB
playbooks/infra-state.mdmarkdown13.7 KB
playbooks/logs-s3.mdmarkdown29.5 KB
playbooks/logs.mdmarkdown21.3 KB
playbooks/metrics.mdmarkdown39.0 KB
playbooks/other-backends.mdmarkdown7.2 KB
playbooks/traces.mdmarkdown16.2 KB
playbooks/waf.mdmarkdown17.2 KB
templates/incident.mdmarkdown3.9 KB
templates/ledger.mdmarkdown9.7 KB
templates/lesson.mdmarkdown2.7 KB
templates/profile.mdmarkdown4.5 KB
templates/report.mdmarkdown6.5 KB

analytics/lib/s3logs/awscli.py

View on GitHub
"""The only ways this module talks to AWS, both through the `aws` CLI with shell=False:

* `aws s3api list-objects-v2`, read-only listing, and
* `aws configure export-credentials [--profile P] --format process`, which turns any
  profile (SSO included, which s5cmd cannot load itself) into short-lived keys for s5cmd.

argv is built from a fixed allow-list of shapes (`_check_argv` refuses everything else).
There is no generic "run aws" entry point on purpose; downloads go through s5cmd, not here.
The exported keys are secrets: they are returned to the caller and never put in an error,
a hint or a log line (stdout of the export command is never quoted).
"""
from __future__ import annotations

import json
import os
import re
import shutil
import subprocess
from typing import Dict, Iterator, List, NamedTuple, Optional, Tuple

from s3logs import validate
from s3logs.validate import Refused

_TIMEOUT_S = 600
_EXPORT_TIMEOUT_S = 60
_MAX_PAGES = 10000
_PAGE_SIZE = "1000"
_ALLOWED_OPTIONS = frozenset(
    ["--bucket", "--prefix", "--profile", "--region", "--output", "--starting-token", "--page-size"])
# Same shapes config.py enforces; checked again here so this module is safe on its own.
_PROFILE_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.@+-]{0,127}")
_REGION_RE = re.compile(r"[a-z]{2}(?:-[a-z]+)+-[0-9]")
_TOKEN_RE = re.compile(r"[A-Za-z0-9+/=_.-]{1,4096}")
_EXPIRED_MARKERS = ("ExpiredToken", "Token has expired", "SSO session", "SSO Token",
                    "UnauthorizedException")
_CRED_VALUE_RE = re.compile(r"[\x21-\x7e]{1,16384}")
_EXPORT_HINT = "aws configure export-credentials needs AWS CLI v2.9 or newer and a working profile"
_CONTROL_RE = re.compile(r"[\x00-\x08\x0b-\x1f\x7f]")


class Entry(NamedTuple):
    key: str
    size: int
    etag: str


class AwsError(Exception):
    """aws exited non-zero (or could not run). `hint` is a command for the user, if any."""

    def __init__(self, msg: str, hint: Optional[str] = None):
        super().__init__(msg)
        self.msg = msg
        self.hint = hint

    def __str__(self) -> str:
        return self.msg


def _check_export_argv(rest: List[str]) -> None:
    """`[--profile P] --format process` and nothing else, in that order."""
    if rest == ["--format", "process"]:
        return
    if len(rest) == 4 and rest[0] == "--profile" and rest[2:] == ["--format", "process"] \
            and _PROFILE_RE.fullmatch(rest[1]):
        return
    raise Refused("only aws configure export-credentials [--profile P] --format process is allowed")


def _check_argv(argv: List[str]) -> None:
    """Refuse any argv that is not `aws s3api list-objects-v2` plus allow-listed options, or
    exactly `aws configure export-credentials [--profile P] --format process`."""
    if argv[:3] == ["aws", "configure", "export-credentials"]:
        _check_export_argv(argv[3:])
        return
    if argv[:3] != ["aws", "s3api", "list-objects-v2"]:
        raise Refused("only aws s3api list-objects-v2 and aws configure export-credentials are allowed")
    rest = argv[3:]
    if len(rest) % 2:
        raise Refused("unexpected aws argument")
    seen = set()
    for i in range(0, len(rest), 2):
        name, value = rest[i], rest[i + 1]
        if name not in _ALLOWED_OPTIONS or name in seen:
            raise Refused("aws option %s is not allowed" % name)
        seen.add(name)
        if value.startswith("-"):
            raise Refused("aws option value may not start with '-'")
    if not {"--bucket", "--prefix", "--output", "--page-size"} <= seen:
        raise Refused("aws argv is incomplete")
    if rest[rest.index("--output") + 1] != "json":
        raise Refused("only --output json is allowed")


def _build_argv(bucket: str, prefix: str, profile: Optional[str], region: Optional[str],
                token: Optional[str]) -> List[str]:
    validate.bucket(bucket)
    validate.safe_key(prefix)
    if profile is not None and not _PROFILE_RE.fullmatch(profile):
        raise Refused("invalid AWS profile name")
    if region is not None and not _REGION_RE.fullmatch(region):
        raise Refused("invalid AWS region")
    if token is not None and not _TOKEN_RE.fullmatch(token):
        raise Refused("invalid pagination token")
    argv = ["aws", "s3api", "list-objects-v2", "--bucket", bucket, "--prefix", prefix]
    if profile:
        argv += ["--profile", profile]
    if region:
        argv += ["--region", region]
    argv += ["--output", "json", "--page-size", _PAGE_SIZE]
    if token:
        argv += ["--starting-token", token]
    _check_argv(argv)
    return argv


def _clean(text: str, limit: int = 400) -> str:
    text = _CONTROL_RE.sub(" ", text).strip()
    text = " ".join(text.split())
    return text if len(text) <= limit else text[:limit] + "..."


def _hint_for(stderr: str, profile: Optional[str]) -> Optional[str]:
    if any(marker in stderr for marker in _EXPIRED_MARKERS):
        return "! aws sso login --profile %s" % profile if profile else "! aws sso login"
    if "AccessDenied" in stderr:
        return "check that the role may s3:ListBucket on this bucket and prefix"
    return None


def _exec(argv: List[str], timeout: int, timeout_hint: str,
          region: Optional[str] = None) -> "subprocess.CompletedProcess[str]":
    """Run an already-checked aws argv: shell=False, no stdin, no pager, no prompt."""
    exe = shutil.which("aws")
    if not exe:
        raise AwsError("the aws CLI was not found on PATH", "install AWS CLI v2 and re-run")
    env = dict(os.environ)
    env["AWS_PAGER"] = ""            # never open a pager
    env["AWS_CLI_AUTO_PROMPT"] = "off"
    if region:
        env["AWS_REGION"] = region
    try:
        return subprocess.run([exe] + argv[1:], stdin=subprocess.DEVNULL, capture_output=True,
                              text=True, errors="replace", timeout=timeout, env=env, shell=False)
    except subprocess.TimeoutExpired:
        raise AwsError("aws timed out after %d seconds" % timeout, timeout_hint) from None
    except OSError as e:
        raise AwsError("could not run aws: %s" % (e.strerror or e))


def _run(argv: List[str], profile: Optional[str]) -> str:
    r = _exec(argv, _TIMEOUT_S, "re-run with a narrower day range, or check the network/prefix")
    if r.returncode != 0:
        raise AwsError("aws exited %d: %s" % (r.returncode, _clean(r.stderr) or "no error text"),
                       _hint_for(r.stderr, profile))
    return r.stdout


def _parse_page(text: str) -> Tuple[List[Entry], Optional[str]]:
    """One CLI response -> entries and the next token (if any). Empty output = no objects."""
    if not text.strip():
        return [], None
    try:
        doc = json.loads(text)
    except ValueError:
        raise AwsError("aws printed output that is not JSON")
    if not isinstance(doc, dict):
        raise AwsError("aws printed an unexpected listing shape")
    entries = []
    for obj in doc.get("Contents") or []:
        key, size, etag = (obj.get("Key"), obj.get("Size"), obj.get("ETag")) if isinstance(obj, dict) \
            else (None, None, None)
        if not isinstance(key, str) or isinstance(size, bool) or not isinstance(size, int) \
                or not isinstance(etag, str):
            raise AwsError("aws printed an unexpected listing shape")
        entries.append(Entry(key, size, etag.strip('"')))
    token = doc.get("NextToken")
    return entries, token if isinstance(token, str) and token else None


def list_objects(bucket: str, prefix: str, profile: Optional[str], region: Optional[str]) -> Iterator[Entry]:
    """Yield every object under bucket/prefix. Raises Refused for bad input, AwsError for aws failures."""
    token = None
    for _ in range(_MAX_PAGES):
        argv = _build_argv(bucket, prefix, profile, region, token)
        entries, token = _parse_page(_run(argv, profile))
        for e in entries:
            yield e
        if not token:
            return
    raise AwsError("listing did not finish after %d pages" % _MAX_PAGES)


def export_credentials(profile: Optional[str], region: Optional[str]) -> Dict[str, str]:
    """Short-lived keys for `profile` as AWS_* variables, via `aws configure export-credentials`.

    s5cmd cannot load SSO profiles, the aws CLI can. The result is a secret: callers put it in
    a child environment only. Any failure is an AwsError whose text never includes stdout
    (which holds the keys); stderr is cleaned like the other helpers."""
    if profile is not None and not _PROFILE_RE.fullmatch(profile):
        raise Refused("invalid AWS profile name")
    if region is not None and not _REGION_RE.fullmatch(region):
        raise Refused("invalid AWS region")
    argv = ["aws", "configure", "export-credentials"]
    if profile:
        argv += ["--profile", profile]
    argv += ["--format", "process"]
    _check_argv(argv)
    timeout_hint = ("the profile's credential lookup timed out; check the network or run "
                    "aws configure export-credentials%s" % (" --profile %s" % profile if profile else ""))
    r = _exec(argv, _EXPORT_TIMEOUT_S, timeout_hint, region)
    if r.returncode != 0:
        hint = _hint_for(r.stderr, profile)
        if hint is None or hint.startswith("check that"):
            hint = _EXPORT_HINT
        raise AwsError("aws exited %d: %s" % (r.returncode, _clean(r.stderr) or "no error text"), hint)
    try:
        doc = json.loads(r.stdout)
    except ValueError:
        doc = None
    out = {}  # type: Dict[str, str]
    if isinstance(doc, dict) and doc.get("Version") == 1 and not isinstance(doc.get("Version"), bool):
        for src, dst, required in (("AccessKeyId", "AWS_ACCESS_KEY_ID", True),
                                   ("SecretAccessKey", "AWS_SECRET_ACCESS_KEY", True),
                                   ("SessionToken", "AWS_SESSION_TOKEN", False)):
            value = doc.get(src)
            if value is None and not required:
                continue
            if not isinstance(value, str) or not _CRED_VALUE_RE.fullmatch(value):
                out = {}
                break
            out[dst] = value
    if "AWS_ACCESS_KEY_ID" not in out or "AWS_SECRET_ACCESS_KEY" not in out:
        raise AwsError("aws configure export-credentials printed no usable credentials", _EXPORT_HINT)
    return out
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall