Skip to content

culiops/culiops-skills / skills/sre-investigate

sre-investigate

Unverified

FileKindSize
SKILL.mdskill md71.5 KB
access/standalone.mdmarkdown19.0 KB
agents/evidence.mdmarkdown8.1 KB
agents/helper.mdmarkdown3.9 KB
analytics/bin/logs-fetchexecutablescript229 B
analytics/bin/logs-inventoryexecutablescript209 B
analytics/bin/logs-statusexecutablescript192 B
analytics/bin/sqlqexecutablescript212 B
analytics/lib/s3logs/__init__.pyscript62 B
analytics/lib/s3logs/awscli.pyscript10.0 KB
analytics/lib/s3logs/budget.pyscript9.6 KB
analytics/lib/s3logs/cli_fetch.pyscript34.0 KB
analytics/lib/s3logs/cli_inventory.pyscript7.0 KB
analytics/lib/s3logs/cli_sqlq.pyscript18.4 KB
analytics/lib/s3logs/cli_status.pyscript3.0 KB
analytics/lib/s3logs/config.pyscript7.4 KB
analytics/lib/s3logs/convert.pyscript23.6 KB
analytics/lib/s3logs/duck.pyscript11.4 KB
analytics/lib/s3logs/layout.pyscript8.8 KB
analytics/lib/s3logs/paths.pyscript4.9 KB
analytics/lib/s3logs/redact.pyscript22.1 KB
analytics/lib/s3logs/s5.pyscript9.1 KB
analytics/lib/s3logs/sqlguard.pyscript11.8 KB
analytics/lib/s3logs/validate.pyscript4.4 KB
analytics/logs.env.exampleother4.2 KB
analytics/sql/alb/by_minute.sqlother918 B
analytics/sql/alb/lb_vs_target_status.sqlother629 B
analytics/sql/alb/status_by_target.sqlother887 B
analytics/sql/alb/summary.sqlother1.2 KB
analytics/sql/alb/target_latency_pct.sqlother1.4 KB
analytics/sql/alb/top_clients.sqlother429 B
analytics/sql/alb/top_paths_5xx.sqlother673 B
analytics/sql/cloudfront/aborted.sqlother1.1 KB
analytics/sql/cloudfront/by_day.sqlother499 B
analytics/sql/cloudfront/by_hour.sqlother489 B
analytics/sql/cloudfront/by_object.sqlother544 B
analytics/sql/cloudfront/bytes_per_viewer.sqlother1.1 KB
analytics/sql/cloudfront/cache_results.sqlother823 B
analytics/sql/cloudfront/client_concentration.sqlother707 B
analytics/sql/cloudfront/edges.sqlother434 B
analytics/sql/cloudfront/object_by_day.sqlother599 B
analytics/sql/cloudfront/range_requests.sqlother528 B
analytics/sql/cloudfront/referer_pages.sqlother393 B
analytics/sql/cloudfront/status_codes.sqlother342 B
analytics/sql/cloudfront/summary.sqlother719 B
analytics/sql/cloudfront/top_clients.sqlother353 B
analytics/sql/cloudfront/top_objects.sqlother349 B
analytics/sql/cloudfront/top_referers.sqlother492 B
analytics/sql/cloudfront/user_agents.sqlother533 B
analytics/sql/waf/asns.sqlother378 B
analytics/sql/waf/by_day.sqlother453 B
analytics/sql/waf/by_minute.sqlother314 B
analytics/sql/waf/client_keys.sqlother443 B
analytics/sql/waf/clients.sqlother521 B
analytics/sql/waf/compare_days.sqlother924 B
analytics/sql/waf/concentration.sqlother775 B
analytics/sql/waf/count_rules.sqlother556 B
analytics/sql/waf/countries.sqlother373 B
analytics/sql/waf/fingerprints.sqlother424 B
analytics/sql/waf/header_shape.sqlother425 B
analytics/sql/waf/labels.sqlother347 B
analytics/sql/waf/matched_data.sqlother530 B
analytics/sql/waf/rules_by_minute.sqlother497 B
analytics/sql/waf/summary.sqlother735 B
analytics/sql/waf/top_rules.sqlother361 B
analytics/sql/waf/uris.sqlother402 B
analytics/sql/waf/user_agents.sqlother557 B
analytics/tests/conftest.pyscript4.7 KB
analytics/tests/gen_fixtures.pyscript19.7 KB
analytics/tests/stubs/awsexecutablescript4.9 KB
analytics/tests/stubs/s5cmdexecutablescript4.5 KB
analytics/tests/test_awscli.pyscript16.0 KB
analytics/tests/test_budget.pyscript13.3 KB
analytics/tests/test_config.pyscript9.1 KB
analytics/tests/test_convert.pyscript25.9 KB
analytics/tests/test_duck.pyscript10.6 KB
analytics/tests/test_fetch.pyscript42.8 KB
analytics/tests/test_gen_fixtures.pyscript1.9 KB
analytics/tests/test_inventory.pyscript12.9 KB
analytics/tests/test_layout.pyscript9.8 KB
analytics/tests/test_named_queries.pyscript26.7 KB
analytics/tests/test_paths.pyscript3.2 KB
analytics/tests/test_redact.pyscript32.0 KB
analytics/tests/test_s5.pyscript17.8 KB
analytics/tests/test_sqlguard.pyscript10.8 KB
analytics/tests/test_sqlq.pyscript37.2 KB
analytics/tests/test_status.pyscript5.8 KB
analytics/tests/test_validate.pyscript2.9 KB
environment.template.mdmarkdown7.1 KB
playbooks/changes.mdmarkdown37.2 KB
playbooks/errors.mdmarkdown10.4 KB
playbooks/external.mdmarkdown15.4 KB
playbooks/infra-state.mdmarkdown13.7 KB
playbooks/logs-s3.mdmarkdown29.5 KB
playbooks/logs.mdmarkdown21.3 KB
playbooks/metrics.mdmarkdown39.0 KB
playbooks/other-backends.mdmarkdown7.2 KB
playbooks/traces.mdmarkdown16.2 KB
playbooks/waf.mdmarkdown17.2 KB
templates/incident.mdmarkdown3.9 KB
templates/ledger.mdmarkdown9.7 KB
templates/lesson.mdmarkdown2.7 KB
templates/profile.mdmarkdown4.5 KB
templates/report.mdmarkdown6.5 KB

analytics/lib/s3logs/config.py

View on GitHub
"""Parse logs.env. The file is parsed line by line, never sourced by a shell.

Caps and every other setting come from the file or the defaults below, never
from environment variables.
"""
from __future__ import annotations

import re
from dataclasses import dataclass, field
from decimal import Decimal, InvalidOperation
from pathlib import Path
from typing import Dict, Optional, Tuple

from s3logs import paths
from s3logs.validate import Refused, bucket, dataset_name, pos_int, safe_key

DATASET_TYPES = ("cloudfront", "alb", "waf")

DEFAULT_MAX_MB = 200
DEFAULT_RUN_MAX_MB = 1024
DEFAULT_DAY_MAX_MB = 2048
DEFAULT_MARKER_TTL_S = 1800
DEFAULT_IDENTITY_COOKIES = ("session", "sessionid", "sid", "jsessionid", "phpsessid", "connect.sid")
DEFAULT_IDENTITY_HEADERS = ("authorization", "x-api-key")
DEFAULT_SENSITIVE_PARAMS = (
    "token", "access_token", "id_token", "refresh_token", "code", "password", "passwd", "pwd",
    "secret", "key", "api_key", "apikey", "signature", "sig", "auth", "session", "sessionid",
    "email", "phone",
)

_MAX_FILE_BYTES = 64 * 1024
_LINE_RE = re.compile(r"([A-Za-z][A-Za-z0-9_-]*)=([A-Za-z0-9_.,:/{}@+=-]*)")
_COMMENT_RE = re.compile(r"(?:^|\s)#.*$")
_REGION_RE = re.compile(r"[a-z]{2}(?:-[a-z]+)+-[0-9]")
_PROFILE_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.@+-]{0,127}")
_LIST_ITEM_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{0,63}")
_TOKEN_RE = re.compile(r"\{([^{}]*)\}")
_DATE_TOKENS = ("{yyyy}", "{MM}", "{dd}")
_ALL_TOKENS = ("yyyy", "MM", "dd", "HH")

_SCALAR_KEYS = (
    "AWS_PROFILE", "AWS_REGION", "PRICE_PER_GB", "PRICE_PER_1K_GET", "MAX_MB", "RUN_MAX_MB",
    "DAY_MAX_MB", "MARKER_TTL_S", "WAF_IDENTITY_COOKIES", "WAF_IDENTITY_HEADERS", "SENSITIVE_PARAMS",
)


class ConfigError(Exception):
    def __init__(self, line: int, msg: str):
        super().__init__(line, msg)
        self.line = line
        self.msg = msg

    def __str__(self) -> str:
        return "line %d: %s" % (self.line, self.msg) if self.line > 0 else self.msg


@dataclass(frozen=True)
class Dataset:
    name: str
    type: str
    bucket: str
    template: str


@dataclass(frozen=True)
class Config:
    aws_profile: Optional[str] = None
    aws_region: Optional[str] = None
    datasets: Dict[str, Dataset] = field(default_factory=dict)
    price_per_gb: Optional[Decimal] = None
    price_per_1k_get: Optional[Decimal] = None
    max_mb: int = DEFAULT_MAX_MB
    run_max_mb: int = DEFAULT_RUN_MAX_MB
    day_max_mb: int = DEFAULT_DAY_MAX_MB
    marker_ttl_s: int = DEFAULT_MARKER_TTL_S
    identity_cookies: Tuple[str, ...] = DEFAULT_IDENTITY_COOKIES
    identity_headers: Tuple[str, ...] = DEFAULT_IDENTITY_HEADERS
    sensitive_params: frozenset = frozenset(DEFAULT_SENSITIVE_PARAMS)


def default_path() -> Path:
    return paths.config_path()


def _decimal(value: str) -> Decimal:
    try:
        d = Decimal(value)
    except InvalidOperation:
        raise Refused("expected a decimal number, got %r" % (value,))
    if not d.is_finite() or d < 0:
        raise Refused("expected a non-negative decimal number, got %r" % (value,))
    return d


def _name_list(value: str) -> Tuple[str, ...]:
    items = [i.strip().lower() for i in value.split(",")]
    if not value or any(not _LIST_ITEM_RE.fullmatch(i) for i in items):
        raise Refused("expected a comma-separated list of names (a-z 0-9 _ . -), got %r" % (value,))
    return tuple(items)


def _dataset(name: str, value: str) -> Dataset:
    typ, sep, rest = value.partition(":")
    if not sep or typ not in DATASET_TYPES:
        raise Refused("dataset type must be one of %s (got %r)" % ("|".join(DATASET_TYPES), typ))
    if not rest.startswith("s3://"):
        raise Refused("dataset location must look like s3://<bucket>/<template>")
    bkt, sep, template = rest[len("s3://"):].partition("/")
    if not sep or not template:
        raise Refused("dataset location needs a template after the bucket")
    if "://" in template:
        raise Refused("template must not contain '://'")
    bucket(bkt)
    for tok in _TOKEN_RE.findall(template):
        if tok not in _ALL_TOKENS:
            raise Refused("unknown template token {%s} (allowed: {yyyy} {MM} {dd} {HH})" % tok)
    if "{" in _TOKEN_RE.sub("", template) or "}" in _TOKEN_RE.sub("", template):
        raise Refused("unbalanced braces in template")
    for tok in _DATE_TOKENS:
        if tok not in template:
            raise Refused("template must contain %s" % tok)
    if typ == "cloudfront" and "{HH}" not in template:
        raise Refused("a cloudfront template must contain {HH} (CloudFront log files are named per hour)")
    safe_key(_TOKEN_RE.sub("0", template))
    return Dataset(name, typ, bkt, template)


def _apply(values: dict, key: str, value: str) -> None:
    if key == "AWS_PROFILE":
        if value and not _PROFILE_RE.fullmatch(value):
            raise Refused("invalid AWS_PROFILE %r" % (value,))
        values["aws_profile"] = value or None
    elif key == "AWS_REGION":
        if value and not _REGION_RE.fullmatch(value):
            raise Refused("invalid AWS_REGION %r" % (value,))
        values["aws_region"] = value or None
    elif key == "PRICE_PER_GB":
        values["price_per_gb"] = _decimal(value)
    elif key == "PRICE_PER_1K_GET":
        values["price_per_1k_get"] = _decimal(value)
    elif key == "MAX_MB":
        values["max_mb"] = pos_int(value, key)
    elif key == "RUN_MAX_MB":
        values["run_max_mb"] = pos_int(value, key)
    elif key == "DAY_MAX_MB":
        values["day_max_mb"] = pos_int(value, key)
    elif key == "MARKER_TTL_S":
        values["marker_ttl_s"] = pos_int(value, key)
    elif key == "WAF_IDENTITY_COOKIES":
        values["identity_cookies"] = _name_list(value)
    elif key == "WAF_IDENTITY_HEADERS":
        values["identity_headers"] = _name_list(value)
    elif key == "SENSITIVE_PARAMS":
        values["sensitive_params"] = frozenset(_name_list(value))


def load(path: Optional[Path] = None) -> Config:
    """Read and strictly parse the config file. Raises ConfigError(line, msg)."""
    p = Path(path) if path is not None else default_path()
    try:
        if not p.is_file():
            raise ConfigError(0, "%s is not a regular file" % (p,))
        raw = p.read_bytes()
        if len(raw) > _MAX_FILE_BYTES:
            raise ConfigError(0, "config file is larger than %d bytes" % _MAX_FILE_BYTES)
        text = raw.decode("utf-8")
    except UnicodeDecodeError:
        raise ConfigError(0, "config file is not valid UTF-8")
    except OSError as e:
        raise ConfigError(0, "cannot read config file: %s" % (e.strerror or e,))

    values: dict = {}
    datasets: Dict[str, Dataset] = {}
    seen = set()
    for lineno, line in enumerate(text.splitlines(), start=1):
        line = _COMMENT_RE.sub("", line).strip()
        if not line:
            continue
        m = _LINE_RE.fullmatch(line)
        if not m:
            raise ConfigError(lineno, "not a KEY=value line (values may use only A-Za-z0-9 _ . , : / { } @ + = -)")
        key, value = m.group(1), m.group(2)
        if key in seen:
            raise ConfigError(lineno, "duplicate key %s" % key)
        try:
            if key.startswith("DATASET_"):
                name = dataset_name(key[len("DATASET_"):])
                datasets[name] = _dataset(name, value)
            elif key in _SCALAR_KEYS:
                _apply(values, key, value)
            else:
                raise ConfigError(lineno, "unknown key %s" % key)
        except Refused as e:
            raise ConfigError(lineno, "%s: %s" % (key, e))
        seen.add(key)
    return Config(datasets=datasets, **values)
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall