Skip to content

culiops/culiops-skills / skills/sre-investigate

sre-investigate

Unverified

FileKindSize
SKILL.mdskill md71.5 KB
access/standalone.mdmarkdown19.0 KB
agents/evidence.mdmarkdown8.1 KB
agents/helper.mdmarkdown3.9 KB
analytics/bin/logs-fetchexecutablescript229 B
analytics/bin/logs-inventoryexecutablescript209 B
analytics/bin/logs-statusexecutablescript192 B
analytics/bin/sqlqexecutablescript212 B
analytics/lib/s3logs/__init__.pyscript62 B
analytics/lib/s3logs/awscli.pyscript10.0 KB
analytics/lib/s3logs/budget.pyscript9.6 KB
analytics/lib/s3logs/cli_fetch.pyscript34.0 KB
analytics/lib/s3logs/cli_inventory.pyscript7.0 KB
analytics/lib/s3logs/cli_sqlq.pyscript18.4 KB
analytics/lib/s3logs/cli_status.pyscript3.0 KB
analytics/lib/s3logs/config.pyscript7.4 KB
analytics/lib/s3logs/convert.pyscript23.6 KB
analytics/lib/s3logs/duck.pyscript11.4 KB
analytics/lib/s3logs/layout.pyscript8.8 KB
analytics/lib/s3logs/paths.pyscript4.9 KB
analytics/lib/s3logs/redact.pyscript22.1 KB
analytics/lib/s3logs/s5.pyscript9.1 KB
analytics/lib/s3logs/sqlguard.pyscript11.8 KB
analytics/lib/s3logs/validate.pyscript4.4 KB
analytics/logs.env.exampleother4.2 KB
analytics/sql/alb/by_minute.sqlother918 B
analytics/sql/alb/lb_vs_target_status.sqlother629 B
analytics/sql/alb/status_by_target.sqlother887 B
analytics/sql/alb/summary.sqlother1.2 KB
analytics/sql/alb/target_latency_pct.sqlother1.4 KB
analytics/sql/alb/top_clients.sqlother429 B
analytics/sql/alb/top_paths_5xx.sqlother673 B
analytics/sql/cloudfront/aborted.sqlother1.1 KB
analytics/sql/cloudfront/by_day.sqlother499 B
analytics/sql/cloudfront/by_hour.sqlother489 B
analytics/sql/cloudfront/by_object.sqlother544 B
analytics/sql/cloudfront/bytes_per_viewer.sqlother1.1 KB
analytics/sql/cloudfront/cache_results.sqlother823 B
analytics/sql/cloudfront/client_concentration.sqlother707 B
analytics/sql/cloudfront/edges.sqlother434 B
analytics/sql/cloudfront/object_by_day.sqlother599 B
analytics/sql/cloudfront/range_requests.sqlother528 B
analytics/sql/cloudfront/referer_pages.sqlother393 B
analytics/sql/cloudfront/status_codes.sqlother342 B
analytics/sql/cloudfront/summary.sqlother719 B
analytics/sql/cloudfront/top_clients.sqlother353 B
analytics/sql/cloudfront/top_objects.sqlother349 B
analytics/sql/cloudfront/top_referers.sqlother492 B
analytics/sql/cloudfront/user_agents.sqlother533 B
analytics/sql/waf/asns.sqlother378 B
analytics/sql/waf/by_day.sqlother453 B
analytics/sql/waf/by_minute.sqlother314 B
analytics/sql/waf/client_keys.sqlother443 B
analytics/sql/waf/clients.sqlother521 B
analytics/sql/waf/compare_days.sqlother924 B
analytics/sql/waf/concentration.sqlother775 B
analytics/sql/waf/count_rules.sqlother556 B
analytics/sql/waf/countries.sqlother373 B
analytics/sql/waf/fingerprints.sqlother424 B
analytics/sql/waf/header_shape.sqlother425 B
analytics/sql/waf/labels.sqlother347 B
analytics/sql/waf/matched_data.sqlother530 B
analytics/sql/waf/rules_by_minute.sqlother497 B
analytics/sql/waf/summary.sqlother735 B
analytics/sql/waf/top_rules.sqlother361 B
analytics/sql/waf/uris.sqlother402 B
analytics/sql/waf/user_agents.sqlother557 B
analytics/tests/conftest.pyscript4.7 KB
analytics/tests/gen_fixtures.pyscript19.7 KB
analytics/tests/stubs/awsexecutablescript4.9 KB
analytics/tests/stubs/s5cmdexecutablescript4.5 KB
analytics/tests/test_awscli.pyscript16.0 KB
analytics/tests/test_budget.pyscript13.3 KB
analytics/tests/test_config.pyscript9.1 KB
analytics/tests/test_convert.pyscript25.9 KB
analytics/tests/test_duck.pyscript10.6 KB
analytics/tests/test_fetch.pyscript42.8 KB
analytics/tests/test_gen_fixtures.pyscript1.9 KB
analytics/tests/test_inventory.pyscript12.9 KB
analytics/tests/test_layout.pyscript9.8 KB
analytics/tests/test_named_queries.pyscript26.7 KB
analytics/tests/test_paths.pyscript3.2 KB
analytics/tests/test_redact.pyscript32.0 KB
analytics/tests/test_s5.pyscript17.8 KB
analytics/tests/test_sqlguard.pyscript10.8 KB
analytics/tests/test_sqlq.pyscript37.2 KB
analytics/tests/test_status.pyscript5.8 KB
analytics/tests/test_validate.pyscript2.9 KB
environment.template.mdmarkdown7.1 KB
playbooks/changes.mdmarkdown37.2 KB
playbooks/errors.mdmarkdown10.4 KB
playbooks/external.mdmarkdown15.4 KB
playbooks/infra-state.mdmarkdown13.7 KB
playbooks/logs-s3.mdmarkdown29.5 KB
playbooks/logs.mdmarkdown21.3 KB
playbooks/metrics.mdmarkdown39.0 KB
playbooks/other-backends.mdmarkdown7.2 KB
playbooks/traces.mdmarkdown16.2 KB
playbooks/waf.mdmarkdown17.2 KB
templates/incident.mdmarkdown3.9 KB
templates/ledger.mdmarkdown9.7 KB
templates/lesson.mdmarkdown2.7 KB
templates/profile.mdmarkdown4.5 KB
templates/report.mdmarkdown6.5 KB

analytics/lib/s3logs/paths.py

View on GitHub
"""Fixed locations. The only environment input is HOME (via Path.home())."""
from __future__ import annotations

import os
import re
import stat
import time
from pathlib import Path
from typing import List, Optional

from s3logs.validate import Refused

_CONFIG_SUB = Path(".config/culiops/sre-investigate")
_STATE_SUB = Path(".local/share/culiops/sre-investigate")


def config_dir() -> Path:
    return Path.home() / _CONFIG_SUB


def config_path() -> Path:
    return config_dir() / "logs.env"


def marker_path() -> Path:
    """The over-cap override marker. Only the user ever creates it."""
    return config_dir() / "over-cap.allow"


def state_root() -> Path:
    return Path.home() / _STATE_SUB


def _private_dir(p: Path, mode: int) -> None:
    if p.is_symlink():
        raise Refused("%s is a symlink; refusing to use it as private state" % (p,))
    if p.exists() and not p.is_dir():
        raise Refused("%s exists and is not a directory" % (p,))
    if not p.exists():
        try:
            p.mkdir(mode=mode)
        except FileExistsError:
            # Lost a creation race with another process; re-check what is there now.
            if p.is_symlink() or not p.is_dir():
                raise Refused("%s exists and is not a private directory" % (p,))
    os.chmod(str(p), mode)


def ensure_private_dirs() -> None:
    """Create the state tree: dirs 0700, staging/ 0300 (write+search, no listing)."""
    root = state_root()
    root.parent.mkdir(parents=True, exist_ok=True)
    _private_dir(root, 0o700)
    for name in ("budgets", "manifests", "data"):
        _private_dir(root / name, 0o700)
    _private_dir(root / "staging", 0o300)


def manifest_path(dataset: str, day_from: str, day_to: str) -> Path:
    """manifests/<dataset>/<from>_<to>.json under the state root (arguments pre-validated)."""
    return state_root() / "manifests" / dataset / ("%s_%s.json" % (day_from, day_to))


def private_subdir(parent: Path, name: str) -> Path:
    """Create parent/name (0700) if missing, refusing a symlink or non-directory there."""
    p = parent / name
    _private_dir(p, 0o700)
    return p


# ---- removal helpers (staging sweep, purge) --------------------------------------------------

STAGING_STALE_S = 24 * 60 * 60
# What logs-fetch itself creates under staging/: its batch dir and convert's private dirs.
_STAGING_NAME_RE = re.compile(r"(?:[0-9a-f]{32}|conv-[0-9a-f]{16})")


def _empty_dir(fd: int) -> None:
    """Delete everything inside the open directory `fd`, never following a symlink."""
    for name in os.listdir(fd):
        st = os.lstat(name, dir_fd=fd)
        if stat.S_ISDIR(st.st_mode):
            child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
            try:
                os.fchmod(child, 0o700)
                _empty_dir(child)
            finally:
                os.close(child)
            os.rmdir(name, dir_fd=fd)
        else:
            os.unlink(name, dir_fd=fd)


def rmtree_force(path: Path) -> None:
    """Remove a directory tree even when parts of it are read-only (dataset dirs are 0555).

    Every directory is opened with O_NOFOLLOW and made writable before it is emptied, and
    entries are removed relative to the open directory, so a symlink anywhere in the tree is
    unlinked, never followed. A symlink at `path` itself is unlinked and its target left alone.
    """
    p = str(path)
    try:
        st = os.lstat(p)
    except FileNotFoundError:
        return
    if not stat.S_ISDIR(st.st_mode):
        os.unlink(p)
        return
    fd = os.open(p, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
    try:
        os.fchmod(fd, 0o700)
        _empty_dir(fd)
    finally:
        os.close(fd)
    os.rmdir(p)


def sweep_staging(max_age_s: int = STAGING_STALE_S, now: Optional[float] = None) -> List[str]:
    """Remove batch dirs in staging/ not touched for `max_age_s` (default one day).

    staging/ is mode 0300 (no listing), so it is made 0700 only for the listing and removals
    and put back to 0300 in `finally`. Only directories named like the ones logging creates are
    removed; symlinks and anything else are left alone. Returns the removed names.
    """
    root = state_root() / "staging"
    try:
        st = os.lstat(str(root))
    except FileNotFoundError:
        return []
    if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode):
        raise Refused("%s is not a private directory" % (root,))
    cutoff = (time.time() if now is None else now) - max_age_s
    removed = []  # type: List[str]
    os.chmod(str(root), 0o700)
    try:
        for name in sorted(os.listdir(str(root))):
            if not _STAGING_NAME_RE.fullmatch(name):
                continue
            child = root / name
            cst = os.lstat(str(child))
            if not stat.S_ISDIR(cst.st_mode) or cst.st_mtime > cutoff:
                continue
            rmtree_force(child)
            removed.append(name)
    finally:
        os.chmod(str(root), 0o300)
    return removed
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall