analyticsengineering/ae-toolkit / skills/rename-model
rename-model
VerifiedCoordinates name-only renames of analytical models, columns, relations, sources, metrics, semantic objects, and every repository reference while preserving the data contract. Use for atomic internal column renames and cross-layer reference updates; use migration-plan when behavior/schema changes or a published consumer interface needs compatibility and phased cutover.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 8.3 KB |
| references/diagrams.md | markdown | 3.2 KB |
| references/operating-rules.md | markdown | 23.3 KB |
| references/privacy.md | markdown | 2.0 KB |
| references/severity-model.md | markdown | 1.5 KB |
| references/terminology.md | markdown | 1.9 KB |
| references/warehouse-portability.md | markdown | 4.9 KB |
| scripts/infer_conventions.py | script | 55.6 KB |
| scripts/inspect_dbt.py | script | 20.2 KB |
| scripts/inspect_git.py | script | 8.5 KB |
| scripts/inspect_repo.py | script | 13.5 KB |
| scripts/project_context.py | script | 27.4 KB |
| scripts/redact_secrets.py | script | 15.2 KB |
scripts/redact_secrets.py
View on GitHub#!/usr/bin/env python3
"""Detect and redact obvious secrets without modifying source files."""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable, Sequence
DISCLAIMER = (
"Secret detection is best-effort and may miss secrets or flag non-secrets; "
"do not treat this result as a complete security audit."
)
PLACEHOLDERS = {
"",
"changeme",
"example",
"password",
"redacted",
"replace_me",
"secret",
"test",
"token",
"your_api_key",
"your_token",
}
@dataclass(frozen=True)
class SecretMatch:
"""An internal secret span; its raw value is intentionally not serializable."""
kind: str
start: int
end: int
confidence: str
@dataclass(frozen=True)
class PatternSpec:
kind: str
pattern: re.Pattern[str]
confidence: str = "high"
group: str | int = "secret"
PATTERNS = (
PatternSpec(
"private_key",
re.compile(
r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----"
r"[\s\S]*?"
r"-----END (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----"
),
group=0,
),
PatternSpec(
"service_account_private_key",
re.compile(
r'(?i)"private_key"\s*:\s*"(?P<secret>(?:\\.|[^"\\]){20,})"'
),
),
PatternSpec(
"service_account_private_key_id",
re.compile(r'(?i)"private_key_id"\s*:\s*"(?P<secret>[a-f0-9]{20,})"'),
),
PatternSpec(
"github_token",
re.compile(
r"(?<![A-Za-z0-9_])(?P<secret>gh(?:p|o|u|s|r)_[A-Za-z0-9_]{30,})(?![A-Za-z0-9_])"
),
),
PatternSpec(
"slack_token",
re.compile(
r"(?<![A-Za-z0-9-])(?P<secret>xox(?:a|b|p|r|s)-[A-Za-z0-9-]{10,})(?![A-Za-z0-9-])"
),
),
PatternSpec(
"aws_access_key_id",
re.compile(r"(?<![A-Z0-9])(?P<secret>(?:AKIA|ASIA)[A-Z0-9]{16})(?![A-Z0-9])"),
),
PatternSpec(
"google_api_key",
re.compile(r"(?<![A-Za-z0-9_-])(?P<secret>AIza[A-Za-z0-9_-]{35})(?![A-Za-z0-9_-])"),
),
PatternSpec(
"stripe_secret_key",
re.compile(
r"(?<![A-Za-z0-9_])(?P<secret>(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,})(?![A-Za-z0-9_])"
),
),
PatternSpec(
"jwt",
re.compile(
r"(?<![A-Za-z0-9_-])(?P<secret>eyJ[A-Za-z0-9_-]{8,}\."
r"[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})(?![A-Za-z0-9_-])"
),
confidence="medium",
),
PatternSpec(
"authorization_bearer_token",
re.compile(r"(?i)\bBearer\s+(?P<secret>[A-Za-z0-9._~+/=-]{12,})"),
),
PatternSpec(
"credential_in_url",
re.compile(
r"(?i)\b[a-z][a-z0-9+.-]*://[^/\s:@]+:(?P<secret>[^@\s/]{6,})@"
),
),
PatternSpec(
"password_or_token_assignment",
re.compile(
r"""(?ix)
["']?\b(?:password|passwd|pwd|secret|client_secret|api_?key|
access_?token|auth_?token|refresh_?token|token|
private_?key|aws_secret_access_key)
\b["']?\s*[:=]\s*
(?:
["'](?P<quoted_secret>[^"'\r\n]{6,})["']
|
(?P<unquoted_secret>[^\s,;#}\]]{6,})
)
"""
),
confidence="medium",
group="dynamic",
),
)
class RedactionError(ValueError):
"""Raised for expected redaction input errors."""
class JsonArgumentParser(argparse.ArgumentParser):
"""Emit parse failures as JSON for automation-friendly CLIs."""
def error(self, message: str) -> None:
print(
json.dumps({"ok": False, "error": message, "error_type": "ArgumentError"}),
file=sys.stderr,
)
raise SystemExit(2)
# Code that fetches a credential rather than containing one. `api_key=os.environ[...]`
# is the recommended way to handle a secret, and flagging it trains people to
# ignore the scanner on exactly the files where it should be trusted.
CREDENTIAL_LOOKUP = re.compile(
r"""(?ix)^\s*(?:
os\.environ
| os\.getenv
| environ
| getenv
| process\.env
| config\b
| settings\b
| secrets?\.
| keyring\.
| vault\b
| \w*get_secret\w*
| input\s*\(
)"""
)
def _looks_like_placeholder(value: str, *, quoted: bool = True) -> bool:
normalized = value.strip().strip("\"'").lower()
if (
normalized in PLACEHOLDERS
or normalized.startswith(("${", "{{", "<"))
or normalized.endswith(("}}", ">"))
or set(normalized) <= {"*", "x", "X", "-"}
or bool(CREDENTIAL_LOOKUP.match(value.strip()))
):
return True
# An unquoted value with expression structure is code that produces a value
# rather than a value: a subscript such as tokens[index], a call such as
# uuid.uuid4().hex, and an attribute path such as self.config all read a
# credential rather than containing one. A literal secret written without
# quotes has no call, subscript, or dotted path, and the unquoted formats
# that do occur -- hex digests, base64 keys, JWTs -- have dedicated
# high-confidence patterns of their own.
if quoted:
return False
stripped = value.strip()
return bool(
"(" in stripped
or "[" in stripped
or re.fullmatch(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+", stripped)
)
def _span_for_match(spec: PatternSpec, match: re.Match[str]) -> tuple[int, int]:
if spec.group == "dynamic":
group = "quoted_secret" if match.groupdict().get("quoted_secret") else "unquoted_secret"
return match.span(group)
return match.span(spec.group)
def find_secrets(text: str) -> list[SecretMatch]:
"""Find non-overlapping likely-secret spans without returning secret values."""
candidates: list[SecretMatch] = []
for spec in PATTERNS:
for match in spec.pattern.finditer(text):
start, end = _span_for_match(spec, match)
if start < 0 or end <= start:
continue
if spec.kind == "password_or_token_assignment" and _looks_like_placeholder(
text[start:end],
quoted=bool(match.groupdict().get("quoted_secret")),
):
continue
candidates.append(SecretMatch(spec.kind, start, end, spec.confidence))
# Prefer longer/high-confidence spans, then present findings in source order.
confidence_rank = {"high": 0, "medium": 1, "low": 2}
selected: list[SecretMatch] = []
for candidate in sorted(
candidates,
key=lambda item: (
confidence_rank.get(item.confidence, 9),
-(item.end - item.start),
item.start,
),
):
if any(
candidate.start < existing.end and existing.start < candidate.end
for existing in selected
):
continue
selected.append(candidate)
return sorted(selected, key=lambda item: (item.start, item.end, item.kind))
def _line_and_column(text: str, offset: int) -> tuple[int, int]:
line = text.count("\n", 0, offset) + 1
last_newline = text.rfind("\n", 0, offset)
column = offset + 1 if last_newline < 0 else offset - last_newline
return line, column
def findings_for_text(text: str) -> list[dict[str, Any]]:
"""Return safe metadata about likely secrets, never their values."""
findings = []
for match in find_secrets(text):
line, column = _line_and_column(text, match.start)
end_line, end_column = _line_and_column(text, match.end)
findings.append(
{
"kind": match.kind,
"confidence": match.confidence,
"line": line,
"column": column,
"end_line": end_line,
"end_column": end_column,
"value_emitted": False,
}
)
return findings
def detect_secrets(text: str) -> list[dict[str, Any]]:
"""Public alias returning safe, JSON-serializable secret findings."""
return findings_for_text(text)
# Keys whose presence is itself the finding: a populated `password` field is a
# credential regardless of whether its value matches any pattern.
SENSITIVE_KEY = re.compile(
r"(?:^|_)(?:api_?key|access_?token|refresh_?token|client_?secret|password|"
r"passwd|private_?key|authorization|cookie|secret|credential)(?:$|_)",
re.IGNORECASE,
)
def scan_structure(value: Any, path: str = "$") -> list[dict[str, Any]]:
"""Scan a decoded JSON structure for credentials, by key and by value.
The cloud gate carried its own smaller copy of these patterns, so a token
format added here was still transmitted there. One scanner means one answer,
and it keeps reporting locations rather than values.
"""
findings: list[dict[str, Any]] = []
if isinstance(value, dict):
for key, child in value.items():
key_text = str(key)
child_path = f"{path}.{key_text}"
if SENSITIVE_KEY.search(key_text) and child not in ("", None, [], {}):
if not (isinstance(child, str) and _looks_like_placeholder(child)):
findings.append(
{
"path": child_path,
"kind": "sensitive_key",
"confidence": "high",
"value_emitted": False,
}
)
continue
findings.extend(scan_structure(child, child_path))
elif isinstance(value, list):
for index, child in enumerate(value):
findings.extend(scan_structure(child, f"{path}[{index}]"))
elif isinstance(value, str):
for finding in findings_for_text(value):
findings.append(
{
"path": path,
"kind": finding["kind"],
"confidence": finding["confidence"],
"value_emitted": False,
}
)
return findings
def redact_text(text: str, replacement: str = "[REDACTED]") -> tuple[str, list[dict[str, Any]]]:
"""Return redacted text and safe finding metadata."""
matches = find_secrets(text)
redacted = text
for match in reversed(matches):
marker = replacement
if "{kind}" in replacement:
marker = replacement.replace("{kind}", match.kind)
redacted = redacted[: match.start] + marker + redacted[match.end :]
return redacted, findings_for_text(text)
def inspect_text(
text: str,
*,
source: str = "text",
emit_content: bool = True,
replacement: str = "[REDACTED]",
) -> dict[str, Any]:
"""Inspect one text input and optionally return its redacted form."""
redacted, findings = redact_text(text, replacement)
result: dict[str, Any] = {
"source": source,
"finding_count": len(findings),
"findings": findings,
}
if emit_content:
result["redacted_content"] = redacted
return result
def redact_files(
paths: Iterable[str | Path],
*,
emit_content: bool = True,
replacement: str = "[REDACTED]",
encoding: str = "utf-8",
) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:
"""Read and inspect files without writing to them."""
inputs: list[dict[str, Any]] = []
errors: list[dict[str, str]] = []
for raw_path in paths:
path = Path(raw_path).expanduser()
try:
if not path.is_file():
raise OSError("path does not exist or is not a regular file")
text = path.read_text(encoding=encoding)
inputs.append(
inspect_text(
text,
source=str(path.resolve()),
emit_content=emit_content,
replacement=replacement,
)
)
except (OSError, UnicodeError) as exc:
errors.append({"source": str(path), "error": str(exc)})
return inputs, errors
def _report(inputs: list[dict[str, Any]], errors: list[dict[str, str]]) -> dict[str, Any]:
return {
"ok": not errors,
"best_effort": True,
"disclaimer": DISCLAIMER,
"total_findings": sum(item["finding_count"] for item in inputs),
"inputs": inputs,
"errors": errors,
"inputs_modified": False,
}
def _parser() -> argparse.ArgumentParser:
parser = JsonArgumentParser(description=__doc__)
parser.add_argument("files", nargs="*", help="Files to inspect read-only")
source = parser.add_mutually_exclusive_group()
source.add_argument("--stdin", action="store_true", help="Read content from stdin")
source.add_argument("--json-text", help="Inspect literal JSON text")
parser.add_argument(
"--findings-only",
action="store_true",
help="Do not include redacted content in the JSON report",
)
parser.add_argument(
"--replacement",
default="[REDACTED:{kind}]",
help="Replacement marker; {kind} expands to the finding kind",
)
parser.add_argument("--encoding", default="utf-8", help="File encoding")
parser.add_argument("--pretty", action="store_true", help="Indent JSON output")
return parser
def main(argv: Sequence[str] | None = None) -> int:
args = _parser().parse_args(argv)
emit_content = not args.findings_only
inputs: list[dict[str, Any]] = []
errors: list[dict[str, str]] = []
if args.files and (args.stdin or args.json_text is not None):
errors.append(
{
"source": "arguments",
"error": "provide files, --stdin, or --json-text, not a combination",
}
)
elif args.json_text is not None:
try:
json.loads(args.json_text)
except json.JSONDecodeError as exc:
errors.append(
{
"source": "json-text",
"error": f"invalid JSON at line {exc.lineno}, column {exc.colno}",
}
)
else:
inputs.append(
inspect_text(
args.json_text,
source="json-text",
emit_content=emit_content,
replacement=args.replacement,
)
)
elif args.files:
inputs, errors = redact_files(
args.files,
emit_content=emit_content,
replacement=args.replacement,
encoding=args.encoding,
)
elif args.stdin or not sys.stdin.isatty():
inputs.append(
inspect_text(
sys.stdin.read(),
source="stdin",
emit_content=emit_content,
replacement=args.replacement,
)
)
else:
errors.append(
{"source": "arguments", "error": "provide file(s), --stdin, or --json-text"}
)
report = _report(inputs, errors)
stream = sys.stdout if report["ok"] else sys.stderr
print(
json.dumps(report, indent=2 if args.pretty else None, sort_keys=args.pretty),
file=stream,
)
return 0 if report["ok"] else 1
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |