analyticsengineering/ae-toolkit / skills/freshness-investigation
freshness-investigation
VerifiedInvestigates whether analytical data is ready by separating configured schedules and freshness expectations from locally observed timestamps and run artifacts. Use when data appears stale, late, incomplete, or unavailable, and before raising a freshness threshold or silencing a failing check; it explains timing evidence and gaps without inventing runtime history or querying external systems.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 8.2 KB |
| references/diagrams.md | markdown | 3.2 KB |
| references/operating-rules.md | markdown | 23.3 KB |
| references/privacy.md | markdown | 2.0 KB |
| references/severity-model.md | markdown | 1.5 KB |
| references/terminology.md | markdown | 1.9 KB |
| references/warehouse-portability.md | markdown | 4.9 KB |
| scripts/infer_conventions.py | script | 55.6 KB |
| scripts/inspect_dbt.py | script | 20.2 KB |
| scripts/inspect_git.py | script | 8.5 KB |
| scripts/inspect_repo.py | script | 13.5 KB |
| scripts/project_context.py | script | 27.4 KB |
| scripts/redact_secrets.py | script | 15.2 KB |
scripts/inspect_repo.py
View on GitHub#!/usr/bin/env python3
"""Statically detect analytics-engineering technologies in a repository."""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any, Iterable, Sequence
SKIP_DIRECTORIES = {
".git",
".hg",
".svn",
".mypy_cache",
".pytest_cache",
".ruff_cache",
".tox",
".venv",
"__pycache__",
"build",
"dist",
"node_modules",
"target",
"venv",
}
CONFIG_NAMES = {
".env",
".env.example",
".python-version",
"airflow.cfg",
"dbt_project.yml",
"dbt_project.yaml",
"docker-compose.yml",
"docker-compose.yaml",
"environment.yml",
"packages.yml",
"packages.yaml",
"poetry.lock",
"profiles.yml",
"profiles.yaml",
"pyproject.toml",
"requirements.txt",
"setup.cfg",
"tox.ini",
}
WAREHOUSE_PATTERNS = {
"bigquery": [
re.compile(r"\b(?:from|import)\s+google\.cloud(?:\s+import\s+|\.)bigquery\b", re.I),
re.compile(r"\bgoogle-cloud-bigquery\b", re.I),
re.compile(r"\btype\s*:\s*bigquery\b", re.I),
re.compile(r"\bBigQueryOperator\b", re.I),
re.compile(r"`[\w-]+\.[\w-]+\.[\w$-]+`"),
],
"snowflake": [
re.compile(r"\b(?:from|import)\s+snowflake(?:\.connector)?\b", re.I),
re.compile(r"\bsnowflake-connector-python\b", re.I),
re.compile(r"\btype\s*:\s*snowflake\b", re.I),
],
"redshift": [
re.compile(r"\b(?:redshift_connector|redshift-connector)\b", re.I),
re.compile(r"\btype\s*:\s*redshift\b", re.I),
],
"databricks": [
re.compile(r"\b(?:from|import)\s+databricks\b", re.I),
re.compile(r"\bdatabricks-sql-connector\b", re.I),
re.compile(r"\btype\s*:\s*databricks\b", re.I),
],
"postgres": [
re.compile(r"\b(?:from|import)\s+psycopg2?\b", re.I),
re.compile(r"\b(?:psycopg2?-binary|postgresql://)\b", re.I),
re.compile(r"\btype\s*:\s*postgres\b", re.I),
],
"duckdb": [
re.compile(r"\b(?:from|import)\s+duckdb\b", re.I),
re.compile(r"(?m)^\s*duckdb(?:==|>=|~=|$)", re.I),
re.compile(r"\btype\s*:\s*duckdb\b", re.I),
],
}
class RepoInspectionError(ValueError):
"""Raised for expected repository inspection errors."""
class JsonArgumentParser(argparse.ArgumentParser):
"""Emit parse failures as JSON for automation-friendly CLIs."""
def error(self, message: str) -> None:
print(
json.dumps({"ok": False, "error": message, "error_type": "ArgumentError"}),
file=sys.stderr,
)
raise SystemExit(2)
def _walk_files(root: Path) -> Iterable[Path]:
for current, directories, filenames in os.walk(root, followlinks=False):
directories[:] = sorted(
directory
for directory in directories
if directory not in SKIP_DIRECTORIES
and not (directory.startswith(".") and directory != ".ae")
)
for filename in sorted(filenames):
yield Path(current, filename)
def _read_text(path: Path, max_bytes: int = 2_000_000) -> str:
try:
with path.open("rb") as handle:
raw = handle.read(max_bytes + 1)
except OSError:
return ""
if b"\0" in raw:
return ""
return raw[:max_bytes].decode("utf-8", errors="replace")
def _relative(path: Path, root: Path) -> str:
return path.relative_to(root).as_posix()
def validate_exercise_config(value: Any) -> dict[str, Any]:
"""Validate the portable, intentionally small .ae/exercise.json schema."""
errors: list[str] = []
warnings: list[str] = []
if not isinstance(value, dict):
return {
"valid": False,
"errors": ["root must be a JSON object"],
"warnings": [],
}
course_fields = {"exercise_id", "module_id", "lesson_id", "skills", "course_url"}
course_schema = bool(course_fields & value.keys())
required = (
course_fields | {"mode"}
if course_schema
else {
"id",
"mode",
"constraints",
"rubric",
"allowed_tools",
"expected_artifacts",
"tasks",
}
)
for field in sorted(required - value.keys()):
errors.append(f"missing required field: {field}")
for field in (
"id",
"exercise_id",
"module_id",
"lesson_id",
"course_url",
"rubric",
):
if field in value and (
not isinstance(value[field], str) or not value[field].strip()
):
errors.append(f"{field} must be a non-empty string")
if "mode" in value:
if not isinstance(value["mode"], str) or value["mode"].lower() not in {
"work",
"learn",
"assessment",
}:
errors.append("mode must be work, learn, or assessment")
if "skills" in value:
if not isinstance(value["skills"], list) or not all(
isinstance(item, str) and item.strip() for item in value["skills"]
):
errors.append("skills must be an array of non-empty strings")
if "course_url" in value and isinstance(value["course_url"], str):
if not re.match(r"^https://", value["course_url"], re.IGNORECASE):
warnings.append("course_url should use HTTPS")
optional_list_fields = (
"constraints",
"allowed_tools",
"expected_artifacts",
)
for field in optional_list_fields:
if field in value and (
not isinstance(value[field], list)
or not all(isinstance(item, str) and item.strip() for item in value[field])
):
errors.append(f"{field} must be an array of non-empty strings")
if "tasks" in value:
if not isinstance(value["tasks"], list) or not value["tasks"]:
errors.append("tasks must be a non-empty array")
else:
for index, task in enumerate(value["tasks"]):
if not isinstance(task, dict):
errors.append(f"tasks[{index}] must be an object")
continue
for field in ("id", "prompt"):
if not isinstance(task.get(field), str) or not task[field].strip():
errors.append(
f"tasks[{index}].{field} must be a non-empty string"
)
return {"valid": not errors, "errors": errors, "warnings": warnings}
def _inspect_exercise(root: Path) -> dict[str, Any]:
path = root / ".ae" / "exercise.json"
if not path.is_file():
return {"present": False, "path": ".ae/exercise.json"}
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
return {
"present": True,
"path": ".ae/exercise.json",
"valid": False,
"errors": [f"invalid JSON: {exc}"],
"warnings": [],
}
validation = validate_exercise_config(value)
metadata_fields = (
"exercise_id",
"module_id",
"lesson_id",
"mode",
"skills",
"course_url",
)
return {
"present": True,
"path": ".ae/exercise.json",
**validation,
"declared_fields": sorted(value) if isinstance(value, dict) else [],
"metadata": (
{field: value[field] for field in metadata_fields if field in value}
if isinstance(value, dict)
else {}
),
}
def inspect_repo(path: str | Path = ".") -> dict[str, Any]:
"""Inspect a directory and return deterministic technology signals."""
root = Path(path).expanduser().resolve()
if not root.is_dir():
raise RepoInspectionError(f"Repository path is not a directory: {root}")
files = list(_walk_files(root))
relative_files = [_relative(file, root) for file in files]
extension_counts: Counter[str] = Counter(
file.suffix.lower() or "[no extension]" for file in files
)
sql_files = [file for file in files if file.suffix.lower() == ".sql"]
python_files = [file for file in files if file.suffix.lower() == ".py"]
yaml_files = [file for file in files if file.suffix.lower() in {".yml", ".yaml"}]
config_files = sorted(
_relative(file, root)
for file in files
if file.name.lower() in CONFIG_NAMES
or file.name.lower().endswith((".toml", ".ini", ".cfg"))
)
warehouse_evidence: dict[str, set[str]] = defaultdict(set)
airflow_evidence: set[str] = set()
dbt_evidence: set[str] = set()
python_signals: Counter[str] = Counter()
sql_signals: Counter[str] = Counter()
candidates = [
file
for file in files
if file.resolve() != Path(__file__).resolve()
if file.suffix.lower()
in {".py", ".sql", ".yml", ".yaml", ".toml", ".txt", ".cfg", ".ini"}
or file.name in {"Dockerfile"}
]
for file in candidates:
text = _read_text(file)
if not text:
continue
relative = _relative(file, root)
for warehouse, patterns in WAREHOUSE_PATTERNS.items():
if any(pattern.search(text) for pattern in patterns):
warehouse_evidence[warehouse].add(relative)
if file.suffix.lower() == ".py":
for name, pattern in {
"pandas": r"\b(?:import|from)\s+pandas\b",
"polars": r"\b(?:import|from)\s+polars\b",
"pyspark": r"\b(?:import|from)\s+pyspark\b",
"sqlalchemy": r"\b(?:import|from)\s+sqlalchemy\b",
"jupyter_notebook_code": r"\bget_ipython\s*\(",
}.items():
if re.search(pattern, text):
python_signals[name] += 1
if re.search(r"\bfrom\s+airflow\b|\bimport\s+airflow\b", text):
airflow_evidence.add(relative)
if re.search(r"\bDAG\s*\(|@dag\b", text):
airflow_evidence.add(relative)
if file.suffix.lower() == ".sql":
for name, pattern in {
"jinja": r"(?:\{\{|\{%)",
"dbt_ref": r"\bref\s*\(",
"dbt_source": r"\bsource\s*\(",
"select_star": r"\bselect\s+(?:\w+\.)?\*",
"merge": r"\bmerge\s+into\b",
"qualify": r"\bqualify\b",
}.items():
if re.search(pattern, text, re.I):
sql_signals[name] += 1
if re.search(r"\b(?:ref|source|config)\s*\(", text, re.I):
dbt_evidence.add(relative)
project_files = {
file.name: _relative(file, root)
for file in files
if file.name in {"dbt_project.yml", "dbt_project.yaml"}
}
dbt_evidence.update(project_files.values())
airflow_configs = [
relative
for relative in relative_files
if Path(relative).name in {"airflow.cfg", "webserver_config.py"}
]
airflow_evidence.update(airflow_configs)
orchestration = {
"airflow": {
"detected": bool(airflow_evidence),
"evidence_files": sorted(airflow_evidence)[:100],
}
}
warehouses = {
name: {
"detected": bool(paths),
"evidence_files": sorted(paths)[:100],
}
for name, paths in sorted(warehouse_evidence.items())
}
for expected in ("bigquery", "snowflake", "redshift", "databricks", "postgres", "duckdb"):
warehouses.setdefault(expected, {"detected": False, "evidence_files": []})
return {
"ok": True,
"root": str(root),
"file_count": len(files),
"file_types": dict(sorted(extension_counts.items())),
"config_files": config_files,
"technologies": {
"dbt": {
"detected": bool(dbt_evidence),
"project_file": next(iter(sorted(project_files.values())), None),
"evidence_files": sorted(dbt_evidence)[:100],
},
"sql": {
"detected": bool(sql_files),
"file_count": len(sql_files),
"signals": dict(sorted(sql_signals.items())),
},
"python": {
"detected": bool(python_files),
"file_count": len(python_files),
"signals": dict(sorted(python_signals.items())),
},
"bigquery": warehouses["bigquery"],
"airflow": orchestration["airflow"],
},
"warehouses": dict(sorted(warehouses.items())),
"orchestration": orchestration,
"yaml_file_count": len(yaml_files),
"exercise": _inspect_exercise(root),
"inspection_note": "Detection is static and heuristic; inspected files were not modified.",
}
def _parser() -> argparse.ArgumentParser:
parser = JsonArgumentParser(description=__doc__)
parser.add_argument("path", nargs="?", default=".", help="Repository directory")
parser.add_argument("--pretty", action="store_true", help="Indent JSON output")
return parser
def main(argv: Sequence[str] | None = None) -> int:
args = _parser().parse_args(argv)
try:
result = inspect_repo(args.path)
except (RepoInspectionError, OSError, ValueError) as exc:
print(
json.dumps({"ok": False, "error": str(exc), "error_type": type(exc).__name__}),
file=sys.stderr,
)
return 1
print(json.dumps(result, indent=2 if args.pretty else None, sort_keys=args.pretty))
return 0
if __name__ == "__main__":
raise SystemExit(main())
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |