bestdeejay-design/agent-skills / skills/csv-pro
csv-pro
VerifiedПрофилирование CSV-файлов: типы колонок, статистика, аномалии. Скрипт csv_pro.py читает CSV (файл --input или stdin --stdin), определяет разделитель (по умолчанию ';', затем ','), для каждой колонки считает тип (int/float/str/date), min/max/mean, пропуски, уникальные значения и топ-3 частых. Ищет аномалии: нулевая дисперсия, >95% пустых, дубликаты строк, строки длиннее 1000 символов, выбросы (значение ≥ 5×IQR от медианы). Вывод: markdown-таблица (по умолчанию), JSON или HTML с графиками. Триггеры: 'csv profile', 'профиль csv', 'анализ csv', 'csv анализ', 'аномалии csv', 'csv anomalies', 'профилирование csv', 'что в csv'.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 5.2 KB |
| references/canonical-patterns.md | markdown | 7.9 KB |
| scripts/csv_pro.py | script | 11.7 KB |
| skill.json | config | 1.7 KB |
scripts/csv_pro.py
View on GitHub#!/usr/bin/env python3
"""csv_pro.py — профиль CSV: типы колонок, статистика, аномалии.
Читает CSV (файл --input или stdin --stdin), определяет разделитель
(по умолчанию ';', затем ','), для каждой колонки считает тип
(int/float/str/date), min/max/mean, пропуски, уникальные значения и топ-3
частых. Ищет аномалии: нулевая дисперсия, >95% пустых, дубликаты строк,
строки длиннее 1000 символов, выбросы (значение >= 5×IQR от медианы).
Вывод: markdown-таблица (по умолчанию) или JSON (--output json).
Примеры:
python3 csv_pro.py --input data.csv
python3 csv_pro.py --input data.csv --output json
cat data.csv | python3 csv_pro.py --stdin
"""
import argparse
import csv
import json
import statistics
import sys
from collections import Counter
from datetime import datetime
DATE_FORMATS = ("%Y-%m-%d", "%d.%m.%Y", "%Y/%m/%d")
LONG_STRING = 1000
MISSING_RATIO = 0.95
SPIKE_IQR = 5.0
def load_lines(path):
if path:
with open(path, "rb") as f:
raw = f.readlines()
else:
raw = sys.stdin.buffer.readlines()
lines, skipped = [], 0
for b in raw:
try:
lines.append(b.decode("utf-8-sig"))
except UnicodeDecodeError:
skipped += 1
return lines, skipped
def detect_delimiter(lines):
for ln in lines:
if ln.strip():
return ";" if ln.count(";") > ln.count(",") else ","
return ","
def parse_value(raw):
s = raw.strip()
if not s:
return None
try:
int(s)
return "int"
except ValueError:
pass
try:
float(s)
return "float"
except ValueError:
pass
for fmt in DATE_FORMATS:
try:
datetime.strptime(s, fmt)
return "date"
except ValueError:
pass
try:
datetime.fromisoformat(s)
return "date"
except ValueError:
return "str"
def is_header(row):
return bool(row) and all(parse_value(c) in ("str", "date") for c in row)
def analyze_column(name, values, missing, total):
types = {t for t in (parse_value(v) for v in values) if t}
if types == {"int"}:
ctype = "int"
elif types and types <= {"int", "float"}:
ctype = "float"
elif types == {"date"}:
ctype = "date"
else:
ctype = "str"
nums = []
if ctype in ("int", "float"):
for v in values:
try:
nums.append(float(v))
except ValueError:
pass
prof = {
"name": name,
"type": ctype,
"missing": missing,
"unique": len(set(values)),
"top3": [list(t) for t in Counter(values).most_common(3)],
}
if nums:
prof["min"] = min(nums)
prof["max"] = max(nums)
prof["mean"] = statistics.mean(nums)
anomalies = []
if len(nums) >= 2 and min(nums) == max(nums):
anomalies.append({"type": "zero_variance", "column": name,
"detail": f"все значения = {min(nums):g}"})
if total and missing / total > MISSING_RATIO:
anomalies.append({"type": "mostly_missing", "column": name,
"detail": f"{missing}/{total} пустых"})
long = sum(1 for v in values if len(v) > LONG_STRING)
if long:
anomalies.append({"type": "long_strings", "column": name,
"detail": f"{long} значений длиннее {LONG_STRING} символов"})
if len(nums) >= 4:
med = statistics.median(nums)
q1, _, q3 = statistics.quantiles(nums, n=4)
iqr = q3 - q1
if iqr > 0:
spikes = sum(1 for v in nums if abs(v - med) >= SPIKE_IQR * iqr)
if spikes:
anomalies.append({"type": "spikes", "column": name,
"detail": f"{spikes} значений >= 5xIQR от медианы"})
return prof, anomalies
def fmt_num(v):
return f"{v:g}" if isinstance(v, float) else str(v)
def short(s, n=40):
s = str(s)
return s if len(s) <= n else s[: n - 1] + "…"
def esc(s):
return str(s).replace("|", "\\|")
def render_md(meta, profiles, anomalies):
out = [f"# Профиль CSV: `{meta['file']}`", ""]
out.append(f"- Строк: {meta['rows']}")
out.append(f"- Колонок: {meta['columns']}")
out.append(f"- Разделитель: `{meta['delimiter']}`")
if meta["skipped_lines"]:
out.append(f"- Не декодировано строк: {meta['skipped_lines']}")
out += ["", "## Колонки", "",
"| Колонка | Тип | Уникальных | Пустых | Min | Max | Mean | Топ-3 частых |",
"|---------|-----|-----------|--------|-----|-----|------|--------------|"]
for p in profiles:
top3 = ", ".join(f"{short(v)} ({n})" for v, n in p["top3"])
out.append(f"| {esc(p['name'])} | {p['type']} | {p['unique']} | {p['missing']} "
f"| {fmt_num(p.get('min', ''))} | {fmt_num(p.get('max', ''))} "
f"| {fmt_num(p.get('mean', ''))} | {top3} |")
out += ["", "## Аномалии", ""]
if not anomalies:
out.append("Аномалий не найдено.")
else:
for a in anomalies:
col = f"`{a['column']}`: " if a["column"] else ""
out.append(f"- {col}{a['type']} — {a['detail']}")
return "\n".join(out)
def render_json(meta, profiles, anomalies):
return json.dumps({
"file": meta["file"],
"rows": meta["rows"],
"columns": meta["columns"],
"delimiter": meta["delimiter"],
"skipped_lines": meta["skipped_lines"],
"columns": profiles,
"anomalies": anomalies,
}, ensure_ascii=False, indent=2)
def render_html(meta, profiles, anomalies):
presets = {
"mono": {"bg": "#F0EFEB", "ink": "#1C1C1A", "muted": "#8F8E88",
"grid": "#DEDDD6", "data": ["#1C1C1A", "#4A4944", "#6A6963", "#8F8E88", "#B0AFA9"]},
}
p = presets["mono"]
numeric = [pr for pr in profiles if pr["type"] in ("int", "float")]
categorical = [pr for pr in profiles if pr["type"] in ("str", "date")]
html = f"""<!DOCTYPE html>
<html lang="ru">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Профиль CSV: {meta['file']}</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
<style>
* {{ margin: 0; padding: 0; box-sizing: border-box; }}
body {{ font-family: Inter, -apple-system, sans-serif; background: {p['bg']}; color: {p['ink']}; padding: 24px; }}
.card {{ background: {p['bg']}; border-radius: 24px; padding: 28px; margin-bottom: 20px; }}
h1 {{ font-size: 24px; font-weight: 700; margin-bottom: 8px; }}
h2 {{ font-size: 16px; font-weight: 700; margin-bottom: 12px; }}
.meta {{ font-size: 12px; color: {p['muted']}; margin-bottom: 24px; }}
.grid {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(400px, 1fr)); gap: 20px; }}
.chart-container {{ position: relative; height: 250px; }}
.source {{ font-size: 10px; text-transform: uppercase; letter-spacing: 0.08em; color: {p['muted']}; margin-top: 12px; }}
.anomaly {{ font-size: 13px; padding: 8px 12px; background: #fff3cd; border-radius: 8px; margin-bottom: 8px; }}
@media (prefers-reduced-motion: reduce) {{ * {{ animation: none !important; transition: none !important; }} }}
</style>
</head>
<body>
<h1>Профиль CSV</h1>
<div class="meta">{meta['rows']} строк · {meta['columns']} колонок · разделитель: {meta['delimiter']}</div>
<div class="grid">
"""
for i, pr in enumerate(numeric[:6]):
vals = []
for row_data in profiles:
if row_data["name"] == pr["name"] and "min" in pr:
vals = [pr["min"], pr["max"], pr["mean"]]
break
color = p["data"][i % len(p["data"])]
html += f"""
<div class="card">
<h2>{pr['name']}</h2>
<div class="chart-container"><canvas id="csv-hist{i}"></canvas></div>
<div class="source">{pr['type']} · уникальных: {pr['unique']}</div>
</div>
"""
html += """
</div>
<div class="card">
<h2>Аномалии</h2>
"""
if anomalies:
for a in anomalies:
html += f' <div class="anomaly">{a["type"]}: {a["detail"]}</div>\n'
else:
html += ' <div style="font-size:14px">Аномалий не найдено</div>\n'
html += f"""
<div class="source">csv-pro · agent-skills</div>
</div>
</body></html>"""
return html
def main():
ap = argparse.ArgumentParser(description="CSV profile: types, stats, anomalies")
ap.add_argument("--input", help="input CSV file")
ap.add_argument("--stdin", action="store_true", help="read CSV from stdin")
ap.add_argument("--output", choices=["md", "json", "html"], default="md",
help="output format (default: md)")
ap.add_argument("--delimiter", default=None,
help="CSV delimiter (default: auto-detect ';' then ',')")
args = ap.parse_args()
if not args.input and not args.stdin:
ap.error("need --input or --stdin")
if args.delimiter and len(args.delimiter) != 1:
ap.error("--delimiter must be a single character")
csv.field_size_limit(10_000_000)
path = args.input
try:
lines, skipped = load_lines(path)
except OSError as e:
sys.stderr.write(f"csv_pro: не удалось прочитать {path}: {e}\n")
sys.exit(1)
if skipped:
sys.stderr.write(f"csv_pro: пропущено {skipped} строк (не UTF-8)\n")
delim = args.delimiter or detect_delimiter(lines)
try:
rows = [r for r in csv.reader(lines, delimiter=delim) if any(c.strip() for c in r)]
except csv.Error as e:
sys.stderr.write(f"csv_pro: ошибка разбора CSV: {e}\n")
sys.exit(1)
meta = {"file": path or "<stdin>", "rows": 0, "columns": 0,
"delimiter": delim, "skipped_lines": skipped}
if not rows:
if args.output == "json":
print(render_json(meta, [], []))
else:
print(f"# Профиль CSV: `{meta['file']}`\n\nФайл пуст: нет данных для анализа.")
return
has_header = is_header(rows[0])
names = rows[0] if has_header else [f"Column{i+1}" for i in range(len(rows[0]))]
data = rows[1:] if has_header else rows
meta["rows"] = len(data)
meta["columns"] = len(names)
if not data:
if args.output == "json":
print(render_json(meta, [], []))
else:
print(f"# Профиль CSV: `{meta['file']}`\n\nТолько заголовок, данных нет.")
return
profiles, anomalies = [], []
for i, name in enumerate(names):
values, missing = [], 0
for row in data:
cell = row[i] if i < len(row) else ""
if cell.strip():
values.append(cell)
else:
missing += 1
prof, anoms = analyze_column(name, values, missing, len(data))
profiles.append(prof)
anomalies.extend(anoms)
dup = len(data) - len({tuple(r) for r in data})
if dup:
anomalies.append({"type": "duplicate_rows", "column": None,
"detail": f"{dup} дубликатов строк"})
if args.output == "json":
print(render_json(meta, profiles, anomalies))
elif args.output == "html":
print(render_html(meta, profiles, anomalies))
else:
print(render_md(meta, profiles, anomalies))
if __name__ == "__main__":
main()Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| deprecation-and-migrationaddyosmani/agent-skillsManages deprecation and migration. Use when removing old systems, APIs, or features. Use when migrating users from one implementation to another. Use when migrating a database schema in production, such as renaming or dropping a column without downtime (expand/contract). Use when deciding whether to maintain or sunset existing code. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| performance-optimizationaddyosmani/agent-skillsOptimizes application performance across frontend, backend, queries, and databases. Use when performance requirements exist, when you suspect performance regressions, when Core Web Vitals or load times need improvement, when N+1 query patterns need fixing, or when profiling reveals bottlenecks. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |
| agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |