foryourhealth111-pixel/Vibe-Skills / bundled/skills/preprocessing-data-with-automated-pipelines
preprocessing-data-with-automated-pipelines
VerifiedDesign and implement repeatable preprocessing pipelines for cleaning, encoding, transforming, and validating ML input data.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 1.5 KB |
| assets/README.md | markdown | 309 B |
| assets/example_data.csv | other | 1.3 KB |
| references/README.md | markdown | 437 B |
| scripts/README.md | markdown | 426 B |
| scripts/handle_errors.py | script | 10.1 KB |
| scripts/pipeline.py | script | 11.5 KB |
| scripts/transform_data.py | script | 13.1 KB |
| scripts/validate_data.py | script | 7.5 KB |
scripts/handle_errors.py
View on GitHub#!/usr/bin/env python3
"""
Error handling script for data preprocessing pipeline.
Manages and logs errors during preprocessing including:
- Exception tracking
- Error categorization
- Logging to files
- Error statistics
- Recovery recommendations
"""
import argparse
import json
import sys
from pathlib import Path
from datetime import datetime
from typing import Any, Dict, List, Optional
import traceback
class ErrorHandler:
"""Handles and logs errors in preprocessing pipeline."""
ERROR_CATEGORIES = {
'validation': 'Data validation errors',
'transformation': 'Data transformation errors',
'io': 'File I/O errors',
'type': 'Type conversion errors',
'missing': 'Missing data errors',
'duplicate': 'Duplicate data errors',
'schema': 'Schema mismatch errors',
'unknown': 'Unknown errors',
}
def __init__(self, log_file: Optional[str] = None):
"""
Initialize error handler.
Args:
log_file: Path to log file (optional)
"""
self.log_file = log_file
self.errors = []
self.error_stats = {cat: 0 for cat in self.ERROR_CATEGORIES.keys()}
self.session_id = datetime.now().isoformat()
def log_error(
self,
error: Exception,
category: str = 'unknown',
context: Optional[Dict[str, Any]] = None,
stack_trace: bool = True
) -> Dict[str, Any]:
"""
Log an error.
Args:
error: Exception object
category: Error category
context: Additional context data
stack_trace: Include stack trace
Returns:
Error record dictionary
"""
if category not in self.ERROR_CATEGORIES:
category = 'unknown'
error_record = {
'timestamp': datetime.now().isoformat(),
'category': category,
'message': str(error),
'type': type(error).__name__,
'context': context or {},
}
if stack_trace:
error_record['stack_trace'] = traceback.format_exc()
self.errors.append(error_record)
self.error_stats[category] += 1
# Write to log file if configured
if self.log_file:
self._write_to_log(error_record)
return error_record
def _write_to_log(self, error_record: Dict[str, Any]) -> None:
"""Write error record to log file."""
try:
path = Path(self.log_file)
path.parent.mkdir(parents=True, exist_ok=True)
with open(self.log_file, 'a', encoding='utf-8') as f:
f.write(json.dumps(error_record) + '\n')
except Exception as e:
print(f"Failed to write to log file: {str(e)}", file=sys.stderr)
def validate_data_integrity(
self,
data: List[Dict[str, Any]],
max_errors: int = 100
) -> Dict[str, Any]:
"""
Validate data integrity and log issues.
Args:
data: List of data rows
max_errors: Maximum errors to report
Returns:
Integrity report
"""
integrity_issues = {
'empty_rows': [],
'null_values': [],
'duplicates': [],
'type_mismatches': [],
}
seen_rows = set()
error_count = 0
for idx, row in enumerate(data):
if error_count >= max_errors:
break
# Check for empty rows
if not row:
integrity_issues['empty_rows'].append(idx)
error_count += 1
continue
# Check for null values
null_fields = [k for k, v in row.items() if v is None or v == '']
if null_fields:
if len(integrity_issues['null_values']) < max_errors:
integrity_issues['null_values'].append({
'row': idx,
'fields': null_fields
})
error_count += 1
# Check for duplicates
row_str = json.dumps(row, sort_keys=True, default=str)
if row_str in seen_rows:
if len(integrity_issues['duplicates']) < max_errors:
integrity_issues['duplicates'].append(idx)
error_count += 1
else:
seen_rows.add(row_str)
return {
'total_rows': len(data),
'issues': integrity_issues,
'issue_count': sum(len(v) for v in integrity_issues.values()),
}
def get_error_summary(self) -> Dict[str, Any]:
"""Get error summary."""
return {
'session_id': self.session_id,
'total_errors': len(self.errors),
'error_stats': self.error_stats,
'errors': self.errors[-100:], # Last 100 errors
}
def generate_recovery_recommendations(self) -> List[str]:
"""Generate recovery recommendations based on errors."""
recommendations = []
if self.error_stats['validation'] > 0:
recommendations.append(
"Validation errors detected. Review data against schema constraints."
)
if self.error_stats['missing'] > 0:
recommendations.append(
"Missing values detected. Consider imputation strategies (mean, median, forward-fill)."
)
if self.error_stats['duplicate'] > 0:
recommendations.append(
"Duplicate records detected. Remove duplicates or keep unique identifiers."
)
if self.error_stats['type'] > 0:
recommendations.append(
"Type conversion errors detected. Verify field types match expected schema."
)
if self.error_stats['io'] > 0:
recommendations.append(
"I/O errors detected. Check file permissions and storage availability."
)
if not recommendations:
recommendations.append("No errors detected. Data appears clean.")
return recommendations
def save_error_report(self, output_file: str) -> None:
"""
Save comprehensive error report.
Args:
output_file: Output file path
"""
try:
report = {
'metadata': {
'session_id': self.session_id,
'generated_at': datetime.now().isoformat(),
},
'summary': self.get_error_summary(),
'recommendations': self.generate_recovery_recommendations(),
'categories': self.ERROR_CATEGORIES,
}
path = Path(output_file)
path.parent.mkdir(parents=True, exist_ok=True)
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(report, f, indent=2)
except Exception as e:
print(f"Failed to save error report: {str(e)}", file=sys.stderr)
sys.exit(1)
def main():
"""Main entry point."""
parser = argparse.ArgumentParser(
description='Manage and log errors during data preprocessing pipeline'
)
parser.add_argument(
'action',
choices=['test', 'analyze', 'summary'],
help='Action to perform'
)
parser.add_argument(
'-l', '--log-file',
help='Path to error log file',
default=None
)
parser.add_argument(
'-d', '--data-file',
help='Path to data file for integrity checking'
)
parser.add_argument(
'-o', '--output',
help='Output file for error report'
)
parser.add_argument(
'-v', '--verbose',
action='store_true',
help='Print detailed error information'
)
args = parser.parse_args()
try:
handler = ErrorHandler(log_file=args.log_file)
if args.action == 'test':
# Test error logging
try:
raise ValueError("Test validation error")
except ValueError as e:
handler.log_error(e, category='validation', context={'test': True})
try:
raise KeyError("Test missing field")
except KeyError as e:
handler.log_error(e, category='missing', context={'field': 'test'})
print("Test errors logged successfully")
elif args.action == 'analyze':
# Analyze data file for integrity issues
if not args.data_file:
print("Error: --data-file required for analyze action", file=sys.stderr)
sys.exit(1)
path = Path(args.data_file)
if not path.exists():
print(f"Error: File not found: {args.data_file}", file=sys.stderr)
sys.exit(1)
# Load data
data = []
try:
if path.suffix.lower() == '.json':
with open(args.data_file, 'r') as f:
content = json.load(f)
data = content if isinstance(content, list) else [content]
else:
print(f"Error: Unsupported file format: {path.suffix}", file=sys.stderr)
sys.exit(1)
except json.JSONDecodeError as e:
handler.log_error(e, category='io')
integrity_report = handler.validate_data_integrity(data)
if args.verbose:
print(json.dumps(integrity_report, indent=2))
elif args.action == 'summary':
# Generate error summary
summary = handler.get_error_summary()
recommendations = handler.generate_recovery_recommendations()
report = {
'summary': summary,
'recommendations': recommendations,
}
if args.output:
handler.save_error_report(args.output)
print(f"Error report saved to: {args.output}")
else:
print(json.dumps(report, indent=2))
sys.exit(0)
except Exception as e:
print(f"Error: {str(e)}", file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| writing-skillsobra/superpowersUse when creating new skills, editing existing skills, or verifying skills work before deployment | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| xlsxanthropics/skillsUse this skill any time a spreadsheet file is the primary input or output. This means any task where the user wants to: open, read, edit, or fix an existing .xlsx, .xlsm, .xltx, .csv, or .tsv file (e.g., adding columns, computing formulas, formatting, charting, cleaning messy data); create a new spreadsheet from scratch or from other data sources; or convert between tabular file formats. Trigger especially when the user references a spreadsheet file by name or path — even casually (like "the xlsx in my downloads") — and wants something done to it or produced from it. Also trigger for cleaning or restructuring messy tabular data files (malformed rows, misplaced headers, junk data) into proper spreadsheets. The deliverable must be a spreadsheet file. Do NOT trigger when the primary deliverable is a Word document, HTML report, standalone Python script, database pipeline, or Google Sheets API integration, even if tabular data is involved. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| shipping-and-launchaddyosmani/agent-skillsPrepares production launches. Use when preparing to deploy to production, or when asking what needs to be in place before shipping. Use when you need a pre-launch checklist, when setting up monitoring, when planning a staged rollout, or when you need a rollback strategy. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| childhoodthedotmack/claude-memBabysit a shipped release after merge, publication, or deployment. Verify published packages, production behavior, installed plugins, and late CI or user reports until the observation window is clean; use babysit for PR readiness before release. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |