foryourhealth111-pixel/Vibe-Skills / bundled/skills/preprocessing-data-with-automated-pipelines
preprocessing-data-with-automated-pipelines
VerifiedDesign and implement repeatable preprocessing pipelines for cleaning, encoding, transforming, and validating ML input data.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 1.5 KB |
| assets/README.md | markdown | 309 B |
| assets/example_data.csv | other | 1.3 KB |
| references/README.md | markdown | 437 B |
| scripts/README.md | markdown | 426 B |
| scripts/handle_errors.py | script | 10.1 KB |
| scripts/pipeline.py | script | 11.5 KB |
| scripts/transform_data.py | script | 13.1 KB |
| scripts/validate_data.py | script | 7.5 KB |
scripts/validate_data.py
View on GitHub#!/usr/bin/env python3
"""
Data validation script for preprocessing pipeline.
Validates data against predefined schemas or rules including:
- Required fields presence
- Data type correctness
- Value ranges and constraints
- Missing value handling
- Duplicates detection
"""
import argparse
import json
import sys
from pathlib import Path
from typing import Any, Dict, List, Tuple
import csv
class DataValidator:
"""Validates data against schemas and rules."""
def __init__(self, schema_file: str = None):
"""
Initialize validator.
Args:
schema_file: Path to JSON schema file (optional)
"""
self.schema = {}
if schema_file and Path(schema_file).exists():
with open(schema_file, 'r') as f:
self.schema = json.load(f)
self.errors = []
self.warnings = []
def validate_file(self, file_path: str) -> bool:
"""
Validate data file.
Args:
file_path: Path to data file (CSV or JSON)
Returns:
True if valid, False otherwise
"""
try:
path = Path(file_path)
if not path.exists():
self.errors.append(f"File not found: {file_path}")
return False
if path.suffix.lower() == '.csv':
return self._validate_csv(str(path))
elif path.suffix.lower() == '.json':
return self._validate_json(str(path))
else:
self.errors.append(f"Unsupported file format: {path.suffix}")
return False
except Exception as e:
self.errors.append(f"Validation error: {str(e)}")
return False
def _validate_csv(self, file_path: str) -> bool:
"""Validate CSV file structure and content."""
try:
with open(file_path, 'r', encoding='utf-8') as f:
reader = csv.DictReader(f)
if not reader.fieldnames:
self.errors.append("CSV file is empty or has no headers")
return False
# Check schema fields if defined
if self.schema:
required_fields = self.schema.get('required_fields', [])
for field in required_fields:
if field not in reader.fieldnames:
self.errors.append(f"Missing required field: {field}")
return False
row_count = 0
for row_count, row in enumerate(reader, start=1):
if not self._validate_row(row, reader.fieldnames):
self.errors.append(f"Invalid data at row {row_count}")
if len(self.errors) > 10: # Limit errors
self.warnings.append("... (more errors truncated)")
break
if row_count == 0:
self.errors.append("CSV file contains no data rows")
return False
return len(self.errors) == 0
except Exception as e:
self.errors.append(f"CSV validation failed: {str(e)}")
return False
def _validate_json(self, file_path: str) -> bool:
"""Validate JSON file structure and content."""
try:
with open(file_path, 'r', encoding='utf-8') as f:
data = json.load(f)
if isinstance(data, list):
if not data:
self.errors.append("JSON array is empty")
return False
for idx, item in enumerate(data):
if not isinstance(item, dict):
self.errors.append(f"Row {idx} is not a dictionary")
return False
if not self._validate_row(item, item.keys()):
self.errors.append(f"Invalid data at row {idx}")
if len(self.errors) > 10:
self.warnings.append("... (more errors truncated)")
break
elif isinstance(data, dict):
if not data:
self.warnings.append("JSON object is empty")
else:
self.errors.append("JSON must be an object or array")
return False
return len(self.errors) == 0
except json.JSONDecodeError as e:
self.errors.append(f"Invalid JSON: {str(e)}")
return False
except Exception as e:
self.errors.append(f"JSON validation failed: {str(e)}")
return False
def _validate_row(self, row: Dict[str, Any], fields: List[str]) -> bool:
"""Validate individual row against schema."""
if not self.schema:
# No schema defined, just check for basic issues
for key, value in row.items():
if value is None or (isinstance(value, str) and not value.strip()):
self.warnings.append(f"Empty value for field: {key}")
return True
field_types = self.schema.get('field_types', {})
for field, expected_type in field_types.items():
if field in row:
if not self._validate_type(row[field], expected_type):
self.errors.append(
f"Type mismatch for field '{field}': "
f"expected {expected_type}, got {type(row[field]).__name__}"
)
return False
return True
def _validate_type(self, value: Any, expected_type: str) -> bool:
"""Validate value type."""
if value is None:
return True
type_map = {
'string': str,
'int': int,
'float': (int, float),
'bool': bool,
'number': (int, float),
}
if expected_type not in type_map:
return True
expected = type_map[expected_type]
return isinstance(value, expected)
def get_report(self) -> Dict[str, Any]:
"""Get validation report."""
return {
'valid': len(self.errors) == 0,
'errors': self.errors,
'warnings': self.warnings,
'error_count': len(self.errors),
'warning_count': len(self.warnings),
}
def main():
"""Main entry point."""
parser = argparse.ArgumentParser(
description='Validate data against predefined schemas or rules'
)
parser.add_argument(
'data_file',
help='Path to data file (CSV or JSON)'
)
parser.add_argument(
'-s', '--schema',
help='Path to JSON schema file for validation',
default=None
)
parser.add_argument(
'-v', '--verbose',
action='store_true',
help='Print detailed validation report'
)
parser.add_argument(
'-o', '--output',
help='Save validation report to JSON file',
default=None
)
args = parser.parse_args()
# Validate data file
validator = DataValidator(schema_file=args.schema)
is_valid = validator.validate_file(args.data_file)
report = validator.get_report()
# Output report
if args.verbose or not is_valid:
print(json.dumps(report, indent=2))
# Save report if requested
if args.output:
with open(args.output, 'w') as f:
json.dump(report, f, indent=2)
print(f"Validation report saved to: {args.output}")
# Exit with appropriate code
sys.exit(0 if is_valid else 1)
if __name__ == '__main__':
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| writing-skillsobra/superpowersUse when creating new skills, editing existing skills, or verifying skills work before deployment | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| xlsxanthropics/skillsUse this skill any time a spreadsheet file is the primary input or output. This means any task where the user wants to: open, read, edit, or fix an existing .xlsx, .xlsm, .xltx, .csv, or .tsv file (e.g., adding columns, computing formulas, formatting, charting, cleaning messy data); create a new spreadsheet from scratch or from other data sources; or convert between tabular file formats. Trigger especially when the user references a spreadsheet file by name or path — even casually (like "the xlsx in my downloads") — and wants something done to it or produced from it. Also trigger for cleaning or restructuring messy tabular data files (malformed rows, misplaced headers, junk data) into proper spreadsheets. The deliverable must be a spreadsheet file. Do NOT trigger when the primary deliverable is a Word document, HTML report, standalone Python script, database pipeline, or Google Sheets API integration, even if tabular data is involved. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| shipping-and-launchaddyosmani/agent-skillsPrepares production launches. Use when preparing to deploy to production, or when asking what needs to be in place before shipping. Use when you need a pre-launch checklist, when setting up monitoring, when planning a staged rollout, or when you need a rollback strategy. | Stars: 119k | Updated | Claude CodeCodex | Verified | Install |
| childhoodthedotmack/claude-memBabysit a shipped release after merge, publication, or deployment. Verify published packages, production behavior, installed plugins, and late CI or user reports until the observation window is clean; use babysit for PR readiness before release. | Stars: 118k | Updated | Claude CodeCodex | Verified | Install |