Skip to content

foryourhealth111-pixel/Vibe-Skills / bundled/skills/preprocessing-data-with-automated-pipelines

preprocessing-data-with-automated-pipelines

Verified

Design and implement repeatable preprocessing pipelines for cleaning, encoding, transforming, and validating ML input data.

FileKindSize
SKILL.mdskill md1.5 KB
assets/README.mdmarkdown309 B
assets/example_data.csvother1.3 KB
references/README.mdmarkdown437 B
scripts/README.mdmarkdown426 B
scripts/handle_errors.pyscript10.1 KB
scripts/pipeline.pyscript11.5 KB
scripts/transform_data.pyscript13.1 KB
scripts/validate_data.pyscript7.5 KB

scripts/validate_data.py

View on GitHub
#!/usr/bin/env python3
"""
Data validation script for preprocessing pipeline.

Validates data against predefined schemas or rules including:
- Required fields presence
- Data type correctness
- Value ranges and constraints
- Missing value handling
- Duplicates detection
"""

import argparse
import json
import sys
from pathlib import Path
from typing import Any, Dict, List, Tuple
import csv


class DataValidator:
    """Validates data against schemas and rules."""

    def __init__(self, schema_file: str = None):
        """
        Initialize validator.

        Args:
            schema_file: Path to JSON schema file (optional)
        """
        self.schema = {}
        if schema_file and Path(schema_file).exists():
            with open(schema_file, 'r') as f:
                self.schema = json.load(f)
        self.errors = []
        self.warnings = []

    def validate_file(self, file_path: str) -> bool:
        """
        Validate data file.

        Args:
            file_path: Path to data file (CSV or JSON)

        Returns:
            True if valid, False otherwise
        """
        try:
            path = Path(file_path)
            if not path.exists():
                self.errors.append(f"File not found: {file_path}")
                return False

            if path.suffix.lower() == '.csv':
                return self._validate_csv(str(path))
            elif path.suffix.lower() == '.json':
                return self._validate_json(str(path))
            else:
                self.errors.append(f"Unsupported file format: {path.suffix}")
                return False
        except Exception as e:
            self.errors.append(f"Validation error: {str(e)}")
            return False

    def _validate_csv(self, file_path: str) -> bool:
        """Validate CSV file structure and content."""
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                reader = csv.DictReader(f)
                if not reader.fieldnames:
                    self.errors.append("CSV file is empty or has no headers")
                    return False

                # Check schema fields if defined
                if self.schema:
                    required_fields = self.schema.get('required_fields', [])
                    for field in required_fields:
                        if field not in reader.fieldnames:
                            self.errors.append(f"Missing required field: {field}")
                            return False

                row_count = 0
                for row_count, row in enumerate(reader, start=1):
                    if not self._validate_row(row, reader.fieldnames):
                        self.errors.append(f"Invalid data at row {row_count}")
                        if len(self.errors) > 10:  # Limit errors
                            self.warnings.append("... (more errors truncated)")
                            break

                if row_count == 0:
                    self.errors.append("CSV file contains no data rows")
                    return False

                return len(self.errors) == 0
        except Exception as e:
            self.errors.append(f"CSV validation failed: {str(e)}")
            return False

    def _validate_json(self, file_path: str) -> bool:
        """Validate JSON file structure and content."""
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                data = json.load(f)

            if isinstance(data, list):
                if not data:
                    self.errors.append("JSON array is empty")
                    return False

                for idx, item in enumerate(data):
                    if not isinstance(item, dict):
                        self.errors.append(f"Row {idx} is not a dictionary")
                        return False
                    if not self._validate_row(item, item.keys()):
                        self.errors.append(f"Invalid data at row {idx}")
                        if len(self.errors) > 10:
                            self.warnings.append("... (more errors truncated)")
                            break

            elif isinstance(data, dict):
                if not data:
                    self.warnings.append("JSON object is empty")
            else:
                self.errors.append("JSON must be an object or array")
                return False

            return len(self.errors) == 0
        except json.JSONDecodeError as e:
            self.errors.append(f"Invalid JSON: {str(e)}")
            return False
        except Exception as e:
            self.errors.append(f"JSON validation failed: {str(e)}")
            return False

    def _validate_row(self, row: Dict[str, Any], fields: List[str]) -> bool:
        """Validate individual row against schema."""
        if not self.schema:
            # No schema defined, just check for basic issues
            for key, value in row.items():
                if value is None or (isinstance(value, str) and not value.strip()):
                    self.warnings.append(f"Empty value for field: {key}")
            return True

        field_types = self.schema.get('field_types', {})
        for field, expected_type in field_types.items():
            if field in row:
                if not self._validate_type(row[field], expected_type):
                    self.errors.append(
                        f"Type mismatch for field '{field}': "
                        f"expected {expected_type}, got {type(row[field]).__name__}"
                    )
                    return False
        return True

    def _validate_type(self, value: Any, expected_type: str) -> bool:
        """Validate value type."""
        if value is None:
            return True

        type_map = {
            'string': str,
            'int': int,
            'float': (int, float),
            'bool': bool,
            'number': (int, float),
        }

        if expected_type not in type_map:
            return True

        expected = type_map[expected_type]
        return isinstance(value, expected)

    def get_report(self) -> Dict[str, Any]:
        """Get validation report."""
        return {
            'valid': len(self.errors) == 0,
            'errors': self.errors,
            'warnings': self.warnings,
            'error_count': len(self.errors),
            'warning_count': len(self.warnings),
        }


def main():
    """Main entry point."""
    parser = argparse.ArgumentParser(
        description='Validate data against predefined schemas or rules'
    )
    parser.add_argument(
        'data_file',
        help='Path to data file (CSV or JSON)'
    )
    parser.add_argument(
        '-s', '--schema',
        help='Path to JSON schema file for validation',
        default=None
    )
    parser.add_argument(
        '-v', '--verbose',
        action='store_true',
        help='Print detailed validation report'
    )
    parser.add_argument(
        '-o', '--output',
        help='Save validation report to JSON file',
        default=None
    )

    args = parser.parse_args()

    # Validate data file
    validator = DataValidator(schema_file=args.schema)
    is_valid = validator.validate_file(args.data_file)
    report = validator.get_report()

    # Output report
    if args.verbose or not is_valid:
        print(json.dumps(report, indent=2))

    # Save report if requested
    if args.output:
        with open(args.output, 'w') as f:
            json.dump(report, f, indent=2)
        print(f"Validation report saved to: {args.output}")

    # Exit with appropriate code
    sys.exit(0 if is_valid else 1)


if __name__ == '__main__':
    main()
writing-skillsobra/superpowersUse when creating new skills, editing existing skills, or verifying skills work before deploymentStars: 322kUpdated Claude CodeCodexVerifiedInstall
xlsxanthropics/skillsUse this skill any time a spreadsheet file is the primary input or output. This means any task where the user wants to: open, read, edit, or fix an existing .xlsx, .xlsm, .xltx, .csv, or .tsv file (e.g., adding columns, computing formulas, formatting, charting, cleaning messy data); create a new spreadsheet from scratch or from other data sources; or convert between tabular file formats. Trigger especially when the user references a spreadsheet file by name or path — even casually (like "the xlsx in my downloads") — and wants something done to it or produced from it. Also trigger for cleaning or restructuring messy tabular data files (malformed rows, misplaced headers, junk data) into proper spreadsheets. The deliverable must be a spreadsheet file. Do NOT trigger when the primary deliverable is a Word document, HTML report, standalone Python script, database pipeline, or Google Sheets API integration, even if tabular data is involved.Stars: 198kUpdated Claude CodeCodexVerifiedInstall
shipping-and-launchaddyosmani/agent-skillsPrepares production launches. Use when preparing to deploy to production, or when asking what needs to be in place before shipping. Use when you need a pre-launch checklist, when setting up monitoring, when planning a staged rollout, or when you need a rollback strategy.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
childhoodthedotmack/claude-memBabysit a shipped release after merge, publication, or deployment. Verify published packages, production behavior, installed plugins, and late CI or user reports until the observation window is clean; use babysit for PR readiness before release.Stars: 118kUpdated Claude CodeCodexVerifiedInstall