Skip to content

HabitaNexus/monorepo / skills/system-reliability-engineering/logging-log-aggregation

logging-log-aggregation

Verified

scripts/python/structured_logger.py

View on GitHub
#!/usr/bin/env python3
"""
Structured Logger for Python

Structured logging with JSON output for centralized logging systems.

Usage:
    from structured_logger import get_logger

    logger = get_logger(service='my-service')
    logger.info('User created', extra={
        'user_id': '12345',
        'trace_id': request.headers.get('X-Trace-Id'),
        'http_method': request.method,
        'http_path': request.path,
        'http_status': 201,
        'duration_ms': 45,
    })
"""

import json
import logging
import sys
from datetime import datetime
from typing import Dict, Any, Optional


class StructuredFormatter(logging.Formatter):
    """JSON formatter for structured logging."""

    def format(self, record: logging.LogRecord) -> str:
        """Format log record as JSON."""
        log_data = {
            'timestamp': datetime.utcnow().isoformat() + 'Z',
            'level': record.levelname,
            'message': record.getMessage(),
            'module': record.module,
            'function': record.funcName,
            'line': record.lineno,
        }

        # Add extra fields from record
        for key, value in record.__dict__.items():
            if key not in [
                'name', 'msg', 'args', 'created', 'filename', 'funcName',
                'levelname', 'levelno', 'lineno', 'module', 'msecs',
                'message', 'pathname', 'process', 'processName', 'relativeCreated',
                'thread', 'threadName', 'exc_info', 'exc_text', 'stack_info'
            ]:
                log_data[key] = value

        # Add exception info if present
        if record.exc_info:
            log_data['exception'] = self.formatException(record.exc_info)

        # Add stack trace if present
        if record.stack_info:
            log_data['stack'] = self.formatStack(record.stack_info)

        return json.dumps(log_data)


def get_logger(
    name: Optional[str] = None,
    service: Optional[str] = None,
    environment: Optional[str] = None,
    version: Optional[str] = None,
    level: str = 'INFO'
) -> logging.Logger:
    """
    Get a structured logger instance.

    Args:
        name: Logger name (default: __name__)
        service: Service name (default: from env or 'application')
        environment: Environment (default: from env or 'development')
        version: Application version (default: from env or '1.0.0')
        level: Log level (default: 'INFO')

    Returns:
        Configured logger instance
    """
    import os

    logger = logging.getLogger(name or __name__)
    logger.setLevel(getattr(logging, level.upper(), logging.INFO))

    # Clear existing handlers
    logger.handlers.clear()

    # Create formatter
    formatter = StructuredFormatter()

    # Console handler
    console_handler = logging.StreamHandler(sys.stdout)
    console_handler.setFormatter(formatter)
    logger.addHandler(console_handler)

    # File handler for errors
    error_handler = logging.FileHandler('logs/error.log')
    error_handler.setLevel(logging.ERROR)
    error_handler.setFormatter(formatter)
    logger.addHandler(error_handler)

    # File handler for all logs
    file_handler = logging.FileHandler('logs/combined.log')
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)

    # Add default metadata
    service_name = service or os.getenv('SERVICE_NAME', 'application')
    env = environment or os.getenv('ENVIRONMENT', 'development')
    app_version = version or os.getenv('APP_VERSION', '1.0.0')

    # Add adapter to inject default metadata
    class ContextAdapter(logging.LoggerAdapter):
        def process(self, msg, kwargs):
            kwargs.setdefault('extra', {})
            kwargs['extra'].setdefault('service', service_name)
            kwargs['extra'].setdefault('environment', env)
            kwargs['extra'].setdefault('version', app_version)
            return msg, kwargs

    return ContextAdapter(logger, {})


# Convenience functions
def log_user_event(logger: logging.Logger, event: str, user_id: str, **metadata):
    """Log a user event."""
    logger.info(event, extra={
        'user_id': user_id,
        **metadata,
    })


def log_http_request(
    logger: logging.Logger,
    method: str,
    path: str,
    status: int,
    duration_ms: float,
    trace_id: Optional[str] = None,
    span_id: Optional[str] = None,
    **metadata
):
    """Log an HTTP request."""
    logger.info('HTTP request', extra={
        'http_method': method,
        'http_path': path,
        'http_status': status,
        'duration_ms': duration_ms,
        'trace_id': trace_id,
        'span_id': span_id,
        **metadata,
    })


def log_error(logger: logging.Logger, error: Exception, **context):
    """Log an error with context."""
    logger.error('Error occurred', extra={
        'error': str(error),
        'error_type': type(error).__name__,
        **context,
    }, exc_info=True)


def log_database_query(
    logger: logging.Logger,
    query: str,
    duration_ms: float,
    success: bool = True
):
    """Log a database query."""
    level = logging.INFO if success else logging.ERROR
    logger.log(level, 'Database query', extra={
        'query': query[:200],  # Truncate long queries
        'duration_ms': duration_ms,
        'success': success,
    })


# Default logger instance
logger = get_logger()


# CLI usage
if __name__ == '__main__':
    import os

    # Create logs directory
    os.makedirs('logs', exist_ok=True)

    # Example usage
    logger.info('Application started', extra={
        'port': os.getenv('PORT', '3000'),
        'python_version': sys.version,
    })

    logger.info('User created', extra={
        'user_id': '12345',
        'email': 'user@example.com',
        'trace_id': 'abc-123',
        'span_id': 'span-456',
        'http_method': 'POST',
        'http_path': '/api/users',
        'http_status': 201,
        'duration_ms': 45,
    })

    logger.error('Database connection failed', extra={
        'error': 'Connection timeout',
        'database': 'users-db',
        'retry_attempt': 3,
        'trace_id': 'abc-123',
    })
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall