Skip to content

borghei/Claude-Skills / engineering/browser-automation

browser-automation

Verified

Generates polite, rate-limited scraper and form-submission scripts (requests, no JS) and audits automation code for blocking and overload risks. Use when writing a permitted scraper, scripting a form post, or reviewing a crawler.

scripts/scraping_toolkit.py

View on GitHub
#!/usr/bin/env python3
"""
Scraping Toolkit - Generate web scraping code with rate limiting and best practices.

Analyzes target URLs and generates production-ready scraping scripts with proper
rate limiting, error handling, data extraction, and anti-detection measures.

License: MIT
"""

import argparse
import json
import os
import re
import sys
from dataclasses import dataclass, asdict, field
from pathlib import Path
from typing import List, Dict, Optional
from urllib.parse import urlparse, urljoin
import textwrap


RATE_STRATEGIES = {
    "aggressive": {"min_delay": 0.5, "max_delay": 1.5, "concurrent": 5, "description": "Fast, for own APIs or test environments"},
    "normal": {"min_delay": 1.0, "max_delay": 3.0, "concurrent": 3, "description": "Balanced speed and politeness"},
    "polite": {"min_delay": 3.0, "max_delay": 7.0, "concurrent": 1, "description": "Respectful, minimizes server impact"},
    "gentle": {"min_delay": 5.0, "max_delay": 15.0, "concurrent": 1, "description": "Slowest, for small or fragile sites"},
}

# Generated scrapers identify themselves honestly; replace the name, URL and contact before use.
DEFAULT_USER_AGENT = "ExampleCrawler/1.0 (+https://example.com/bot; bot-owner@example.com)"


@dataclass
class ScrapingConfig:
    """Configuration for generated scraper."""
    target_url: str
    strategy: str
    output_format: str  # csv, json, jsonl
    selectors: Dict[str, str] = field(default_factory=dict)
    pagination: bool = False
    respect_robots: bool = True
    max_pages: int = 100
    proxy_support: bool = False


def generate_scraper_script(config: ScrapingConfig) -> str:
    """Generate a complete scraping script."""
    strategy = RATE_STRATEGIES[config.strategy]
    parsed = urlparse(config.target_url)
    domain = parsed.netloc

    script = textwrap.dedent(f'''\
        #!/usr/bin/env python3
        """
        Web Scraper for {domain}
        Generated by Scraping Toolkit

        Strategy: {config.strategy} ({strategy["description"]})
        Rate: {strategy["min_delay"]}-{strategy["max_delay"]}s between requests
        """

        import csv
        import json
        import logging
        import os
        import random
        import sys
        import time
        from dataclasses import dataclass, asdict
        from datetime import datetime
        from pathlib import Path
        from typing import List, Dict, Optional, Generator
        from urllib.parse import urljoin, urlparse
        from urllib.robotparser import RobotFileParser

        # Configure logging
        logging.basicConfig(
            level=logging.INFO,
            format="%(asctime)s [%(levelname)s] %(message)s",
        )
        logger = logging.getLogger(__name__)

        # --- Configuration ---
        TARGET_URL = "{config.target_url}"
        DOMAIN = "{domain}"
        MIN_DELAY = {strategy["min_delay"]}
        MAX_DELAY = {strategy["max_delay"]}
        MAX_PAGES = {config.max_pages}
        MAX_RETRIES = 3
        BACKOFF_FACTOR = 2
        RESPECT_ROBOTS = {config.respect_robots}
        OUTPUT_FORMAT = "{config.output_format}"

        # Identify the bot and give site owners a way to reach you. Edit before running.
        USER_AGENT = "{DEFAULT_USER_AGENT}"

        # --- Rate Limiter ---
        class RateLimiter:
            """Enforces delays between requests with jitter."""

            def __init__(self, min_delay: float, max_delay: float):
                self.min_delay = min_delay
                self.max_delay = max_delay
                self.last_request = 0.0

            def wait(self):
                elapsed = time.time() - self.last_request
                delay = random.uniform(self.min_delay, self.max_delay)
                remaining = delay - elapsed
                if remaining > 0:
                    time.sleep(remaining)
                self.last_request = time.time()


        # --- Robots.txt Checker ---
        class RobotsChecker:
            """Check robots.txt compliance."""

            def __init__(self, base_url: str):
                self.parser = RobotFileParser()
                self.base_url = base_url
                self.loaded = False

            def load(self):
                try:
                    robots_url = urljoin(self.base_url, "/robots.txt")
                    self.parser.set_url(robots_url)
                    self.parser.read()
                    self.loaded = True
                    logger.info("Loaded robots.txt from %s", robots_url)
                except Exception as e:
                    logger.warning("Could not load robots.txt: %s", e)
                    self.loaded = True  # Allow if can't load

            def can_fetch(self, url: str) -> bool:
                if not self.loaded:
                    self.load()
                return self.parser.can_fetch(USER_AGENT, url)


        # --- Session Manager ---
        class ScraperSession:
            """
            Manages HTTP sessions with retry logic.
            NOTE: This is a template. Install requests library: pip install requests
            """

            def __init__(self):
                try:
                    import requests
                    self.session = requests.Session()
                    self.session.headers.update({{
                        "User-Agent": USER_AGENT,
                        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                        "Accept-Language": "en-US,en;q=0.9",
                        "Accept-Encoding": "gzip, deflate, br",
                        "Connection": "keep-alive",
                    }})
                    self.has_requests = True
                except ImportError:
                    logger.error("requests library not installed. Run: pip install requests")
                    self.has_requests = False

            def get(self, url: str, retries: int = MAX_RETRIES) -> Optional[str]:
                if not self.has_requests:
                    return None
                import requests

                for attempt in range(retries):
                    try:
                        response = self.session.get(url, timeout=30)

                        if response.status_code == 200:
                            return response.text
                        elif response.status_code == 429:
                            wait = BACKOFF_FACTOR ** (attempt + 2)
                            logger.warning("Rate limited (429). Waiting %ds...", wait)
                            time.sleep(wait)
                        elif response.status_code >= 500:
                            wait = BACKOFF_FACTOR ** attempt
                            logger.warning("Server error %d. Retry in %ds...", response.status_code, wait)
                            time.sleep(wait)
                        else:
                            logger.error("HTTP %d for %s", response.status_code, url)
                            return None
                    except requests.RequestException as e:
                        logger.error("Request failed: %s", e)
                        if attempt < retries - 1:
                            time.sleep(BACKOFF_FACTOR ** attempt)

                return None


        # --- Data Writer ---
        class DataWriter:
            """Write scraped data to file."""

            def __init__(self, output_path: str, fmt: str):
                self.output_path = output_path
                self.fmt = fmt
                self.count = 0

                if fmt == "csv":
                    self._file = open(output_path, "w", newline="", encoding="utf-8")
                    self._writer = None  # initialized on first write
                elif fmt == "jsonl":
                    self._file = open(output_path, "w", encoding="utf-8")
                elif fmt == "json":
                    self._items = []

            def write(self, item: dict):
                self.count += 1
                if self.fmt == "csv":
                    if self._writer is None:
                        self._writer = csv.DictWriter(self._file, fieldnames=item.keys())
                        self._writer.writeheader()
                    self._writer.writerow(item)
                elif self.fmt == "jsonl":
                    self._file.write(json.dumps(item, ensure_ascii=False) + "\\n")
                elif self.fmt == "json":
                    self._items.append(item)

            def close(self):
                if self.fmt == "json":
                    with open(self.output_path, "w", encoding="utf-8") as f:
                        json.dumps(self._items, f, indent=2, ensure_ascii=False)
                elif hasattr(self, "_file"):
                    self._file.close()
                logger.info("Wrote %d items to %s", self.count, self.output_path)


        # --- Main Scraper ---
        def scrape():
            """Main scraping function - customize the extraction logic below."""
            rate_limiter = RateLimiter(MIN_DELAY, MAX_DELAY)
            robots = RobotsChecker(TARGET_URL) if RESPECT_ROBOTS else None
            session = ScraperSession()

            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            output_file = f"scraped_{{DOMAIN}}_{{timestamp}}.{{OUTPUT_FORMAT}}"
            writer = DataWriter(output_file, OUTPUT_FORMAT)

            logger.info("Starting scrape of %s", TARGET_URL)
            logger.info("Strategy: {config.strategy}, Delay: %.1f-%.1fs", MIN_DELAY, MAX_DELAY)

            url = TARGET_URL
            pages_scraped = 0

            while url and pages_scraped < MAX_PAGES:
                # Check robots.txt
                if robots and not robots.can_fetch(url):
                    logger.warning("Blocked by robots.txt: %s", url)
                    break

                # Rate limit
                rate_limiter.wait()

                # Fetch page
                html = session.get(url)
                if html is None:
                    logger.error("Failed to fetch: %s", url)
                    break

                pages_scraped += 1
                logger.info("Page %d: %s (%d chars)", pages_scraped, url, len(html))

                # =====================================================
                # TODO: CUSTOMIZE EXTRACTION LOGIC BELOW
                # Use BeautifulSoup, lxml, or regex to extract data
                # Example with BeautifulSoup:
                #   from bs4 import BeautifulSoup
                #   soup = BeautifulSoup(html, "html.parser")
                #   for item in soup.select(".item-class"):
                #       writer.write({{"title": item.text, "url": url}})
                # =====================================================

                item = {{
                    "url": url,
                    "page": pages_scraped,
                    "content_length": len(html),
                    "scraped_at": datetime.now().isoformat(),
                }}
                writer.write(item)

                # =====================================================
                # TODO: CUSTOMIZE PAGINATION LOGIC
                # Example: find next page link
                #   next_link = soup.select_one("a.next-page")
                #   url = urljoin(url, next_link["href"]) if next_link else None
                # =====================================================
                url = None  # Stop after first page by default

            writer.close()
            logger.info("Scraping complete. %d pages processed.", pages_scraped)


        if __name__ == "__main__":
            scrape()
    ''')
    return script


def generate_config_summary(config: ScrapingConfig) -> Dict:
    """Generate a summary of the scraping configuration."""
    strategy = RATE_STRATEGIES[config.strategy]
    parsed = urlparse(config.target_url)
    return {
        "target": {
            "url": config.target_url,
            "domain": parsed.netloc,
            "scheme": parsed.scheme,
        },
        "rate_limiting": {
            "strategy": config.strategy,
            "min_delay_seconds": strategy["min_delay"],
            "max_delay_seconds": strategy["max_delay"],
            "max_concurrent": strategy["concurrent"],
            "description": strategy["description"],
        },
        "output": {
            "format": config.output_format,
        },
        "safety": {
            "respect_robots_txt": config.respect_robots,
            "max_pages": config.max_pages,
            "retry_with_backoff": True,
            "identifying_user_agent": DEFAULT_USER_AGENT,
        },
    }


def format_human_output(config: ScrapingConfig, script: str) -> str:
    """Format for human-readable output."""
    strategy = RATE_STRATEGIES[config.strategy]
    lines = []
    lines.append("=" * 60)
    lines.append("SCRAPING TOOLKIT - CODE GENERATOR")
    lines.append("=" * 60)
    lines.append(f"Target: {config.target_url}")
    lines.append(f"Strategy: {config.strategy} ({strategy['description']})")
    lines.append(f"Rate: {strategy['min_delay']}-{strategy['max_delay']}s between requests")
    lines.append(f"Output Format: {config.output_format}")
    lines.append(f"Robots.txt: {'Respected' if config.respect_robots else 'Ignored'}")
    lines.append(f"Max Pages: {config.max_pages}")
    lines.append("")
    lines.append("Features included:")
    lines.append("  [x] Rate limiting with randomized delays")
    lines.append("  [x] Identifying User-Agent (edit the name and contact)")
    lines.append("  [x] Exponential backoff on errors")
    lines.append("  [x] robots.txt compliance")
    lines.append("  [x] Session management with cookies")
    lines.append("  [x] Structured data output")
    lines.append("")
    lines.append("-" * 60)
    lines.append("GENERATED SCRIPT:")
    lines.append("-" * 60)
    lines.append(script)
    return "\n".join(lines)


def main():
    parser = argparse.ArgumentParser(
        description="Scraping Toolkit - Generate web scraping code with rate limiting"
    )
    parser.add_argument("--url", required=True, help="Target URL to scrape")
    parser.add_argument("--strategy", choices=list(RATE_STRATEGIES.keys()), default="polite",
                        help="Rate limiting strategy (default: polite)")
    parser.add_argument("--output", help="Write generated script to file")
    parser.add_argument("--output-format", choices=["csv", "json", "jsonl"], default="json",
                        help="Data output format in generated script (default: json)")
    parser.add_argument("--no-robots", action="store_true",
                        help="Don't include robots.txt checking")
    parser.add_argument("--max-pages", type=int, default=100,
                        help="Maximum pages to scrape (default: 100)")
    parser.add_argument("--format", choices=["human", "json"], default="human",
                        help="Output format for this tool (default: human)")

    args = parser.parse_args()

    config = ScrapingConfig(
        target_url=args.url,
        strategy=args.strategy,
        output_format=args.output_format,
        respect_robots=not args.no_robots,
        max_pages=args.max_pages,
    )

    script = generate_scraper_script(config)

    if args.output:
        Path(args.output).write_text(script)
        print(f"Script written to: {args.output}")

    if args.format == "json":
        summary = generate_config_summary(config)
        summary["generated_script_lines"] = len(script.split("\n"))
        print(json.dumps(summary, indent=2))
    else:
        print(format_human_output(config, script))


if __name__ == "__main__":
    main()
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall