borghei/Claude-Skills / engineering/browser-automation
browser-automation
VerifiedGenerates polite, rate-limited scraper and form-submission scripts (requests, no JS) and audits automation code for blocking and overload risks. Use when writing a permitted scraper, scripting a form post, or reviewing a crawler.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 9.0 KB |
| assets/sample_bare_scraper.py | script | 556 B |
| assets/sample_contact_form.html | other | 666 B |
| references/browser-automation-guide.md | markdown | 2.6 KB |
| scripts/anti_detection_checker.py | script | 9.9 KB |
| scripts/form_automation_builder.py | script | 14.6 KB |
| scripts/scraping_toolkit.py | script | 15.3 KB |
scripts/scraping_toolkit.py
View on GitHub#!/usr/bin/env python3
"""
Scraping Toolkit - Generate web scraping code with rate limiting and best practices.
Analyzes target URLs and generates production-ready scraping scripts with proper
rate limiting, error handling, data extraction, and anti-detection measures.
License: MIT
"""
import argparse
import json
import os
import re
import sys
from dataclasses import dataclass, asdict, field
from pathlib import Path
from typing import List, Dict, Optional
from urllib.parse import urlparse, urljoin
import textwrap
RATE_STRATEGIES = {
"aggressive": {"min_delay": 0.5, "max_delay": 1.5, "concurrent": 5, "description": "Fast, for own APIs or test environments"},
"normal": {"min_delay": 1.0, "max_delay": 3.0, "concurrent": 3, "description": "Balanced speed and politeness"},
"polite": {"min_delay": 3.0, "max_delay": 7.0, "concurrent": 1, "description": "Respectful, minimizes server impact"},
"gentle": {"min_delay": 5.0, "max_delay": 15.0, "concurrent": 1, "description": "Slowest, for small or fragile sites"},
}
# Generated scrapers identify themselves honestly; replace the name, URL and contact before use.
DEFAULT_USER_AGENT = "ExampleCrawler/1.0 (+https://example.com/bot; bot-owner@example.com)"
@dataclass
class ScrapingConfig:
"""Configuration for generated scraper."""
target_url: str
strategy: str
output_format: str # csv, json, jsonl
selectors: Dict[str, str] = field(default_factory=dict)
pagination: bool = False
respect_robots: bool = True
max_pages: int = 100
proxy_support: bool = False
def generate_scraper_script(config: ScrapingConfig) -> str:
"""Generate a complete scraping script."""
strategy = RATE_STRATEGIES[config.strategy]
parsed = urlparse(config.target_url)
domain = parsed.netloc
script = textwrap.dedent(f'''\
#!/usr/bin/env python3
"""
Web Scraper for {domain}
Generated by Scraping Toolkit
Strategy: {config.strategy} ({strategy["description"]})
Rate: {strategy["min_delay"]}-{strategy["max_delay"]}s between requests
"""
import csv
import json
import logging
import os
import random
import sys
import time
from dataclasses import dataclass, asdict
from datetime import datetime
from pathlib import Path
from typing import List, Dict, Optional, Generator
from urllib.parse import urljoin, urlparse
from urllib.robotparser import RobotFileParser
# Configure logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
)
logger = logging.getLogger(__name__)
# --- Configuration ---
TARGET_URL = "{config.target_url}"
DOMAIN = "{domain}"
MIN_DELAY = {strategy["min_delay"]}
MAX_DELAY = {strategy["max_delay"]}
MAX_PAGES = {config.max_pages}
MAX_RETRIES = 3
BACKOFF_FACTOR = 2
RESPECT_ROBOTS = {config.respect_robots}
OUTPUT_FORMAT = "{config.output_format}"
# Identify the bot and give site owners a way to reach you. Edit before running.
USER_AGENT = "{DEFAULT_USER_AGENT}"
# --- Rate Limiter ---
class RateLimiter:
"""Enforces delays between requests with jitter."""
def __init__(self, min_delay: float, max_delay: float):
self.min_delay = min_delay
self.max_delay = max_delay
self.last_request = 0.0
def wait(self):
elapsed = time.time() - self.last_request
delay = random.uniform(self.min_delay, self.max_delay)
remaining = delay - elapsed
if remaining > 0:
time.sleep(remaining)
self.last_request = time.time()
# --- Robots.txt Checker ---
class RobotsChecker:
"""Check robots.txt compliance."""
def __init__(self, base_url: str):
self.parser = RobotFileParser()
self.base_url = base_url
self.loaded = False
def load(self):
try:
robots_url = urljoin(self.base_url, "/robots.txt")
self.parser.set_url(robots_url)
self.parser.read()
self.loaded = True
logger.info("Loaded robots.txt from %s", robots_url)
except Exception as e:
logger.warning("Could not load robots.txt: %s", e)
self.loaded = True # Allow if can't load
def can_fetch(self, url: str) -> bool:
if not self.loaded:
self.load()
return self.parser.can_fetch(USER_AGENT, url)
# --- Session Manager ---
class ScraperSession:
"""
Manages HTTP sessions with retry logic.
NOTE: This is a template. Install requests library: pip install requests
"""
def __init__(self):
try:
import requests
self.session = requests.Session()
self.session.headers.update({{
"User-Agent": USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}})
self.has_requests = True
except ImportError:
logger.error("requests library not installed. Run: pip install requests")
self.has_requests = False
def get(self, url: str, retries: int = MAX_RETRIES) -> Optional[str]:
if not self.has_requests:
return None
import requests
for attempt in range(retries):
try:
response = self.session.get(url, timeout=30)
if response.status_code == 200:
return response.text
elif response.status_code == 429:
wait = BACKOFF_FACTOR ** (attempt + 2)
logger.warning("Rate limited (429). Waiting %ds...", wait)
time.sleep(wait)
elif response.status_code >= 500:
wait = BACKOFF_FACTOR ** attempt
logger.warning("Server error %d. Retry in %ds...", response.status_code, wait)
time.sleep(wait)
else:
logger.error("HTTP %d for %s", response.status_code, url)
return None
except requests.RequestException as e:
logger.error("Request failed: %s", e)
if attempt < retries - 1:
time.sleep(BACKOFF_FACTOR ** attempt)
return None
# --- Data Writer ---
class DataWriter:
"""Write scraped data to file."""
def __init__(self, output_path: str, fmt: str):
self.output_path = output_path
self.fmt = fmt
self.count = 0
if fmt == "csv":
self._file = open(output_path, "w", newline="", encoding="utf-8")
self._writer = None # initialized on first write
elif fmt == "jsonl":
self._file = open(output_path, "w", encoding="utf-8")
elif fmt == "json":
self._items = []
def write(self, item: dict):
self.count += 1
if self.fmt == "csv":
if self._writer is None:
self._writer = csv.DictWriter(self._file, fieldnames=item.keys())
self._writer.writeheader()
self._writer.writerow(item)
elif self.fmt == "jsonl":
self._file.write(json.dumps(item, ensure_ascii=False) + "\\n")
elif self.fmt == "json":
self._items.append(item)
def close(self):
if self.fmt == "json":
with open(self.output_path, "w", encoding="utf-8") as f:
json.dumps(self._items, f, indent=2, ensure_ascii=False)
elif hasattr(self, "_file"):
self._file.close()
logger.info("Wrote %d items to %s", self.count, self.output_path)
# --- Main Scraper ---
def scrape():
"""Main scraping function - customize the extraction logic below."""
rate_limiter = RateLimiter(MIN_DELAY, MAX_DELAY)
robots = RobotsChecker(TARGET_URL) if RESPECT_ROBOTS else None
session = ScraperSession()
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
output_file = f"scraped_{{DOMAIN}}_{{timestamp}}.{{OUTPUT_FORMAT}}"
writer = DataWriter(output_file, OUTPUT_FORMAT)
logger.info("Starting scrape of %s", TARGET_URL)
logger.info("Strategy: {config.strategy}, Delay: %.1f-%.1fs", MIN_DELAY, MAX_DELAY)
url = TARGET_URL
pages_scraped = 0
while url and pages_scraped < MAX_PAGES:
# Check robots.txt
if robots and not robots.can_fetch(url):
logger.warning("Blocked by robots.txt: %s", url)
break
# Rate limit
rate_limiter.wait()
# Fetch page
html = session.get(url)
if html is None:
logger.error("Failed to fetch: %s", url)
break
pages_scraped += 1
logger.info("Page %d: %s (%d chars)", pages_scraped, url, len(html))
# =====================================================
# TODO: CUSTOMIZE EXTRACTION LOGIC BELOW
# Use BeautifulSoup, lxml, or regex to extract data
# Example with BeautifulSoup:
# from bs4 import BeautifulSoup
# soup = BeautifulSoup(html, "html.parser")
# for item in soup.select(".item-class"):
# writer.write({{"title": item.text, "url": url}})
# =====================================================
item = {{
"url": url,
"page": pages_scraped,
"content_length": len(html),
"scraped_at": datetime.now().isoformat(),
}}
writer.write(item)
# =====================================================
# TODO: CUSTOMIZE PAGINATION LOGIC
# Example: find next page link
# next_link = soup.select_one("a.next-page")
# url = urljoin(url, next_link["href"]) if next_link else None
# =====================================================
url = None # Stop after first page by default
writer.close()
logger.info("Scraping complete. %d pages processed.", pages_scraped)
if __name__ == "__main__":
scrape()
''')
return script
def generate_config_summary(config: ScrapingConfig) -> Dict:
"""Generate a summary of the scraping configuration."""
strategy = RATE_STRATEGIES[config.strategy]
parsed = urlparse(config.target_url)
return {
"target": {
"url": config.target_url,
"domain": parsed.netloc,
"scheme": parsed.scheme,
},
"rate_limiting": {
"strategy": config.strategy,
"min_delay_seconds": strategy["min_delay"],
"max_delay_seconds": strategy["max_delay"],
"max_concurrent": strategy["concurrent"],
"description": strategy["description"],
},
"output": {
"format": config.output_format,
},
"safety": {
"respect_robots_txt": config.respect_robots,
"max_pages": config.max_pages,
"retry_with_backoff": True,
"identifying_user_agent": DEFAULT_USER_AGENT,
},
}
def format_human_output(config: ScrapingConfig, script: str) -> str:
"""Format for human-readable output."""
strategy = RATE_STRATEGIES[config.strategy]
lines = []
lines.append("=" * 60)
lines.append("SCRAPING TOOLKIT - CODE GENERATOR")
lines.append("=" * 60)
lines.append(f"Target: {config.target_url}")
lines.append(f"Strategy: {config.strategy} ({strategy['description']})")
lines.append(f"Rate: {strategy['min_delay']}-{strategy['max_delay']}s between requests")
lines.append(f"Output Format: {config.output_format}")
lines.append(f"Robots.txt: {'Respected' if config.respect_robots else 'Ignored'}")
lines.append(f"Max Pages: {config.max_pages}")
lines.append("")
lines.append("Features included:")
lines.append(" [x] Rate limiting with randomized delays")
lines.append(" [x] Identifying User-Agent (edit the name and contact)")
lines.append(" [x] Exponential backoff on errors")
lines.append(" [x] robots.txt compliance")
lines.append(" [x] Session management with cookies")
lines.append(" [x] Structured data output")
lines.append("")
lines.append("-" * 60)
lines.append("GENERATED SCRIPT:")
lines.append("-" * 60)
lines.append(script)
return "\n".join(lines)
def main():
parser = argparse.ArgumentParser(
description="Scraping Toolkit - Generate web scraping code with rate limiting"
)
parser.add_argument("--url", required=True, help="Target URL to scrape")
parser.add_argument("--strategy", choices=list(RATE_STRATEGIES.keys()), default="polite",
help="Rate limiting strategy (default: polite)")
parser.add_argument("--output", help="Write generated script to file")
parser.add_argument("--output-format", choices=["csv", "json", "jsonl"], default="json",
help="Data output format in generated script (default: json)")
parser.add_argument("--no-robots", action="store_true",
help="Don't include robots.txt checking")
parser.add_argument("--max-pages", type=int, default=100,
help="Maximum pages to scrape (default: 100)")
parser.add_argument("--format", choices=["human", "json"], default="human",
help="Output format for this tool (default: human)")
args = parser.parse_args()
config = ScrapingConfig(
target_url=args.url,
strategy=args.strategy,
output_format=args.output_format,
respect_robots=not args.no_robots,
max_pages=args.max_pages,
)
script = generate_scraper_script(config)
if args.output:
Path(args.output).write_text(script)
print(f"Script written to: {args.output}")
if args.format == "json":
summary = generate_config_summary(config)
summary["generated_script_lines"] = len(script.split("\n"))
print(json.dumps(summary, indent=2))
else:
print(format_human_output(config, script))
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |