alirezarezvani/claude-skills / engineering/universal-scraping-architect/skills/universal-scraping-architect
universal-scraping-architect
VerifiedBuild a scraper or extractor for sites, PDFs or paginated APIs and validate the fields (Firecrawl or local Python). Load whenever the user wants data pulled from pages, files or an API, or a scraper broke.
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 5.0 KB |
| references/firecrawl-technical-guide.md | markdown | 2.7 KB |
| references/local-extraction-patterns.md | markdown | 2.5 KB |
| references/scraping-ethics-security.md | markdown | 2.5 KB |
| requirements.txt | manifest | 300 B |
| scripts/firecrawl_example.py | script | 7.7 KB |
| scripts/local_bs4_example.py | script | 10.0 KB |
| scripts/validate_extraction.py | script | 1.3 KB |
scripts/firecrawl_example.py
View on GitHub"""
Universal Scraping Architect: Firecrawl Example (Path C — Repeatable Deliverable)
===================================================================================
Extracts clean markdown content from a target URL using the Firecrawl SDK.
Demonstrates:
- Safe API key loading from environment
- robots.txt courtesy check (stdlib, no extra dependency)
- Token budget tracking before LLM processing
- Firecrawl quota awareness logging
- Structured error handling
- Clean output saving with validation
This is an editable runner template: tweak the CONFIG block, then run it.
It also supports `--help` and `--sample` so it passes the repo smoke tests
without Firecrawl installed (the SDK is imported lazily inside main()).
Usage:
export FIRECRAWL_API_KEY="fc-YOUR_KEY_HERE" # Linux/macOS
$env:FIRECRAWL_API_KEY = "fc-YOUR_KEY_HERE" # Windows PowerShell
python scripts/firecrawl_example.py
"""
import argparse
import os
import sys
import urllib.robotparser
from datetime import datetime
# =============================================================================
# CONFIG — Edit these for your task
# =============================================================================
TARGET_URL = "https://example.com/research-data"
OUTPUT_FILE = "clean_extraction.md"
LOG_FILE = "firecrawl_run.log"
# Firecrawl / Token settings
FIRECRAWL_API_KEY_ENV = "FIRECRAWL_API_KEY"
USER_AGENT = "UniversalScrapingArchitect/1.0 (contact: your@email.com)"
TOKEN_CONTEXT_LIMIT = 100_000 # Adjust to your model's context window
RESERVED_OUTPUT_TOKENS = 4_000 # Tokens held back for the model's response
RESPECT_ROBOTS = True # Set False only for targets you know are safe to fetch
# =============================================================================
# HELPERS
# =============================================================================
def log(message: str, level: str = "INFO"):
"""Prints a timestamped log line and appends it to the log file."""
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
line = f"[{timestamp}] [{level}] {message}"
print(line)
with open(LOG_FILE, "a", encoding="utf-8") as f:
f.write(line + "\n")
def check_environment() -> str:
"""Loads the Firecrawl API key from the environment. Fails fast if missing."""
api_key = os.getenv(FIRECRAWL_API_KEY_ENV)
if not api_key:
log(
f"Missing environment variable: {FIRECRAWL_API_KEY_ENV}. "
"Set it before running this script.",
level="ERROR"
)
sys.exit(1)
log("API key loaded successfully from environment.")
return api_key
def check_robots(url: str) -> bool:
"""
Courtesy robots.txt check using the stdlib (RFC 9309). Returns True if the
fetch is allowed (or robots.txt is unreachable, which we treat as allowed
but log). The skill mandates this — model the correct behaviour by default.
"""
if not RESPECT_ROBOTS:
log("robots.txt check skipped (RESPECT_ROBOTS=False).", level="WARNING")
return True
try:
parts = url.split("/")
robots_url = f"{parts[0]}//{parts[2]}/robots.txt"
rp = urllib.robotparser.RobotFileParser()
rp.set_url(robots_url)
rp.read()
allowed = rp.can_fetch(USER_AGENT, url)
if not allowed:
log(f"robots.txt DISALLOWS fetching {url}. Aborting.", level="ERROR")
else:
log("robots.txt allows this fetch.")
return allowed
except Exception as e:
log(f"Could not read robots.txt ({e}); proceeding with caution.", level="WARNING")
return True
def estimate_tokens(text: str) -> int:
"""Rough token estimate based on character count (characters / 4)."""
return len(text) // 4
def check_token_budget(text: str) -> bool:
"""
Estimates token usage and warns if over budget.
Returns True if within budget, False if over.
"""
estimated = estimate_tokens(text)
available = TOKEN_CONTEXT_LIMIT - RESERVED_OUTPUT_TOKENS
log(f"Token estimate: {estimated:,} / {available:,} available tokens.")
if estimated > available:
log(
f"OVER TOKEN BUDGET by {estimated - available:,} tokens. "
"Consider chunking the output before passing to an LLM.",
level="WARNING"
)
return False
log("Within token budget.")
return True
def validate_content(content: str) -> bool:
"""Basic validation — checks the response is non-empty."""
if not content or not content.strip():
log("Validation failed: empty content received from Firecrawl.", level="ERROR")
return False
log(f"Validation passed. Content length: {len(content):,} characters.")
return True
def save_output(content: str, path: str):
"""Saves validated content to the output path."""
with open(path, "w", encoding="utf-8") as f:
f.write(content)
log(f"Output saved to: {path}")
# =============================================================================
# MAIN
# =============================================================================
def main():
log("=" * 60)
log("Universal Scraping Architect — Firecrawl Path C")
log("=" * 60)
# Step 1: Environment check
api_key = check_environment()
# Step 2: robots.txt courtesy check (the skill mandates this)
if not check_robots(TARGET_URL):
sys.exit(1)
# Step 3: Initialise Firecrawl client (imported lazily so --help works without the SDK)
from firecrawl import Firecrawl
app = Firecrawl(api_key=api_key)
log(f"Firecrawl client initialised. Target: {TARGET_URL}")
# Step 4: Scrape
# firecrawl-py v2+ API: Firecrawl(...).scrape(url, formats=[...]) returns a document
# object. The v1 FirecrawlApp.scrape_url call is deprecated.
try:
log("Starting scrape...")
doc = app.scrape(TARGET_URL, formats=["markdown"])
markdown_content = getattr(doc, "markdown", "") or ""
except Exception as e:
log(f"Firecrawl scrape failed: {e}", level="ERROR")
log(
"Tip: if this is a quota or auth error, check your FIRECRAWL_API_KEY "
"and run `firecrawl --status` to inspect account state.",
level="WARNING"
)
sys.exit(1)
# Step 5: Validate
if not validate_content(markdown_content):
sys.exit(1)
# Step 6: Token budget check
check_token_budget(markdown_content)
# Step 7: Save clean output
save_output(markdown_content, OUTPUT_FILE)
# Step 8: Final summary
log("=" * 60)
log("EXTRACTION COMPLETE")
log(f" Source URL : {TARGET_URL}")
log(f" Output file : {OUTPUT_FILE}")
log(f" Characters : {len(markdown_content):,}")
log(f" Est. tokens : {estimate_tokens(markdown_content):,}")
log(f" Log file : {LOG_FILE}")
log("=" * 60)
log("Customisable: TARGET_URL, OUTPUT_FILE, TOKEN_CONTEXT_LIMIT, formats.")
def _print_sample():
"""Prints a non-network sample of what this template does (for smoke tests)."""
print("Firecrawl example (template). Edit the CONFIG block, set FIRECRAWL_API_KEY, then run.")
print(f" TARGET_URL = {TARGET_URL}")
print(f" OUTPUT_FILE = {OUTPUT_FILE}")
print(f" TOKEN_CONTEXT_LIMIT = {TOKEN_CONTEXT_LIMIT}")
print(" Pipeline: env-check -> robots.txt -> scrape -> validate -> token-budget -> save")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Firecrawl extraction example (editable runner template).")
parser.add_argument("--sample", action="store_true", help="Print a sample summary and exit (no network).")
args = parser.parse_args()
if args.sample:
_print_sample()
sys.exit(0)
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| receiving-code-reviewobra/superpowersUse when receiving code review feedback, before implementing suggestions, especially if feedback seems unclear or technically questionable - requires technical rigor and verification, not performative agreement or blind implementation | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| requesting-code-reviewobra/superpowersUse when completing tasks, implementing major features, or before merging to verify work meets requirements | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| using-git-worktreesobra/superpowersUse when starting feature work that needs isolation from current workspace or before executing implementation plans - ensures an isolated workspace exists via native tools or git worktree fallback | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| verification-before-completionobra/superpowersUse when about to claim work is complete, fixed, or passing, before committing or creating PRs - requires running verification commands and confirming output before making any success claims; evidence before assertions always | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |