Bhanunamikaze/Agentic-SEO-Skill
seo
VerifiedDeterministic LLM-first SEO audits for websites, blog posts, and GitHub repositories. Use this when the user asks to "perform SEO analysis", "run SEO audit", "analyze SEO", "check technical SEO", "review schema", "Core Web Vitals", "E-E-A-T", "hreflang", "GEO", "AEO", or GitHub repository SEO optimization. For full/page/repo audits, run bundled scripts for evidence and return prioritized, confidence-labeled fixes.
scripts/robots_checker.py
View on GitHub#!/usr/bin/env python3
"""
Parse and analyze robots.txt for SEO and AI crawler management.
Usage:
python robots_checker.py https://example.com
python robots_checker.py https://example.com --json
"""
import argparse
import json
import sys
from urllib.parse import urljoin, urlparse
try:
import requests
except ImportError:
print("Error: requests library required. Install with: pip install requests")
sys.exit(1)
try:
from lib.safe_http import default_headers, safe_get
except ImportError:
from scripts.lib.safe_http import default_headers, safe_get
# AI crawlers to check for explicit management
AI_CRAWLERS = [
"GPTBot",
"ChatGPT-User",
"ClaudeBot",
"PerplexityBot",
"Google-Extended",
"Applebot-Extended",
"Bytespider",
"CCBot",
"anthropic-ai",
"FacebookBot",
"Amazonbot",
]
# Standard crawlers for reference
STANDARD_CRAWLERS = [
"Googlebot",
"Bingbot",
"Yandex",
"Baiduspider",
"DuckDuckBot",
]
def fetch_robots_txt(url: str, timeout: int = 15) -> dict:
"""Fetch and parse robots.txt from a domain."""
parsed = urlparse(url)
if not parsed.scheme:
url = f"https://{url}"
parsed = urlparse(url)
robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
result = {
"url": robots_url,
"status": None,
"raw": None,
"user_agents": {},
"sitemaps": [],
"crawl_delays": {},
"ai_crawler_status": {},
"issues": [],
"error": None,
}
try:
resp = safe_get(robots_url, timeout=timeout, headers=default_headers())
result["status"] = resp.status_code
if resp.status_code == 404:
result["issues"].append("đ´ No robots.txt found â all crawlers allowed by default")
# Still check AI crawlers
for crawler in AI_CRAWLERS:
result["ai_crawler_status"][crawler] = "allowed (no robots.txt)"
return result
if resp.status_code != 200:
result["error"] = f"HTTP {resp.status_code}"
return result
result["raw"] = resp.text
_parse_robots(resp.text, result)
except requests.exceptions.RequestException as e:
result["error"] = str(e)
return result
def _parse_robots(content: str, result: dict):
"""Parse robots.txt content into structured data."""
current_agents = []
for line in content.splitlines():
line = line.strip()
# Skip comments and empty lines
if not line or line.startswith("#"):
continue
# Split on first colon
if ":" not in line:
continue
directive, _, value = line.partition(":")
directive = directive.strip().lower()
value = value.strip()
if directive == "user-agent":
current_agents = [value]
if value not in result["user_agents"]:
result["user_agents"][value] = {"allow": [], "disallow": []}
elif directive == "disallow" and current_agents:
for agent in current_agents:
if agent not in result["user_agents"]:
result["user_agents"][agent] = {"allow": [], "disallow": []}
if value:
result["user_agents"][agent]["disallow"].append(value)
elif directive == "allow" and current_agents:
for agent in current_agents:
if agent not in result["user_agents"]:
result["user_agents"][agent] = {"allow": [], "disallow": []}
result["user_agents"][agent]["allow"].append(value)
elif directive == "sitemap":
result["sitemaps"].append(value)
elif directive == "crawl-delay" and current_agents:
for agent in current_agents:
try:
result["crawl_delays"][agent] = float(value)
except ValueError:
pass
# Analyze AI crawler management
managed_agents = set(result["user_agents"].keys())
for crawler in AI_CRAWLERS:
if crawler in managed_agents:
rules = result["user_agents"][crawler]
if rules["disallow"] and "/" in rules["disallow"]:
result["ai_crawler_status"][crawler] = "fully blocked"
elif rules["disallow"]:
result["ai_crawler_status"][crawler] = f"partially blocked ({len(rules['disallow'])} paths)"
elif rules["allow"]:
result["ai_crawler_status"][crawler] = "explicitly allowed"
else:
result["ai_crawler_status"][crawler] = "declared but no rules"
else:
# Check wildcard rules
if "*" in managed_agents:
wildcard = result["user_agents"]["*"]
if wildcard["disallow"] and "/" in wildcard["disallow"]:
result["ai_crawler_status"][crawler] = "blocked by wildcard (*)"
else:
result["ai_crawler_status"][crawler] = "not managed (inherits * rules)"
else:
result["ai_crawler_status"][crawler] = "not managed (allowed by default)"
# Generate issues
unmanaged = [c for c, s in result["ai_crawler_status"].items()
if "not managed" in s or "allowed by default" in s]
if unmanaged:
result["issues"].append(
f"â ī¸ {len(unmanaged)} AI crawlers not explicitly managed: {', '.join(unmanaged[:5])}"
)
if not result["sitemaps"]:
result["issues"].append("â ī¸ No Sitemap directive found in robots.txt")
def main():
parser = argparse.ArgumentParser(description="Analyze robots.txt for SEO and AI crawlers")
parser.add_argument("url", help="Website URL or domain")
parser.add_argument("--json", "-j", action="store_true", help="Output as JSON")
args = parser.parse_args()
result = fetch_robots_txt(args.url)
if args.json:
# Exclude raw content from JSON for brevity
output = {k: v for k, v in result.items() if k != "raw"}
print(json.dumps(output, indent=2))
return
if result["error"]:
print(f"Error: {result['error']}")
sys.exit(1)
print(f"robots.txt Analysis â {result['url']}")
print("=" * 50)
print(f"Status: {result['status']}")
if result["sitemaps"]:
print(f"\nSitemaps ({len(result['sitemaps'])}):")
for sm in result["sitemaps"]:
print(f" âĸ {sm}")
print(f"\nUser-Agents ({len(result['user_agents'])}):")
for agent, rules in result["user_agents"].items():
allow_count = len(rules["allow"])
disallow_count = len(rules["disallow"])
print(f" {agent}: {disallow_count} disallow, {allow_count} allow")
if result["crawl_delays"]:
print(f"\nCrawl Delays:")
for agent, delay in result["crawl_delays"].items():
print(f" {agent}: {delay}s")
print(f"\nAI Crawler Management:")
for crawler, status in result["ai_crawler_status"].items():
icon = "â
" if "blocked" in status else "â ī¸" if "not managed" in status else "âšī¸"
print(f" {icon} {crawler}: {status}")
if result["issues"]:
print(f"\nIssues ({len(result['issues'])}):")
for issue in result["issues"]:
print(f" {issue}")
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |