Skip to content

Bhanunamikaze/Agentic-SEO-Skill

Deterministic LLM-first SEO audits for websites, blog posts, and GitHub repositories. Use this when the user asks to "perform SEO analysis", "run SEO audit", "analyze SEO", "check technical SEO", "review schema", "Core Web Vitals", "E-E-A-T", "hreflang", "GEO", "AEO", or GitHub repository SEO optimization. For full/page/repo audits, run bundled scripts for evidence and return prioritized, confidence-labeled fixes.

FileKindSize
SKILL.mdskill md19.4 KB
scripts/__init__.pyscript34 B
scripts/a11y_seo_checker.pyscript3.4 KB
scripts/ai_crawler_policy_matrix.pyscript2.2 KB
scripts/analyze_visual.pyexecutablescript6.5 KB
scripts/anchor_text_audit.pyscript7.7 KB
scripts/answer_block_scanner.pyscript3.9 KB
scripts/article_seo.pyscript27.1 KB
scripts/audit_runner.pyscript3.1 KB
scripts/broken_links.pyscript8.3 KB
scripts/cache_compression_checker.pyscript6.7 KB
scripts/canonical_checker.pyscript3.4 KB
scripts/capture_screenshot.pyexecutablescript4.4 KB
scripts/citation_readiness.pyscript5.1 KB
scripts/collection_page_checker.pyscript4.2 KB
scripts/competitor_gap.pyscript10.2 KB
scripts/content_decay_detector.pyscript6.2 KB
scripts/content_intent_matcher.pyscript5.4 KB
scripts/crawl_audit.pyscript7.2 KB
scripts/critical_request_chain.pyscript4.8 KB
scripts/duplicate_content.pyscript14.0 KB
scripts/eeat_signal_checker.pyscript5.5 KB
scripts/entity_checker.pyscript19.4 KB
scripts/env_loader.pyscript3.6 KB
scripts/external_link_quality.pyscript6.6 KB
scripts/faceted_nav_audit.pyscript3.3 KB
scripts/fetch_page.pyexecutablescript3.6 KB
scripts/finding_verifier.pyscript4.9 KB
scripts/font_audit.pyscript5.9 KB
scripts/freshness_checker.pyscript6.2 KB
scripts/generate_report.pyscript80.7 KB
scripts/github_api.pyscript12.8 KB
scripts/github_community_health.pyscript8.9 KB
scripts/github_competitor_research.pyscript17.2 KB
scripts/github_readme_lint.pyscript14.9 KB
scripts/github_repo_audit.pyscript21.0 KB
scripts/github_search_benchmark.pyscript7.7 KB
scripts/github_seo_report.pyscript32.0 KB
scripts/github_traffic_archiver.pyscript6.3 KB
scripts/github_weekly_scorecard.pyscript4.4 KB
scripts/gsc_checker.pyscript11.4 KB
scripts/hreflang_checker.pyscript24.2 KB
scripts/hreflang_sitemap_validator.pyscript6.4 KB
scripts/image_inventory.pyscript2.8 KB
scripts/image_weight_audit.pyscript4.8 KB
scripts/indexability_matrix.pyscript3.5 KB
scripts/indexnow_checker.pyscript9.4 KB
scripts/internal_links.pyscript10.5 KB
scripts/javascript_render_audit.pyscript2.9 KB
scripts/lcp_subparts.pyscript7.3 KB
scripts/lib/__init__.pyscript46 B
scripts/lib/safe_http.pyscript6.1 KB
scripts/lighthouse_runner.pyscript6.0 KB
scripts/link_profile.pyscript12.5 KB
scripts/llms_txt_checker.pyscript7.5 KB
scripts/llms_txt_generator.pyscript2.9 KB
scripts/local_seo_checker.pyscript3.7 KB
scripts/log_file_analyzer.pyscript6.0 KB
scripts/mobile_render_checker.pyscript6.9 KB
scripts/orphan_pages_from_sitemap.pyscript6.2 KB
scripts/pagespeed.pyscript12.9 KB
scripts/parse_html.pyexecutablescript12.2 KB
scripts/pre_commit_seo_check.shexecutablescript2.9 KB
scripts/product_schema_checker.pyscript4.9 KB
scripts/readability.pyscript14.5 KB
scripts/redirect_backlink_reclaim.pyscript5.7 KB
scripts/redirect_checker.pyscript6.2 KB
scripts/reference_freshness.pyexecutablescript2.9 KB
scripts/repo_docs_site_checker.pyscript5.7 KB
scripts/repo_file_inventory.pyscript3.6 KB
scripts/repo_release_seo.pyscript6.6 KB
scripts/repo_social_preview_checker.pyscript4.1 KB
scripts/repo_topic_suggester.pyscript6.3 KB
scripts/review_schema_checker.pyscript4.0 KB
scripts/rich_results_guard.pyscript3.4 KB
scripts/robots_checker.pyscript7.2 KB
scripts/robots_path_tester.pyscript1.8 KB
scripts/schema_diff.pyscript3.1 KB
scripts/schema_required_props.pyscript8.1 KB
scripts/schema_template_generator.pyscript5.4 KB
scripts/security_headers.pyscript6.8 KB
scripts/seo_common.pyscript14.1 KB
scripts/sitemap_checker.pyscript6.5 KB
scripts/sitemap_generator.pyscript3.4 KB
scripts/social_meta.pyscript8.7 KB
scripts/third_party_script_audit.pyscript4.7 KB
scripts/topical_cluster_mapper.pyscript6.4 KB
scripts/url_quality.pyscript2.5 KB
scripts/validate_schema.pyexecutablescript4.3 KB
scripts/validate_skill_inventory.pyexecutablescript4.8 KB
scripts/video_schema_checker.pyscript3.6 KB
scripts/visual_regression_snapshot.pyscript5.3 KB
scripts/x_robots_header_checker.pyscript2.0 KB

scripts/robots_checker.py

View on GitHub
#!/usr/bin/env python3
"""
Parse and analyze robots.txt for SEO and AI crawler management.

Usage:
    python robots_checker.py https://example.com
    python robots_checker.py https://example.com --json
"""

import argparse
import json
import sys
from urllib.parse import urljoin, urlparse

try:
    import requests
except ImportError:
    print("Error: requests library required. Install with: pip install requests")
    sys.exit(1)

try:
    from lib.safe_http import default_headers, safe_get
except ImportError:
    from scripts.lib.safe_http import default_headers, safe_get


# AI crawlers to check for explicit management
AI_CRAWLERS = [
    "GPTBot",
    "ChatGPT-User",
    "ClaudeBot",
    "PerplexityBot",
    "Google-Extended",
    "Applebot-Extended",
    "Bytespider",
    "CCBot",
    "anthropic-ai",
    "FacebookBot",
    "Amazonbot",
]

# Standard crawlers for reference
STANDARD_CRAWLERS = [
    "Googlebot",
    "Bingbot",
    "Yandex",
    "Baiduspider",
    "DuckDuckBot",
]


def fetch_robots_txt(url: str, timeout: int = 15) -> dict:
    """Fetch and parse robots.txt from a domain."""
    parsed = urlparse(url)
    if not parsed.scheme:
        url = f"https://{url}"
        parsed = urlparse(url)

    robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"

    result = {
        "url": robots_url,
        "status": None,
        "raw": None,
        "user_agents": {},
        "sitemaps": [],
        "crawl_delays": {},
        "ai_crawler_status": {},
        "issues": [],
        "error": None,
    }

    try:
        resp = safe_get(robots_url, timeout=timeout, headers=default_headers())
        result["status"] = resp.status_code

        if resp.status_code == 404:
            result["issues"].append("🔴 No robots.txt found — all crawlers allowed by default")
            # Still check AI crawlers
            for crawler in AI_CRAWLERS:
                result["ai_crawler_status"][crawler] = "allowed (no robots.txt)"
            return result

        if resp.status_code != 200:
            result["error"] = f"HTTP {resp.status_code}"
            return result

        result["raw"] = resp.text
        _parse_robots(resp.text, result)

    except requests.exceptions.RequestException as e:
        result["error"] = str(e)

    return result


def _parse_robots(content: str, result: dict):
    """Parse robots.txt content into structured data."""
    current_agents = []

    for line in content.splitlines():
        line = line.strip()

        # Skip comments and empty lines
        if not line or line.startswith("#"):
            continue

        # Split on first colon
        if ":" not in line:
            continue

        directive, _, value = line.partition(":")
        directive = directive.strip().lower()
        value = value.strip()

        if directive == "user-agent":
            current_agents = [value]
            if value not in result["user_agents"]:
                result["user_agents"][value] = {"allow": [], "disallow": []}

        elif directive == "disallow" and current_agents:
            for agent in current_agents:
                if agent not in result["user_agents"]:
                    result["user_agents"][agent] = {"allow": [], "disallow": []}
                if value:
                    result["user_agents"][agent]["disallow"].append(value)

        elif directive == "allow" and current_agents:
            for agent in current_agents:
                if agent not in result["user_agents"]:
                    result["user_agents"][agent] = {"allow": [], "disallow": []}
                result["user_agents"][agent]["allow"].append(value)

        elif directive == "sitemap":
            result["sitemaps"].append(value)

        elif directive == "crawl-delay" and current_agents:
            for agent in current_agents:
                try:
                    result["crawl_delays"][agent] = float(value)
                except ValueError:
                    pass

    # Analyze AI crawler management
    managed_agents = set(result["user_agents"].keys())

    for crawler in AI_CRAWLERS:
        if crawler in managed_agents:
            rules = result["user_agents"][crawler]
            if rules["disallow"] and "/" in rules["disallow"]:
                result["ai_crawler_status"][crawler] = "fully blocked"
            elif rules["disallow"]:
                result["ai_crawler_status"][crawler] = f"partially blocked ({len(rules['disallow'])} paths)"
            elif rules["allow"]:
                result["ai_crawler_status"][crawler] = "explicitly allowed"
            else:
                result["ai_crawler_status"][crawler] = "declared but no rules"
        else:
            # Check wildcard rules
            if "*" in managed_agents:
                wildcard = result["user_agents"]["*"]
                if wildcard["disallow"] and "/" in wildcard["disallow"]:
                    result["ai_crawler_status"][crawler] = "blocked by wildcard (*)"
                else:
                    result["ai_crawler_status"][crawler] = "not managed (inherits * rules)"
            else:
                result["ai_crawler_status"][crawler] = "not managed (allowed by default)"

    # Generate issues
    unmanaged = [c for c, s in result["ai_crawler_status"].items()
                 if "not managed" in s or "allowed by default" in s]
    if unmanaged:
        result["issues"].append(
            f"âš ī¸ {len(unmanaged)} AI crawlers not explicitly managed: {', '.join(unmanaged[:5])}"
        )

    if not result["sitemaps"]:
        result["issues"].append("âš ī¸ No Sitemap directive found in robots.txt")


def main():
    parser = argparse.ArgumentParser(description="Analyze robots.txt for SEO and AI crawlers")
    parser.add_argument("url", help="Website URL or domain")
    parser.add_argument("--json", "-j", action="store_true", help="Output as JSON")

    args = parser.parse_args()
    result = fetch_robots_txt(args.url)

    if args.json:
        # Exclude raw content from JSON for brevity
        output = {k: v for k, v in result.items() if k != "raw"}
        print(json.dumps(output, indent=2))
        return

    if result["error"]:
        print(f"Error: {result['error']}")
        sys.exit(1)

    print(f"robots.txt Analysis — {result['url']}")
    print("=" * 50)
    print(f"Status: {result['status']}")

    if result["sitemaps"]:
        print(f"\nSitemaps ({len(result['sitemaps'])}):")
        for sm in result["sitemaps"]:
            print(f"  â€ĸ {sm}")

    print(f"\nUser-Agents ({len(result['user_agents'])}):")
    for agent, rules in result["user_agents"].items():
        allow_count = len(rules["allow"])
        disallow_count = len(rules["disallow"])
        print(f"  {agent}: {disallow_count} disallow, {allow_count} allow")

    if result["crawl_delays"]:
        print(f"\nCrawl Delays:")
        for agent, delay in result["crawl_delays"].items():
            print(f"  {agent}: {delay}s")

    print(f"\nAI Crawler Management:")
    for crawler, status in result["ai_crawler_status"].items():
        icon = "✅" if "blocked" in status else "âš ī¸" if "not managed" in status else "â„šī¸"
        print(f"  {icon} {crawler}: {status}")

    if result["issues"]:
        print(f"\nIssues ({len(result['issues'])}):")
        for issue in result["issues"]:
            print(f"  {issue}")


if __name__ == "__main__":
    main()
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall