Skip to content

balsm-health/website / .claude/skills/seo

Comprehensive SEO analysis for any website or business type. Full site audits, single-page analysis, technical SEO (crawlability, indexability, Core Web Vitals with INP), schema markup, content quality (E-E-A-T), image optimization, sitemap analysis, and GEO for AI Overviews/ChatGPT/Perplexity. Industry detection for SaaS, e-commerce, local, publishers, agencies. Triggers on: SEO, audit, schema, Core Web Vitals, sitemap, E-E-A-T, AI Overviews, GEO, technical SEO, content quality, page speed.

FileKindSize
LICENSE.txtmarkdown143 B
SKILL.mdskill md16.9 KB
bin/claude-seoexecutablescript1.5 KB
extensions/banana/references/cost-tracking.mdmarkdown1.2 KB
extensions/banana/references/gemini-models.mdmarkdown5.8 KB
extensions/banana/references/mcp-tools.mdmarkdown3.7 KB
extensions/banana/references/post-processing.mdmarkdown5.1 KB
extensions/banana/references/presets.mdmarkdown2.2 KB
extensions/banana/references/prompt-engineering.mdmarkdown20.9 KB
extensions/banana/references/seo-image-presets.mdmarkdown4.0 KB
extensions/banana/scripts/batch.pyexecutablescript3.3 KB
extensions/banana/scripts/cost_tracker.pyexecutablescript7.2 KB
extensions/banana/scripts/edit.pyexecutablescript5.6 KB
extensions/banana/scripts/generate.pyexecutablescript6.4 KB
extensions/banana/scripts/presets.pyexecutablescript4.9 KB
extensions/banana/scripts/setup_mcp.pyexecutablescript4.5 KB
extensions/banana/scripts/validate_setup.pyexecutablescript3.8 KB
hooks/hooks.jsonconfig416 B
hooks/run-python-hook.jsscript1.7 KB
hooks/validate-schema.pyexecutablescript5.9 KB
pdf/google-seo-reference.mdmarkdown8.0 KB
references/backlink-quality.mdmarkdown3.4 KB
references/cwv-thresholds.mdmarkdown6.9 KB
references/eeat-framework.mdmarkdown8.1 KB
references/free-backlink-sources.mdmarkdown5.6 KB
references/local-schema-types.mdmarkdown9.2 KB
references/local-seo-signals.mdmarkdown9.6 KB
references/maps-api-endpoints.mdmarkdown5.9 KB
references/maps-free-apis.mdmarkdown6.0 KB
references/maps-gbp-checklist.mdmarkdown5.9 KB
references/maps-geo-grid.mdmarkdown4.3 KB
references/quality-gates.mdmarkdown5.1 KB
references/schema-types.mdmarkdown7.8 KB
references/thinking-framework.mdmarkdown9.3 KB
requirements.txtmanifest1.5 KB
runtime-plugin.jsonconfig1.5 KB
schema/templates.jsonconfig9.4 KB
scripts/agent_ux_check.pyscript8.2 KB
scripts/analyze_visual.pyexecutablescript7.2 KB
scripts/backlinks_auth.pyscript16.1 KB
scripts/bing_webmaster.pyscript21.8 KB
scripts/capture_screenshot.pyexecutablescript6.4 KB
scripts/commoncrawl_graph.pyscript15.6 KB
scripts/consistency_check.pyscript12.5 KB
scripts/content_humanize.pyscript7.2 KB
scripts/content_quality.pyscript9.7 KB
scripts/content_verify.pyscript7.1 KB
scripts/crux_history.pyscript10.7 KB
scripts/dataforseo_costs.pyscript15.8 KB
scripts/dataforseo_merchant.pyexecutablescript16.7 KB
scripts/dataforseo_normalize.pyexecutablescript14.7 KB
scripts/domain_history.pyscript9.6 KB
scripts/drift_baseline.pyscript13.5 KB
scripts/drift_compare.pyscript22.3 KB
scripts/drift_history.pyscript3.4 KB
scripts/drift_report.pyscript11.0 KB
scripts/fetch_page.pyexecutablescript9.4 KB
scripts/ga4_report.pyscript16.8 KB
scripts/gbp_deprecation_lint.pyexecutablescript4.7 KB
scripts/google_auth.pyscript31.0 KB
scripts/google_report.pyscript97.6 KB
scripts/gsc_inspect.pyscript11.0 KB
scripts/gsc_query.pyscript18.6 KB
scripts/indexing_notify.pyscript10.3 KB
scripts/indexnow_submit.pyscript8.8 KB
scripts/iptc_ai_label.pyscript7.6 KB
scripts/keyword_planner.pyscript10.8 KB
scripts/lcp_subparts.pyscript7.2 KB
scripts/moz_api.pyscript14.7 KB
scripts/nlp_analyze.pyscript12.1 KB
scripts/pagespeed_check.pyscript22.9 KB
scripts/parasite_risk.pyexecutablescript8.6 KB
scripts/parse_html.pyexecutablescript8.8 KB
scripts/portability_check.pyscript7.8 KB
scripts/preload_check.pyscript8.1 KB
scripts/release_sign.pyscript5.6 KB
scripts/render_page.pyscript20.9 KB
scripts/runtime.pyexecutablescript15.1 KB
scripts/schema_ecommerce_validate.pyscript11.8 KB
scripts/schema_generate.pyscript12.0 KB
scripts/seo_updates.pyscript4.8 KB
scripts/sitemap_discovery.pyscript10.0 KB
scripts/sync_flow.pyscript11.9 KB
scripts/ucp_check.pyscript8.4 KB
scripts/unlighthouse_run.pyscript5.6 KB
scripts/url_safety.pyscript22.0 KB
scripts/validate_backlink_report.pyscript11.7 KB
scripts/verify_backlinks.pyscript12.9 KB
scripts/verify_release.pyscript4.8 KB
scripts/youtube_search.pyscript12.3 KB

scripts/sitemap_discovery.py

View on GitHub
#!/usr/bin/env python3
"""Discover sitemaps through robots.txt and common locations safely.

Every network request uses the repository's DNS-pinned URL safety layer. The
helper accepts cross-host Sitemap directives because the sitemap protocol
allows them, but validates each target independently before connecting.
"""

from __future__ import annotations

import argparse
import json
import os
import re
import sys
from urllib.parse import urlparse, urlunparse

import requests
from lxml import etree

_SCRIPTS = os.path.dirname(os.path.abspath(__file__))
if _SCRIPTS not in sys.path:
    sys.path.insert(0, _SCRIPTS)

from url_safety import URLSafetyError, safe_requests_session, validate_url  # noqa: E402

COMMON_PATHS = (
    "/sitemap.xml",
    "/sitemap_index.xml",
    "/sitemap-index.xml",
    "/wp-sitemap.xml",
)
MAX_DECLARED = 16
MAX_ROBOTS_BYTES = 1024 * 1024
MAX_SITEMAP_BYTES = 50 * 1024 * 1024
USER_AGENT = "ClaudeSEO/2.2 SitemapDiscovery"
_SITEMAP_LINE = re.compile(r"^\s*sitemap\s*:\s*(\S+)\s*$", re.IGNORECASE)


def _display_url(url: str) -> tuple[str, bool]:
    """Return a safe-to-display URL with userinfo, query, and fragment removed."""
    parsed = urlparse(url)
    host = parsed.hostname or ""
    try:
        port = parsed.port
    except ValueError:
        port = None
        redacted = True
    else:
        redacted = False
    if port:
        host = f"{host}:{port}"
    clean = urlunparse((parsed.scheme, host, parsed.path or "/", "", "", ""))
    redacted = redacted or bool(parsed.username or parsed.password or parsed.query or parsed.fragment)
    return clean, redacted


def _origin(url: str) -> str:
    if "://" not in url:
        url = f"https://{url}"
    if not validate_url(url):
        raise ValueError("Target must be a public HTTP or HTTPS URL")
    parsed = urlparse(url)
    netloc = parsed.hostname or ""
    try:
        port = parsed.port
    except ValueError as exc:
        raise ValueError("Target URL contains an invalid port") from exc
    if port:
        netloc = f"{netloc}:{port}"
    return urlunparse((parsed.scheme, netloc, "", "", "", ""))


def _bounded_fetch(url: str, max_bytes: int) -> dict:
    """Fetch at most max_bytes after decompression without exposing response text."""
    result = {
        "status_code": None,
        "content": b"",
        "content_type": "",
        "final_url": url,
        "too_large": False,
        "error": None,
    }
    try:
        with safe_requests_session(url) as session:
            response = session.get(
                url,
                headers={"User-Agent": USER_AGENT, "Accept": "application/xml,text/xml,text/plain,*/*"},
                timeout=30,
                allow_redirects=True,
                stream=True,
            )
            result["status_code"] = response.status_code
            result["content_type"] = response.headers.get("Content-Type", "")
            result["final_url"] = response.url
            chunks = []
            size = 0
            for chunk in response.iter_content(chunk_size=65536):
                if not chunk:
                    continue
                size += len(chunk)
                if size > max_bytes:
                    result["too_large"] = True
                    break
                chunks.append(chunk)
            result["content"] = b"".join(chunks)
            response.close()
    except URLSafetyError:
        result["error"] = "URL safety validation failed"
    except requests.exceptions.Timeout:
        result["error"] = "Request timed out"
    except requests.exceptions.RequestException:
        result["error"] = "Request failed"
    return result


def _robots_sitemaps(content: bytes) -> list[str]:
    text = content.decode("utf-8", errors="replace")
    found = []
    for line in text.splitlines():
        match = _SITEMAP_LINE.match(line)
        if match and match.group(1) not in found:
            found.append(match.group(1))
    return found


def _valid_sitemap_url_syntax(value: str) -> bool:
    """Check a text-sitemap entry without resolving or connecting to its host."""
    if not value or any(char.isspace() for char in value) or "\\" in value:
        return False
    try:
        parsed = urlparse(value)
        _ = parsed.port
    except ValueError:
        return False
    return (
        parsed.scheme.lower() in {"http", "https"}
        and bool(parsed.hostname)
        and parsed.username is None
        and parsed.password is None
    )


def _sitemap_kind(content: bytes, content_type: str, url: str) -> tuple[str | None, str | None]:
    if not content.strip():
        return None, "empty response"
    if b"<!DOCTYPE" in content.upper():
        return None, "DOCTYPE is not allowed in sitemap XML"

    stripped = content.lstrip()
    if stripped.startswith(b"<"):
        try:
            parser = etree.XMLParser(
                resolve_entities=False,
                no_network=True,
                load_dtd=False,
                recover=False,
                huge_tree=False,
            )
            root = etree.fromstring(content, parser=parser)
        except (etree.XMLSyntaxError, ValueError):
            return None, "invalid XML"
        local = etree.QName(root).localname.lower()
        if local in {"urlset", "sitemapindex"}:
            return local, None
        if local in {"rss", "feed"}:
            return local, None
        return None, "unsupported XML root"

    if "text/plain" in content_type.lower() or urlparse(url).path.lower().endswith(".txt"):
        lines = [line.strip() for line in content.decode("utf-8", errors="replace").splitlines() if line.strip()]
        if len(lines) > 50000:
            return None, "text sitemap exceeds the 50,000 URL protocol limit"
        # Entries are reported only as a sitemap classification, not fetched.
        # Avoid up to 50,000 DNS lookups here; connection-time validation still
        # applies to every sitemap candidate that discovery fetches.
        if lines and all(_valid_sitemap_url_syntax(line) for line in lines):
            return "text", None
        return None, "invalid text sitemap"
    return None, "response is not a supported sitemap format"


def discover_sitemaps(target_url: str) -> dict:
    result = {
        "target": None,
        "robots_url": None,
        "declared": [],
        "found": [],
        "checked": [],
        "warnings": [],
        "error": None,
    }
    try:
        origin = _origin(target_url)
    except ValueError as exc:
        result["error"] = str(exc)
        return result

    result["target"] = origin
    robots_url = f"{origin}/robots.txt"
    result["robots_url"] = robots_url
    robots = _bounded_fetch(robots_url, MAX_ROBOTS_BYTES)
    declared_raw = []
    if robots["error"]:
        result["warnings"].append("robots.txt could not be fetched safely")
    elif robots["too_large"]:
        result["warnings"].append("robots.txt exceeded the 1 MiB discovery limit")
    elif robots["status_code"] == 200:
        declared_raw = _robots_sitemaps(robots["content"])
    elif robots["status_code"] is not None:
        result["warnings"].append(f"robots.txt returned HTTP {robots['status_code']}")

    if len(declared_raw) > MAX_DECLARED:
        result["warnings"].append(
            f"robots.txt declared more than {MAX_DECLARED} sitemaps; extra entries were not fetched"
        )
        declared_raw = declared_raw[:MAX_DECLARED]

    for item in declared_raw:
        display, redacted = _display_url(item)
        result["declared"].append({"url": display, "query_redacted": redacted})

    candidates = list(declared_raw)
    candidates.extend(f"{origin}{path}" for path in COMMON_PATHS)
    deduped = []
    seen = set()
    for candidate in candidates:
        if candidate not in seen:
            seen.add(candidate)
            deduped.append(candidate)

    target_host = urlparse(origin).hostname
    for candidate in deduped:
        display, query_redacted = _display_url(candidate)
        entry = {
            "url": display,
            "query_redacted": query_redacted,
            "source": "robots.txt" if candidate in declared_raw else "common_path",
            "status_code": None,
            "kind": None,
            "valid": False,
            "error": None,
        }
        if urlparse(candidate).hostname != target_host and candidate in declared_raw:
            entry["cross_host"] = True

        fetched = _bounded_fetch(candidate, MAX_SITEMAP_BYTES)
        entry["status_code"] = fetched["status_code"]
        if fetched["error"]:
            entry["error"] = fetched["error"]
        elif fetched["too_large"]:
            entry["error"] = "Sitemap exceeds the 50 MiB uncompressed protocol limit"
        elif fetched["status_code"] is None or not 200 <= fetched["status_code"] < 300:
            entry["error"] = f"HTTP {fetched['status_code']}" if fetched["status_code"] is not None else "No response"
        else:
            kind, error = _sitemap_kind(
                fetched["content"], fetched["content_type"], fetched["final_url"]
            )
            entry["kind"] = kind
            entry["error"] = error
            entry["valid"] = kind is not None
            if entry["valid"]:
                final_display, final_redacted = _display_url(fetched["final_url"])
                entry["url"] = final_display
                entry["query_redacted"] = query_redacted or final_redacted
                result["found"].append(dict(entry))
        result["checked"].append(entry)

    return result


def main() -> int:
    parser = argparse.ArgumentParser(description="Discover sitemaps safely")
    parser.add_argument("url", help="Public site URL")
    parser.add_argument("--json", action="store_true", help="Output structured JSON")
    args = parser.parse_args()
    result = discover_sitemaps(args.url)
    if args.json:
        print(json.dumps(result, indent=2))
    elif result["error"]:
        print(f"Error: {result['error']}", file=sys.stderr)
    elif result["found"]:
        for item in result["found"]:
            print(f"{item['url']} ({item['kind']})")
    else:
        print("No valid sitemap found")
    return 1 if result["error"] else 0


if __name__ == "__main__":
    raise SystemExit(main())
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall