Bhanunamikaze/Agentic-SEO-Skill
seo
VerifiedDeterministic LLM-first SEO audits for websites, blog posts, and GitHub repositories. Use this when the user asks to "perform SEO analysis", "run SEO audit", "analyze SEO", "check technical SEO", "review schema", "Core Web Vitals", "E-E-A-T", "hreflang", "GEO", "AEO", or GitHub repository SEO optimization. For full/page/repo audits, run bundled scripts for evidence and return prioritized, confidence-labeled fixes.
scripts/orphan_pages_from_sitemap.py
View on GitHub#!/usr/bin/env python3
"""Find sitemap URLs that are not reachable from an internal crawl."""
from __future__ import annotations
import argparse
from collections import deque
from urllib.parse import urlparse
from seo_common import (
discover_sitemap_urls,
fetch_url,
normalize_url,
parse_html,
parse_sitemap_xml,
print_json_or_text,
same_host,
)
def _page_key(url: str) -> str:
parsed = urlparse(normalize_url(url))
path = parsed.path or "/"
if path != "/":
path = path.rstrip("/")
return f"{parsed.scheme}://{parsed.netloc}{path}"
def load_sitemap_urls(site_url: str, sitemap_urls: list[str] | None = None, timeout: int = 15, max_sitemaps: int = 25) -> dict:
queue = list(dict.fromkeys(sitemap_urls or discover_sitemap_urls(site_url, timeout=timeout)))
seen_sitemaps = set()
urls = []
errors = []
while queue and len(seen_sitemaps) < max_sitemaps:
sitemap_url = normalize_url(queue.pop(0), site_url)
if sitemap_url in seen_sitemaps:
continue
seen_sitemaps.add(sitemap_url)
fetched = fetch_url(sitemap_url, timeout=timeout, max_bytes=8_000_000)
if fetched.get("status") != 200 or not fetched.get("text"):
errors.append({"url": sitemap_url, "status": fetched.get("status"), "error": fetched.get("error")})
continue
parsed = parse_sitemap_xml(fetched["text"], sitemap_url)
if parsed.get("error"):
errors.append({"url": sitemap_url, "error": parsed["error"]})
continue
queue.extend(item["loc"] for item in parsed.get("sitemaps", []))
urls.extend(item["loc"] for item in parsed.get("urls", []))
deduped = []
seen = set()
for url in urls:
key = _page_key(url)
if key not in seen:
seen.add(key)
deduped.append(key)
return {"sitemaps_checked": sorted(seen_sitemaps), "urls": deduped, "errors": errors}
def crawl_reachable_pages(start_url: str, depth: int = 2, max_pages: int = 100, timeout: int = 15) -> dict:
start_url = normalize_url(start_url)
queue = deque([(start_url, 0)])
reachable = {}
errors = []
while queue and len(reachable) < max_pages:
url, current_depth = queue.popleft()
key = _page_key(url)
if key in reachable or current_depth > depth:
continue
fetched = fetch_url(url, timeout=timeout, max_bytes=2_000_000)
reachable[key] = {
"url": key,
"status": fetched.get("status"),
"final_url": fetched.get("url"),
"depth": current_depth,
"in_sitemap": False,
}
if fetched.get("status") != 200 or not fetched.get("text"):
errors.append({"url": url, "status": fetched.get("status"), "error": fetched.get("error")})
continue
if current_depth >= depth:
continue
html = parse_html(fetched["text"], fetched.get("url") or url)
for link in html.get("links", []):
href = link.get("href") or ""
if href and same_host(start_url, href):
target = _page_key(href)
if target not in reachable and len(reachable) + len(queue) < max_pages:
queue.append((target, current_depth + 1))
return {"pages": reachable, "errors": errors}
def find_orphan_pages(site_url: str, sitemap_urls: list[str] | None = None, depth: int = 2, max_pages: int = 100, timeout: int = 15) -> dict:
site_url = normalize_url(site_url)
sitemap = load_sitemap_urls(site_url, sitemap_urls=sitemap_urls, timeout=timeout)
crawl = crawl_reachable_pages(site_url, depth=depth, max_pages=max_pages, timeout=timeout)
sitemap_set = set(sitemap["urls"])
reachable_set = set(crawl["pages"])
for page in crawl["pages"].values():
page["in_sitemap"] = page["url"] in sitemap_set
orphan_urls = sorted(sitemap_set - reachable_set)
discovered_not_in_sitemap = sorted(reachable_set - sitemap_set)
issues = []
if orphan_urls:
issues.append({"severity": "warning", "type": "sitemap_orphans", "count": len(orphan_urls), "message": "Sitemap URLs were not reached by crawl"})
if discovered_not_in_sitemap:
issues.append({"severity": "info", "type": "crawl_only_pages", "count": len(discovered_not_in_sitemap), "message": "Crawled URLs are not present in sitemap"})
return {
"site": site_url,
"summary": {
"sitemaps_checked": len(sitemap["sitemaps_checked"]),
"sitemap_urls": len(sitemap_set),
"reachable_pages": len(reachable_set),
"orphan_pages": len(orphan_urls),
"discovered_not_in_sitemap": len(discovered_not_in_sitemap),
},
"sitemaps_checked": sitemap["sitemaps_checked"],
"orphan_pages": orphan_urls,
"discovered_not_in_sitemap": discovered_not_in_sitemap,
"reachable_pages": list(crawl["pages"].values()),
"issues": issues,
"errors": {"sitemap": sitemap["errors"], "crawl": crawl["errors"]},
}
def main() -> None:
parser = argparse.ArgumentParser(description="Find sitemap URLs not reachable from an internal crawl")
parser.add_argument("site", help="Website URL")
parser.add_argument("--sitemap", action="append", help="Explicit sitemap URL; can be repeated")
parser.add_argument("--depth", type=int, default=2, help="Internal crawl depth (default: 2)")
parser.add_argument("--max-pages", type=int, default=100, help="Maximum crawl pages (default: 100)")
parser.add_argument("--timeout", type=int, default=15)
parser.add_argument("--json", "-j", action="store_true", help="Output JSON")
args = parser.parse_args()
result = find_orphan_pages(args.site, sitemap_urls=args.sitemap, depth=args.depth, max_pages=args.max_pages, timeout=args.timeout)
lines = [
f"Orphan page check for {result['site']}",
(
f"Sitemap URLs: {result['summary']['sitemap_urls']} "
f"Reachable pages: {result['summary']['reachable_pages']} "
f"Orphans: {result['summary']['orphan_pages']}"
),
]
lines.extend(f"Orphan: {url}" for url in result["orphan_pages"][:25])
print_json_or_text(result, args.json, lines)
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |