Bhanunamikaze/Agentic-SEO-Skill
seo
VerifiedDeterministic LLM-first SEO audits for websites, blog posts, and GitHub repositories. Use this when the user asks to "perform SEO analysis", "run SEO audit", "analyze SEO", "check technical SEO", "review schema", "Core Web Vitals", "E-E-A-T", "hreflang", "GEO", "AEO", or GitHub repository SEO optimization. For full/page/repo audits, run bundled scripts for evidence and return prioritized, confidence-labeled fixes.
scripts/duplicate_content.py
View on GitHub#!/usr/bin/env python3
"""
Duplicate & Thin Content Detector
Detects near-duplicate pages and thin content across a site using
MinHash / Jaccard similarity and word-count thresholds.
Usage:
python duplicate_content.py https://example.com --depth 2 --json
python duplicate_content.py https://example.com --threshold 0.85
"""
import argparse
import hashlib
import json
import re
import sys
import time
from collections import defaultdict
from urllib.parse import urljoin, urlparse
try:
from bs4 import BeautifulSoup
except ImportError:
print("Error: beautifulsoup4 required. Install with: pip install beautifulsoup4")
sys.exit(1)
try:
from lib.safe_http import safe_get
except ImportError:
from scripts.lib.safe_http import safe_get
# Quality gates from resources/references/quality-gates.md
THIN_CONTENT_THRESHOLDS = {
"blog_post": 1500,
"landing_page": 800,
"product_page": 300,
"location_page": 350,
"default": 300,
}
# ---------------------------------------------------------------------------
# Fetch & extract
# ---------------------------------------------------------------------------
def fetch_page(url: str, timeout: int = 12) -> str:
try:
resp = safe_get(url, timeout=timeout)
ct = resp.headers.get("Content-Type", "")
if "text/html" not in ct:
return ""
return resp.text
except Exception:
return ""
def extract_text(html: str) -> str:
"""Extract visible body text, stripping nav/footer/scripts."""
soup = BeautifulSoup(html, "html.parser")
for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
tag.decompose()
body = soup.find("body")
if not body:
return ""
return body.get_text(separator=" ", strip=True)
def extract_internal_links(html: str, base_url: str) -> list:
"""Extract internal links from a page."""
soup = BeautifulSoup(html, "html.parser")
base_domain = urlparse(base_url).netloc
links = []
for a in soup.find_all("a", href=True):
href = a["href"]
if href.startswith("#") or href.startswith("javascript:") or href.startswith("mailto:"):
continue
full = urljoin(base_url, href)
parsed = urlparse(full)
if parsed.netloc == base_domain and parsed.scheme in ("http", "https"):
clean = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
if clean.endswith("/"):
clean = clean[:-1]
links.append(clean)
return list(set(links))
# ---------------------------------------------------------------------------
# Shingle-based MinHash for near-duplicate detection
# ---------------------------------------------------------------------------
def shingle(text: str, k: int = 5) -> set:
"""Create k-word shingles from text."""
words = re.findall(r"\b[a-z]+\b", text.lower())
if len(words) < k:
return {" ".join(words)}
return {" ".join(words[i:i+k]) for i in range(len(words) - k + 1)}
def minhash_signature(shingles: set, num_hashes: int = 100) -> list:
"""Compute MinHash signature for a set of shingles."""
sig = []
for i in range(num_hashes):
min_hash = float("inf")
for s in shingles:
h = int(hashlib.md5(f"{i}:{s}".encode()).hexdigest(), 16)
if h < min_hash:
min_hash = h
sig.append(min_hash)
return sig
def jaccard_from_minhash(sig1: list, sig2: list) -> float:
"""Estimate Jaccard similarity from two MinHash signatures."""
if not sig1 or not sig2:
return 0.0
matches = sum(1 for a, b in zip(sig1, sig2) if a == b)
return matches / len(sig1)
def exact_hash(text: str) -> str:
"""SHA-256 of normalized text for exact duplicate detection."""
normalized = re.sub(r"\s+", " ", text.lower().strip())
return hashlib.sha256(normalized.encode()).hexdigest()
def _robots_directives(value: str | None) -> set[str]:
"""Normalize a robots directive string into lowercase tokens."""
if not value:
return set()
return {token.strip().lower() for token in re.split(r"[\s,;]+", value) if token.strip()}
def _robots_content_has_noindex(content: str | None) -> bool:
directives = _robots_directives(content)
return "noindex" in directives or "none" in directives
def html_has_noindex(html: str) -> bool:
"""Return True when retained HTML contains a robots noindex directive."""
if not html:
return False
soup = BeautifulSoup(html, "html.parser")
for meta in soup.find_all("meta"):
name = (meta.get("name") or "").strip().lower()
http_equiv = (meta.get("http-equiv") or "").strip().lower()
content = meta.get("content")
if name in {"robots", "googlebot", "bingbot"} and _robots_content_has_noindex(content):
return True
if http_equiv == "x-robots-tag" and _robots_content_has_noindex(content):
return True
return False
def page_is_noindex(data: dict) -> bool:
"""Detect noindex from explicit fields first, then retained HTML."""
if data.get("noindex") is True:
return True
for key in ("meta_robots", "robots", "x_robots_tag", "x-robots-tag"):
if _robots_content_has_noindex(data.get(key)):
return True
return html_has_noindex(data.get("html") or "")
# ---------------------------------------------------------------------------
# Crawl
# ---------------------------------------------------------------------------
def crawl_site(start_url: str, max_pages: int = 50, depth: int = 2) -> dict:
"""
Crawl a site starting from start_url.
Returns {url: {"text": str, "word_count": int, "html": str}}.
"""
visited = {}
queue = [(start_url, 0)]
seen = {start_url}
while queue and len(visited) < max_pages:
url, d = queue.pop(0)
time.sleep(0.5) # polite delay
html = fetch_page(url)
if not html:
continue
text = extract_text(html)
word_count = len(re.findall(r"\b\w+\b", text))
visited[url] = {
"text": text,
"word_count": word_count,
"html": html,
"noindex": html_has_noindex(html),
}
if d < depth:
for link in extract_internal_links(html, url):
if link not in seen and len(seen) < max_pages * 3:
seen.add(link)
queue.append((link, d + 1))
return visited
# ---------------------------------------------------------------------------
# Analysis
# ---------------------------------------------------------------------------
def detect_duplicates(pages: dict, similarity_threshold: float = 0.85) -> dict:
"""
Detect exact and near-duplicate pages.
Returns report with exact dupes, near-dupes, and thin content.
"""
# Step 1: Exact duplicates (hash comparison)
hash_groups = defaultdict(list)
signatures = {}
for url, data in pages.items():
text = data["text"]
if not text.strip():
continue
h = exact_hash(text)
hash_groups[h].append(url)
# MinHash signature for near-duplicate comparison
s = shingle(text)
if s:
signatures[url] = minhash_signature(s)
exact_dupes = []
for h, urls in hash_groups.items():
if len(urls) > 1:
indexable_urls = [url for url in urls if not page_is_noindex(pages[url])]
noindex_urls = [url for url in urls if page_is_noindex(pages[url])]
if len(indexable_urls) > 1:
exact_dupes.append({
"type": "exact_duplicate",
"severity": "Critical",
"urls": indexable_urls,
"finding": f"{len(indexable_urls)} indexable pages have identical content.",
"fix": "Consolidate into a single canonical page and redirect duplicates with 301.",
})
if noindex_urls:
exact_dupes.append({
"type": "exact_duplicate",
"severity": "Info",
"urls": urls,
"noindex_urls": noindex_urls,
"finding": "Identical content includes at least one noindex page.",
"fix": "No action required for duplicate-content risk while noindex is intentional.",
})
# Step 2: Near-duplicates (MinHash Jaccard)
near_dupes = []
urls = list(signatures.keys())
checked = set()
for i in range(len(urls)):
for j in range(i + 1, len(urls)):
pair = (urls[i], urls[j])
if pair in checked:
continue
checked.add(pair)
sim = jaccard_from_minhash(signatures[urls[i]], signatures[urls[j]])
if sim >= similarity_threshold:
# Skip if already in exact dupes
if any(urls[i] in ed["urls"] and urls[j] in ed["urls"] for ed in exact_dupes):
continue
noindex_in_pair = page_is_noindex(pages[urls[i]]) or page_is_noindex(pages[urls[j]])
near_dupes.append({
"type": "near_duplicate",
"severity": "Info" if noindex_in_pair else "Warning",
"similarity": round(sim, 3),
"url_a": urls[i],
"url_b": urls[j],
"word_count_a": pages[urls[i]]["word_count"],
"word_count_b": pages[urls[j]]["word_count"],
"noindex_in_pair": noindex_in_pair,
"finding": (
f"Pages are {sim:.0%} similar, but at least one page is noindex."
if noindex_in_pair
else f"Pages are {sim:.0%} similar ā likely near-duplicate content."
),
"fix": (
"No action required for duplicate-content risk while noindex is intentional."
if noindex_in_pair
else "Differentiate content significantly, or set one as canonical and noindex the other."
),
})
# Step 3: Thin content
thin_pages = []
for url, data in pages.items():
if page_is_noindex(data):
continue
wc = data["word_count"]
threshold = THIN_CONTENT_THRESHOLDS["default"]
if wc < threshold:
thin_pages.append({
"type": "thin_content",
"severity": "Warning" if wc >= 100 else "Critical",
"url": url,
"word_count": wc,
"threshold": threshold,
"finding": f"Only {wc} words (minimum: {threshold}).",
"fix": f"Expand content to at least {threshold} words of substantive, unique content, or noindex if low-value.",
})
return {
"pages_analyzed": len(pages),
"exact_duplicates": exact_dupes,
"near_duplicates": near_dupes,
"thin_content": thin_pages,
"summary": {
"exact_duplicate_groups": len(exact_dupes),
"near_duplicate_pairs": len(near_dupes),
"thin_pages": len(thin_pages),
"avg_word_count": round(
sum(p["word_count"] for p in pages.values()) / max(1, len(pages))
),
},
}
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(
description="Duplicate & Thin Content Detector (MinHash / Jaccard similarity)"
)
parser.add_argument("url", help="Start URL to crawl")
parser.add_argument("--depth", type=int, default=2, help="Crawl depth (default: 2)")
parser.add_argument("--max-pages", type=int, default=50, help="Max pages to crawl (default: 50)")
parser.add_argument("--threshold", type=float, default=0.85,
help="Jaccard similarity threshold for near-duplicates (default: 0.85)")
parser.add_argument("--json", action="store_true", help="Output as JSON")
args = parser.parse_args()
print(f"Crawling {args.url} (depth={args.depth}, max={args.max_pages})...", file=sys.stderr)
pages = crawl_site(args.url, max_pages=args.max_pages, depth=args.depth)
print(f"Crawled {len(pages)} pages. Analyzing...", file=sys.stderr)
report = detect_duplicates(pages, similarity_threshold=args.threshold)
if args.json:
print(json.dumps(report, indent=2))
return
print(f"\nDuplicate & Thin Content Report")
print("=" * 60)
print(f"Pages Analyzed : {report['pages_analyzed']}")
print(f"Avg Word Count : {report['summary']['avg_word_count']}")
if report["exact_duplicates"]:
print(f"\nExact Duplicates ({report['summary']['exact_duplicate_groups']} groups):")
for group in report["exact_duplicates"]:
print(f" š“ {len(group['urls'])} identical pages:")
for url in group["urls"]:
print(f" - {url}")
print(f" Fix: {group['fix']}")
if report["near_duplicates"]:
print(f"\nNear-Duplicates ({report['summary']['near_duplicate_pairs']} pairs):")
for pair in report["near_duplicates"]:
print(f" ā ļø {pair['similarity']:.0%} similar:")
print(f" A: {pair['url_a']} ({pair['word_count_a']} words)")
print(f" B: {pair['url_b']} ({pair['word_count_b']} words)")
print(f" Fix: {pair['fix']}")
if report["thin_content"]:
print(f"\nThin Content ({report['summary']['thin_pages']} pages):")
for page in sorted(report["thin_content"], key=lambda x: x["word_count"]):
icon = "š“" if page["severity"] == "Critical" else "ā ļø"
print(f" {icon} {page['url']} ā {page['word_count']} words (min: {page['threshold']})")
if not report["exact_duplicates"] and not report["near_duplicates"] and not report["thin_content"]:
print("\nā
No duplicate or thin content issues detected.")
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 471k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |