Skip to content

AgriciDaniel/claude-blog / skills/blog-google

blog-google

Verified

Credentialed Google API calls for a blog: PageSpeed Insights, CrUX history, Search Console performance and URL Inspection, GA4 organic traffic, Cloud NLP entities, YouTube video search, Google Ads Keyword Planner, and the Indexing API (job and livestream pages only). Uses the user's Google credentials, quotas, and sometimes billing, and asks for consent before the first call. Runs only on an explicit request: /blog google, "run PageSpeed on this blog URL", "pull my Search Console data", "inspect this URL's index status", "set up Google API credentials". Shares config with claude-seo at ~/.config/claude-seo/google-api.json.

scripts/gsc_inspect.py

View on GitHub
#!/usr/bin/env python3
"""
Google Search Console URL Inspection API helper.

Inspects URLs for indexing status, canonical selection, crawl info,
mobile usability, and rich results. Supports single URL and batch mode.

Usage:
    python gsc_inspect.py https://example.com/page --site-url sc-domain:example.com
    python gsc_inspect.py --batch urls.txt --site-url sc-domain:example.com
    python gsc_inspect.py https://example.com/page --json
"""

import argparse
import datetime as dt
import json
import sys
import time
from pathlib import Path
from typing import Optional
from urllib.parse import urlparse

try:
    from google_auth import (
        describe_google_api_error, execute_with_retries, get_oauth_credentials,
        load_config, safe_exception_text, setup_hint,
    )
except ImportError:
    import os
    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
    from google_auth import (
        describe_google_api_error, execute_with_retries, get_oauth_credentials,
        load_config, safe_exception_text, setup_hint,
    )

try:
    from googleapiclient.discovery import build
except ImportError:
    print(f"Error: google-api-python-client required. {setup_hint()}", file=sys.stderr)
    sys.exit(1)

GSC_SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly"]

# Daily limit per site
DAILY_LIMIT = 2000
QPM_LIMIT = 600
CANONICAL_REEVALUATION_DAYS = 14


def _path_contains_symlink(path: Path, root: Path) -> bool:
    """Return True if path or any existing parent below root is a symlink."""
    current = path if path.exists() or path.is_symlink() else path.parent
    current = current.absolute()
    root = root.absolute()
    while current != current.parent:
        if current.is_symlink():
            return True
        if current == root:
            return False
        current = current.parent
    return False


def _resolve_batch_file(path_value: str) -> Path:
    """Resolve a batch file under the current working directory."""
    root = Path.cwd().resolve()
    candidate = Path(path_value).expanduser()
    if not candidate.is_absolute():
        candidate = root / candidate
    if _path_contains_symlink(candidate, root):
        raise ValueError("Batch file must not use symlinks")
    try:
        resolved = candidate.resolve(strict=True)
    except FileNotFoundError as exc:
        raise ValueError(f"Batch file not found: {path_value}") from exc
    try:
        resolved.relative_to(root)
    except ValueError as exc:
        raise ValueError(f"Refusing to read batch file outside working directory: {path_value}") from exc
    if not resolved.is_file():
        raise ValueError(f"Batch path is not a regular file: {path_value}")
    return resolved


def _validate_url(url: str) -> str:
    """Validate an http or https URL before calling Google APIs."""
    clean = str(url).strip()
    parsed = urlparse(clean)
    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
        raise ValueError(f"Invalid URL: {url}")
    return clean


def _load_batch_urls(path_value: str) -> list[str]:
    """Load and validate URLs from a cwd-local batch file."""
    path = _resolve_batch_file(path_value)
    urls = []
    with open(path, "r", encoding="utf-8") as f:
        for line_number, line in enumerate(f, 1):
            clean = line.strip()
            if not clean:
                continue
            try:
                urls.append(_validate_url(clean))
            except ValueError as exc:
                raise ValueError(f"Invalid URL on line {line_number}: {clean}") from exc
    return urls


def _exit_error(message: str, as_json: bool) -> None:
    """Emit a CLI error in the requested format."""
    if as_json:
        print(json.dumps({"error": message}, indent=2))
    else:
        print(f"Error: {message}", file=sys.stderr)
    sys.exit(1)


def _build_inspection_service():
    """Build the Search Console v1 service for URL Inspection."""
    credentials = get_oauth_credentials(GSC_SCOPES)
    if not credentials:
        return None
    try:
        return build("searchconsole", "v1", credentials=credentials)
    except Exception as e:
        print(f"Error building service: {safe_exception_text(e)}", file=sys.stderr)
        return None


def _parse_fix_date(value: str | dt.date | None) -> dt.date | None:
    """Parse an optional declared canonical-fix date."""
    if value is None or value == "":
        return None
    if isinstance(value, dt.datetime):
        return value.date()
    if isinstance(value, dt.date):
        return value
    try:
        return dt.date.fromisoformat(str(value))
    except ValueError as exc:
        raise ValueError("canonical fix date must use YYYY-MM-DD") from exc


def classify_canonical_reevaluation(
    *,
    inspection_url: str,
    google_canonical: str | None,
    user_canonical: str | None,
    fix_date: str | dt.date,
    expected_canonical: str | None = None,
    today: dt.date | None = None,
) -> dict:
    """Classify a declared canonical fix without replacing Google's verdict.

    Pending status requires all three facts: the inspected markup now declares
    the expected canonical, Google still selects a different canonical, and the
    declared fix is no more than 14 days old.
    """
    parsed_fix_date = _parse_fix_date(fix_date)
    if parsed_fix_date is None:
        raise ValueError("canonical fix date is required")
    expected = _validate_url(expected_canonical or inspection_url)
    current_date = today or dt.datetime.now(dt.timezone.utc).date()
    age_days = (current_date - parsed_fix_date).days
    implementation_correct = user_canonical == expected
    selection_pending = bool(google_canonical) and google_canonical != expected
    pending = (
        implementation_correct
        and selection_pending
        and 0 <= age_days <= CANONICAL_REEVALUATION_DAYS
    )
    if pending:
        classification = "PENDING_REEVALUATION"
    elif google_canonical == expected and user_canonical == expected:
        classification = "MATCH"
    elif age_days < 0:
        classification = "INVALID_FUTURE_FIX_DATE"
    else:
        classification = "MISMATCH"
    return {
        "classification": classification,
        "pending_reevaluation": pending,
        "fix_date": parsed_fix_date.isoformat(),
        "age_days": age_days,
        "expected_canonical": expected,
        "implementation_correct": implementation_correct,
    }


def inspect_url(
    inspection_url: str,
    site_url: str,
    language_code: str = "en",
    service=None,
    canonical_fix_date: str | dt.date | None = None,
    expected_canonical: str | None = None,
    today: dt.date | None = None,
) -> dict:
    """
    Inspect a single URL via the GSC URL Inspection API.

    Args:
        inspection_url: The URL to inspect.
        site_url: The GSC property (e.g., 'sc-domain:example.com').
        language_code: Language for localized messages (default: 'en').

    Returns:
        Dictionary with inspection results including index status,
        crawl info, canonical, mobile usability, and rich results.
    """
    result = {
        "url": inspection_url,
        "property": site_url,
        "index_status": None,
        "crawl_info": None,
        "canonical": None,
        "mobile_usability": None,
        "rich_results": None,
        "verdict": None,
        "error": None,
    }
    try:
        inspection_url = _validate_url(inspection_url)
        result["url"] = inspection_url
    except ValueError as e:
        result["error"] = str(e)
        return result
    try:
        parsed_fix_date = _parse_fix_date(canonical_fix_date)
        if expected_canonical is not None:
            expected_canonical = _validate_url(expected_canonical)
    except ValueError as e:
        result["error"] = str(e)
        return result

    service = service or _build_inspection_service()
    if not service:
        result["error"] = "Could not build GSC service. Check service account credentials."
        return result

    body = {
        "inspectionUrl": inspection_url,
        "siteUrl": site_url,
        "languageCode": language_code,
    }

    try:
        # 429 is surfaced at once: sleeping and retrying every URL of a batch
        # against an exhausted quota only burns time (batch_inspect stops).
        response = execute_with_retries(
            service.urlInspection().index().inspect(body=body), retry_quota=False
        )
    except Exception as e:
        status = getattr(getattr(e, "resp", None), "status", None)
        if isinstance(status, str) and status.isdigit():
            status = int(status)
        if status == 403:
            result["error"] = describe_google_api_error(
                e, 'URL Inspection', "Check the service account's URL Inspection access in Google Search Console for this property."
            )
        elif status == 429:
            result["quota_exceeded"] = True
            result["error"] = (
                f"Rate limit exceeded. URL Inspection: {QPM_LIMIT} QPM / {DAILY_LIMIT} QPD per site."
            )
        elif status == 400:
            result["error"] = (
                f"Invalid request. Ensure the URL '{inspection_url}' belongs to "
                f"property '{site_url}'."
            )
        else:
            result["error"] = describe_google_api_error(e, 'URL Inspection')
        return result

    ir = response.get("inspectionResult", {})

    # Index status
    idx = ir.get("indexStatusResult", {})
    result["verdict"] = idx.get("verdict", "VERDICT_UNSPECIFIED")
    result["index_status"] = {
        "verdict": idx.get("verdict"),
        "coverage_state": idx.get("coverageState"),
        "robots_txt_state": idx.get("robotsTxtState"),
        "indexing_state": idx.get("indexingState"),
        "page_fetch_state": idx.get("pageFetchState"),
        "last_crawl_time": idx.get("lastCrawlTime"),
        "crawled_as": idx.get("crawledAs"),
        "referring_urls": idx.get("referringUrls", []),
    }

    # Canonical
    result["canonical"] = {
        "google_canonical": idx.get("googleCanonical"),
        "user_canonical": idx.get("userCanonical"),
        "match": idx.get("googleCanonical") == idx.get("userCanonical")
        if idx.get("googleCanonical") and idx.get("userCanonical") else None,
    }
    if parsed_fix_date is not None:
        result["canonical"].update(
            classify_canonical_reevaluation(
                inspection_url=inspection_url,
                google_canonical=idx.get("googleCanonical"),
                user_canonical=idx.get("userCanonical"),
                fix_date=parsed_fix_date,
                expected_canonical=expected_canonical,
                today=today,
            )
        )

    # Mobile usability (deprecated April 2023 but may still return data)
    mu = ir.get("mobileUsabilityResult", {})
    if mu:
        result["mobile_usability"] = {
            "verdict": mu.get("verdict"),
            "issues": [
                {"type": issue.get("issueType"), "message": issue.get("message")}
                for issue in mu.get("issues", [])
            ],
        }

    # Rich results
    rr = ir.get("richResultsResult", {})
    if rr:
        result["rich_results"] = {
            "verdict": rr.get("verdict"),
            "detected_items": [
                {
                    "type": item.get("richResultType"),
                    "items": [
                        {"name": i.get("name"), "issues": i.get("issues", [])}
                        for i in item.get("items", [])
                    ],
                }
                for item in rr.get("detectedItems", [])
            ],
        }

    return result


def batch_inspect(
    urls: list,
    site_url: str,
    delay: float = 1.0,
    language_code: str = "en",
    canonical_fix_date: str | dt.date | None = None,
) -> dict:
    """
    Batch inspect multiple URLs with rate limiting.

    Args:
        urls: List of URLs to inspect.
        site_url: GSC property.
        delay: Seconds between requests (default: 1.0 for safety).
        language_code: Language code.

    Returns:
        Dictionary with results list and summary.
    """
    result = {
        "property": site_url,
        "total": len(urls),
        "results": [],
        "summary": {
            "pass": 0,
            "fail": 0,
            "neutral": 0,
            "error": 0,
        },
        "warning": None,
        "stopped_on_quota": False,
        "not_inspected": [],
        "error": None,
    }

    try:
        urls = [_validate_url(url) for url in urls if str(url).strip()]
    except ValueError as e:
        result["error"] = str(e)
        result["total"] = 0
        return result

    result["total"] = len(urls)

    if len(urls) > DAILY_LIMIT:
        # A warning, not an error: the batch still runs for the first DAILY_LIMIT URLs.
        result["warning"] = (
            f"Batch size ({len(urls)}) exceeds daily limit ({DAILY_LIMIT}). "
            f"Only the first {DAILY_LIMIT} URLs will be processed."
        )
        result["not_inspected"] = list(urls[DAILY_LIMIT:])
        urls = urls[:DAILY_LIMIT]

    service = _build_inspection_service()
    if not service:
        result["error"] = "Could not build GSC service. Check service account credentials."
        return result

    for i, url in enumerate(urls):
        url = url.strip()
        if not url:
            continue

        print(f"Inspecting [{i + 1}/{len(urls)}]: {url}", file=sys.stderr)

        inspection = inspect_url(
            url,
            site_url,
            language_code,
            service=service,
            canonical_fix_date=canonical_fix_date,
        )
        result["results"].append(inspection)

        verdict = inspection.get("verdict", "")
        if inspection.get("error"):
            result["summary"]["error"] += 1
        elif verdict == "PASS":
            result["summary"]["pass"] += 1
        elif verdict == "FAIL":
            result["summary"]["fail"] += 1
        else:
            result["summary"]["neutral"] += 1

        if inspection.get("quota_exceeded"):
            result["stopped_on_quota"] = True
            result["not_inspected"] = list(urls[i + 1:]) + result["not_inspected"]
            result["warning"] = (
                f"Stopped on quota/rate limit after {i + 1} of {len(urls)} URLs; "
                f"{len(result['not_inspected'])} URL(s) were not inspected. "
                "Wait for the limit to reset and rerun with the remaining URLs."
            )
            break

        # Rate limiting
        if i < len(urls) - 1:
            time.sleep(delay)

    return result


def main():
    parser = argparse.ArgumentParser(
        description="Google Search Console URL Inspection API helper"
    )
    parser.add_argument("url", nargs="?", help="URL to inspect")
    parser.add_argument(
        "--site-url", "-s",
        help="GSC property (e.g., sc-domain:example.com). Uses default from config if not specified.",
    )
    parser.add_argument(
        "--batch", "-b",
        help="File with URLs to inspect (one per line)",
    )
    parser.add_argument(
        "--delay",
        type=float,
        default=1.0,
        help="Delay between batch requests in seconds (default: 1.0)",
    )
    parser.add_argument("--json", "-j", action="store_true", help="Output as JSON")
    parser.add_argument(
        "--canonical-fix-date",
        help=(
            "Declared date a canonical implementation was fixed (YYYY-MM-DD). "
            "Adds a pending-reevaluation classification without replacing the "
            "Google URL Inspection verdict."
        ),
    )
    parser.add_argument(
        "--expected-canonical",
        help=(
            "Expected canonical URL for single-URL mode. Defaults to the "
            "inspected URL."
        ),
    )

    args = parser.parse_args()
    if args.batch and args.expected_canonical:
        _exit_error("--expected-canonical is only valid for single-URL mode", args.json)

    # Resolve site URL
    site_url = args.site_url
    if not site_url:
        config = load_config()
        site_url = config.get("default_property")
    if not site_url:
        print("Error: No site URL specified. Use --site-url or set default_property in config.", file=sys.stderr)
        sys.exit(1)

    if args.batch:
        # Batch mode
        try:
            urls = _load_batch_urls(args.batch)
        except (OSError, ValueError) as e:
            _exit_error(f"Error reading batch file: {e}", args.json)

        result = batch_inspect(
            urls,
            site_url,
            delay=args.delay,
            canonical_fix_date=args.canonical_fix_date,
        )
    elif args.url:
        result = inspect_url(
            args.url,
            site_url,
            canonical_fix_date=args.canonical_fix_date,
            expected_canonical=args.expected_canonical,
        )
    else:
        parser.print_help()
        sys.exit(1)

    if args.json:
        print(json.dumps(result, indent=2))
    else:
        if args.batch:
            summary = result.get("summary", {})
            print(f"=== URL Inspection Batch Results ===")
            print(f"Property: {site_url}")
            print(f"Total: {result.get('total', 0)} | Pass: {summary.get('pass', 0)} | Fail: {summary.get('fail', 0)} | Errors: {summary.get('error', 0)}")
            if result.get("warning"):
                print(f"Warning: {result['warning']}")
            print()
            for r in result.get("results", []):
                verdict = r.get("verdict", "?")
                status = {"PASS": "OK", "FAIL": "FAIL", "NEUTRAL": "--"}.get(verdict, "ERR")
                print(f"  [{status}] {r.get('url')}")
                if r.get("error"):
                    print(f"       Error: {r['error']}")
                elif verdict == "FAIL":
                    idx = r.get("index_status", {})
                    print(f"       Coverage: {idx.get('coverage_state')} | Fetch: {idx.get('page_fetch_state')}")
        else:
            if result.get("error"):
                print(f"Error: {result['error']}", file=sys.stderr)
                sys.exit(1)

            verdict = result.get("verdict", "?")
            print(f"=== URL Inspection: {result.get('url')} ===")
            print(f"Verdict: {verdict}")

            idx = result.get("index_status", {})
            if idx:
                print(f"\nIndex Status:")
                print(f"  Coverage: {idx.get('coverage_state')}")
                print(f"  Robots.txt: {idx.get('robots_txt_state')}")
                print(f"  Indexing: {idx.get('indexing_state')}")
                print(f"  Page Fetch: {idx.get('page_fetch_state')}")
                print(f"  Last Crawl: {idx.get('last_crawl_time', 'N/A')}")
                print(f"  Crawled As: {idx.get('crawled_as')}")

            canon = result.get("canonical", {})
            if canon:
                print(f"\nCanonical:")
                print(f"  Google: {canon.get('google_canonical', 'N/A')}")
                print(f"  User: {canon.get('user_canonical', 'N/A')}")
                match = canon.get("match")
                if match is not None:
                    print(f"  Match: {'Yes' if match else 'MISMATCH'}")
                if canon.get("classification"):
                    print(f"  Reevaluation: {canon['classification']}")

            rr = result.get("rich_results")
            if rr and rr.get("detected_items"):
                print(f"\nRich Results: {rr.get('verdict')}")
                for item in rr.get("detected_items", []):
                    print(f"  Type: {item.get('type')}")


if __name__ == "__main__":
    main()
receiving-code-reviewobra/superpowersUse when receiving code review feedback, before implementing suggestions, especially if feedback seems unclear or technically questionable - requires technical rigor and verification, not performative agreement or blind implementationStars: 322kUpdated Claude CodeCodexVerifiedInstall
requesting-code-reviewobra/superpowersUse when completing tasks, implementing major features, or before merging to verify work meets requirementsStars: 322kUpdated Claude CodeCodexVerifiedInstall
using-git-worktreesobra/superpowersUse when starting feature work that needs isolation from current workspace or before executing implementation plans - ensures an isolated workspace exists via native tools or git worktree fallbackStars: 322kUpdated Claude CodeCodexVerifiedInstall
verification-before-completionobra/superpowersUse when about to claim work is complete, fixed, or passing, before committing or creating PRs - requires running verification commands and confirming output before making any success claims; evidence before assertions alwaysStars: 322kUpdated Claude CodeCodexVerifiedInstall