Skip to content

notque/vexjoy-agent / skills/content/content

content

Unverified

Content operations: editorial calendar, marketing, publishing, social media management.

FileKindSize
SKILL.mdskill md11.5 KB
references/at-protocol-api.mdmarkdown7.6 KB
references/at-protocol-preferred-patterns.mdmarkdown8.5 KB
references/batch-edit-examples.mdmarkdown6.9 KB
references/batch-edit-operation-types.mdmarkdown7.0 KB
references/batch-edit-regex-patterns.mdmarkdown5.4 KB
references/batch-edit-safety-checklist.mdmarkdown6.0 KB
references/batch-edit.mdmarkdown9.9 KB
references/calendar-format.mdmarkdown3.5 KB
references/campaign-planning.mdmarkdown17.5 KB
references/classification-prompt.mdmarkdown5.9 KB
references/content-engine.mdmarkdown8.7 KB
references/content-engine/error-handling.mdmarkdown7.8 KB
references/content-engine/phase-playbook.mdmarkdown8.3 KB
references/content-engine/platform-specs.mdmarkdown10.9 KB
references/content-strategy.mdmarkdown14.5 KB
references/email-sequences.mdmarkdown15.8 KB
references/error-handling.mdmarkdown8.2 KB
references/headlines.mdmarkdown5.9 KB
references/image-audit-alt-text-examples.mdmarkdown6.8 KB
references/image-audit-format-selection.mdmarkdown5.8 KB
references/image-audit-report-templates.mdmarkdown3.8 KB
references/image-audit-size-guidelines.mdmarkdown4.2 KB
references/image-audit.mdmarkdown10.4 KB
references/link-audit-false-positives.mdmarkdown4.1 KB
references/link-audit-fix-strategies.mdmarkdown7.1 KB
references/link-audit-graph-metrics.mdmarkdown5.0 KB
references/link-audit.mdmarkdown7.8 KB
references/llm-marketing-failure-modes.mdmarkdown18.7 KB
references/marketing.mdmarkdown20.4 KB
references/metrics.mdmarkdown7.4 KB
references/news-collection.mdmarkdown5.2 KB
references/operations.mdmarkdown5.8 KB
references/outline-examples.mdmarkdown11.2 KB
references/outline-structure-templates.mdmarkdown6.4 KB
references/outline.mdmarkdown8.9 KB
references/pagespeed-insights.mdmarkdown10.4 KB
references/pipeline-stages.mdmarkdown3.3 KB
references/pre-publish-checklist.mdmarkdown7.4 KB
references/pre-publish-hugo-frontmatter.mdmarkdown5.3 KB
references/pre-publish-seo-guidelines.mdmarkdown4.1 KB
references/pre-publish.mdmarkdown10.8 KB
references/reddit-moderate.mdmarkdown7.9 KB
references/script-commands.mdmarkdown3.7 KB
references/seo-audit.mdmarkdown16.4 KB
references/seo-guidelines.mdmarkdown5.5 KB
references/seo-keyword-placement.mdmarkdown6.0 KB
references/seo-title-patterns.mdmarkdown5.6 KB
references/seo.mdmarkdown8.9 KB
references/series-planner.mdmarkdown11.7 KB
references/series-planner/cadence-guidelines.mdmarkdown6.8 KB
references/series-planner/cross-linking.mdmarkdown7.2 KB
references/series-planner/output-format.mdmarkdown3.8 KB
references/series-planner/series-types.mdmarkdown7.6 KB
references/taxonomy-audit-report-format.mdmarkdown3.2 KB
references/taxonomy-consolidation-rules.mdmarkdown6.7 KB
references/taxonomy-examples.mdmarkdown9.7 KB
references/taxonomy-guidelines.mdmarkdown5.5 KB
references/taxonomy.mdmarkdown9.0 KB
references/topic-brainstormer.mdmarkdown10.1 KB
references/topic-brainstormer/content-filter.mdmarkdown7.1 KB
references/topic-brainstormer/priority-scoring.mdmarkdown8.1 KB
references/topic-brainstormer/topic-sources.mdmarkdown7.6 KB
references/wordpress-upload.mdmarkdown11.2 KB
references/x-api.mdmarkdown7.8 KB
scripts/publish/link_scanner.pyscript21.4 KB
scripts/publish/pagespeed.pyscript15.3 KB
scripts/publish/tests/test_load_env_file.pyscript3.8 KB
scripts/publish/tests/test_wordpress_inline_urls.pyscript4.4 KB
scripts/publish/tests/test_wordpress_media_upload.pyscript1.5 KB
scripts/publish/tests/test_wordpress_scraper.pyscript4.1 KB
scripts/publish/wordpress-edit-post.pyscript21.2 KB
scripts/publish/wordpress-media-upload.pyscript8.7 KB
scripts/publish/wordpress-scraper.pyscript23.4 KB
scripts/publish/wordpress-upload.pyscript44.2 KB
scripts/reddit-moderate/reddit-automod-cron.shscript3.9 KB
scripts/reddit-moderate/reddit-mod.pyscript83.0 KB
scripts/reddit-moderate/tests/test_reddit_mod.pyscript31.3 KB

scripts/publish/wordpress-scraper.py

View on GitHub
#!/usr/bin/env python3
"""WordPress.com blog scraper — fetches all posts via the public REST API and saves as markdown.

Scrapes posts from any WordPress.com-hosted blog using the WordPress.com public API
(public-api.wordpress.com/rest/v1.1/). Converts HTML content to markdown and saves
each post as a separate .md file with YAML frontmatter.

Usage:
    python3 skills/content/content/scripts/publish/wordpress-scraper.py \\
        --source https://example-blog.wordpress.com --output /tmp/articles/ --human
    python3 skills/content/content/scripts/publish/wordpress-scraper.py \\
        --source https://example-blog.wordpress.com --output /tmp/articles/ --limit 5 --human
    python3 skills/content/content/scripts/publish/wordpress-scraper.py \\
        --source https://example-blog.wordpress.com --output /tmp/articles/ \\
        --download-images --human

Exit codes:
    0 = success
    1 = error
"""

from __future__ import annotations

import argparse
import json
import re
import sys
import time
import urllib.parse
from html import unescape
from pathlib import Path
from typing import Any

import requests

# ---------------------------------------------------------------------------
# Rate limiting
# ---------------------------------------------------------------------------

_last_request_time: float = 0.0
RATE_LIMIT_SECONDS: float = 1.0
MAX_IMAGE_BYTES: int = 25 * 1024 * 1024


def _rate_limited_get(url: str, params: dict[str, Any] | None = None, timeout: int = 30) -> requests.Response:
    """Send a GET request with rate limiting (1 request per second).

    Args:
        url: Request URL.
        params: Query parameters.
        timeout: Request timeout in seconds.

    Returns:
        Response object.
    """
    global _last_request_time
    elapsed = time.monotonic() - _last_request_time
    if elapsed < RATE_LIMIT_SECONDS:
        time.sleep(RATE_LIMIT_SECONDS - elapsed)
    _last_request_time = time.monotonic()
    return requests.get(url, params=params, timeout=timeout)


# ---------------------------------------------------------------------------
# WordPress.com Public API fetching
# ---------------------------------------------------------------------------

WPCOM_API_BASE = "https://public-api.wordpress.com/rest/v1.1/sites"


def _site_id(source_url: str) -> str:
    """Extract the site identifier from a WordPress.com blog URL.

    Args:
        source_url: Blog URL (e.g., https://example-blog.wordpress.com).

    Returns:
        Site hostname for API calls (e.g., example-blog.wordpress.com).
    """
    parsed = urllib.parse.urlparse(source_url)
    return parsed.netloc or parsed.path.strip("/")


def fetch_all_posts(
    source_url: str,
    post_type: str = "post",
    limit: int | None = None,
    human: bool = False,
) -> list[dict[str, Any]]:
    """Fetch all posts or pages from the WordPress.com public API.

    Uses offset-based pagination with the /posts/ endpoint.

    Args:
        source_url: Blog URL (e.g., https://example-blog.wordpress.com).
        post_type: Content type — 'post' or 'page'.
        limit: Maximum number of items to fetch. None for all.
        human: If True, print progress to stderr.

    Returns:
        List of post/page dicts from the API.
    """
    site = _site_id(source_url)
    api_url = f"{WPCOM_API_BASE}/{site}/posts/"
    all_items: list[dict[str, Any]] = []
    offset = 0
    per_page = 100

    while True:
        page_num = (offset // per_page) + 1
        if human:
            print(f"  Fetching {post_type}s page {page_num}...", file=sys.stderr)

        params: dict[str, Any] = {
            "number": per_page,
            "offset": offset,
            "type": post_type,
            "status": "publish",
        }

        try:
            response = _rate_limited_get(api_url, params=params)
        except requests.exceptions.RequestException as e:
            print(f"Error: request failed for {post_type}s at offset {offset}: {e}", file=sys.stderr)
            break

        if response.status_code != 200:
            print(f"Error: HTTP {response.status_code} fetching {post_type}s at offset {offset}", file=sys.stderr)
            break

        data = response.json()
        posts = data.get("posts", [])
        total_found = data.get("found", 0)

        if not posts:
            break

        all_items.extend(posts)

        if limit is not None and len(all_items) >= limit:
            all_items = all_items[:limit]
            break

        offset += len(posts)
        if offset >= total_found:
            break

    return all_items


# ---------------------------------------------------------------------------
# HTML to Markdown conversion
# ---------------------------------------------------------------------------


def _strip_html_tags(text: str) -> str:
    """Remove all HTML tags from text.

    Args:
        text: HTML string.

    Returns:
        Plain text with tags removed.
    """
    return re.sub(r"<[^>]+>", "", text)


def html_to_markdown(html: str) -> str:
    """Convert HTML content to markdown.

    Handles common WordPress HTML elements: headings, paragraphs, links,
    images, bold, italic, lists, blockquotes, and iframes.

    Args:
        html: HTML content string.

    Returns:
        Markdown-formatted string.
    """
    if not html:
        return ""

    text = html

    # Normalize line endings
    text = text.replace("\r\n", "\n").replace("\r", "\n")

    # Handle <br> tags before block processing
    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)

    # --- Block-level elements ---

    # Headings (h1-h6)
    for level in range(6, 0, -1):
        prefix = "#" * level
        text = re.sub(
            rf"<h{level}[^>]*>(.*?)</h{level}>",
            lambda m, p=prefix: f"\n\n{p} {_strip_html_tags(m.group(1)).strip()}\n\n",
            text,
            flags=re.IGNORECASE | re.DOTALL,
        )

    # Blockquotes
    text = re.sub(
        r"<blockquote[^>]*>(.*?)</blockquote>",
        lambda m: "\n\n> " + _strip_html_tags(m.group(1)).strip().replace("\n", "\n> ") + "\n\n",
        text,
        flags=re.IGNORECASE | re.DOTALL,
    )

    # Ordered list items
    text = re.sub(
        r"<ol[^>]*>(.*?)</ol>",
        lambda m: _convert_ordered_list(m.group(1)),
        text,
        flags=re.IGNORECASE | re.DOTALL,
    )

    # Unordered list items
    text = re.sub(
        r"<ul[^>]*>(.*?)</ul>",
        lambda m: _convert_unordered_list(m.group(1)),
        text,
        flags=re.IGNORECASE | re.DOTALL,
    )

    # --- Inline elements (before stripping remaining tags) ---

    # Images — extract src and alt (handle both orderings of attributes)
    text = re.sub(
        r'<img[^>]*\ssrc=["\']([^"\']+)["\'][^>]*\salt=["\']([^"\']*)["\'][^>]*/?>',
        r"![\2](\1)",
        text,
        flags=re.IGNORECASE,
    )
    text = re.sub(
        r'<img[^>]*\salt=["\']([^"\']*)["\'][^>]*\ssrc=["\']([^"\']+)["\'][^>]*/?>',
        r"![\1](\2)",
        text,
        flags=re.IGNORECASE,
    )
    # Images with src only (no alt)
    text = re.sub(
        r'<img[^>]*\ssrc=["\']([^"\']+)["\'][^>]*/?>',
        r"![](\1)",
        text,
        flags=re.IGNORECASE,
    )

    # Links
    text = re.sub(
        r'<a[^>]*\shref=["\']([^"\']+)["\'][^>]*>(.*?)</a>',
        lambda m: f"[{_strip_html_tags(m.group(2)).strip()}]({m.group(1)})",
        text,
        flags=re.IGNORECASE | re.DOTALL,
    )

    # Bold
    text = re.sub(r"<(?:strong|b)>(.*?)</(?:strong|b)>", r"**\1**", text, flags=re.IGNORECASE | re.DOTALL)

    # Italic
    text = re.sub(r"<(?:em|i)>(.*?)</(?:em|i)>", r"*\1*", text, flags=re.IGNORECASE | re.DOTALL)

    # Iframes (YouTube embeds etc.) — preserve URL
    text = re.sub(
        r'<iframe[^>]*\ssrc=["\']([^"\']+)["\'][^>]*>.*?</iframe>',
        r"\n\n\1\n\n",
        text,
        flags=re.IGNORECASE | re.DOTALL,
    )

    # Paragraphs — convert to double newlines
    text = re.sub(r"<p[^>]*>", "\n\n", text, flags=re.IGNORECASE)
    text = re.sub(r"</p>", "\n\n", text, flags=re.IGNORECASE)

    # Divs and figures — treat as block separators
    block_tags = r"</?(?:div|figure|figcaption|section|article|aside|header|footer|nav)[^>]*>"
    text = re.sub(block_tags, "\n\n", text, flags=re.IGNORECASE)

    # Horizontal rules
    text = re.sub(r"<hr[^>]*/?>", "\n\n---\n\n", text, flags=re.IGNORECASE)

    # WordPress caption shortcodes
    text = re.sub(r"\[caption[^\]]*\](.*?)\[/caption\]", r"\1", text, flags=re.IGNORECASE | re.DOTALL)

    # Strip any remaining HTML tags
    text = _strip_html_tags(text)

    # Decode HTML entities
    text = unescape(text)

    # Clean up whitespace
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = re.sub(r" {2,}", " ", text)
    lines = [line.rstrip() for line in text.split("\n")]
    text = "\n".join(lines)
    text = text.strip()

    return text


def _convert_unordered_list(list_html: str) -> str:
    """Convert <li> items inside a <ul> to markdown unordered list.

    Args:
        list_html: Inner HTML of the <ul> element.

    Returns:
        Markdown list string.
    """
    items = re.findall(r"<li[^>]*>(.*?)</li>", list_html, flags=re.IGNORECASE | re.DOTALL)
    md_items = [f"- {_strip_html_tags(item).strip()}" for item in items]
    return "\n\n" + "\n".join(md_items) + "\n\n"


def _convert_ordered_list(list_html: str) -> str:
    """Convert <li> items inside an <ol> to markdown ordered list.

    Args:
        list_html: Inner HTML of the <ol> element.

    Returns:
        Markdown list string.
    """
    items = re.findall(r"<li[^>]*>(.*?)</li>", list_html, flags=re.IGNORECASE | re.DOTALL)
    md_items = [f"{i}. {_strip_html_tags(item).strip()}" for i, item in enumerate(items, 1)]
    return "\n\n" + "\n".join(md_items) + "\n\n"


# ---------------------------------------------------------------------------
# Image downloading
# ---------------------------------------------------------------------------


def extract_image_urls(html: str) -> list[str]:
    """Extract all image URLs from HTML content.

    Args:
        html: HTML content string.

    Returns:
        List of unique image URLs found in <img> tags.
    """
    urls = re.findall(r'<img[^>]*\ssrc=["\']([^"\']+)["\']', html, flags=re.IGNORECASE)
    # Deduplicate while preserving order
    seen: set[str] = set()
    unique: list[str] = []
    for url in urls:
        if url not in seen:
            seen.add(url)
            unique.append(url)
    return unique


def download_image(url: str, output_dir: Path, human: bool = False) -> str | None:
    """Download an image and return the local filename.

    Args:
        url: Image URL to download.
        output_dir: Directory to save images into.
        human: If True, print progress to stderr.

    Returns:
        Local filename (not full path) on success, None on failure.
    """
    output_dir.mkdir(parents=True, exist_ok=True)

    # Derive filename from URL
    parsed = urllib.parse.urlparse(url)
    filename = Path(parsed.path).name
    if not filename:
        filename = "image.jpg"

    # Ensure unique filename
    local_path = output_dir / filename
    counter = 1
    while local_path.exists():
        stem = Path(filename).stem
        suffix = Path(filename).suffix
        local_path = output_dir / f"{stem}_{counter}{suffix}"
        counter += 1

    try:
        response = _rate_limited_get(url, timeout=30)
        if response.status_code == 200:
            # Check Content-Length header first, then actual body size.
            content_length = response.headers.get("Content-Length")
            if content_length is not None and int(content_length) > MAX_IMAGE_BYTES:
                if human:
                    print(f"    Skipped (too large: {content_length} bytes): {url}", file=sys.stderr)
                return None
            if len(response.content) > MAX_IMAGE_BYTES:
                if human:
                    print(f"    Skipped (too large: {len(response.content)} bytes): {url}", file=sys.stderr)
                return None
            local_path.write_bytes(response.content)
            if human:
                print(f"    Downloaded: {local_path.name}", file=sys.stderr)
            return local_path.name
        else:
            if human:
                print(f"    Failed to download {url}: HTTP {response.status_code}", file=sys.stderr)
            return None
    except requests.exceptions.RequestException as e:
        if human:
            print(f"    Failed to download {url}: {e}", file=sys.stderr)
        return None


def download_and_replace_images(
    html: str,
    markdown: str,
    output_dir: Path,
    human: bool = False,
) -> str:
    """Download all images from HTML content and replace URLs in markdown.

    Args:
        html: Original HTML content (for extracting image URLs).
        markdown: Converted markdown content.
        output_dir: Base output directory (images saved to output_dir/images/).
        human: If True, print progress to stderr.

    Returns:
        Markdown with image URLs replaced with local paths.
    """
    image_urls = extract_image_urls(html)
    if not image_urls:
        return markdown

    images_dir = output_dir / "images"
    updated_markdown = markdown

    for url in image_urls:
        local_name = download_image(url, images_dir, human=human)
        if local_name:
            updated_markdown = updated_markdown.replace(url, f"images/{local_name}")

    return updated_markdown


# ---------------------------------------------------------------------------
# Post processing and file output
# ---------------------------------------------------------------------------


def get_featured_image_url(post: dict[str, Any]) -> str | None:
    """Extract featured image URL from a WordPress.com API post.

    Args:
        post: Post dict from the WordPress.com public API.

    Returns:
        Featured image URL or None.
    """
    # WordPress.com API puts featured image URL directly on the post
    featured = post.get("featured_image")
    if featured:
        return featured

    # Fallback: post_thumbnail object
    thumbnail = post.get("post_thumbnail")
    if thumbnail and isinstance(thumbnail, dict):
        return thumbnail.get("URL")

    return None


def get_author_name(post: dict[str, Any]) -> str:
    """Extract author display name from a WordPress.com API post.

    Args:
        post: Post dict from the WordPress.com public API.

    Returns:
        Author display name, or "Unknown" if not available.
    """
    author = post.get("author", {})
    if isinstance(author, dict):
        # Prefer first_name + last_name, fall back to name field
        first = author.get("first_name", "")
        last = author.get("last_name", "")
        if first or last:
            return f"{first} {last}".strip()
        return author.get("name", "Unknown")
    return "Unknown"


def extract_categories(post: dict[str, Any]) -> list[str]:
    """Extract category names from a WordPress.com API post.

    WordPress.com API returns categories as an object keyed by slug.

    Args:
        post: Post dict from the WordPress.com public API.

    Returns:
        List of category name strings.
    """
    cats = post.get("categories", {})
    if isinstance(cats, dict):
        return [unescape(cat.get("name", slug)) for slug, cat in cats.items()]
    return []


def extract_tags(post: dict[str, Any]) -> list[str]:
    """Extract tag names from a WordPress.com API post.

    WordPress.com API returns tags as an object keyed by slug.

    Args:
        post: Post dict from the WordPress.com public API.

    Returns:
        List of tag name strings.
    """
    tags = post.get("tags", {})
    if isinstance(tags, dict):
        return [unescape(tag.get("name", slug)) for slug, tag in tags.items()]
    return []


def build_frontmatter(post: dict[str, Any], source_url: str) -> str:
    """Build YAML frontmatter string for a post.

    Args:
        post: Post dict from the WordPress.com public API.
        source_url: Blog base URL for reference.

    Returns:
        YAML frontmatter string including opening/closing --- delimiters.
    """
    title = unescape(post.get("title", "Untitled"))
    # Collapse control characters (CR, LF, tabs, etc.) to a single space so
    # a crafted title cannot inject extra YAML keys.
    title = re.sub(r"[\x00-\x1f\x7f]+", " ", title)
    # Escape quotes in title for YAML safety
    title = title.replace('"', '\\"')

    date = post.get("date", "")
    slug = post.get("slug", "")
    author = get_author_name(post)
    featured_image = get_featured_image_url(post)

    categories = extract_categories(post)
    tags = extract_tags(post)

    original_url = post.get("URL", f"{source_url.rstrip('/')}/{slug}/")

    lines = [
        "---",
        f'title: "{title}"',
        f'date: "{date}"',
        f'author: "{author}"',
    ]

    if categories:
        cat_list = ", ".join(f'"{c}"' for c in categories)
        lines.append(f"categories: [{cat_list}]")

    if tags:
        tag_list = ", ".join(f'"{t}"' for t in tags)
        lines.append(f"tags: [{tag_list}]")

    lines.append(f'slug: "{slug}"')
    lines.append(f'original_url: "{original_url}"')

    if featured_image:
        lines.append(f'featured_image: "{featured_image}"')

    lines.append("---")
    return "\n".join(lines)


def post_to_filename(post: dict[str, Any]) -> str:
    """Generate a filename for a post based on date and slug.

    Format: YYYY-MM-DD-slug.md

    Args:
        post: Post dict from the WordPress.com public API.

    Returns:
        Filename string.
    """
    date_str = post.get("date", "")[:10]  # YYYY-MM-DD
    slug = post.get("slug", "untitled")

    # Sanitize slug for filesystem safety
    slug = re.sub(r"[^\w\-]", "-", slug)
    slug = re.sub(r"-{2,}", "-", slug).strip("-")

    if date_str:
        return f"{date_str}-{slug}.md"
    return f"{slug}.md"


def save_post(
    post: dict[str, Any],
    output_dir: Path,
    source_url: str,
    download_images: bool = False,
    human: bool = False,
) -> str | None:
    """Convert a post to markdown and save to disk.

    Args:
        post: Post dict from the WordPress.com public API.
        output_dir: Directory to write markdown files.
        source_url: Blog base URL.
        download_images: If True, download images and rewrite URLs.
        human: If True, print progress to stderr.

    Returns:
        Filename of saved post, or None on failure.
    """
    filename = post_to_filename(post)
    file_path = output_dir / filename

    # WordPress.com API: content is a direct string field
    html_content = post.get("content", "")
    markdown_body = html_to_markdown(html_content)

    # Download images if requested
    if download_images:
        markdown_body = download_and_replace_images(html_content, markdown_body, output_dir, human=human)

        # Also download featured image
        featured_url = get_featured_image_url(post)
        if featured_url:
            images_dir = output_dir / "images"
            download_image(featured_url, images_dir, human=human)

    frontmatter = build_frontmatter(post, source_url)
    full_content = f"{frontmatter}\n\n{markdown_body}\n"

    file_path.write_text(full_content, encoding="utf-8")
    return filename


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------


def main() -> int:
    """CLI entry point for WordPress scraper."""
    parser = argparse.ArgumentParser(
        description="Scrape all posts from a WordPress.com blog and save as markdown files",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__,
    )
    parser.add_argument(
        "--source",
        required=True,
        help="WordPress.com blog URL (e.g., https://example-blog.wordpress.com)",
    )
    parser.add_argument("--output", required=True, help="Directory to save markdown files")
    parser.add_argument("--limit", type=int, default=None, help="Maximum number of posts to scrape (for testing)")
    parser.add_argument(
        "--download-images", action="store_true", help="Download images and rewrite URLs to local paths"
    )
    parser.add_argument("--skip-existing", action="store_true", help="Skip posts that already have a markdown file")
    parser.add_argument("--human", action="store_true", help="Human-readable progress output instead of JSON")

    args = parser.parse_args()

    source_url = args.source.rstrip("/")
    output_dir = Path(args.output)
    output_dir.mkdir(parents=True, exist_ok=True)

    if args.human:
        print(f"Scraping: {source_url}", file=sys.stderr)
        print(f"Output:   {output_dir}", file=sys.stderr)
        if args.limit:
            print(f"Limit:    {args.limit} posts", file=sys.stderr)
        print(file=sys.stderr)

    # Fetch all posts
    if args.human:
        print("Fetching posts...", file=sys.stderr)

    posts = fetch_all_posts(source_url, post_type="post", limit=args.limit, human=args.human)

    if args.human:
        print(f"  Found {len(posts)} posts", file=sys.stderr)
        print(file=sys.stderr)

    # Also fetch pages
    if args.human:
        print("Fetching pages...", file=sys.stderr)

    pages = fetch_all_posts(source_url, post_type="page", limit=None, human=args.human)

    if args.human:
        print(f"  Found {len(pages)} pages", file=sys.stderr)
        print(file=sys.stderr)

    # Process all items
    all_items = posts + pages
    saved: list[str] = []
    skipped: list[str] = []
    errors: list[str] = []

    if args.human:
        print(f"Processing {len(all_items)} items...", file=sys.stderr)

    for i, post in enumerate(all_items, 1):
        title = unescape(post.get("title", "Untitled"))
        filename = post_to_filename(post)

        # Skip existing files if requested
        if args.skip_existing and (output_dir / filename).exists():
            skipped.append(filename)
            if args.human:
                print(f"  [{i}/{len(all_items)}] SKIP: {title}", file=sys.stderr)
            continue

        if args.human:
            print(f"  [{i}/{len(all_items)}] {title}", file=sys.stderr)

        try:
            saved_name = save_post(
                post=post,
                output_dir=output_dir,
                source_url=source_url,
                download_images=args.download_images,
                human=args.human,
            )
            if saved_name:
                saved.append(saved_name)
            else:
                errors.append(f"Failed to save: {title}")
        except Exception as e:
            errors.append(f"{title}: {e}")
            if args.human:
                print(f"    ERROR: {e}", file=sys.stderr)

    # Output summary
    result: dict[str, Any] = {
        "status": "success" if not errors else "partial",
        "source": source_url,
        "output_dir": str(output_dir),
        "posts_found": len(posts),
        "pages_found": len(pages),
        "saved": len(saved),
        "skipped": len(skipped),
        "errors": len(errors),
        "files": saved,
    }

    if errors:
        result["error_details"] = errors

    if args.human:
        print(file=sys.stderr)
        print("Done!", file=sys.stderr)
        print(f"  Posts found:  {len(posts)}", file=sys.stderr)
        print(f"  Pages found:  {len(pages)}", file=sys.stderr)
        print(f"  Saved:        {len(saved)}", file=sys.stderr)
        if skipped:
            print(f"  Skipped:      {len(skipped)}", file=sys.stderr)
        if errors:
            print(f"  Errors:       {len(errors)}", file=sys.stderr)
            for err in errors:
                print(f"    - {err}", file=sys.stderr)
    else:
        print(json.dumps(result, indent=2))

    return 0 if not errors else 1


if __name__ == "__main__":
    sys.exit(main())
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 197kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall