Skip to content

gooseworks-ai/goose-skills / skills/shared/reddit-post-finder

reddit-post-finder

Verified

Scrape and search Reddit posts using Apify. Use when you need to find Reddit discussions, track competitor mentions, monitor product feedback, discover pain points, or analyze subreddit content. Supports keyword filtering, time-based searches, and subreddit-specific queries.

FileKindSize
SKILL.mdskill md4.8 KB
references/apify-config.mdmarkdown3.2 KB
scripts/search_reddit.pyscript9.1 KB
skill.meta.jsonconfig147 B

scripts/search_reddit.py

View on GitHub
#!/usr/bin/env python3
"""
Search Reddit posts using Apify Reddit Scraper Lite.
Supports keyword filtering, subreddit filtering, and time range filtering.

Usage:
  python3 search_reddit.py --subreddit growthhacking --days 7 --max-posts 20
  python3 search_reddit.py --subreddit "growthhacking,gtmengineering" --days 7 --sort top --time week
  python3 search_reddit.py --subreddit LLMDevs --keywords "Langfuse,Arize" --days 30
"""

import json
import os
import sys
import argparse
import requests
import time as time_mod
from datetime import datetime, timedelta, timezone


ACTOR_ID = "trudax~reddit-scraper-lite"

GOOSEWORKS_API_BASE = os.environ.get("GOOSEWORKS_API_BASE", "https://api.gooseworks.ai")
GOOSEWORKS_API_KEY = os.environ.get("GOOSEWORKS_API_KEY")

if GOOSEWORKS_API_KEY:
    BASE_URL = f"{GOOSEWORKS_API_BASE}/v1/proxy/apify"
else:
    BASE_URL = "https://api.apify.com/v2"


def get_token(cli_token=None):
    """Get API token from CLI arg, GOOSEWORKS_API_KEY, or APIFY_API_TOKEN env var."""
    token = cli_token or GOOSEWORKS_API_KEY or os.environ.get("APIFY_API_TOKEN")
    if not token:
        print("Error: Set GOOSEWORKS_API_KEY or APIFY_API_TOKEN env var.", file=sys.stderr)
        sys.exit(1)
    return token


def build_subreddit_urls(subreddits, sort="top", time="week"):
    """
    Build full Reddit URLs for each subreddit.

    The Apify actor requires startUrls with full URLs, not bare subreddit names.
    Sort and time are embedded in the URL path/query.

    Args:
        subreddits: List of subreddit names (without r/ prefix)
        sort: "hot", "top", "new", or "rising"
        time: "hour", "day", "week", "month", "year", "all" (only used with "top" sort)

    Returns:
        List of {"url": "..."} dicts for the actor's startUrls parameter
    """
    urls = []
    for sub in subreddits:
        sub = sub.strip().strip("/").replace("r/", "").replace("https://www.reddit.com/r/", "")
        if sort == "top":
            url = f"https://www.reddit.com/r/{sub}/top/?t={time}"
        elif sort in ("hot", "new", "rising"):
            url = f"https://www.reddit.com/r/{sub}/{sort}/"
        else:
            url = f"https://www.reddit.com/r/{sub}/top/?t={time}"
        urls.append({"url": url})
    return urls


def run_apify_actor(token, start_urls, max_posts=100, timeout=300):
    """
    Run the Apify Reddit Scraper Lite actor and return results.

    Args:
        token: Apify API token
        start_urls: List of {"url": "..."} dicts
        max_posts: Maximum posts to scrape
        timeout: Max seconds to wait for the run to complete

    Returns:
        List of post dicts from the actor's dataset
    """
    run_input = {
        "startUrls": start_urls,
        "maxItems": max_posts,
    }

    # Start the actor run
    print(f"Starting Apify actor run...", file=sys.stderr)
    resp = requests.post(
        f"{BASE_URL}/acts/{ACTOR_ID}/runs",
        json=run_input,
        params={"token": token},
    )
    resp.raise_for_status()
    run_data = resp.json()
    run_id = run_data["data"]["id"]
    print(f"Run started (ID: {run_id})", file=sys.stderr)

    # Poll for completion
    deadline = time_mod.time() + timeout
    while time_mod.time() < deadline:
        status_resp = requests.get(
            f"{BASE_URL}/acts/{ACTOR_ID}/runs/{run_id}",
            params={"token": token},
        )
        status_resp.raise_for_status()
        status_data = status_resp.json()
        status = status_data["data"]["status"]

        if status == "SUCCEEDED":
            print("Scraping complete.", file=sys.stderr)
            break
        elif status in ("FAILED", "ABORTED", "TIMED-OUT"):
            print(f"Actor run {status}.", file=sys.stderr)
            raise RuntimeError(f"Actor run {status}: {json.dumps(status_data['data'], indent=2)}")

        print(f"Status: {status}...", file=sys.stderr)
        time_mod.sleep(3)
    else:
        raise TimeoutError(f"Actor run did not complete within {timeout}s")

    # Fetch dataset items
    dataset_id = status_data["data"]["defaultDatasetId"]
    dataset_resp = requests.get(
        f"{BASE_URL}/datasets/{dataset_id}/items",
        params={"token": token, "format": "json"},
    )
    dataset_resp.raise_for_status()
    posts = dataset_resp.json()
    print(f"Fetched {len(posts)} posts.", file=sys.stderr)
    return posts


def filter_posts(posts, keywords=None, days_back=None):
    """
    Client-side filtering by keywords and date range.

    Args:
        posts: List of post dicts from Apify
        keywords: Optional list of keywords (OR logic, case-insensitive)
        days_back: Optional number of days; posts older than this are dropped

    Returns:
        Filtered list of posts
    """
    filtered = posts

    # Date filter
    if days_back is not None:
        cutoff = datetime.now(timezone.utc) - timedelta(days=days_back)
        date_filtered = []
        for p in filtered:
            created = p.get("createdAt") or p.get("created_utc")
            if created is None:
                date_filtered.append(p)
                continue
            if isinstance(created, str):
                try:
                    dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
                except ValueError:
                    date_filtered.append(p)
                    continue
            elif isinstance(created, (int, float)):
                dt = datetime.fromtimestamp(created, tz=timezone.utc)
            else:
                date_filtered.append(p)
                continue
            if dt >= cutoff:
                date_filtered.append(p)
        filtered = date_filtered

    # Keyword filter (OR logic)
    if keywords:
        kw_lower = [k.lower() for k in keywords]
        kw_filtered = []
        for p in filtered:
            text = f"{p.get('title', '')} {p.get('body', '')}".lower()
            if any(kw in text for kw in kw_lower):
                kw_filtered.append(p)
        filtered = kw_filtered

    return filtered


def format_summary(posts):
    """Format posts as a human-readable summary table."""
    lines = []
    lines.append(f"{'#':<4} {'Upvotes':<9} {'Comments':<10} {'Subreddit':<20} {'Title'}")
    lines.append("-" * 100)
    for i, p in enumerate(posts, 1):
        title = p.get("title", "")[:60]
        upvotes = p.get("upVotes", 0)
        comments = p.get("numberOfComments", 0)
        sub = p.get("communityName", "")
        lines.append(f"{i:<4} {upvotes:<9} {comments:<10} r/{sub:<18} {title}")
    return "\n".join(lines)


def main():
    parser = argparse.ArgumentParser(
        description="Search Reddit posts using Apify",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  # Top posts from r/growthhacking in last week
  %(prog)s --subreddit growthhacking --days 7 --sort top --time week

  # Hot posts from multiple subreddits
  %(prog)s --subreddit "growthhacking,gtmengineering" --days 7 --sort hot

  # Search with keyword filtering
  %(prog)s --subreddit LLMDevs --keywords "Langfuse,Arize" --days 30

  # Human-readable summary
  %(prog)s --subreddit growthhacking --days 7 --output summary
""",
    )

    parser.add_argument("--subreddit", required=True,
                        help="Subreddit name(s), comma-separated (e.g. 'growthhacking,gtmengineering')")
    parser.add_argument("--keywords", help="Keywords to filter for (comma-separated, OR logic)")
    parser.add_argument("--days", type=int, default=30, help="How many days back to include (default: 30)")
    parser.add_argument("--max-posts", type=int, default=50, help="Max posts to scrape per subreddit (default: 50)")
    parser.add_argument("--sort", choices=["hot", "top", "new", "rising"], default="top",
                        help="Sort order (default: top)")
    parser.add_argument("--time", choices=["hour", "day", "week", "month", "year", "all"], default="week",
                        help="Time window for 'top' sort (default: week)")
    parser.add_argument("--token", help="Apify API token (or set APIFY_API_TOKEN env var)")
    parser.add_argument("--output", choices=["json", "summary"], default="json",
                        help="Output format (default: json)")
    parser.add_argument("--timeout", type=int, default=300,
                        help="Max seconds to wait for Apify run (default: 300)")

    args = parser.parse_args()

    token = get_token(args.token)

    # Parse subreddits
    subreddits = [s.strip() for s in args.subreddit.split(",") if s.strip()]

    # Build URLs
    start_urls = build_subreddit_urls(subreddits, sort=args.sort, time=args.time)
    print(f"Scraping {len(subreddits)} subreddit(s): {', '.join(f'r/{s}' for s in subreddits)}", file=sys.stderr)

    # Run actor
    posts = run_apify_actor(token, start_urls, max_posts=args.max_posts, timeout=args.timeout)

    # Parse keywords
    keywords = None
    if args.keywords:
        keywords = [k.strip() for k in args.keywords.split(",")]

    # Filter
    posts = filter_posts(posts, keywords=keywords, days_back=args.days)

    # Sort by upvotes descending
    posts.sort(key=lambda p: p.get("upVotes", 0), reverse=True)

    # Output
    if args.output == "summary":
        print(format_summary(posts))
    else:
        print(json.dumps(posts, indent=2))


if __name__ == "__main__":
    main()
receiving-code-reviewobra/superpowersUse when receiving code review feedback, before implementing suggestions, especially if feedback seems unclear or technically questionable - requires technical rigor and verification, not performative agreement or blind implementationStars: 322kUpdated Claude CodeCodexVerifiedInstall
requesting-code-reviewobra/superpowersUse when completing tasks, implementing major features, or before merging to verify work meets requirementsStars: 322kUpdated Claude CodeCodexVerifiedInstall
using-git-worktreesobra/superpowersUse when starting feature work that needs isolation from current workspace or before executing implementation plans - ensures an isolated workspace exists via native tools or git worktree fallbackStars: 322kUpdated Claude CodeCodexVerifiedInstall
verification-before-completionobra/superpowersUse when about to claim work is complete, fixed, or passing, before committing or creating PRs - requires running verification commands and confirming output before making any success claims; evidence before assertions alwaysStars: 322kUpdated Claude CodeCodexVerifiedInstall