Skip to content

Parsa-29/loop-config / skills/last30days

last30days

Flagged

Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web.

FileKindSize
SKILL.mdskill md127.7 KB
agents/openai.yamlconfig448 B
assets/aging-portrait.jpegbinary2.7 MB
assets/claude-code-rap.mp3binary2.2 MB
assets/dog-as-human.pngbinary2.3 MB
assets/dog-original.jpegbinary3.8 MB
assets/swimmom-mockup.jpegbinary2.6 MB
references/save-html-brief.mdmarkdown5.8 KB
scripts/briefing.pyscript8.1 KB
scripts/build-skill.shexecutablescript1.3 KB
scripts/compare.shexecutablescript2.2 KB
scripts/evaluate_search_quality.pyscript19.3 KB
scripts/last30days.pyscript43.2 KB
scripts/lib/__init__.pyscript29 B
scripts/lib/bird_x.pyscript17.5 KB
scripts/lib/bluesky.pyscript12.1 KB
scripts/lib/categories.pyscript7.5 KB
scripts/lib/chrome_cookies.pyscript10.7 KB
scripts/lib/cluster.pyscript10.8 KB
scripts/lib/competitors.pyscript7.2 KB
scripts/lib/cookie_extract.pyscript14.1 KB
scripts/lib/dates.pyscript3.1 KB
scripts/lib/dedupe.pyscript3.2 KB
scripts/lib/digg.pyscript13.0 KB
scripts/lib/entity_extract.pyscript4.1 KB
scripts/lib/env.pyscript25.3 KB
scripts/lib/fanout.pyscript3.1 KB
scripts/lib/fusion.pyscript8.4 KB
scripts/lib/github.pyscript36.5 KB
scripts/lib/grounding.pyscript11.4 KB
scripts/lib/hackernews.pyscript11.3 KB
scripts/lib/html_render.pyscript17.4 KB
scripts/lib/http.pyscript11.4 KB
scripts/lib/instagram.pyscript18.4 KB
scripts/lib/log.pyscript831 B
scripts/lib/normalize.pyscript19.4 KB
scripts/lib/perplexity.pyscript4.7 KB
scripts/lib/pinterest.pyscript5.2 KB
scripts/lib/pipeline.pyscript44.8 KB
scripts/lib/planner.pyscript30.5 KB
scripts/lib/polymarket.pyscript29.6 KB
scripts/lib/preflight.pyscript4.4 KB
scripts/lib/providers.pyscript15.4 KB
scripts/lib/quality_nudge.pyscript13.1 KB
scripts/lib/query.pyscript3.9 KB
scripts/lib/reddit.pyscript24.1 KB
scripts/lib/reddit_enrich.pyscript9.3 KB
scripts/lib/reddit_keyless.pyscript9.9 KB
scripts/lib/reddit_listing.pyscript6.2 KB
scripts/lib/reddit_public.pyscript9.2 KB
scripts/lib/reddit_rss.pyscript7.6 KB
scripts/lib/reddit_shreddit.pyscript6.3 KB
scripts/lib/relevance.pyscript5.8 KB
scripts/lib/render.pyscript69.9 KB
scripts/lib/rerank.pyscript16.7 KB
scripts/lib/resolve.pyscript12.2 KB
scripts/lib/safari_cookies.pyscript5.6 KB
scripts/lib/schema.pyscript10.9 KB
scripts/lib/setup_wizard.pyscript18.3 KB
scripts/lib/signals.pyscript9.5 KB
scripts/lib/skill_meta.pyscript1.2 KB
scripts/lib/snippet.pyscript1.5 KB
scripts/lib/subproc.pyscript2.7 KB
scripts/lib/threads.pyscript6.6 KB
scripts/lib/tiktok.pyscript20.5 KB
scripts/lib/truthsocial.pyscript5.1 KB
scripts/lib/ui.pyscript25.3 KB
scripts/lib/xai_x.pyscript7.3 KB
scripts/lib/xiaohongshu_api.pyscript5.2 KB
scripts/lib/xquik.pyscript6.7 KB
scripts/lib/xurl_x.pyscript5.2 KB
scripts/lib/youtube_yt.pyscript36.5 KB
scripts/setup-keychain.shexecutablescript3.2 KB
scripts/store.pyscript31.1 KB
scripts/test-v1-vs-v2.shexecutablescript5.8 KB
scripts/test_device_auth.pyscript3.9 KB
scripts/verify_v3.pyscript6.0 KB
scripts/watchlist.pyscript9.1 KB

scripts/lib/reddit_shreddit.py

View on GitHub
"""Keyless Reddit comment enrichment via shreddit /svc endpoints.

Reddit's ``{thread}.json`` endpoint now returns HTTP 403. The shreddit partial
endpoint ``/svc/shreddit/comments/r/{sub}/t3_{id}`` still serves HTTP 200 HTML
with no API key, embedding each comment as a ``<shreddit-comment>`` custom
element whose start-tag attributes carry ``score`` / ``author`` / ``created`` /
``permalink``, and whose body lives in a ``<div id="{thingId}-post-rtjson-content">``
block. This module parses that markup into top comments, matching the
``top_comments`` / ``comment_insights`` shape produced by ``reddit_enrich`` so
the renderer is unaffected.

Limitation: the comments endpoint carries the real comment count
(``total-comments``) but not the post's upvote score, so post-level ``score``
cannot be recovered keylessly here (ScrapeCreators backup still provides it).
"""

import html as _html
import re
import sys
from datetime import datetime
from typing import Any, Dict, List, Optional

from . import http
from . import reddit_enrich

# Up to N posts enriched per run, by depth (mirrors reddit_public.ENRICH_LIMITS).
ENRICH_LIMITS = {
    "quick": 3,
    "default": 5,
    "deep": 8,
}

# Max comments returned per post (independent of how many posts get enriched).
MAX_COMMENTS = 10

SVC_TIMEOUT = 12

# Match the exact <shreddit-comment> element start tag, not <shreddit-comment-tree>
# or <shreddit-comment-tree-stats> (lookahead requires whitespace or '>').
_COMMENT_START = re.compile(r"<shreddit-comment(?=[\s>])[^>]*>")
_TOTAL_COMMENTS = re.compile(r'total-comments="(\d+)"')
_PARA = re.compile(r"<p[^>]*>(.*?)</p>", re.S)
_TAG = re.compile(r"<[^>]+>")
_WS = re.compile(r"\s+")
_NEXT_RTJSON = re.compile(r'id="t1_[A-Za-z0-9]+-(?:comment|post)-rtjson-content"')


def _log(msg: str) -> None:
    sys.stderr.write(f"[RedditShreddit] {msg}\n")
    sys.stderr.flush()


def extract_post_ref(url: str) -> Optional[tuple]:
    """Return (subreddit, post_id) from a Reddit thread URL, or None."""
    m = re.search(r"/r/([^/]+)/comments/([A-Za-z0-9]+)", url or "")
    if not m:
        return None
    return m.group(1), m.group(2)


def _svc_url(subreddit: str, post_id: str) -> str:
    # sort=top guarantees Reddit front-loads the highest-scored comments on the
    # first page, so the true top comments are captured even on huge threads
    # (we still re-sort by score locally as a backstop).
    return (
        f"https://www.reddit.com/svc/shreddit/comments/r/{subreddit}/t3_{post_id}"
        f"?sort=top"
    )


def _attr(tag: str, name: str) -> str:
    m = re.search(rf'\b{name}="([^"]*)"', tag)
    return _html.unescape(m.group(1)) if m else ""


def _iso_to_date(value: str) -> Optional[str]:
    if not value:
        return None
    try:
        return datetime.fromisoformat(value.strip()).date().isoformat()
    except (ValueError, TypeError):
        return None


def _body_for(html_text: str, thing_id: str) -> str:
    """Extract a comment's text body, anchored on its unique thingId.

    The body div id embeds the comment's thingId, so this assigns body→comment
    correctly even for nested replies. The slice is bounded by the next
    comment's rtjson anchor to avoid swallowing child-comment text.
    """
    if not thing_id:
        return ""
    anchor = f'id="{thing_id}-post-rtjson-content"'
    idx = html_text.find(anchor)
    if idx == -1:
        return ""
    window = html_text[idx + len(anchor): idx + len(anchor) + 8000]
    nxt = _NEXT_RTJSON.search(window)
    if nxt:
        window = window[: nxt.start()]
    paras = _PARA.findall(window)
    if not paras:
        return ""
    text = " ".join(_TAG.sub("", p) for p in paras)
    return _WS.sub(" ", _html.unescape(text)).strip()


def parse_comments(html_text: str, limit: int = MAX_COMMENTS) -> List[Dict[str, Any]]:
    """Parse <shreddit-comment> elements into scored comment dicts (sorted desc)."""
    comments: List[Dict[str, Any]] = []
    for m in _COMMENT_START.finditer(html_text or ""):
        tag = m.group(0)
        author = _attr(tag, "author") or "[deleted]"
        if author in ("[deleted]", "[removed]"):
            continue
        thing_id = _attr(tag, "thingId")
        body = _body_for(html_text, thing_id)
        if not body or body in ("[deleted]", "[removed]"):
            continue
        try:
            score = int(_attr(tag, "score") or 0)
        except ValueError:
            score = 0
        permalink = _attr(tag, "permalink")
        comments.append({
            "score": score,
            "author": author,
            "body": body[:300],
            "excerpt": body[:200],
            "permalink": permalink,
            "date": _iso_to_date(_attr(tag, "created")),
            "url": f"https://reddit.com{permalink}" if permalink else "",
        })

    comments.sort(key=lambda c: c.get("score", 0), reverse=True)
    return comments[:limit]


def _total_comments(html_text: str) -> Optional[int]:
    m = _TOTAL_COMMENTS.search(html_text or "")
    return int(m.group(1)) if m else None


def fetch_comments(
    post_url: str,
    timeout: int = SVC_TIMEOUT,
) -> Dict[str, Any]:
    """Fetch and parse top comments for a Reddit post via the shreddit endpoint.

    Args:
        post_url: Reddit thread URL (…/r/{sub}/comments/{id}/…)
        timeout: HTTP timeout in seconds

    Returns:
        Dict with 'top_comments' (list, reddit_enrich shape), 'comment_insights'
        (list[str]), and 'num_comments' (int or None). Empty/None on any
        failure — never raises, so the caller can fall through to SC backup.
    """
    ref = extract_post_ref(post_url)
    if not ref:
        return {"top_comments": [], "comment_insights": [], "num_comments": None}
    sub, post_id = ref

    html_text = http.get_text(_svc_url(sub, post_id), timeout=timeout, accept="text/html")
    if not html_text:
        return {"top_comments": [], "comment_insights": [], "num_comments": None}

    comments = parse_comments(html_text, limit=MAX_COMMENTS)
    insights = reddit_enrich.extract_comment_insights(comments)
    return {
        "top_comments": [
            {
                "score": c["score"],
                "date": c["date"],
                "author": c["author"],
                "excerpt": c["excerpt"],
                "url": c["url"],
            }
            for c in comments
        ],
        "comment_insights": insights,
        "num_comments": _total_comments(html_text),
    }
add-product-e2e-evalpaperclipai/paperclipAdd or extend a Paperclip full-stack runner E2E workflow, fixture, matcher, or report evidence path for local or Daytona execution.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
add-runner-evalpaperclipai/paperclipAdd or extend a Paperclip Runner protocol evaluation definition, roster, assertion, or report fixture with provenance and narrow validation.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
diagnose-why-work-stoppedpaperclipai/paperclipCompatibility entry point for failed, stalled, looping, or over-recovered Paperclip work. Use fix-failed-paperclip-run for the full diagnosis, regression, repair, real verification, and green-PR workflow; honor an explicitly requested diagnosis-only scope.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
fix-failed-paperclip-runpaperclipai/paperclipDiagnose and fix failed, stuck, slow, or incorrectly blocked Paperclip tasks and runs. Use with a task/run URL or failure report to trace the root cause, prove a regression red then green, check second-order effects, verify the real workflow, and deliver a PR with passing checks.Stars: 115kUpdated Claude CodeCodexVerifiedInstall