Skip to content

Parsa-29/loop-config / skills/last30days

last30days

Flagged

Research what people actually say about any topic in the last 30 days. Pulls posts and engagement from Reddit, X, YouTube, TikTok, Hacker News, Polymarket, GitHub, and the web.

FileKindSize
SKILL.mdskill md127.7 KB
agents/openai.yamlconfig448 B
assets/aging-portrait.jpegbinary2.7 MB
assets/claude-code-rap.mp3binary2.2 MB
assets/dog-as-human.pngbinary2.3 MB
assets/dog-original.jpegbinary3.8 MB
assets/swimmom-mockup.jpegbinary2.6 MB
references/save-html-brief.mdmarkdown5.8 KB
scripts/briefing.pyscript8.1 KB
scripts/build-skill.shexecutablescript1.3 KB
scripts/compare.shexecutablescript2.2 KB
scripts/evaluate_search_quality.pyscript19.3 KB
scripts/last30days.pyscript43.2 KB
scripts/lib/__init__.pyscript29 B
scripts/lib/bird_x.pyscript17.5 KB
scripts/lib/bluesky.pyscript12.1 KB
scripts/lib/categories.pyscript7.5 KB
scripts/lib/chrome_cookies.pyscript10.7 KB
scripts/lib/cluster.pyscript10.8 KB
scripts/lib/competitors.pyscript7.2 KB
scripts/lib/cookie_extract.pyscript14.1 KB
scripts/lib/dates.pyscript3.1 KB
scripts/lib/dedupe.pyscript3.2 KB
scripts/lib/digg.pyscript13.0 KB
scripts/lib/entity_extract.pyscript4.1 KB
scripts/lib/env.pyscript25.3 KB
scripts/lib/fanout.pyscript3.1 KB
scripts/lib/fusion.pyscript8.4 KB
scripts/lib/github.pyscript36.5 KB
scripts/lib/grounding.pyscript11.4 KB
scripts/lib/hackernews.pyscript11.3 KB
scripts/lib/html_render.pyscript17.4 KB
scripts/lib/http.pyscript11.4 KB
scripts/lib/instagram.pyscript18.4 KB
scripts/lib/log.pyscript831 B
scripts/lib/normalize.pyscript19.4 KB
scripts/lib/perplexity.pyscript4.7 KB
scripts/lib/pinterest.pyscript5.2 KB
scripts/lib/pipeline.pyscript44.8 KB
scripts/lib/planner.pyscript30.5 KB
scripts/lib/polymarket.pyscript29.6 KB
scripts/lib/preflight.pyscript4.4 KB
scripts/lib/providers.pyscript15.4 KB
scripts/lib/quality_nudge.pyscript13.1 KB
scripts/lib/query.pyscript3.9 KB
scripts/lib/reddit.pyscript24.1 KB
scripts/lib/reddit_enrich.pyscript9.3 KB
scripts/lib/reddit_keyless.pyscript9.9 KB
scripts/lib/reddit_listing.pyscript6.2 KB
scripts/lib/reddit_public.pyscript9.2 KB
scripts/lib/reddit_rss.pyscript7.6 KB
scripts/lib/reddit_shreddit.pyscript6.3 KB
scripts/lib/relevance.pyscript5.8 KB
scripts/lib/render.pyscript69.9 KB
scripts/lib/rerank.pyscript16.7 KB
scripts/lib/resolve.pyscript12.2 KB
scripts/lib/safari_cookies.pyscript5.6 KB
scripts/lib/schema.pyscript10.9 KB
scripts/lib/setup_wizard.pyscript18.3 KB
scripts/lib/signals.pyscript9.5 KB
scripts/lib/skill_meta.pyscript1.2 KB
scripts/lib/snippet.pyscript1.5 KB
scripts/lib/subproc.pyscript2.7 KB
scripts/lib/threads.pyscript6.6 KB
scripts/lib/tiktok.pyscript20.5 KB
scripts/lib/truthsocial.pyscript5.1 KB
scripts/lib/ui.pyscript25.3 KB
scripts/lib/xai_x.pyscript7.3 KB
scripts/lib/xiaohongshu_api.pyscript5.2 KB
scripts/lib/xquik.pyscript6.7 KB
scripts/lib/xurl_x.pyscript5.2 KB
scripts/lib/youtube_yt.pyscript36.5 KB
scripts/setup-keychain.shexecutablescript3.2 KB
scripts/store.pyscript31.1 KB
scripts/test-v1-vs-v2.shexecutablescript5.8 KB
scripts/test_device_auth.pyscript3.9 KB
scripts/verify_v3.pyscript6.0 KB
scripts/watchlist.pyscript9.1 KB

scripts/lib/dedupe.py

View on GitHub
"""Within-source near-duplicate detection."""

from __future__ import annotations

import re

from . import schema

STOPWORDS = frozenset(
    {
        "the",
        "a",
        "an",
        "to",
        "for",
        "how",
        "is",
        "in",
        "of",
        "on",
        "and",
        "with",
        "from",
        "by",
        "at",
        "this",
        "that",
        "it",
        "what",
        "are",
        "do",
        "can",
    }
)


def normalize_text(text: str) -> str:
    text = re.sub(r"[^\w\s]", " ", text.lower())
    return re.sub(r"\s+", " ", text).strip()


def _ngrams_of_normalized(norm: str, n: int = 3) -> set[str]:
    if len(norm) < n:
        return {norm} if norm else set()
    return {norm[index:index + n] for index in range(len(norm) - n + 1)}


def get_ngrams(text: str, n: int = 3) -> set[str]:
    return _ngrams_of_normalized(normalize_text(text), n)


def jaccard_similarity(left: set[str], right: set[str]) -> float:
    if not left or not right:
        return 0.0
    union = left | right
    if not union:
        return 0.0
    return len(left & right) / len(union)


def token_jaccard(text_a: str, text_b: str) -> float:
    tokens_a = {
        token
        for token in normalize_text(text_a).split()
        if len(token) > 1 and token not in STOPWORDS
    }
    tokens_b = {
        token
        for token in normalize_text(text_b).split()
        if len(token) > 1 and token not in STOPWORDS
    }
    return jaccard_similarity(tokens_a, tokens_b)


def hybrid_similarity(text_a: str, text_b: str) -> float:
    return max(
        jaccard_similarity(get_ngrams(text_a), get_ngrams(text_b)),
        token_jaccard(text_a, text_b),
    )


def _tokenize(normalized: str) -> frozenset[str]:
    return frozenset(
        tok for tok in normalized.split()
        if len(tok) > 1 and tok not in STOPWORDS
    )


class _PreparedText:
    """Pre-computed text representations for fast repeated similarity checks."""

    __slots__ = ("ngrams", "tokens")

    def __init__(self, raw: str) -> None:
        norm = normalize_text(raw)
        self.ngrams = _ngrams_of_normalized(norm)
        self.tokens = _tokenize(norm)


def prepared_similarity(a: _PreparedText, b: _PreparedText) -> float:
    return max(
        jaccard_similarity(a.ngrams, b.ngrams),
        jaccard_similarity(a.tokens, b.tokens),
    )


def item_text(item: schema.SourceItem) -> str:
    parts = [item.title, item.body, item.author or "", item.container or ""]
    return " ".join(part for part in parts if part).strip()


def dedupe_items(items: list[schema.SourceItem], threshold: float = 0.7) -> list[schema.SourceItem]:
    """Remove near-duplicates while keeping earlier, better-scored items."""
    kept: list[schema.SourceItem] = []
    kept_prepared: list[_PreparedText] = []
    for item in items:
        text = item_text(item)
        if not text:
            kept.append(item)
            continue
        prep = _PreparedText(text)
        is_duplicate = False
        for existing_prep in kept_prepared:
            if prepared_similarity(prep, existing_prep) >= threshold:
                is_duplicate = True
                break
        if not is_duplicate:
            kept.append(item)
            kept_prepared.append(prep)
    return kept
add-product-e2e-evalpaperclipai/paperclipAdd or extend a Paperclip full-stack runner E2E workflow, fixture, matcher, or report evidence path for local or Daytona execution.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
add-runner-evalpaperclipai/paperclipAdd or extend a Paperclip Runner protocol evaluation definition, roster, assertion, or report fixture with provenance and narrow validation.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
diagnose-why-work-stoppedpaperclipai/paperclipCompatibility entry point for failed, stalled, looping, or over-recovered Paperclip work. Use fix-failed-paperclip-run for the full diagnosis, regression, repair, real verification, and green-PR workflow; honor an explicitly requested diagnosis-only scope.Stars: 115kUpdated Claude CodeCodexVerifiedInstall
fix-failed-paperclip-runpaperclipai/paperclipDiagnose and fix failed, stuck, slow, or incorrectly blocked Paperclip tasks and runs. Use with a task/run URL or failure report to trace the root cause, prove a regression red then green, check second-order effects, verify the real workflow, and deliver a PR with passing checks.Stars: 115kUpdated Claude CodeCodexVerifiedInstall