Skip to content

IvanCampos/agents / skills/opml-reader

opml-reader

Verified

Store OPML files and fetch latest posts across RSS/Atom feeds. Use when a user provides an OPML URL or file path, wants it saved under ~/Code/Files, or wants the newest entries aggregated across feeds (with optional JSON/TSV output, dedupe, and concurrency controls).

FileKindSize
SKILL.mdskill md1.8 KB
agents/openai.yamlconfig224 B
scripts/opml_reader.pyexecutablescript6.3 KB

scripts/opml_reader.py

View on GitHub
#!/usr/bin/env python3
import argparse
import calendar
import json
import os
import socket
import sys
import urllib.request
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime

try:
    import feedparser
except Exception:
    feedparser = None

USER_AGENT = "Mozilla/5.0 (opml-reader; +https://example.com)"


def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)


def ensure_feedparser():
    if feedparser is None:
        eprint("feedparser is required. Install in a venv:")
        eprint("  python3 -m venv ~/Code/sandbox/.venv-opml-reader")
        eprint("  ~/Code/sandbox/.venv-opml-reader/bin/pip install feedparser")
        eprint("Then run using:")
        eprint("  ~/Code/sandbox/.venv-opml-reader/bin/python scripts/opml_reader.py ...")
        sys.exit(2)


def parse_opml(opml_path):
    root = ET.parse(opml_path).getroot()
    feeds = []
    for outline in root.findall('.//outline'):
        url = outline.attrib.get('xmlUrl') or outline.attrib.get('xmlurl')
        if url:
            title = outline.attrib.get('title') or outline.attrib.get('text') or outline.attrib.get('name') or 'Unknown'
            feeds.append((title.strip(), url.strip()))
    return feeds


def fetch_url(url, timeout):
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        return resp.read()


def to_utc(dt):
    if dt.tzinfo is None:
        return dt.replace(tzinfo=timezone.utc)
    return dt.astimezone(timezone.utc)


def parse_date(entry):
    if getattr(entry, 'published_parsed', None):
        return datetime.fromtimestamp(calendar.timegm(entry.published_parsed), tz=timezone.utc)
    if getattr(entry, 'updated_parsed', None):
        return datetime.fromtimestamp(calendar.timegm(entry.updated_parsed), tz=timezone.utc)

    for attr in ("published", "updated", "date"):
        value = getattr(entry, attr, None)
        if not value:
            continue
        try:
            dt = parsedate_to_datetime(value)
            return to_utc(dt)
        except Exception:
            pass
        try:
            dt = datetime.fromisoformat(value)
            return to_utc(dt)
        except Exception:
            pass
    return None


def fetch_feed(title, url, timeout):
    try:
        data = fetch_url(url, timeout)
    except Exception:
        return []
    try:
        parsed = feedparser.parse(data)
    except Exception:
        return []

    items = []
    for entry in parsed.entries:
        dt = parse_date(entry)
        if not dt:
            continue
        items.append({
            "feed": title,
            "title": (getattr(entry, "title", "") or "").strip(),
            "link": (getattr(entry, "link", "") or "").strip(),
            "date": dt,
        })
    return items


def output_items(items, fmt):
    if fmt == "json":
        payload = [
            {"title": i["title"], "feed": i["feed"], "date": i["date"].isoformat(), "link": i["link"]}
            for i in items
        ]
        print(json.dumps(payload, indent=2))
        return
    if fmt == "tsv":
        print("title\tfeed\tdate\tlink")
        for i in items:
            print(f"{i['title']}\t{i['feed']}\t{i['date'].isoformat()}\t{i['link']}")
        return

    for idx, i in enumerate(items, 1):
        print(f"{idx}. {i['title']} | {i['feed']} | {i['date'].isoformat()} | {i['link']}")


def cmd_store(args):
    dest = os.path.expanduser(args.dest)
    if os.path.exists(dest) and not args.overwrite:
        eprint(f"Refusing to overwrite existing file: {dest}")
        sys.exit(1)
    os.makedirs(os.path.dirname(dest), exist_ok=True)
    data = fetch_url(args.url, args.timeout)
    with open(dest, "wb") as f:
        f.write(data)
    print(dest)


def cmd_latest(args):
    ensure_feedparser()
    opml_path = os.path.expanduser(args.opml)
    feeds = parse_opml(opml_path)

    socket.setdefaulttimeout(args.timeout)
    entries = []

    with ThreadPoolExecutor(max_workers=args.workers) as exe:
        futures = {exe.submit(fetch_feed, title, url, args.timeout): (title, url) for title, url in feeds}
        for fut in as_completed(futures):
            try:
                entries.extend(fut.result())
            except Exception:
                continue

    if args.dedupe:
        seen = set()
        unique = []
        for item in entries:
            key = item["link"] or (item["feed"], item["title"], item["date"].isoformat())
            if key in seen:
                continue
            seen.add(key)
            unique.append(item)
        entries = unique

    entries.sort(key=lambda x: x["date"], reverse=True)
    output_items(entries[: args.limit], args.format)


def build_parser():
    parser = argparse.ArgumentParser(description="Store OPML files and fetch latest posts across their feeds.")
    sub = parser.add_subparsers(dest="cmd", required=True)

    store = sub.add_parser("store", help="Download an OPML URL to a local path")
    store.add_argument("url", help="OPML URL")
    store.add_argument("dest", help="Destination path (e.g., ~/Code/Files/feeds.opml)")
    store.add_argument("--timeout", type=int, default=10, help="Download timeout in seconds")
    store.add_argument("--overwrite", action="store_true", help="Overwrite destination if it exists")
    store.set_defaults(func=cmd_store)

    latest = sub.add_parser("latest", help="List latest posts across feeds in an OPML file")
    latest.add_argument("opml", help="Path to OPML file")
    latest.add_argument("--limit", type=int, default=5, help="Number of posts to return")
    latest.add_argument("--format", choices=["text", "json", "tsv"], default="text", help="Output format")
    latest.add_argument("--timeout", type=int, default=8, help="Per-feed fetch timeout in seconds")
    latest.add_argument("--workers", type=int, default=12, help="Concurrent fetch workers")
    latest.add_argument("--dedupe", action="store_true", default=True, help="Deduplicate posts by link")
    latest.add_argument("--no-dedupe", dest="dedupe", action="store_false", help="Disable dedupe")
    latest.set_defaults(func=cmd_latest)

    return parser


def main():
    parser = build_parser()
    args = parser.parse_args()
    args.func(args)


if __name__ == "__main__":
    main()
receiving-code-reviewobra/superpowersUse when receiving code review feedback, before implementing suggestions, especially if feedback seems unclear or technically questionable - requires technical rigor and verification, not performative agreement or blind implementationStars: 322kUpdated Claude CodeCodexVerifiedInstall
requesting-code-reviewobra/superpowersUse when completing tasks, implementing major features, or before merging to verify work meets requirementsStars: 322kUpdated Claude CodeCodexVerifiedInstall
using-git-worktreesobra/superpowersUse when starting feature work that needs isolation from current workspace or before executing implementation plans - ensures an isolated workspace exists via native tools or git worktree fallbackStars: 322kUpdated Claude CodeCodexVerifiedInstall
verification-before-completionobra/superpowersUse when about to claim work is complete, fixed, or passing, before committing or creating PRs - requires running verification commands and confirming output before making any success claims; evidence before assertions alwaysStars: 322kUpdated Claude CodeCodexVerifiedInstall