JimLiu/baoyu-skills / skills/baoyu-url-to-markdown
baoyu-url-to-markdown
VerifiedFetch any URL and convert to markdown using baoyu-fetch CLI (Chrome CDP with site-specific adapters). Built-in adapters for X/Twitter, YouTube transcripts, Hacker News threads, and generic pages via Defuddle. Handles login/CAPTCHA via interaction wait modes. Use when user wants to save a webpage as markdown.
scripts/lib/extract/html-extractor.ts
View on GitHubimport { Readability } from "@mozilla/readability";
import { JSDOM } from "jsdom";
import type { ExtractedDocument } from "./document";
function getMetaContent(document: Document, selectors: string[]): string | undefined {
for (const selector of selectors) {
const value = document.querySelector(selector)?.getAttribute("content")?.trim();
if (value) {
return value;
}
}
return undefined;
}
export function extractDocumentFromHtml(input: {
url: string;
html: string;
adapter?: string;
}): ExtractedDocument {
const dom = new JSDOM(input.html, { url: input.url });
const document = dom.window.document;
const canonicalUrl =
document.querySelector('link[rel="canonical"]')?.getAttribute("href")?.trim() ??
getMetaContent(document, ['meta[property="og:url"]']);
const siteName = getMetaContent(document, [
'meta[property="og:site_name"]',
'meta[name="application-name"]',
]);
const metadataAuthor = getMetaContent(document, [
'meta[name="author"]',
'meta[property="article:author"]',
'meta[name="twitter:creator"]',
]);
const publishedAt = getMetaContent(document, [
'meta[property="article:published_time"]',
'meta[name="pubdate"]',
'meta[name="date"]',
'meta[itemprop="datePublished"]',
]);
const article = new Readability(document).parse();
const title =
article?.title?.trim() ||
getMetaContent(document, ['meta[property="og:title"]']) ||
document.title.trim() ||
undefined;
const summary =
article?.excerpt?.trim() ||
getMetaContent(document, [
'meta[name="description"]',
'meta[property="og:description"]',
'meta[name="twitter:description"]',
]);
const contentHtml =
article?.content?.trim() ||
document.querySelector("main")?.innerHTML?.trim() ||
document.body?.innerHTML?.trim() ||
"";
const author = article?.byline?.trim() || metadataAuthor;
return {
url: input.url,
canonicalUrl,
title,
author,
siteName,
publishedAt,
summary,
adapter: input.adapter ?? "generic",
metadata: {
language: document.documentElement.lang || undefined,
},
content: contentHtml ? [{ type: "html", html: contentHtml }] : [],
};
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |