JimLiu/baoyu-skills / skills/baoyu-url-to-markdown
baoyu-url-to-markdown
VerifiedFetch any URL and convert to markdown using baoyu-fetch CLI (Chrome CDP with site-specific adapters). Built-in adapters for X/Twitter, YouTube transcripts, Hacker News threads, and generic pages via Defuddle. Handles login/CAPTCHA via interaction wait modes. Use when user wants to save a webpage as markdown.
scripts/lib/adapters/generic/index.ts
View on GitHubimport type { Adapter } from "../types";
import { detectInteractionGate } from "../../browser/interaction-gates";
import { captureNormalizedPageSnapshot } from "../../browser/page-snapshot";
import { convertHtmlToMarkdown } from "../../extract/html-to-markdown";
export const genericAdapter: Adapter = {
name: "generic",
match() {
return true;
},
async process(context) {
context.log.info(`Loading ${context.input.url.toString()} with generic adapter`);
await context.browser.goto(context.input.url.toString(), context.timeoutMs);
try {
await context.network.waitForIdle({
idleMs: 1_200,
timeoutMs: Math.min(context.timeoutMs, 15_000),
});
} catch {
context.log.debug("Network idle timed out on initial load; continuing.");
}
await context.browser.scrollToEnd({ maxSteps: 4, delayMs: 300 });
try {
await context.network.waitForIdle({
idleMs: 900,
timeoutMs: Math.min(context.timeoutMs, 10_000),
});
} catch {
context.log.debug("Network idle timed out after scrolling; continuing.");
}
const interaction = await detectInteractionGate(context.browser);
if (interaction) {
return {
status: "needs_interaction",
interaction,
};
}
const snapshot = await captureNormalizedPageSnapshot(context.browser);
const converted = await convertHtmlToMarkdown(snapshot.html, snapshot.finalUrl, {
enableRemoteMarkdownFallback: context.outputFormat === "markdown",
preserveBase64Images: context.downloadMedia,
});
const document = {
url: snapshot.finalUrl,
canonicalUrl: converted.metadata.canonicalUrl,
title: converted.metadata.title,
author: converted.metadata.author,
siteName: converted.metadata.siteName,
publishedAt: converted.metadata.publishedAt,
summary: converted.metadata.summary,
adapter: "generic",
metadata: {
coverImage: converted.metadata.coverImage,
language: converted.metadata.language,
capturedAt: converted.metadata.capturedAt,
conversionMethod: converted.conversionMethod,
fallbackReason: converted.fallbackReason,
kind: "generic/article",
},
content: converted.markdown ? [{ type: "markdown" as const, markdown: converted.markdown }] : [],
};
return {
status: "ok",
document,
media: converted.media,
};
},
};
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |