Skip to content

JimLiu/baoyu-skills / skills/baoyu-url-to-markdown

baoyu-url-to-markdown

Verified

Fetch any URL and convert to markdown using baoyu-fetch CLI (Chrome CDP with site-specific adapters). Built-in adapters for X/Twitter, YouTube transcripts, Hacker News threads, and generic pages via Defuddle. Handles login/CAPTCHA via interaction wait modes. Use when user wants to save a webpage as markdown.

FileKindSize
SKILL.mdskill md6.1 KB
references/adapters.mdmarkdown2.5 KB
references/config/first-time-setup.mdmarkdown1.9 KB
references/quality-gate.mdmarkdown2.1 KB
scripts/baoyu-fetchexecutablescript122 B
scripts/bun.lockother34.6 KB
scripts/lib/adapters/generic/index.tsscript2.4 KB
scripts/lib/adapters/hn/index.tsscript10.2 KB
scripts/lib/adapters/index.tsscript834 B
scripts/lib/adapters/types.tsscript2.1 KB
scripts/lib/adapters/x/article.tsscript12.4 KB
scripts/lib/adapters/x/index.tsscript4.0 KB
scripts/lib/adapters/x/login.tsscript2.2 KB
scripts/lib/adapters/x/match.tsscript297 B
scripts/lib/adapters/x/payloads.tsscript1.6 KB
scripts/lib/adapters/x/session.tsscript1.4 KB
scripts/lib/adapters/x/shared.tsscript12.7 KB
scripts/lib/adapters/x/single.tsscript2.5 KB
scripts/lib/adapters/x/thread-loader.tsscript8.9 KB
scripts/lib/adapters/x/thread.tsscript9.0 KB
scripts/lib/adapters/x/types.tsscript600 B
scripts/lib/adapters/youtube/index.tsscript898 B
scripts/lib/adapters/youtube/transcript.tsscript13.0 KB
scripts/lib/adapters/youtube/utils.tsscript6.5 KB
scripts/lib/browser/cdp-client.tsscript6.7 KB
scripts/lib/browser/chrome-launcher.tsscript5.7 KB
scripts/lib/browser/cookie-sidecar.tsscript2.9 KB
scripts/lib/browser/interaction-gates.tsscript4.6 KB
scripts/lib/browser/network-journal.tsscript7.0 KB
scripts/lib/browser/page-snapshot.tsscript3.3 KB
scripts/lib/browser/profile.tsscript5.5 KB
scripts/lib/browser/session.tsscript4.5 KB
scripts/lib/cli.tsexecutablescript6.9 KB
scripts/lib/commands/convert.tsscript17.0 KB
scripts/lib/extract/document.tsscript846 B
scripts/lib/extract/html-cleaner.tsscript10.8 KB
scripts/lib/extract/html-extractor.tsscript2.2 KB
scripts/lib/extract/html-to-markdown.tsscript20.8 KB
scripts/lib/extract/markdown-renderer.tsscript4.6 KB
scripts/lib/media/default-downloader.tsscript5.1 KB
scripts/lib/media/markdown-media.tsscript13.3 KB
scripts/lib/media/media-utils.tsscript6.1 KB
scripts/lib/media/types.tsscript710 B
scripts/lib/types/defuddle-node.d.tsscript605 B
scripts/lib/types/shims.d.tsscript422 B
scripts/lib/utils/logger.tsscript645 B
scripts/lib/utils/url.tsscript300 B
scripts/package.jsonmanifest557 B

scripts/lib/extract/markdown-renderer.ts

View on GitHub
import TurndownService from "turndown";
import { gfm } from "turndown-plugin-gfm";
import { normalizeMarkdownMediaLinks } from "../media/markdown-media";
import type { ContentBlock, ExtractedDocument } from "./document";

const turndownService = new TurndownService({
  codeBlockStyle: "fenced",
  headingStyle: "atx",
  bulletListMarker: "-",
});

turndownService.use(gfm);

function renderBlock(block: ContentBlock): string {
  switch (block.type) {
    case "paragraph":
      return block.text.trim();
    case "heading":
      return `${"#".repeat(Math.min(Math.max(block.depth, 1), 6))} ${block.text.trim()}`;
    case "list":
      return block.items
        .map((item, index) => (block.ordered ? `${index + 1}. ${item.trim()}` : `- ${item.trim()}`))
        .join("\n");
    case "quote":
      return block.text
        .split("\n")
        .map((line) => `> ${line}`)
        .join("\n");
    case "code":
      return `\`\`\`${block.language ?? ""}\n${block.code.trimEnd()}\n\`\`\``;
    case "image":
      return `![${block.alt ?? ""}](${block.url})`;
    case "html":
      return turndownService.turndown(block.html).trim();
    case "markdown":
      return block.markdown.trim();
  }
}

function isDefinedValue(value: unknown): boolean {
  return value !== undefined && value !== null && value !== "";
}

function renderFrontmatterValue(value: unknown): string {
  if (typeof value === "string") {
    if (value.includes("\n")) {
      return `|-\n${value
        .replace(/\r\n/g, "\n")
        .split("\n")
        .map((line) => `  ${line}`)
        .join("\n")}`;
    }
    return JSON.stringify(value);
  }
  if (typeof value === "number" || typeof value === "boolean") {
    return String(value);
  }
  return JSON.stringify(value);
}

function renderFrontmatter(document: ExtractedDocument): string {
  const fields = new Map<string, unknown>();
  const preferredOrder = [
    "title",
    "url",
    "requestedUrl",
    "author",
    "authorName",
    "authorUsername",
    "authorUrl",
    "coverImage",
    "siteName",
    "publishedAt",
    "summary",
    "adapter",
  ];

  fields.set("title", document.title);
  fields.set("url", document.canonicalUrl ?? document.url);
  fields.set("requestedUrl", document.requestedUrl ?? document.url);
  fields.set("author", document.author);
  fields.set("siteName", document.siteName);
  fields.set("publishedAt", document.publishedAt);
  fields.set("summary", document.summary);
  fields.set("adapter", document.adapter);

  for (const [key, value] of Object.entries(document.metadata ?? {})) {
    if (!fields.has(key)) {
      fields.set(key, value);
    }
  }

  const orderedKeys = [
    ...preferredOrder.filter((key) => fields.has(key)),
    ...Array.from(fields.keys()).filter((key) => !preferredOrder.includes(key)).sort(),
  ];

  const lines = orderedKeys
    .map((key) => [key, fields.get(key)] as const)
    .filter(([, value]) => isDefinedValue(value))
    .map(([key, value]) => `${key}: ${renderFrontmatterValue(value)}`);

  if (lines.length === 0) {
    return "";
  }

  return `---\n${lines.join("\n")}\n---`;
}

function cleanMarkdown(markdown: string): string {
  return normalizeMarkdownMediaLinks(markdown.replace(/\n{3,}/g, "\n\n").trim());
}

function normalizeComparableTitle(value: string): string {
  return value
    .trim()
    .toLowerCase()
    .replace(/^>\s*/, "")
    .replace(/^#+\s+/, "")
    .replace(/(?:\.{3}|…)\s*$/, "");
}

function bodyStartsWithTitle(body: string, title: string): boolean {
  const firstMeaningfulLine = body
    .replace(/\r\n/g, "\n")
    .split("\n")
    .map((line) => line.trim())
    .find((line) => line && !/^!?\[[^\]]*\]\([^)]+\)$/.test(line));

  if (!firstMeaningfulLine) {
    return false;
  }

  const comparableTitle = normalizeComparableTitle(title);
  const comparableFirstLine = normalizeComparableTitle(firstMeaningfulLine);
  if (!comparableTitle || !comparableFirstLine) {
    return false;
  }

  return (
    comparableFirstLine === comparableTitle ||
    comparableFirstLine.startsWith(comparableTitle) ||
    comparableTitle.startsWith(comparableFirstLine)
  );
}

export function renderMarkdown(document: ExtractedDocument): string {
  const sections: string[] = [];
  const frontmatter = renderFrontmatter(document);

  if (frontmatter) {
    sections.push(frontmatter);
  }

  const body = document.content
    .map((block) => renderBlock(block))
    .filter(Boolean)
    .join("\n\n");

  if (document.title && !bodyStartsWithTitle(body, document.title)) {
    sections.push(`# ${document.title}`);
  }

  if (body) {
    sections.push(body);
  }

  return cleanMarkdown(sections.join("\n\n"));
}
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall