Skip to content

JimLiu/baoyu-skills / skills/baoyu-url-to-markdown

baoyu-url-to-markdown

Verified

Fetch any URL and convert to markdown using baoyu-fetch CLI (Chrome CDP with site-specific adapters). Built-in adapters for X/Twitter, YouTube transcripts, Hacker News threads, and generic pages via Defuddle. Handles login/CAPTCHA via interaction wait modes. Use when user wants to save a webpage as markdown.

FileKindSize
SKILL.mdskill md6.1 KB
references/adapters.mdmarkdown2.5 KB
references/config/first-time-setup.mdmarkdown1.9 KB
references/quality-gate.mdmarkdown2.1 KB
scripts/baoyu-fetchexecutablescript122 B
scripts/bun.lockother34.6 KB
scripts/lib/adapters/generic/index.tsscript2.4 KB
scripts/lib/adapters/hn/index.tsscript10.2 KB
scripts/lib/adapters/index.tsscript834 B
scripts/lib/adapters/types.tsscript2.1 KB
scripts/lib/adapters/x/article.tsscript12.4 KB
scripts/lib/adapters/x/index.tsscript4.0 KB
scripts/lib/adapters/x/login.tsscript2.2 KB
scripts/lib/adapters/x/match.tsscript297 B
scripts/lib/adapters/x/payloads.tsscript1.6 KB
scripts/lib/adapters/x/session.tsscript1.4 KB
scripts/lib/adapters/x/shared.tsscript12.7 KB
scripts/lib/adapters/x/single.tsscript2.5 KB
scripts/lib/adapters/x/thread-loader.tsscript8.9 KB
scripts/lib/adapters/x/thread.tsscript9.0 KB
scripts/lib/adapters/x/types.tsscript600 B
scripts/lib/adapters/youtube/index.tsscript898 B
scripts/lib/adapters/youtube/transcript.tsscript13.0 KB
scripts/lib/adapters/youtube/utils.tsscript6.5 KB
scripts/lib/browser/cdp-client.tsscript6.7 KB
scripts/lib/browser/chrome-launcher.tsscript5.7 KB
scripts/lib/browser/cookie-sidecar.tsscript2.9 KB
scripts/lib/browser/interaction-gates.tsscript4.6 KB
scripts/lib/browser/network-journal.tsscript7.0 KB
scripts/lib/browser/page-snapshot.tsscript3.3 KB
scripts/lib/browser/profile.tsscript5.5 KB
scripts/lib/browser/session.tsscript4.5 KB
scripts/lib/cli.tsexecutablescript6.9 KB
scripts/lib/commands/convert.tsscript17.0 KB
scripts/lib/extract/document.tsscript846 B
scripts/lib/extract/html-cleaner.tsscript10.8 KB
scripts/lib/extract/html-extractor.tsscript2.2 KB
scripts/lib/extract/html-to-markdown.tsscript20.8 KB
scripts/lib/extract/markdown-renderer.tsscript4.6 KB
scripts/lib/media/default-downloader.tsscript5.1 KB
scripts/lib/media/markdown-media.tsscript13.3 KB
scripts/lib/media/media-utils.tsscript6.1 KB
scripts/lib/media/types.tsscript710 B
scripts/lib/types/defuddle-node.d.tsscript605 B
scripts/lib/types/shims.d.tsscript422 B
scripts/lib/utils/logger.tsscript645 B
scripts/lib/utils/url.tsscript300 B
scripts/package.jsonmanifest557 B

scripts/lib/media/default-downloader.ts

View on GitHub
import path from "node:path";
import { mkdir, writeFile } from "node:fs/promises";
import {
  buildFileName,
  isDataUri,
  normalizeContentType,
  normalizeMediaUrl,
  resolveExtensionFromContentType,
  resolveExtensionFromUrl,
  resolveMediaKind,
  resolveOutputExtension,
  toPosixPath,
} from "./media-utils";
import type { MediaAsset, MediaDownloadRequest, MediaDownloadResult, MediaKind } from "./types";

const DOWNLOAD_USER_AGENT =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";

function parseBase64DataUri(rawUrl: string): { contentType: string; bytes: Buffer } | null {
  const match = rawUrl.match(/^data:([^;,]+);base64,([A-Za-z0-9+/=\s]+)$/i);
  if (!match?.[1] || !match[2]) {
    return null;
  }

  const contentType = normalizeContentType(match[1]);
  if (!contentType) {
    return null;
  }

  try {
    const bytes = Buffer.from(match[2].replace(/\s+/g, ""), "base64");
    if (bytes.length === 0) {
      return null;
    }
    return { contentType, bytes };
  } catch {
    return null;
  }
}

function dedupeMedia(media: MediaAsset[]): MediaAsset[] {
  const deduped: MediaAsset[] = [];
  const seen = new Set<string>();
  for (const item of media) {
    const normalizedUrl = normalizeMediaUrl(item.url);
    if (!normalizedUrl || seen.has(normalizedUrl)) {
      continue;
    }
    seen.add(normalizedUrl);
    deduped.push({
      ...item,
      url: normalizedUrl,
    });
  }
  return deduped;
}

function toRelativePath(fromDir: string, absoluteTarget: string): string {
  const relative = path.relative(fromDir, absoluteTarget) || path.basename(absoluteTarget);
  return toPosixPath(relative);
}

export async function downloadMediaAssets(
  request: MediaDownloadRequest,
): Promise<MediaDownloadResult> {
  const dedupedMedia = dedupeMedia(request.media);
  const absoluteOutputPath = path.resolve(request.outputPath);
  const markdownDir = path.dirname(absoluteOutputPath);
  const baseDir = request.mediaDir ? path.resolve(request.mediaDir) : markdownDir;
  const replacements: MediaDownloadResult["replacements"] = [];

  let downloadedImages = 0;
  let downloadedVideos = 0;

  for (const asset of dedupedMedia) {
    try {
      let sourceUrl = normalizeMediaUrl(asset.url);
      let contentType = "";
      let extension: string | undefined;
      let kind: MediaKind | undefined;
      let bytes: Buffer | null = null;

      if (isDataUri(asset.url)) {
        const parsed = parseBase64DataUri(asset.url);
        if (!parsed) {
          request.log.warn(`Skipping unsupported embedded media: ${asset.url.slice(0, 32)}...`);
          continue;
        }

        contentType = parsed.contentType;
        extension =
          resolveExtensionFromContentType(contentType) ??
          resolveExtensionFromUrl(asset.fileNameHint ?? "");
        kind = resolveMediaKind(sourceUrl, contentType, extension, asset.kind);
        bytes = parsed.bytes;
      } else {
        const response = await fetch(sourceUrl, {
          method: "GET",
          redirect: "follow",
          headers: {
            "user-agent": DOWNLOAD_USER_AGENT,
            ...(asset.headers ?? {}),
          },
        });

        if (!response.ok) {
          request.log.warn(`Skipping media (${response.status}): ${asset.url}`);
          continue;
        }

        sourceUrl = normalizeMediaUrl(response.url || sourceUrl);
        contentType = normalizeContentType(response.headers.get("content-type"));
        extension =
          resolveExtensionFromUrl(sourceUrl) ??
          resolveExtensionFromUrl(asset.url) ??
          resolveExtensionFromUrl(asset.fileNameHint ?? "");
        kind = resolveMediaKind(sourceUrl, contentType, extension, asset.kind);
        bytes = Buffer.from(await response.arrayBuffer());
      }

      if (!kind || !bytes) {
        request.log.debug(`Skipping media with unresolved kind: ${asset.url}`);
        continue;
      }

      const outputExtension = resolveOutputExtension(contentType, extension, kind);
      const nextIndex = kind === "image" ? downloadedImages + 1 : downloadedVideos + 1;
      const dirName = kind === "image" ? "imgs" : "videos";
      const targetDir = path.join(baseDir, dirName);
      await mkdir(targetDir, { recursive: true });

      const fileName = buildFileName(kind, nextIndex, sourceUrl, outputExtension, asset.fileNameHint);
      const absolutePath = path.join(targetDir, fileName);
      await writeFile(absolutePath, bytes);

      replacements.push({
        url: asset.url,
        localPath: toRelativePath(markdownDir, absolutePath),
        absolutePath,
        kind,
      });

      if (kind === "image") {
        downloadedImages = nextIndex;
      } else {
        downloadedVideos = nextIndex;
      }
    } catch (error) {
      const message = error instanceof Error ? error.message : String(error);
      request.log.warn(`Failed to download media ${asset.url}: ${message}`);
    }
  }

  return {
    replacements,
    downloadedImages,
    downloadedVideos,
    imageDir: downloadedImages > 0 ? path.join(baseDir, "imgs") : null,
    videoDir: downloadedVideos > 0 ? path.join(baseDir, "videos") : null,
  };
}
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 196kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 196kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 196kUpdated Claude CodeCodexVerifiedInstall