Skip to content

LeoYeAI/agent-master-skills / skills/baoyu-url-to-markdown

baoyu-url-to-markdown

Verified

Fetch any URL and convert to markdown using Chrome CDP. Saves the rendered HTML snapshot alongside the markdown, uses an upgraded Defuddle pipeline with better web-component handling and YouTube transcript extraction, and automatically falls back to the pre-Defuddle HTML-to-Markdown pipeline when needed. If local browser capture fails entirely, it can fall back to the hosted defuddle.md API. Supports two modes - auto-capture on page load, or wait for user signal (for pages requiring login). Use when user wants to save a webpage as markdown.

scripts/media-localizer.ts

View on GitHub
import path from "node:path";
import { mkdir, writeFile } from "node:fs/promises";

type MediaKind = "image" | "video";
type MediaHint = "image" | "unknown";
type MediaSource = "remote" | "data";

type MarkdownLinkCandidate = {
  url: string;
  hint: MediaHint;
  source: MediaSource;
};

export type LocalizeMarkdownMediaOptions = {
  markdownPath: string;
  log?: (message: string) => void;
};

export type LocalizeMarkdownMediaResult = {
  markdown: string;
  downloadedImages: number;
  downloadedVideos: number;
  imageDir: string | null;
  videoDir: string | null;
};

const MARKDOWN_LINK_RE =
  /(!?\[[^\]\n]*\])\((<)?((?:https?:\/\/[^)\s>]+)|(?:data:[^)>\s]+))(>)?\)/g;
const FRONTMATTER_COVER_RE = /^(coverImage:\s*")((?:https?:\/\/[^"]+)|(?:data:[^"]+))(")/m;

const IMAGE_EXTENSIONS = new Set([
  "jpg",
  "jpeg",
  "png",
  "webp",
  "gif",
  "bmp",
  "avif",
  "heic",
  "heif",
  "svg",
]);

const VIDEO_EXTENSIONS = new Set(["mp4", "m4v", "mov", "webm", "mkv"]);

const MIME_EXTENSION_MAP: Record<string, string> = {
  "image/jpeg": "jpg",
  "image/jpg": "jpg",
  "image/png": "png",
  "image/webp": "webp",
  "image/gif": "gif",
  "image/bmp": "bmp",
  "image/avif": "avif",
  "image/heic": "heic",
  "image/heif": "heif",
  "image/svg+xml": "svg",
  "video/mp4": "mp4",
  "video/webm": "webm",
  "video/quicktime": "mov",
  "video/x-m4v": "m4v",
};

const DOWNLOAD_USER_AGENT =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";

function normalizeContentType(raw: string | null): string {
  return raw?.split(";")[0]?.trim().toLowerCase() ?? "";
}

function normalizeExtension(raw: string | undefined | null): string | undefined {
  if (!raw) return undefined;
  const trimmed = raw.replace(/^\./, "").trim().toLowerCase();
  if (!trimmed) return undefined;
  if (trimmed === "jpeg") return "jpg";
  if (trimmed === "jpg") return "jpg";
  return trimmed;
}

function resolveExtensionFromUrl(rawUrl: string): string | undefined {
  try {
    const parsed = new URL(rawUrl);
    const extFromPath = normalizeExtension(path.posix.extname(parsed.pathname));
    if (extFromPath) return extFromPath;
    const extFromFormat = normalizeExtension(parsed.searchParams.get("format"));
    if (extFromFormat) return extFromFormat;
  } catch {
    return undefined;
  }
  return undefined;
}

function resolveExtensionFromContentType(contentType: string): string | undefined {
  return normalizeExtension(MIME_EXTENSION_MAP[contentType]);
}

function resolveKindFromContentType(contentType: string): MediaKind | undefined {
  if (!contentType) return undefined;
  if (contentType.startsWith("image/")) return "image";
  if (contentType.startsWith("video/")) return "video";
  return undefined;
}

function resolveKindFromExtension(ext: string | undefined): MediaKind | undefined {
  if (!ext) return undefined;
  if (IMAGE_EXTENSIONS.has(ext)) return "image";
  if (VIDEO_EXTENSIONS.has(ext)) return "video";
  return undefined;
}

function resolveMediaKind(
  rawUrl: string,
  contentType: string,
  extension: string | undefined,
  hint: MediaHint
): MediaKind | undefined {
  const kindFromType = resolveKindFromContentType(contentType);
  if (kindFromType) return kindFromType;

  const kindFromExtension = resolveKindFromExtension(extension);
  if (kindFromExtension) return kindFromExtension;

  if (contentType && contentType !== "application/octet-stream") {
    return undefined;
  }

  return hint === "image" ? "image" : undefined;
}

function resolveOutputExtension(
  contentType: string,
  extension: string | undefined,
  kind: MediaKind
): string {
  const extFromMime = resolveExtensionFromContentType(contentType);
  if (extFromMime) return extFromMime;

  const normalizedExt = normalizeExtension(extension);
  if (normalizedExt) return normalizedExt;

  return kind === "video" ? "mp4" : "jpg";
}

function safeDecodeURIComponent(value: string): string {
  try {
    return decodeURIComponent(value);
  } catch {
    return value;
  }
}

function sanitizeFileSegment(input: string): string {
  return input
    .replace(/[^a-zA-Z0-9_-]+/g, "-")
    .replace(/-+/g, "-")
    .replace(/^[-_]+|[-_]+$/g, "")
    .slice(0, 48);
}

function resolveFileStem(rawUrl: string, extension: string): string {
  if (isDataUri(rawUrl)) {
    return "";
  }

  try {
    const parsed = new URL(rawUrl);
    const base = path.posix.basename(parsed.pathname);
    if (!base) return "";
    const decodedBase = safeDecodeURIComponent(base);
    const normalizedExt = normalizeExtension(extension);
    const stripExt = normalizedExt ? new RegExp(`\\.${normalizedExt}$`, "i") : null;
    const rawStem = stripExt ? decodedBase.replace(stripExt, "") : decodedBase;
    return sanitizeFileSegment(rawStem);
  } catch {
    return "";
  }
}

function buildFileName(kind: MediaKind, index: number, sourceUrl: string, extension: string): string {
  const stem = resolveFileStem(sourceUrl, extension);
  const prefix = kind === "image" ? "img" : "video";
  const serial = String(index).padStart(3, "0");
  const suffix = stem ? `-${stem}` : "";
  return `${prefix}-${serial}${suffix}.${extension}`;
}

function isDataUri(value: string): boolean {
  return value.startsWith("data:");
}

function parseBase64DataUri(rawUrl: string): { contentType: string; bytes: Buffer } | null {
  const match = rawUrl.match(/^data:([^;,]+);base64,([A-Za-z0-9+/=\s]+)$/i);
  if (!match?.[1] || !match[2]) return null;

  const contentType = normalizeContentType(match[1]);
  if (!contentType) return null;

  try {
    const bytes = Buffer.from(match[2].replace(/\s+/g, ""), "base64");
    if (bytes.length === 0) return null;
    return { contentType, bytes };
  } catch {
    return null;
  }
}

function collectMarkdownLinkCandidates(markdown: string): MarkdownLinkCandidate[] {
  const candidates: MarkdownLinkCandidate[] = [];
  const seen = new Set<string>();

  const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
  if (fmMatch) {
    const coverMatch = fmMatch[1]?.match(FRONTMATTER_COVER_RE);
    if (coverMatch?.[2] && !seen.has(coverMatch[2])) {
      seen.add(coverMatch[2]);
      candidates.push({
        url: coverMatch[2],
        hint: "image",
        source: isDataUri(coverMatch[2]) ? "data" : "remote",
      });
    }
  }

  MARKDOWN_LINK_RE.lastIndex = 0;
  let match: RegExpExecArray | null;
  while ((match = MARKDOWN_LINK_RE.exec(markdown))) {
    const label = match[1] ?? "";
    const rawUrl = match[3] ?? "";
    if (!rawUrl || seen.has(rawUrl)) continue;
    seen.add(rawUrl);
    candidates.push({
      url: rawUrl,
      hint: label.startsWith("![") ? "image" : "unknown",
      source: isDataUri(rawUrl) ? "data" : "remote",
    });
  }

  return candidates;
}

function rewriteMarkdownMediaLinks(markdown: string, replacements: Map<string, string>): string {
  if (replacements.size === 0) return markdown;
  MARKDOWN_LINK_RE.lastIndex = 0;

  let result = markdown.replace(MARKDOWN_LINK_RE, (full, label, _openAngle, rawUrl) => {
    const localPath = replacements.get(rawUrl);
    if (!localPath) return full;
    return `${label}(${localPath})`;
  });

  result = result.replace(FRONTMATTER_COVER_RE, (full, prefix, rawUrl, suffix) => {
    const localPath = replacements.get(rawUrl);
    if (!localPath) return full;
    return `${prefix}${localPath}${suffix}`;
  });

  return result;
}

export async function localizeMarkdownMedia(
  markdown: string,
  options: LocalizeMarkdownMediaOptions
): Promise<LocalizeMarkdownMediaResult> {
  const log = options.log ?? (() => {});
  const markdownDir = path.dirname(options.markdownPath);
  const candidates = collectMarkdownLinkCandidates(markdown);

  if (candidates.length === 0) {
    return {
      markdown,
      downloadedImages: 0,
      downloadedVideos: 0,
      imageDir: null,
      videoDir: null,
    };
  }

  const replacements = new Map<string, string>();
  let downloadedImages = 0;
  let downloadedVideos = 0;

  for (const candidate of candidates) {
    try {
      let sourceUrl = candidate.url;
      let contentType = "";
      let extension: string | undefined;
      let kind: MediaKind | undefined;
      let bytes: Buffer | null = null;

      if (candidate.source === "data") {
        const parsed = parseBase64DataUri(candidate.url);
        if (!parsed) {
          log("[url-to-markdown] Skip embedded media: unsupported or invalid data URI");
          continue;
        }

        contentType = parsed.contentType;
        extension = resolveExtensionFromContentType(contentType);
        kind = resolveMediaKind(sourceUrl, contentType, extension, candidate.hint);
        bytes = parsed.bytes;
      } else {
        const response = await fetch(candidate.url, {
          method: "GET",
          redirect: "follow",
          headers: {
            "user-agent": DOWNLOAD_USER_AGENT,
          },
        });

        if (!response.ok) {
          log(`[url-to-markdown] Skip media (${response.status}): ${candidate.url}`);
          continue;
        }

        sourceUrl = response.url || candidate.url;
        contentType = normalizeContentType(response.headers.get("content-type"));
        extension = resolveExtensionFromUrl(sourceUrl) ?? resolveExtensionFromUrl(candidate.url);
        kind = resolveMediaKind(sourceUrl, contentType, extension, candidate.hint);
        bytes = Buffer.from(await response.arrayBuffer());
      }

      if (!kind || !bytes) {
        continue;
      }

      const outputExtension = resolveOutputExtension(contentType, extension, kind);
      const nextIndex = kind === "image" ? downloadedImages + 1 : downloadedVideos + 1;
      const dirName = kind === "image" ? "imgs" : "videos";
      const targetDir = path.join(markdownDir, dirName);
      await mkdir(targetDir, { recursive: true });

      const fileName = buildFileName(kind, nextIndex, sourceUrl, outputExtension);
      const absolutePath = path.join(targetDir, fileName);
      const relativePath = path.posix.join(dirName, fileName);
      await writeFile(absolutePath, bytes);
      replacements.set(candidate.url, relativePath);

      if (kind === "image") {
        downloadedImages = nextIndex;
      } else {
        downloadedVideos = nextIndex;
      }
    } catch (error) {
      const message = error instanceof Error ? error.message : String(error ?? "");
      log(`[url-to-markdown] Failed to download media ${candidate.url}: ${message}`);
    }
  }

  return {
    markdown: rewriteMarkdownMediaLinks(markdown, replacements),
    downloadedImages,
    downloadedVideos,
    imageDir: downloadedImages > 0 ? path.join(markdownDir, "imgs") : null,
    videoDir: downloadedVideos > 0 ? path.join(markdownDir, "videos") : null,
  };
}

export function countRemoteMedia(markdown: string): { images: number; videos: number; hasCoverImage: boolean } {
  const fmMatch = markdown.match(/^---\n([\s\S]*?)\n---/);
  const hasCoverImage = !!(fmMatch?.[1]?.match(FRONTMATTER_COVER_RE)?.[2]);
  const candidates = collectMarkdownLinkCandidates(markdown);
  let images = 0;
  let videos = 0;
  for (const c of candidates) {
    if (c.source !== "remote") continue;
    const ext = resolveExtensionFromUrl(c.url);
    const kind = resolveKindFromExtension(ext);
    if (kind === "video") {
      videos++;
    } else if (kind === "image" || c.hint === "image") {
      images++;
    }
  }
  return { images, videos, hasCoverImage };
}
writing-skillsobra/superpowersUse when creating new skills, editing existing skills, or verifying skills work before deploymentStars: 322kUpdated Claude CodeCodexVerifiedInstall
xlsxanthropics/skillsUse this skill any time a spreadsheet file is the primary input or output. This means any task where the user wants to: open, read, edit, or fix an existing .xlsx, .xlsm, .xltx, .csv, or .tsv file (e.g., adding columns, computing formulas, formatting, charting, cleaning messy data); create a new spreadsheet from scratch or from other data sources; or convert between tabular file formats. Trigger especially when the user references a spreadsheet file by name or path — even casually (like "the xlsx in my downloads") — and wants something done to it or produced from it. Also trigger for cleaning or restructuring messy tabular data files (malformed rows, misplaced headers, junk data) into proper spreadsheets. The deliverable must be a spreadsheet file. Do NOT trigger when the primary deliverable is a Word document, HTML report, standalone Python script, database pipeline, or Google Sheets API integration, even if tabular data is involved.Stars: 198kUpdated Claude CodeCodexVerifiedInstall
shipping-and-launchaddyosmani/agent-skillsPrepares production launches. Use when preparing to deploy to production, or when asking what needs to be in place before shipping. Use when you need a pre-launch checklist, when setting up monitoring, when planning a staged rollout, or when you need a rollback strategy.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
childhoodthedotmack/claude-memBabysit a shipped release after merge, publication, or deployment. Verify published packages, production behavior, installed plugins, and late CI or user reports until the observation window is clean; use babysit for PR readiness before release.Stars: 118kUpdated Claude CodeCodexVerifiedInstall