Skip to content

JimLiu/baoyu-skills / skills/baoyu-url-to-markdown

baoyu-url-to-markdown

Verified

Fetch any URL and convert to markdown using baoyu-fetch CLI (Chrome CDP with site-specific adapters). Built-in adapters for X/Twitter, YouTube transcripts, Hacker News threads, and generic pages via Defuddle. Handles login/CAPTCHA via interaction wait modes. Use when user wants to save a webpage as markdown.

FileKindSize
SKILL.mdskill md6.1 KB
references/adapters.mdmarkdown2.5 KB
references/config/first-time-setup.mdmarkdown1.9 KB
references/quality-gate.mdmarkdown2.1 KB
scripts/baoyu-fetchexecutablescript122 B
scripts/bun.lockother34.6 KB
scripts/lib/adapters/generic/index.tsscript2.4 KB
scripts/lib/adapters/hn/index.tsscript10.2 KB
scripts/lib/adapters/index.tsscript834 B
scripts/lib/adapters/types.tsscript2.1 KB
scripts/lib/adapters/x/article.tsscript12.4 KB
scripts/lib/adapters/x/index.tsscript4.0 KB
scripts/lib/adapters/x/login.tsscript2.2 KB
scripts/lib/adapters/x/match.tsscript297 B
scripts/lib/adapters/x/payloads.tsscript1.6 KB
scripts/lib/adapters/x/session.tsscript1.4 KB
scripts/lib/adapters/x/shared.tsscript12.7 KB
scripts/lib/adapters/x/single.tsscript2.5 KB
scripts/lib/adapters/x/thread-loader.tsscript8.9 KB
scripts/lib/adapters/x/thread.tsscript9.0 KB
scripts/lib/adapters/x/types.tsscript600 B
scripts/lib/adapters/youtube/index.tsscript898 B
scripts/lib/adapters/youtube/transcript.tsscript13.0 KB
scripts/lib/adapters/youtube/utils.tsscript6.5 KB
scripts/lib/browser/cdp-client.tsscript6.7 KB
scripts/lib/browser/chrome-launcher.tsscript5.7 KB
scripts/lib/browser/cookie-sidecar.tsscript2.9 KB
scripts/lib/browser/interaction-gates.tsscript4.6 KB
scripts/lib/browser/network-journal.tsscript7.0 KB
scripts/lib/browser/page-snapshot.tsscript3.3 KB
scripts/lib/browser/profile.tsscript5.5 KB
scripts/lib/browser/session.tsscript4.5 KB
scripts/lib/cli.tsexecutablescript6.9 KB
scripts/lib/commands/convert.tsscript17.0 KB
scripts/lib/extract/document.tsscript846 B
scripts/lib/extract/html-cleaner.tsscript10.8 KB
scripts/lib/extract/html-extractor.tsscript2.2 KB
scripts/lib/extract/html-to-markdown.tsscript20.8 KB
scripts/lib/extract/markdown-renderer.tsscript4.6 KB
scripts/lib/media/default-downloader.tsscript5.1 KB
scripts/lib/media/markdown-media.tsscript13.3 KB
scripts/lib/media/media-utils.tsscript6.1 KB
scripts/lib/media/types.tsscript710 B
scripts/lib/types/defuddle-node.d.tsscript605 B
scripts/lib/types/shims.d.tsscript422 B
scripts/lib/utils/logger.tsscript645 B
scripts/lib/utils/url.tsscript300 B
scripts/package.jsonmanifest557 B

scripts/lib/media/media-utils.ts

View on GitHub
import path from "node:path";
import type { MediaKind } from "./types";

const IMAGE_EXTENSIONS = new Set([
  "jpg",
  "jpeg",
  "png",
  "webp",
  "gif",
  "bmp",
  "avif",
  "heic",
  "heif",
  "svg",
]);

const VIDEO_EXTENSIONS = new Set(["mp4", "m4v", "mov", "webm", "mkv"]);

const MIME_EXTENSION_MAP: Record<string, string> = {
  "image/jpeg": "jpg",
  "image/jpg": "jpg",
  "image/png": "png",
  "image/webp": "webp",
  "image/gif": "gif",
  "image/bmp": "bmp",
  "image/avif": "avif",
  "image/heic": "heic",
  "image/heif": "heif",
  "image/svg+xml": "svg",
  "video/mp4": "mp4",
  "video/webm": "webm",
  "video/quicktime": "mov",
  "video/x-m4v": "m4v",
};

export function normalizeContentType(raw: string | null): string {
  return raw?.split(";")[0]?.trim().toLowerCase() ?? "";
}

export function normalizeExtension(raw: string | undefined | null): string | undefined {
  if (!raw) {
    return undefined;
  }
  const trimmed = raw.replace(/^\./, "").trim().toLowerCase();
  if (!trimmed) {
    return undefined;
  }
  if (trimmed === "jpeg" || trimmed === "jpg") {
    return "jpg";
  }
  return trimmed;
}

export function resolveExtensionFromUrl(rawUrl: string): string | undefined {
  try {
    const parsed = new URL(rawUrl);
    const extFromPath = normalizeExtension(path.posix.extname(parsed.pathname));
    if (extFromPath) {
      return extFromPath;
    }
    const extFromFormat = normalizeExtension(parsed.searchParams.get("format"));
    if (extFromFormat) {
      return extFromFormat;
    }
  } catch {
    return undefined;
  }
  return undefined;
}

export function resolveExtensionFromContentType(contentType: string): string | undefined {
  return normalizeExtension(MIME_EXTENSION_MAP[contentType]);
}

export function resolveKindFromContentType(contentType: string): MediaKind | undefined {
  if (!contentType) {
    return undefined;
  }
  if (contentType.startsWith("image/")) {
    return "image";
  }
  if (contentType.startsWith("video/")) {
    return "video";
  }
  return undefined;
}

export function resolveKindFromExtension(extension: string | undefined): MediaKind | undefined {
  if (!extension) {
    return undefined;
  }
  if (IMAGE_EXTENSIONS.has(extension)) {
    return "image";
  }
  if (VIDEO_EXTENSIONS.has(extension)) {
    return "video";
  }
  return undefined;
}

export function resolveMediaKind(
  rawUrl: string,
  contentType: string,
  extension: string | undefined,
  hint?: MediaKind,
): MediaKind | undefined {
  const kindFromType = resolveKindFromContentType(contentType);
  if (kindFromType) {
    return kindFromType;
  }

  const kindFromExtension = resolveKindFromExtension(extension);
  if (kindFromExtension) {
    return kindFromExtension;
  }

  if (contentType && contentType !== "application/octet-stream") {
    return undefined;
  }

  if (hint) {
    return hint;
  }

  if (rawUrl.startsWith("data:image/")) {
    return "image";
  }

  if (rawUrl.startsWith("data:video/")) {
    return "video";
  }

  return undefined;
}

export function resolveOutputExtension(
  contentType: string,
  extension: string | undefined,
  kind: MediaKind,
): string {
  const fromMime = resolveExtensionFromContentType(contentType);
  if (fromMime) {
    return fromMime;
  }
  const normalized = normalizeExtension(extension);
  if (normalized) {
    return normalized;
  }
  return kind === "video" ? "mp4" : "jpg";
}

export function isDataUri(value: string): boolean {
  return value.startsWith("data:");
}

export function safeDecodeURIComponent(value: string): string {
  try {
    return decodeURIComponent(value);
  } catch {
    return value;
  }
}

function extractEmbeddedUrl(value: string): string | undefined {
  const encodedMatch = value.match(/https?%3A%2F%2F.+$/i)?.[0];
  if (encodedMatch) {
    const decoded = safeDecodeURIComponent(encodedMatch);
    try {
      return new URL(decoded).href;
    } catch {
      return undefined;
    }
  }

  const literalMatch = value.match(/https?:\/\/.+$/i)?.[0];
  if (!literalMatch) {
    return undefined;
  }

  try {
    return new URL(literalMatch).href;
  } catch {
    return undefined;
  }
}

export function normalizeMediaUrl(rawUrl: string): string {
  if (isDataUri(rawUrl)) {
    return rawUrl;
  }

  try {
    const parsed = new URL(rawUrl);
    const hostname = parsed.hostname.toLowerCase();

    if (hostname === "substackcdn.com" || hostname.endsWith(".substackcdn.com")) {
      const embeddedUrl = extractEmbeddedUrl(`${parsed.pathname}${parsed.search}`);
      if (embeddedUrl) {
        return embeddedUrl;
      }
    }

    return parsed.href;
  } catch {
    return rawUrl;
  }
}

export function sanitizeFileSegment(input: string): string {
  return input
    .replace(/[^a-zA-Z0-9_-]+/g, "-")
    .replace(/-+/g, "-")
    .replace(/^[-_]+|[-_]+$/g, "")
    .slice(0, 48);
}

export function resolveFileStem(rawUrl: string, extension: string, fileNameHint?: string): string {
  const hintBase = fileNameHint?.trim();
  if (hintBase) {
    const parsed = path.posix.parse(hintBase);
    const stem = parsed.name || parsed.base;
    return sanitizeFileSegment(stem);
  }

  if (isDataUri(rawUrl)) {
    return "";
  }

  try {
    const parsed = new URL(rawUrl);
    const base = path.posix.basename(parsed.pathname);
    if (!base) {
      return "";
    }
    const decodedBase = safeDecodeURIComponent(base);
    const normalizedExtension = normalizeExtension(extension);
    const stripExtension = normalizedExtension ? new RegExp(`\\.${normalizedExtension}$`, "i") : null;
    const rawStem = stripExtension ? decodedBase.replace(stripExtension, "") : decodedBase;
    return sanitizeFileSegment(rawStem);
  } catch {
    return "";
  }
}

export function buildFileName(
  kind: MediaKind,
  index: number,
  sourceUrl: string,
  extension: string,
  fileNameHint?: string,
): string {
  const stem = resolveFileStem(sourceUrl, extension, fileNameHint);
  const prefix = kind === "image" ? "img" : "video";
  const serial = String(index).padStart(3, "0");
  const suffix = stem ? `-${stem}` : "";
  return `${prefix}-${serial}${suffix}.${extension}`;
}

export function toPosixPath(value: string): string {
  return value.split(path.sep).join(path.posix.sep);
}
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall