Skip to content

JimLiu/baoyu-skills / skills/baoyu-url-to-markdown

baoyu-url-to-markdown

Verified

Fetch any URL and convert to markdown using baoyu-fetch CLI (Chrome CDP with site-specific adapters). Built-in adapters for X/Twitter, YouTube transcripts, Hacker News threads, and generic pages via Defuddle. Handles login/CAPTCHA via interaction wait modes. Use when user wants to save a webpage as markdown.

FileKindSize
SKILL.mdskill md6.1 KB
references/adapters.mdmarkdown2.5 KB
references/config/first-time-setup.mdmarkdown1.9 KB
references/quality-gate.mdmarkdown2.1 KB
scripts/baoyu-fetchexecutablescript122 B
scripts/bun.lockother34.6 KB
scripts/lib/adapters/generic/index.tsscript2.4 KB
scripts/lib/adapters/hn/index.tsscript10.2 KB
scripts/lib/adapters/index.tsscript834 B
scripts/lib/adapters/types.tsscript2.1 KB
scripts/lib/adapters/x/article.tsscript12.4 KB
scripts/lib/adapters/x/index.tsscript4.0 KB
scripts/lib/adapters/x/login.tsscript2.2 KB
scripts/lib/adapters/x/match.tsscript297 B
scripts/lib/adapters/x/payloads.tsscript1.6 KB
scripts/lib/adapters/x/session.tsscript1.4 KB
scripts/lib/adapters/x/shared.tsscript12.7 KB
scripts/lib/adapters/x/single.tsscript2.5 KB
scripts/lib/adapters/x/thread-loader.tsscript8.9 KB
scripts/lib/adapters/x/thread.tsscript9.0 KB
scripts/lib/adapters/x/types.tsscript600 B
scripts/lib/adapters/youtube/index.tsscript898 B
scripts/lib/adapters/youtube/transcript.tsscript13.0 KB
scripts/lib/adapters/youtube/utils.tsscript6.5 KB
scripts/lib/browser/cdp-client.tsscript6.7 KB
scripts/lib/browser/chrome-launcher.tsscript5.7 KB
scripts/lib/browser/cookie-sidecar.tsscript2.9 KB
scripts/lib/browser/interaction-gates.tsscript4.6 KB
scripts/lib/browser/network-journal.tsscript7.0 KB
scripts/lib/browser/page-snapshot.tsscript3.3 KB
scripts/lib/browser/profile.tsscript5.5 KB
scripts/lib/browser/session.tsscript4.5 KB
scripts/lib/cli.tsexecutablescript6.9 KB
scripts/lib/commands/convert.tsscript17.0 KB
scripts/lib/extract/document.tsscript846 B
scripts/lib/extract/html-cleaner.tsscript10.8 KB
scripts/lib/extract/html-extractor.tsscript2.2 KB
scripts/lib/extract/html-to-markdown.tsscript20.8 KB
scripts/lib/extract/markdown-renderer.tsscript4.6 KB
scripts/lib/media/default-downloader.tsscript5.1 KB
scripts/lib/media/markdown-media.tsscript13.3 KB
scripts/lib/media/media-utils.tsscript6.1 KB
scripts/lib/media/types.tsscript710 B
scripts/lib/types/defuddle-node.d.tsscript605 B
scripts/lib/types/shims.d.tsscript422 B
scripts/lib/utils/logger.tsscript645 B
scripts/lib/utils/url.tsscript300 B
scripts/package.jsonmanifest557 B

scripts/lib/browser/chrome-launcher.ts

View on GitHub
import { launch, type LaunchedChrome } from "chrome-launcher";
import WebSocket from "ws";
import type { Logger } from "../utils/logger";
import {
  cleanChromeLockArtifacts,
  ensureChromeProfileDir,
  findChromeProcessUsingProfile,
  findExistingChromeDebugPort,
  hasChromeLockArtifacts,
  listChromeProfileEntries,
  resolveChromeProfileDir,
  shouldRetryChromeLaunchRecovery,
} from "./profile";

interface ChromeVersionResponse {
  webSocketDebuggerUrl: string;
}

export interface ChromeConnectOptions {
  cdpUrl?: string;
  browserPath?: string;
  headless?: boolean;
  logger?: Logger;
  profileDir?: string;
}

export interface ChromeConnection {
  browserWsUrl: string;
  origin?: string;
  port?: number;
  profileDir?: string;
  launched: boolean;
  close(): Promise<void>;
}

async function fetchJson<T>(url: string): Promise<T> {
  const response = await fetch(url);
  if (!response.ok) {
    throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`);
  }
  return (await response.json()) as T;
}

async function connectToHttpEndpoint(origin: string): Promise<ChromeConnection> {
  const normalizedOrigin = origin.replace(/\/$/, "");
  const version = await fetchJson<ChromeVersionResponse>(`${normalizedOrigin}/json/version`);
  return {
    browserWsUrl: version.webSocketDebuggerUrl,
    origin: normalizedOrigin,
    port: Number(new URL(normalizedOrigin).port || 80),
    launched: false,
    async close() {
      // Reused external Chrome, nothing to close here.
    },
  };
}

async function tryReuseChrome(profileDir: string, logger?: Logger): Promise<ChromeConnection | null> {
  const port = await findExistingChromeDebugPort({ profileDir });
  if (!port) {
    return null;
  }

  const origin = `http://127.0.0.1:${port}`;
  try {
    const connection = await connectToHttpEndpoint(origin);
    logger?.info(`Reusing Chrome debugger at ${origin} for profile ${profileDir}`);
    return {
      ...connection,
      profileDir,
    };
  } catch {
    // Debugger disappeared between detection and connect.
  }
  return null;
}

async function launchFreshChrome(
  profileDir: string,
  options: Pick<ChromeConnectOptions, "browserPath" | "headless">,
): Promise<ChromeConnection> {
  let launchedChrome: LaunchedChrome | null = null;
  try {
    launchedChrome = await launch({
      chromePath: options.browserPath,
      userDataDir: profileDir,
      chromeFlags: [
        "--disable-background-networking",
        "--disable-default-apps",
        "--disable-popup-blocking",
        "--disable-sync",
        "--no-first-run",
        "--no-default-browser-check",
        "--remote-allow-origins=*",
        ...(!options.headless ? ["--no-startup-window"] : []),
        ...(options.headless ? ["--headless=new"] : []),
      ],
    });

    const origin = `http://127.0.0.1:${launchedChrome.port}`;
    const version = await fetchJson<ChromeVersionResponse>(`${origin}/json/version`);

    const chrome = launchedChrome;
    return {
      browserWsUrl: version.webSocketDebuggerUrl,
      origin,
      port: launchedChrome.port,
      profileDir,
      launched: true,
      async close() {
        if (!chrome) return;
        await gracefulCloseChrome(chrome, origin);
      },
    };
  } catch (error) {
    launchedChrome?.kill();
    throw error;
  }
}

async function gracefulCloseChrome(chrome: LaunchedChrome, origin: string): Promise<void> {
  try {
    const resp = await fetch(`${origin}/json/version`);
    const { webSocketDebuggerUrl } = (await resp.json()) as ChromeVersionResponse;
    if (webSocketDebuggerUrl) {
      const ws = await new Promise<WebSocket>((resolve, reject) => {
        const socket = new WebSocket(webSocketDebuggerUrl);
        socket.once("open", () => resolve(socket));
        socket.once("error", reject);
      });
      const id = 1;
      ws.send(JSON.stringify({ id, method: "Browser.close" }));
      await new Promise<void>((resolve) => {
        const timer = setTimeout(() => { ws.close(); resolve(); }, 5_000);
        ws.once("close", () => { clearTimeout(timer); resolve(); });
      });
      const exited = await new Promise<boolean>((resolve) => {
        if (chrome.pid && !isProcessAlive(chrome.pid)) { resolve(true); return; }
        const timer = setTimeout(() => resolve(false), 3_000);
        chrome.process.once("exit", () => { clearTimeout(timer); resolve(true); });
      });
      if (exited) return;
    }
  } catch {}
  chrome.kill();
}

function isProcessAlive(pid: number): boolean {
  try { process.kill(pid, 0); return true; } catch { return false; }
}

export async function connectChrome(options: ChromeConnectOptions): Promise<ChromeConnection> {
  if (options.cdpUrl) {
    if (options.cdpUrl.startsWith("ws://") || options.cdpUrl.startsWith("wss://")) {
      return {
        browserWsUrl: options.cdpUrl,
        launched: false,
        async close() {},
      };
    }
    return connectToHttpEndpoint(options.cdpUrl);
  }

  const profileDir = ensureChromeProfileDir(resolveChromeProfileDir(options.profileDir));
  const reused = await tryReuseChrome(profileDir, options.logger);
  if (reused) {
    return reused;
  }

  options.logger?.warn(`No running Chrome debugger found for profile ${profileDir}. Launching Chrome with that profile.`);
  try {
    return await launchFreshChrome(profileDir, options);
  } catch (error) {
    const entries = await listChromeProfileEntries(profileDir);
    const shouldRetry = shouldRetryChromeLaunchRecovery({
      hasLockArtifacts: hasChromeLockArtifacts(entries),
      hasLiveOwner: findChromeProcessUsingProfile(profileDir),
    });
    if (!shouldRetry) {
      throw error;
    }

    options.logger?.warn(`Chrome launch failed with stale profile locks. Cleaning ${profileDir} and retrying once.`);
    cleanChromeLockArtifacts(profileDir);
    return await launchFreshChrome(profileDir, options);
  }
}
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall