Skip to content

JimLiu/baoyu-skills / skills/baoyu-url-to-markdown

baoyu-url-to-markdown

Verified

Fetch any URL and convert to markdown using baoyu-fetch CLI (Chrome CDP with site-specific adapters). Built-in adapters for X/Twitter, YouTube transcripts, Hacker News threads, and generic pages via Defuddle. Handles login/CAPTCHA via interaction wait modes. Use when user wants to save a webpage as markdown.

FileKindSize
SKILL.mdskill md6.1 KB
references/adapters.mdmarkdown2.5 KB
references/config/first-time-setup.mdmarkdown1.9 KB
references/quality-gate.mdmarkdown2.1 KB
scripts/baoyu-fetchexecutablescript122 B
scripts/bun.lockother34.6 KB
scripts/lib/adapters/generic/index.tsscript2.4 KB
scripts/lib/adapters/hn/index.tsscript10.2 KB
scripts/lib/adapters/index.tsscript834 B
scripts/lib/adapters/types.tsscript2.1 KB
scripts/lib/adapters/x/article.tsscript12.4 KB
scripts/lib/adapters/x/index.tsscript4.0 KB
scripts/lib/adapters/x/login.tsscript2.2 KB
scripts/lib/adapters/x/match.tsscript297 B
scripts/lib/adapters/x/payloads.tsscript1.6 KB
scripts/lib/adapters/x/session.tsscript1.4 KB
scripts/lib/adapters/x/shared.tsscript12.7 KB
scripts/lib/adapters/x/single.tsscript2.5 KB
scripts/lib/adapters/x/thread-loader.tsscript8.9 KB
scripts/lib/adapters/x/thread.tsscript9.0 KB
scripts/lib/adapters/x/types.tsscript600 B
scripts/lib/adapters/youtube/index.tsscript898 B
scripts/lib/adapters/youtube/transcript.tsscript13.0 KB
scripts/lib/adapters/youtube/utils.tsscript6.5 KB
scripts/lib/browser/cdp-client.tsscript6.7 KB
scripts/lib/browser/chrome-launcher.tsscript5.7 KB
scripts/lib/browser/cookie-sidecar.tsscript2.9 KB
scripts/lib/browser/interaction-gates.tsscript4.6 KB
scripts/lib/browser/network-journal.tsscript7.0 KB
scripts/lib/browser/page-snapshot.tsscript3.3 KB
scripts/lib/browser/profile.tsscript5.5 KB
scripts/lib/browser/session.tsscript4.5 KB
scripts/lib/cli.tsexecutablescript6.9 KB
scripts/lib/commands/convert.tsscript17.0 KB
scripts/lib/extract/document.tsscript846 B
scripts/lib/extract/html-cleaner.tsscript10.8 KB
scripts/lib/extract/html-extractor.tsscript2.2 KB
scripts/lib/extract/html-to-markdown.tsscript20.8 KB
scripts/lib/extract/markdown-renderer.tsscript4.6 KB
scripts/lib/media/default-downloader.tsscript5.1 KB
scripts/lib/media/markdown-media.tsscript13.3 KB
scripts/lib/media/media-utils.tsscript6.1 KB
scripts/lib/media/types.tsscript710 B
scripts/lib/types/defuddle-node.d.tsscript605 B
scripts/lib/types/shims.d.tsscript422 B
scripts/lib/utils/logger.tsscript645 B
scripts/lib/utils/url.tsscript300 B
scripts/package.jsonmanifest557 B

scripts/lib/browser/cdp-client.ts

View on GitHub
import { EventEmitter } from "node:events";
import WebSocket from "ws";

type JsonObject = Record<string, unknown>;

interface CdpPendingCommand {
  resolve(value: unknown): void;
  reject(error: unknown): void;
  method: string;
}

interface CdpErrorShape {
  message?: string;
}

interface CdpCommandResult<T> {
  result?: T;
  error?: CdpErrorShape;
}

interface CreatePageSessionOptions {
  initialUrl?: string;
  visible?: boolean;
}

export class TargetSession extends EventEmitter {
  constructor(
    private readonly client: CdpClient,
    public readonly targetId: string,
    public readonly sessionId: string,
  ) {
    super();
  }

  async send<T>(method: string, params: JsonObject = {}): Promise<T> {
    return this.client.sendSessionCommand<T>(this.sessionId, method, params);
  }

  handleEvent(method: string, params: JsonObject): void {
    this.emit(method, params);
    this.emit("event", { method, params });
  }

  async waitForEvent<T extends JsonObject>(
    method: string,
    predicate?: (params: T) => boolean,
    timeoutMs = 30_000,
  ): Promise<T> {
    return new Promise<T>((resolve, reject) => {
      const timeout = setTimeout(() => {
        this.off(method, listener);
        reject(new Error(`Timed out waiting for ${method}`));
      }, timeoutMs);

      const listener = (params: T): void => {
        if (predicate && !predicate(params)) {
          return;
        }
        clearTimeout(timeout);
        this.off(method, listener);
        resolve(params);
      };

      this.on(method, listener);
    });
  }
}

export class CdpClient {
  private readonly ws: WebSocket;
  private readonly pending = new Map<number, CdpPendingCommand>();
  private readonly sessions = new Map<string, TargetSession>();
  private nextId = 1;

  private constructor(ws: WebSocket) {
    this.ws = ws;
    this.ws.on("message", (raw) => {
      this.handleMessage(raw.toString());
    });
  }

  static async connect(browserWsUrl: string): Promise<CdpClient> {
    const ws = await new Promise<WebSocket>((resolve, reject) => {
      const socket = new WebSocket(browserWsUrl);
      socket.once("open", () => resolve(socket));
      socket.once("error", (error) => reject(error));
    });

    return new CdpClient(ws);
  }

  private handleMessage(rawMessage: string): void {
    const message = JSON.parse(rawMessage) as {
      id?: number;
      sessionId?: string;
      method?: string;
      params?: JsonObject;
      result?: unknown;
      error?: CdpErrorShape;
    };

    if (typeof message.id === "number") {
      const pending = this.pending.get(message.id);
      if (!pending) {
        return;
      }
      this.pending.delete(message.id);
      if (message.error) {
        pending.reject(new Error(`${pending.method}: ${message.error.message ?? "Unknown CDP error"}`));
        return;
      }
      pending.resolve(message.result);
      return;
    }

    if (typeof message.sessionId === "string" && typeof message.method === "string") {
      const session = this.sessions.get(message.sessionId);
      if (session) {
        session.handleEvent(message.method, (message.params ?? {}) as JsonObject);
      }
    }
  }

  private async sendCommand<T>(
    method: string,
    params: JsonObject = {},
    sessionId?: string,
  ): Promise<T> {
    const id = this.nextId;
    this.nextId += 1;

    const payload = sessionId ? { id, method, params, sessionId } : { id, method, params };

    const result = new Promise<T>((resolve, reject) => {
      this.pending.set(id, {
        resolve: (value) => resolve(value as T),
        reject,
        method,
      });
    });

    this.ws.send(JSON.stringify(payload));
    return result;
  }

  async sendBrowserCommand<T>(method: string, params: JsonObject = {}): Promise<T> {
    return this.sendCommand<T>(method, params);
  }

  async sendSessionCommand<T>(sessionId: string, method: string, params: JsonObject = {}): Promise<T> {
    return this.sendCommand<T>(method, params, sessionId);
  }

  private async createPageTarget(initialUrl: string, visible = false): Promise<{ targetId: string }> {
    const attempts: JsonObject[] = visible
      ? [
          {
            url: initialUrl,
            newWindow: true,
            focus: true,
          },
          {
            url: initialUrl,
            focus: true,
          },
          {
            url: initialUrl,
          },
        ]
      : [
          {
            url: initialUrl,
            hidden: true,
          },
          {
            url: initialUrl,
            background: true,
            focus: false,
          },
          {
            url: initialUrl,
          },
        ];

    let lastError: unknown;

    for (const params of attempts) {
      try {
        return await this.sendBrowserCommand<{ targetId: string }>("Target.createTarget", params);
      } catch (error) {
        lastError = error;
      }
    }

    throw lastError instanceof Error ? lastError : new Error("Target.createTarget failed");
  }

  async createPageSession(options: CreatePageSessionOptions = {}): Promise<TargetSession> {
    const initialUrl = options.initialUrl ?? "about:blank";
    const created = await this.createPageTarget(initialUrl, Boolean(options.visible));
    const attached = await this.sendBrowserCommand<{ sessionId: string }>("Target.attachToTarget", {
      targetId: created.targetId,
      flatten: true,
    });

    const session = new TargetSession(this, created.targetId, attached.sessionId);
    this.sessions.set(attached.sessionId, session);

    if (options.visible) {
      await this.sendBrowserCommand("Target.activateTarget", {
        targetId: created.targetId,
      }).catch(() => {});
    }

    await session.send("Page.enable");
    await session.send("Runtime.enable");
    await session.send("DOM.enable");

    if (options.visible) {
      await session.send("Page.bringToFront").catch(() => {});
    }

    return session;
  }

  async closeTarget(targetId: string): Promise<void> {
    try {
      await this.sendBrowserCommand("Target.closeTarget", { targetId });
    } catch {
      // Target may already be gone.
    }
  }

  async close(): Promise<void> {
    await new Promise<void>((resolve) => {
      if (this.ws.readyState === WebSocket.CLOSED) {
        resolve();
        return;
      }
      this.ws.once("close", () => resolve());
      this.ws.close();
    });
  }
}

export async function evaluateRuntime<T>(session: TargetSession, expression: string): Promise<T> {
  const response = await session.send<CdpCommandResult<{ value?: T; description?: string }>>("Runtime.evaluate", {
    expression,
    awaitPromise: true,
    returnByValue: true,
  });

  if (response.error) {
    throw new Error(response.error.message ?? "Runtime.evaluate failed");
  }

  return (response.result?.value as T | undefined) ?? (undefined as T);
}
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall