code-yeongyu/oh-my-openagent / packages/shared-skills/skills/ultimate-browsing
ultimate-browsing
FlaggedReaches web pages the user's browser cannot open: WAF-blocked and bot-scored hosts, platform-native readers (Chinese and social platforms), archive snapshots with provenance, and a browser you own for stealth or throwaway profiles. Not for plain search, an unblocked static fetch, or a page the user's browser can open.
engine/templates/playwright_real_chrome.js
View on GitHub#!/usr/bin/env node
/**
* Generic Playwright fetcher — real Chrome channel (not bundled Chromium).
*
* Usage (driven by engine/executor.py):
* echo '{"url":"...", "profileDir":"/tmp/.p", "waitSelector":"article"}' | node playwright_real_chrome.js
*
* Outputs page HTML to stdout on success; errors to stderr with non-zero exit.
*
* NO-SITE-NAME RULE: this file must never branch on specific hostnames.
* All site specifics come from the JSON input (url, waitSelector).
*
* User setup: ../../references/chrome-stealth.md (engine-local dependencies).
* Chrome must already be installed; optional stealth plugins run only in this script.
* Execute this script from js eval; profileDir must be task-owned or a CLONED profile.
*/
const fs = require("fs")
async function readStdinJson() {
return await new Promise((resolve, reject) => {
let data = ""
process.stdin.on("data", (c) => (data += c))
process.stdin.on("end", () => {
try {
resolve(JSON.parse(data || "{}"))
} catch (e) {
reject(e)
}
})
process.stdin.on("error", reject)
})
}
function describeError(error) {
if (error instanceof Error) {
return `${error.name}: ${error.message}`
}
return String(error)
}
function warnBestEffort(action, error) {
process.stderr.write(`best-effort ${action} failed: ${describeError(error)}\n`)
}
function isMissingTopLevelModule(error, moduleName) {
return (
error instanceof Error &&
error.code === "MODULE_NOT_FOUND" &&
typeof error.message === "string" &&
error.message.includes(`Cannot find module '${moduleName}'`)
)
}
function requireOptionalModule(moduleName) {
let resolvedModule
try {
resolvedModule = require.resolve(moduleName)
} catch (e) {
if (isMissingTopLevelModule(e, moduleName)) {
warnBestEffort(`optional module ${moduleName}`, e)
return null
}
throw e
}
return require(resolvedModule)
}
async function main() {
const args = await readStdinJson()
const url = args.url
if (!url) {
process.stderr.write("missing url\n")
process.exitCode = 2
return
}
const profileDir = args.profileDir || "/tmp/.insane_pw_profile"
const waitSelector = args.waitSelector || null
const timeoutMs = args.timeout || 60000
const headless = args.headless ?? false // Akamai/etc detect headless
const viewport = args.viewport || { width: 1366, height: 900 }
let chromium
const playwrightExtra = requireOptionalModule("playwright-extra")
const stealthPlugin = playwrightExtra ? requireOptionalModule("puppeteer-extra-plugin-stealth") : null
if (playwrightExtra && stealthPlugin) {
chromium = playwrightExtra.addExtra(require("playwright-core").chromium)
const stealth = stealthPlugin()
chromium.use(stealth)
} else {
// Core controls installed Chrome without downloading a managed browser.
;({ chromium } = require("playwright-core"))
}
let ctx
try {
ctx = await chromium.launchPersistentContext(profileDir, {
channel: "chrome", // real Chrome, not bundled Chromium
args: ["--disable-blink-features=AutomationControlled"],
ignoreDefaultArgs: ["--enable-automation"],
headless,
viewport,
})
const page = await ctx.newPage()
const navTimeout = Math.min(timeoutMs, 90000)
// Warmup hop: visit the site root first so Akamai-style bot managers
// can run their JS sensor and set a resolved session cookie. Direct
// landing on a search/deep URL is the classic first-hit rejection pattern.
// Use domcontentloaded (not networkidle) — many SPAs keep analytics/xhr
// open indefinitely and would hit the 90s timeout.
try {
const urlObj = new URL(url)
const rootUrl = `${urlObj.protocol}//${urlObj.host}/`
if (rootUrl !== url) {
await page.goto(rootUrl, { waitUntil: "domcontentloaded", timeout: navTimeout })
await page.waitForTimeout(3500) // let sensor JS finish
}
} catch (e) {
warnBestEffort("warmup navigation", e)
// warmup is best-effort; continue even if it hiccups
}
// Main page — DOM loaded then give the sensor a moment.
await page.goto(url, { waitUntil: "domcontentloaded", timeout: navTimeout })
await page.waitForTimeout(2500)
if (waitSelector) {
try {
await page.waitForSelector(waitSelector, { timeout: Math.min(timeoutMs, 20000) })
} catch (e) {
warnBestEffort("waitSelector", e)
// Selector still missing — try one hard reload in case the first hit
// landed on a challenge page and the sensor has just cleared.
try {
await page.reload({ waitUntil: "domcontentloaded", timeout: navTimeout })
await page.waitForTimeout(2000)
try {
await page.waitForSelector(waitSelector, { timeout: 10000 })
} catch (e2) {
warnBestEffort("retry waitSelector", e2)
// Still no luck — caller validates HTML anyway.
}
} catch (e3) {
warnBestEffort("selector recovery reload", e3)
// reload failed — proceed with whatever we have
}
}
} else {
// Without a positive-proof selector, give the sensor a couple more seconds.
await page.waitForTimeout(2000)
}
const html = await page.content()
process.stdout.write(html)
process.exitCode = 0
return
} catch (e) {
process.stderr.write(`${describeError(e)}\n`)
process.exitCode = 1
return
} finally {
try {
if (ctx) await ctx.close()
} catch (e) {
warnBestEffort("browser context close", e)
}
}
}
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |