Skip to content

code-yeongyu/oh-my-openagent / packages/shared-skills/skills/ultimate-browsing

ultimate-browsing

Flagged

Reaches web pages the user's browser cannot open: WAF-blocked and bot-scored hosts, platform-native readers (Chinese and social platforms), archive snapshots with provenance, and a browser you own for stealth or throwaway profiles. Not for plain search, an unblocked static fetch, or a page the user's browser can open.

FileKindSize
.gitignoreother29 B
ATTRIBUTION.mdmarkdown2.5 KB
SKILL.mdskill md11.9 KB
engine/AGENTS.mdmarkdown10.8 KB
engine/__init__.pyscript668 B
engine/__main__.pyscript5.0 KB
engine/bias_check.pyscript7.1 KB
engine/curl_probe.pyscript2.4 KB
engine/executor.pyscript8.0 KB
engine/fetch_chain.pyscript13.0 KB
engine/referers.pyscript327 B
engine/result_schema.pyscript1.5 KB
engine/summary.pyscript1.2 KB
engine/surrogate.pyscript7.9 KB
engine/surrogates.yamlconfig2.3 KB
engine/templates/knip.workspace.jsoncother1.5 KB
engine/templates/package.jsonmanifest349 B
engine/templates/playwright_mobile_chrome.jsscript3.6 KB
engine/templates/playwright_real_chrome.jsscript5.5 KB
engine/tests/fixtures/amp_redirect_stub.htmlother323 B
engine/tests/fixtures/search_interstitial.htmlother90.4 KB
engine/tests/fixtures/wayback_available.jsonconfig246 B
engine/tests/fixtures/wayback_snapshot.htmlother181.4 KB
engine/tests/test_executor_timeout.pyscript6.2 KB
engine/tests/test_fetch_chain.pyscript2.7 KB
engine/tests/test_playwright_stealth.pyscript3.9 KB
engine/tests/test_playwright_templates.pyscript10.7 KB
engine/tests/test_surrogate.pyscript11.3 KB
engine/tests/test_surrogate_validators.pyscript3.5 KB
engine/url_transforms.pyscript3.2 KB
engine/validators.pyscript9.1 KB
engine/waf_detector.pyscript6.9 KB
engine/waf_profiles.yamlconfig6.2 KB
references/agent-reach/README.mdmarkdown2.2 KB
references/agent-reach/career.mdmarkdown755 B
references/agent-reach/dev.mdmarkdown1.3 KB
references/agent-reach/search.mdmarkdown821 B
references/agent-reach/social.mdmarkdown5.6 KB
references/agent-reach/video.mdmarkdown2.7 KB
references/agent-reach/web.mdmarkdown2.1 KB
references/chrome-stealth.mdmarkdown2.4 KB
references/insane-search/README.mdmarkdown20.2 KB
references/insane-search/cache-archive.mdmarkdown4.1 KB
references/insane-search/fallback.mdmarkdown6.9 KB
references/insane-search/jina.mdmarkdown4.3 KB
references/insane-search/json-api.mdmarkdown3.5 KB
references/insane-search/media.mdmarkdown3.5 KB
references/insane-search/metadata.mdmarkdown2.9 KB
references/insane-search/naver.mdmarkdown3.4 KB
references/insane-search/playwright.mdmarkdown6.8 KB
references/insane-search/public-api.mdmarkdown3.4 KB
references/insane-search/rss.mdmarkdown3.0 KB
references/insane-search/tls-impersonate.mdmarkdown6.5 KB
references/insane-search/twitter.mdmarkdown3.5 KB
scripts/cookie_crypto.pyscript2.9 KB
scripts/cookie_domains.pyscript1.1 KB
scripts/cookie_paths.pyscript3.8 KB
scripts/extract_cookies.pyscript9.4 KB
scripts/tests/test_cookie_domain_filter.pyscript3.5 KB
scripts/tests/test_extract_cookies.pyscript9.4 KB

engine/waf_detector.py

View on GitHub
"""WAF-product detection from a live response.

Returns a *ranking* of (profile_id, confidence) pairs — never a single verdict.
Single-answer detectors cause cascading wrong plans when misfiring (Codex's
critique). Planner consumes the ranking and tries top candidates in order.

All detectors operate on WAF-vendor artifacts (cookies / headers / body
strings) — never site hostnames. See engine/waf_profiles.yaml for the
profile definitions.
"""
from __future__ import annotations

import fnmatch
import os
import re
from dataclasses import dataclass
from typing import Optional

try:
    import yaml  # PyYAML
except ImportError:
    yaml = None


PROFILES_PATH = os.path.join(os.path.dirname(__file__), "waf_profiles.yaml")


# In-code safety net — used when waf_profiles.yaml is missing / invalid
# or PyYAML isn't installed. Keeps fetch() working in a degraded-but-sane
# mode. Must stay site-agnostic (No-Site-Name Rule).
_DEFAULT_PROFILES: dict = {
    "unknown_challenge": {
        "detectors": {},
        "confidence_rules": {"strong": 0, "weak": 0},
        "capabilities_needed": ["needs_js_exec"],
        "tls_impersonate_candidates": [
            ["safari", "chrome", "firefox"],
            ["safari_ios", "chrome_android"],
        ],
        "referer_strategies": ["self_root", "google_search", "none"],
        "url_transform_order": ["original", "mobile_subdomain"],
        "fallback_when_challenge": ["surrogate_wayback", "playwright_mcp", "playwright_real_chrome"],
        "notes": "in-code default — waf_profiles.yaml unavailable",
    },
}


# Module-level sticky error. Readers call `last_load_error()` after each
# `_load_profiles()` call to surface YAML problems in FetchResult.trace.
_LAST_LOAD_ERROR: Optional[str] = None


@dataclass
class DetectionHit:
    profile_id: str
    confidence: float
    signals: list[str]


def last_load_error() -> Optional[str]:
    """Return the most recent profile-loader error (or None if clean)."""
    return _LAST_LOAD_ERROR


def _load_profiles(path: str = PROFILES_PATH) -> dict:
    """Load profiles with graceful fallback.

    Never raises. On any failure (PyYAML missing, file missing, parse error,
    unexpected shape) it returns a copy of `_DEFAULT_PROFILES` and stores
    the reason in `_LAST_LOAD_ERROR` for the caller to surface.
    """
    global _LAST_LOAD_ERROR
    _LAST_LOAD_ERROR = None

    if yaml is None:
        _LAST_LOAD_ERROR = "PyYAML not installed — using in-code default profile"
        return dict(_DEFAULT_PROFILES)
    try:
        with open(path, "r", encoding="utf-8") as f:
            loaded = yaml.safe_load(f) or {}
    except FileNotFoundError:
        _LAST_LOAD_ERROR = f"waf_profiles.yaml not found at {path}"
        return dict(_DEFAULT_PROFILES)
    except yaml.YAMLError as e:
        _LAST_LOAD_ERROR = f"YAML parse error: {type(e).__name__}: {str(e)[:200]}"
        return dict(_DEFAULT_PROFILES)
    except Exception as e:
        _LAST_LOAD_ERROR = f"profile loader: {type(e).__name__}: {str(e)[:200]}"
        return dict(_DEFAULT_PROFILES)

    if not isinstance(loaded, dict) or not any(k for k in loaded if not k.startswith("_")):
        _LAST_LOAD_ERROR = f"waf_profiles.yaml has no usable profiles"
        return dict(_DEFAULT_PROFILES)

    return loaded


def _cookies_dict(resp) -> dict:
    try:
        return {c.name: c.value for c in resp.cookies.jar}
    except Exception:
        try:
            return dict(resp.cookies) if hasattr(resp, "cookies") else {}
        except Exception:
            return {}


def _headers_dict(resp) -> dict:
    try:
        return {k.lower(): v for k, v in dict(resp.headers).items()}
    except Exception:
        return {}


def _match_patterns(haystack_keys: list[str], patterns: list[str]) -> list[str]:
    """Match literal names or fnmatch patterns (for wildcards like `X-Akamai-*`)."""
    hits: list[str] = []
    lowered_keys = [k.lower() for k in haystack_keys]
    for pat in patterns or []:
        pat_l = pat.lower()
        if any(c in pat for c in "*?["):
            for key in lowered_keys:
                if fnmatch.fnmatchcase(key, pat_l):
                    hits.append(pat)
                    break
        else:
            if pat_l in lowered_keys:
                hits.append(pat)
    return hits


def _score_profile(profile_id: str, profile: dict, resp) -> Optional[DetectionHit]:
    """Apply profile detectors to resp. Returns hit or None."""
    if profile_id.startswith("_"):
        return None
    detectors = profile.get("detectors") or {}
    if not detectors and profile_id != "unknown_challenge":
        return None

    cookies = _cookies_dict(resp)
    headers = _headers_dict(resp)
    body = (getattr(resp, "text", "") or "").lower()
    server = headers.get("server", "")

    signals: list[str] = []

    # Cookie detectors
    cookie_pats = detectors.get("cookie") or []
    for hit in _match_patterns(list(cookies.keys()), cookie_pats):
        signals.append(f"cookie:{hit}")

    # Header detectors
    header_pats = detectors.get("header") or []
    for hit in _match_patterns(list(headers.keys()), header_pats):
        signals.append(f"header:{hit}")

    # Server substring
    for needle in detectors.get("server_contains") or []:
        if needle.lower() in server:
            signals.append(f"server:{needle}")

    # Body markers
    for needle in detectors.get("body") or []:
        if needle.lower() in body:
            signals.append(f"body:{needle}")

    if not signals:
        return None

    rules = profile.get("confidence_rules") or {"strong": 2, "weak": 1}
    n = len(signals)
    if n >= rules.get("strong", 2):
        conf = 0.9
    elif n >= rules.get("weak", 1):
        conf = 0.6
    else:
        conf = 0.3

    return DetectionHit(profile_id=profile_id, confidence=conf, signals=signals)


def detect(resp, *, profiles: Optional[dict] = None, min_confidence: float = 0.0) -> list[DetectionHit]:
    """Return ranked list of detection hits (best first).

    When nothing fires, the returned list contains a single `unknown_challenge`
    hit with confidence 0.1 — caller can use its conservative settings.
    """
    if profiles is None:
        profiles = _load_profiles()

    hits: list[DetectionHit] = []
    for profile_id, profile in profiles.items():
        if profile_id.startswith("_"):
            continue
        h = _score_profile(profile_id, profile, resp)
        if h and h.confidence >= min_confidence:
            hits.append(h)

    hits.sort(key=lambda x: x.confidence, reverse=True)

    if not hits:
        hits.append(DetectionHit(
            profile_id="unknown_challenge",
            confidence=0.1,
            signals=["fallback"],
        ))
    return hits


def load_profile(profile_id: str, *, profiles: Optional[dict] = None) -> dict:
    """Get one profile by id, resolving `unknown_challenge` if missing."""
    if profiles is None:
        profiles = _load_profiles()
    return profiles.get(profile_id) or profiles.get("unknown_challenge") or {}
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall