Skip to content

code-yeongyu/oh-my-openagent / packages/shared-skills/skills/ultimate-browsing

ultimate-browsing

Flagged

Reaches web pages the user's browser cannot open: WAF-blocked and bot-scored hosts, platform-native readers (Chinese and social platforms), archive snapshots with provenance, and a browser you own for stealth or throwaway profiles. Not for plain search, an unblocked static fetch, or a page the user's browser can open.

FileKindSize
.gitignoreother29 B
ATTRIBUTION.mdmarkdown2.5 KB
SKILL.mdskill md11.9 KB
engine/AGENTS.mdmarkdown10.8 KB
engine/__init__.pyscript668 B
engine/__main__.pyscript5.0 KB
engine/bias_check.pyscript7.1 KB
engine/curl_probe.pyscript2.4 KB
engine/executor.pyscript8.0 KB
engine/fetch_chain.pyscript13.0 KB
engine/referers.pyscript327 B
engine/result_schema.pyscript1.5 KB
engine/summary.pyscript1.2 KB
engine/surrogate.pyscript7.9 KB
engine/surrogates.yamlconfig2.3 KB
engine/templates/knip.workspace.jsoncother1.5 KB
engine/templates/package.jsonmanifest349 B
engine/templates/playwright_mobile_chrome.jsscript3.6 KB
engine/templates/playwright_real_chrome.jsscript5.5 KB
engine/tests/fixtures/amp_redirect_stub.htmlother323 B
engine/tests/fixtures/search_interstitial.htmlother90.4 KB
engine/tests/fixtures/wayback_available.jsonconfig246 B
engine/tests/fixtures/wayback_snapshot.htmlother181.4 KB
engine/tests/test_executor_timeout.pyscript6.2 KB
engine/tests/test_fetch_chain.pyscript2.7 KB
engine/tests/test_playwright_stealth.pyscript3.9 KB
engine/tests/test_playwright_templates.pyscript10.7 KB
engine/tests/test_surrogate.pyscript11.3 KB
engine/tests/test_surrogate_validators.pyscript3.5 KB
engine/url_transforms.pyscript3.2 KB
engine/validators.pyscript9.1 KB
engine/waf_detector.pyscript6.9 KB
engine/waf_profiles.yamlconfig6.2 KB
references/agent-reach/README.mdmarkdown2.2 KB
references/agent-reach/career.mdmarkdown755 B
references/agent-reach/dev.mdmarkdown1.3 KB
references/agent-reach/search.mdmarkdown821 B
references/agent-reach/social.mdmarkdown5.6 KB
references/agent-reach/video.mdmarkdown2.7 KB
references/agent-reach/web.mdmarkdown2.1 KB
references/chrome-stealth.mdmarkdown2.4 KB
references/insane-search/README.mdmarkdown20.2 KB
references/insane-search/cache-archive.mdmarkdown4.1 KB
references/insane-search/fallback.mdmarkdown6.9 KB
references/insane-search/jina.mdmarkdown4.3 KB
references/insane-search/json-api.mdmarkdown3.5 KB
references/insane-search/media.mdmarkdown3.5 KB
references/insane-search/metadata.mdmarkdown2.9 KB
references/insane-search/naver.mdmarkdown3.4 KB
references/insane-search/playwright.mdmarkdown6.8 KB
references/insane-search/public-api.mdmarkdown3.4 KB
references/insane-search/rss.mdmarkdown3.0 KB
references/insane-search/tls-impersonate.mdmarkdown6.5 KB
references/insane-search/twitter.mdmarkdown3.5 KB
scripts/cookie_crypto.pyscript2.9 KB
scripts/cookie_domains.pyscript1.1 KB
scripts/cookie_paths.pyscript3.8 KB
scripts/extract_cookies.pyscript9.4 KB
scripts/tests/test_cookie_domain_filter.pyscript3.5 KB
scripts/tests/test_extract_cookies.pyscript9.4 KB

engine/tests/test_surrogate.py

View on GitHub
"""Surrogate registry/fetch/ordering tests; registry entries and responses are faked, never live network."""
from __future__ import annotations

import json
import os
import sys
import unittest
from pathlib import Path
from unittest.mock import patch

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))

from engine.result_schema import Attempt  # noqa: E402
from engine.validators import Verdict  # noqa: E402

FIXTURES = Path(__file__).resolve().parent / "fixtures"
TARGET = "https://en.wikipedia.org/wiki/Web_archiving"  # NOTE-BIAS-OK — fixture URL from a third-party surrogate service, not a target site


def _fixture(name: str) -> str:
    return (FIXTURES / name).read_text(encoding="utf-8", errors="replace")


class _Resp:
    def __init__(self, text: str = "", status: int = 200, url: str = "", payload: dict | None = None):
        self.text = text
        self.status_code = status
        self.url = url
        self.cookies = {}
        self.headers = {}
        self._payload = payload

    def json(self):
        if self._payload is None:
            raise ValueError("no json payload")
        return self._payload


def _fail_attempt(phase: str) -> Attempt:
    return Attempt(
        phase=phase,
        executor="curl_cffi",
        url=TARGET,
        url_transform="original",
        impersonate="safari",
        referer="self_root",
        verdict=Verdict.CHALLENGE.value,
    )


class SurrogateRegistry(unittest.TestCase):
    def test_default_registry_contains_wayback(self) -> None:
        from engine import surrogate

        reg = surrogate.load_surrogates()
        self.assertIn("wayback", reg)
        self.assertEqual(reg["wayback"]["kind"], "archive")
        self.assertEqual(reg["wayback"]["trust"], "archive")

    def test_stale_entries_detected(self) -> None:
        from engine import surrogate

        self.assertTrue(surrogate.is_stale({"last_verified": "2025-01-01"}, max_age_days=90))
        self.assertFalse(surrogate.is_stale({"last_verified": "2999-01-01"}, max_age_days=90))


class SurrogateFetch(unittest.TestCase):
    def test_wayback_success_sets_snapshot_provenance(self) -> None:
        from engine import surrogate

        discovery = json.loads(_fixture("wayback_available.json"))

        def fake_http(u: str, **kw):
            if "wayback/available" in u:
                return _Resp(json.dumps(discovery), 200, u, payload=discovery)
            return _Resp(_fixture("wayback_snapshot.html"), 200, u)

        with patch.object(surrogate, "_http_get", side_effect=fake_http):
            atts, meta, content = surrogate.run_surrogate(TARGET, registry=surrogate.load_surrogates(), timeout=5)

        att = atts[-1]
        self.assertEqual(att.phase, "surrogate")
        self.assertEqual(att.executor, "surrogate_wayback")
        self.assertEqual(att.verdict, Verdict.WEAK_OK.value)
        self.assertEqual(meta["provenance"], "snapshot")
        self.assertTrue(meta["snapshot_timestamp"])
        self.assertEqual(meta["trust"], "archive")
        self.assertGreater(len(content), 3000)

    def test_stub_snapshot_is_not_accepted(self) -> None:
        from engine import surrogate

        disc = {"archived_snapshots": {"closest": {"available": True, "url": "https://web.archive.org/web/2024/x", "timestamp": "20240101000000"}}}  # NOTE-BIAS-OK — fixture URL from a third-party surrogate service, not a target site

        def fake_http(u: str, **kw):
            if "wayback/available" in u:
                return _Resp(json.dumps(disc), 200, u, payload=disc)
            return _Resp(_fixture("amp_redirect_stub.html"), 200, u)

        with patch.object(surrogate, "_http_get", side_effect=fake_http):
            atts, _, _ = surrogate.run_surrogate(TARGET, registry={"wayback": surrogate.DEFAULT_SURROGATES["wayback"]}, timeout=5)
        att = atts[-1]
        self.assertNotIn(att.verdict, (Verdict.STRONG_OK.value, Verdict.WEAK_OK.value))

    def test_proxy_skips_without_allow_flag(self) -> None:
        from engine import surrogate

        reg = {"anon_relay": {"kind": "proxy", "trust": "untrusted", "enabled": False,
                              "last_verified": "2999-01-01", "fetch": "https://relay.invalid/{target}"}}  # NOTE-BIAS-OK — fixture URL from a third-party surrogate service, not a target site
        called: list[str] = []

        with patch.object(surrogate, "_http_get", side_effect=lambda u, **kw: (called.append(u), _Resp("<html>" + "y" * 4000 + "</html>", 200, u))[1]):
            atts, _, _ = surrogate.run_surrogate(TARGET, registry=reg, allow_proxy=False, timeout=5)
        self.assertEqual(len(called), 0)

        with patch.object(surrogate, "_http_get", side_effect=lambda u, **kw: (called.append(u), _Resp("<html>" + "y" * 4000 + "</html>", 200, u))[1]):
            atts, meta, _ = surrogate.run_surrogate(TARGET, registry=reg, allow_proxy=True, timeout=5)
        self.assertEqual(len(called), 1)
        att = atts[-1]
        self.assertEqual(att.executor, "surrogate_anon_relay")
        self.assertEqual(meta["trust"], "untrusted")

    def test_proxy_never_carries_credentials(self) -> None:
        from engine import surrogate

        seen: list[dict] = []

        def fake_http(u: str, **kw):
            seen.append(kw.get("headers", {}))
            return _Resp("<html>" + "z" * 4000 + "</html>", 200, u)

        reg = {"anon_relay": {"kind": "proxy", "trust": "untrusted", "enabled": False,
                              "last_verified": "2999-01-01", "fetch": "https://relay.invalid/{target}"}}  # NOTE-BIAS-OK — fixture URL from a third-party surrogate service, not a target site
        os.environ["OMOB_TEST_TOKEN"] = "secret"
        try:
            with patch.object(surrogate, "_http_get", side_effect=fake_http):
                surrogate.run_surrogate(TARGET, registry=reg, allow_proxy=True, timeout=5)
        finally:
            os.environ.pop("OMOB_TEST_TOKEN", None)

        self.assertEqual(len(seen), 1)
        joined = json.dumps(seen[0]).lower()
        self.assertNotIn("secret", joined)
        self.assertNotIn("authorization", joined)
        self.assertNotIn("cookie", joined)


class ChainOrdering(unittest.TestCase):
    def _profile(self) -> dict:
        return {
            "tls_impersonate_candidates": [[]],
            "referer_strategies": [],
            "url_transform_order": ["original"],
            "fallback_when_challenge": ["surrogate_wayback", "playwright_real_chrome"],
        }

    def test_surrogate_success_short_circuits_playwright(self) -> None:
        from engine.fetch_chain import fetch
        from engine import surrogate
        from engine import executor as ex

        def fake_surrogate(*a, **kw):
            att = Attempt(
                phase="surrogate",
                executor="surrogate_wayback",
                url=TARGET,
                url_transform="original",
                impersonate=None,
                referer="",
                verdict=Verdict.WEAK_OK.value,
                body_size=5000,
            )
            return [att], {"provenance": "snapshot", "snapshot_timestamp": "20240101000000", "trust": "archive"}, "<html>content</html>"

        def fail_playwright(*a, **kw):
            raise AssertionError("playwright must not run after surrogate success")

        with patch("engine.fetch_chain._load_profiles", return_value={}), \
                patch("engine.fetch_chain.last_load_error", return_value=None), \
                patch("engine.fetch_chain.run_attempt", side_effect=lambda *a, **kw: (_fail_attempt(str(kw.get("phase", "grid"))), None)), \
                patch("engine.fetch_chain.detect", return_value=[]), \
                patch("engine.fetch_chain.load_profile", side_effect=lambda *a, **kw: self._profile()), \
                patch.object(surrogate, "run_surrogate", side_effect=fake_surrogate), \
                patch.object(ex, "run_playwright_fallback", side_effect=fail_playwright):
            result = fetch(TARGET, enable_playwright=True, max_attempts=1)

        self.assertTrue(result.ok)
        self.assertEqual(result.provenance, "snapshot")
        self.assertEqual(result.snapshot_timestamp, "20240101000000")

    def test_surrogate_still_runs_when_playwright_disabled(self) -> None:
        from engine.fetch_chain import fetch
        from engine import surrogate

        calls: list[str] = []

        def fake_surrogate(*a, **kw):
            calls.append("surrogate")
            att = Attempt(phase="surrogate", executor="surrogate_wayback", url=TARGET,
                          url_transform="original", impersonate=None, referer="",
                          verdict=Verdict.WEAK_OK.value, body_size=5000)
            return [att], {"provenance": "snapshot", "snapshot_timestamp": "20240101000000", "trust": "archive"}, "<html>snap</html>"

        with patch("engine.fetch_chain._load_profiles", return_value={}), \
                patch("engine.fetch_chain.last_load_error", return_value=None), \
                patch("engine.fetch_chain.run_attempt", side_effect=lambda *a, **kw: (_fail_attempt(str(kw.get("phase", "grid"))), None)), \
                patch("engine.fetch_chain.detect", return_value=[]), \
                patch("engine.fetch_chain.load_profile", side_effect=lambda *a, **kw: self._profile()), \
                patch.object(surrogate, "run_surrogate", side_effect=fake_surrogate):
            result = fetch(TARGET, enable_playwright=False, max_attempts=1)

        self.assertEqual(calls, ["surrogate"])
        self.assertTrue(result.ok)
        self.assertEqual(result.provenance, "snapshot")

    def test_playwright_runs_when_surrogate_fails(self) -> None:
        from engine.fetch_chain import fetch
        from engine import surrogate
        from engine import executor as ex

        calls: list[str] = []

        def fake_surrogate(*a, **kw):
            calls.append("surrogate")
            att = Attempt(phase="surrogate", executor="surrogate_wayback", url=TARGET,
                          url_transform="original", impersonate=None, referer="",
                          verdict=Verdict.UNKNOWN.value)
            return [att], {"provenance": "live", "snapshot_timestamp": None, "trust": "origin"}, ""

        def fake_playwright(*a, **kw):
            calls.append("playwright")
            att = Attempt(phase="fallback", executor="playwright_real_chrome", url=TARGET,
                          url_transform="original", impersonate=None, referer="",
                          verdict=Verdict.WEAK_OK.value, body_size=5000)
            return att, "<html>chrome content</html>"

        with patch("engine.fetch_chain._load_profiles", return_value={}), \
                patch("engine.fetch_chain.last_load_error", return_value=None), \
                patch("engine.fetch_chain.run_attempt", side_effect=lambda *a, **kw: (_fail_attempt(str(kw.get("phase", "grid"))), None)), \
                patch("engine.fetch_chain.detect", return_value=[]), \
                patch("engine.fetch_chain.load_profile", side_effect=lambda *a, **kw: self._profile()), \
                patch.object(surrogate, "run_surrogate", side_effect=fake_surrogate), \
                patch.object(ex, "run_playwright_fallback", side_effect=fake_playwright):
            result = fetch(TARGET, enable_playwright=True, max_attempts=1)

        self.assertTrue(result.ok)
        self.assertEqual(result.provenance, "live")
        self.assertEqual(calls, ["surrogate", "playwright"])


if __name__ == "__main__":
    unittest.main()
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 197kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall