Skip to content

stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup

skillopt-setup

Flagged

Set up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.

FileKindSize
SKILL.mdskill md12.9 KB
assets/agent-skills-benchmark/README.mdmarkdown1.7 KB
assets/agent-skills-benchmark/adapter.py.templateother14.4 KB
assets/agent-skills-benchmark/codex_cli_reflector.py.templateother27.5 KB
assets/agent-skills-benchmark/config.codex-cli-all.yamlconfig1.5 KB
assets/agent-skills-benchmark/config.hybrid-codex-target.yamlconfig1.7 KB
assets/agent-skills-benchmark/config.native-provider.yamlconfig1.3 KB
assets/agent-skills-benchmark/dataloader.py.templateother6.6 KB
assets/agent-skills-benchmark/evaluator.py.templateother62.5 KB
assets/agent-skills-benchmark/prompts/analyst_error.mdmarkdown193 B
assets/agent-skills-benchmark/prompts/analyst_error_full_rewrite.mdmarkdown215 B
assets/agent-skills-benchmark/prompts/analyst_success.mdmarkdown183 B
assets/agent-skills-benchmark/prompts/analyst_success_full_rewrite.mdmarkdown157 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_failure.mdmarkdown202 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_success.mdmarkdown186 B
assets/agent-skills-benchmark/rollout.py.templateother143.7 KB
assets/agent-skills-benchmark/schemas/codex_cli_patch.schema.jsonconfig1.2 KB
assets/agent-skills-benchmark/schemas/codex_cli_rollout.schema.jsonconfig854 B
references/adapter-contract.mdmarkdown5.8 KB
references/adoption-policy.mdmarkdown1.0 KB
references/codex-cli-runner.mdmarkdown3.3 KB
references/credential-modes.mdmarkdown5.8 KB
references/data-schema.mdmarkdown7.7 KB
references/local-artifact-audit.mdmarkdown1.9 KB
references/local-openai-gateway.mdmarkdown5.1 KB
references/official-best-practices.mdmarkdown4.1 KB
references/runbook.mdmarkdown20.0 KB
references/troubleshooting.mdmarkdown5.8 KB
scripts/apply-skillopt-best.mjsscript13.9 KB
scripts/audit-skillopt-local-artifacts.mjsscript22.0 KB
scripts/check-skillopt-readiness.mjsscript99.6 KB
scripts/codex-local-openai-chat-gateway.mjsscript51.6 KB
scripts/prepare-local-skillopt-adapter.mjsscript25.4 KB
scripts/prepare-skillopt-split.mjsscript22.9 KB
scripts/probe-codex-cli.mjsscript4.3 KB
scripts/probe-openai-compatible-endpoint.mjsscript7.9 KB
scripts/setup-skillopt-local.mjsscript40.9 KB
scripts/summarize-skillopt-run.mjsscript17.9 KB
scripts/verify-skillopt-run-artifacts.mjsscript7.4 KB

assets/agent-skills-benchmark/evaluator.py.template

View on GitHub
from collections import deque
import errno
import fnmatch
import hashlib
import json
import os
import re
import shutil
import signal
import subprocess
import tempfile
import time
import urllib.parse
from pathlib import Path
from threading import Lock, Thread


JUDGE_PERMISSION_PROFILE = "skillopt_judge"
DRAWIO_PROFILE_STYLE_KEYS = frozenset(
    {
        "designProfile",
        "shape",
        "dataRole",
        "strokeColor",
        "fillColor",
        "gradientColor",
        "gradientDirection",
        "shadow",
        "glass",
        "arcSize",
        "strokeWidth",
        "fontColor",
        "fontSize",
        "profileRole",
    }
)


def _toml_string(value):
    return json.dumps(str(value))


def _path_is_within(path, root):
    try:
        path.relative_to(root)
        return True
    except ValueError:
        return False


def _codex_runtime_read_path(command):
    candidate = Path(str(command or "codex")).expanduser()
    if not candidate.is_absolute():
        resolved = shutil.which(str(candidate))
        if not resolved:
            return None
        candidate = Path(resolved)
    try:
        executable = candidate.resolve(strict=True)
    except OSError:
        return None
    runtime_path = executable
    for parent in executable.parents:
        package_json = parent / "package.json"
        if not package_json.is_file():
            continue
        try:
            package = json.loads(package_json.read_text(encoding="utf-8"))
        except (OSError, ValueError):
            continue
        package_name = str(package.get("name") or "")
        if package_name == "@openai/codex" or package_name.startswith("@openai/codex-"):
            runtime_path = parent
            break
    forbidden_roots = [(Path.home() / ".codex").resolve()]
    if os.environ.get("CODEX_HOME"):
        forbidden_roots.append(Path(os.environ["CODEX_HOME"]).expanduser().resolve())
    if runtime_path in {Path(runtime_path.anchor), Path.home()}:
        return None
    if any(_path_is_within(runtime_path, root) for root in forbidden_roots):
        return None
    return runtime_path


def _text_filesystem_permissions(command):
    entries = ['":minimal"="read"']
    runtime_path = _codex_runtime_read_path(command)
    if runtime_path is not None:
        entries.append(f"{_toml_string(runtime_path)}=\"read\"")
    return "{" + ",".join(entries) + "}"


def _minimal_codex_environment():
    allowed = (
        "HOME",
        "PATH",
        "USER",
        "LOGNAME",
        "SHELL",
        "TMPDIR",
        "TEMP",
        "TMP",
        "LANG",
        "LC_ALL",
        "LC_CTYPE",
        "TERM",
        "COLORTERM",
        "NO_COLOR",
        "CODEX_HOME",
        "SSL_CERT_FILE",
        "SSL_CERT_DIR",
    )
    return {key: os.environ[key] for key in allowed if os.environ.get(key)}


def _toml_inline_string_map(values):
    return "{" + ",".join(
        f"{_toml_string(key)}={_toml_string(value)}" for key, value in values.items()
    ) + "}"


def _shell_environment(workspace, process_env):
    temp_dir = Path(workspace) / ".tmp"
    temp_dir.mkdir(mode=0o700)
    values = {
        "HOME": str(workspace),
        "PATH": process_env.get("PATH", os.defpath),
        "TMPDIR": str(temp_dir),
        "TMP": str(temp_dir),
        "TEMP": str(temp_dir),
    }
    for key in ("LANG", "LC_ALL", "LC_CTYPE", "TERM", "COLORTERM", "NO_COLOR"):
        if process_env.get(key):
            values[key] = process_env[key]
    return values


def _process_group_alive(pgid):
    try:
        os.killpg(pgid, 0)
        return True
    except ProcessLookupError:
        return False
    except OSError:
        return True


def _reap_process_group(process, grace_seconds=1):
    if not _process_group_alive(process.pid):
        return
    try:
        os.killpg(process.pid, signal.SIGTERM)
    except ProcessLookupError:
        return
    deadline = time.monotonic() + grace_seconds
    while _process_group_alive(process.pid) and time.monotonic() < deadline:
        process.poll()
        time.sleep(0.05)
    if _process_group_alive(process.pid):
        try:
            os.killpg(process.pid, signal.SIGKILL)
        except ProcessLookupError:
            return
        deadline = time.monotonic() + 5
        while _process_group_alive(process.pid) and time.monotonic() < deadline:
            process.poll()
            time.sleep(0.05)


_PROMPT_ECHO_MARKER = "[redacted-prompt]"
_PROMPT_ECHO_SEED_CHARS = 16
_PROMPT_ECHO_MAX_UNIQUE_SEEDS = 65_536
_PROMPT_ECHO_MAX_SPANS = 65_536
_PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES = _PROMPT_ECHO_SEED_CHARS * 4
_MAX_STDOUT_CAPTURE_BYTES = 10 * 1024 * 1024
_MAX_STDERR_CAPTURE_BYTES = 64 * 1024
_CAPTURE_CHUNK_BYTES = 64 * 1024
_OUTPUT_TRUNCATION_MARKER = "[output truncated; retained tail]"


class _BoundedCaptureError(OSError):
    pass


class _BoundedByteCapture:
    def __init__(self, limit):
        self.limit = max(1, int(limit))
        self._chunks = deque()
        self._retained_bytes = 0
        self._total_bytes = 0
        self._discarded_tail = b""
        self._lock = Lock()

    def _remember_discarded(self, value):
        recent = value[-_PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES:]
        self._discarded_tail = (
            self._discarded_tail + recent
        )[-_PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES:]

    def append(self, chunk):
        value = bytes(chunk or b"")
        if not value:
            return
        with self._lock:
            self._total_bytes += len(value)
            self._chunks.append(value)
            self._retained_bytes += len(value)
            while self._retained_bytes > self.limit:
                excess = self._retained_bytes - self.limit
                first = self._chunks[0]
                if len(first) <= excess:
                    removed = self._chunks.popleft()
                    self._retained_bytes -= len(first)
                else:
                    removed = first[:excess]
                    self._chunks[0] = first[excess:]
                    self._retained_bytes -= excess
                self._remember_discarded(removed)

    @property
    def total_bytes(self):
        with self._lock:
            return self._total_bytes

    def snapshot(self):
        with self._lock:
            return b"".join(self._chunks), self._total_bytes > self.limit

    def redaction_snapshot(self):
        with self._lock:
            return (
                self._discarded_tail,
                b"".join(self._chunks),
                self._total_bytes > self.limit,
            )


class _BoundedProcessIO:
    def __init__(
        self,
        process,
        prompt,
        stdout_limit=_MAX_STDOUT_CAPTURE_BYTES,
        stderr_limit=_MAX_STDERR_CAPTURE_BYTES,
    ):
        self.process = process
        self.prompt = str(prompt or "").encode("utf-8", errors="replace")
        self.stdout = _BoundedByteCapture(stdout_limit)
        self.stderr = _BoundedByteCapture(stderr_limit)
        self._errors = []
        self._error_lock = Lock()
        self._readers = [
            Thread(
                target=self._read_stream,
                args=("stdout", process.stdout, self.stdout),
                name="skillopt-judge-stdout-drain",
                daemon=True,
            ),
            Thread(
                target=self._read_stream,
                args=("stderr", process.stderr, self.stderr),
                name="skillopt-judge-stderr-drain",
                daemon=True,
            ),
        ]
        self._writer = Thread(
            target=self._write_prompt,
            name="skillopt-judge-stdin-writer",
            daemon=True,
        )
        self._threads = [*self._readers, self._writer]
        self._started_threads = []

    def _record_error(self, label, error):
        with self._error_lock:
            self._errors.append(f"{label}: {type(error).__name__}")

    def _read_stream(self, label, stream, capture):
        try:
            while True:
                chunk = stream.read(_CAPTURE_CHUNK_BYTES)
                if not chunk:
                    return
                capture.append(chunk)
        except Exception as error:
            self._record_error(label, error)
        finally:
            try:
                stream.close()
            except Exception:
                pass

    def _write_prompt(self):
        stream = self.process.stdin
        try:
            view = memoryview(self.prompt)
            while view:
                written = stream.write(view)
                if not written:
                    raise BrokenPipeError("stdin closed before prompt transport completed")
                view = view[written:]
            stream.flush()
        except BrokenPipeError:
            pass
        except OSError as error:
            if error.errno not in {errno.EINVAL, errno.EPIPE}:
                self._record_error("stdin", error)
        except Exception as error:
            self._record_error("stdin", error)
        finally:
            try:
                stream.close()
            except Exception:
                pass

    def start(self):
        for thread in self._threads:
            try:
                thread.start()
            finally:
                if thread.ident is not None and thread not in self._started_threads:
                    self._started_threads.append(thread)

    def close_unstarted_streams(self):
        thread_streams = (
            (self._readers[0], self.process.stdout),
            (self._readers[1], self.process.stderr),
            (self._writer, self.process.stdin),
        )
        for thread, stream in thread_streams:
            if thread in self._started_threads or thread.is_alive():
                continue
            try:
                stream.close()
            except Exception:
                pass

    def close_stopped_streams(self):
        thread_streams = (
            (self._readers[0], self.process.stdout),
            (self._readers[1], self.process.stderr),
            (self._writer, self.process.stdin),
        )
        for thread, stream in thread_streams:
            if thread.is_alive():
                continue
            try:
                stream.close()
            except Exception:
                pass

    def finish(self, timeout=7):
        deadline = time.monotonic() + max(0, timeout)
        for thread in self._started_threads:
            thread.join(timeout=max(0, deadline - time.monotonic()))
        alive = [thread.name for thread in self._started_threads if thread.is_alive()]
        if alive:
            raise _BoundedCaptureError("bounded subprocess I/O threads did not stop")
        with self._error_lock:
            errors = list(self._errors)
        if errors:
            raise _BoundedCaptureError(
                "bounded subprocess I/O failed: " + "; ".join(errors)
            )


def _close_process_pipes(process):
    for stream in (process.stdin, process.stdout, process.stderr):
        try:
            stream.close()
        except Exception:
            pass


def _cleanup_failed_process_io_start(process, process_io):
    cleanup_failed = False
    try:
        if process_io is not None:
            process_io.close_unstarted_streams()
    except Exception:
        cleanup_failed = True
    try:
        _reap_process_group(process, grace_seconds=0)
        try:
            process.wait(timeout=5)
        except subprocess.TimeoutExpired:
            cleanup_failed = True
    except Exception:
        cleanup_failed = True
    if process_io is not None:
        try:
            process_io.finish(timeout=1)
        except Exception:
            cleanup_failed = True
        finally:
            process_io.close_stopped_streams()
    else:
        _close_process_pipes(process)
    try:
        cleanup_failed = (
            process.poll() is None
            or _process_group_alive(process.pid)
            or cleanup_failed
        )
    except Exception:
        cleanup_failed = True
    return cleanup_failed


def _start_bounded_process_io(process, prompt, **kwargs):
    process_io = None
    try:
        process_io = _BoundedProcessIO(process, prompt, **kwargs)
        process_io.start()
        return process_io
    except BaseException as error:
        cleanup_failed = _cleanup_failed_process_io_start(process, process_io)
        if not isinstance(error, Exception):
            raise
        if cleanup_failed:
            raise _BoundedCaptureError(
                "bounded subprocess I/O startup cleanup failed"
            ) from None
        raise _BoundedCaptureError(
            f"bounded subprocess I/O failed to start: {type(error).__name__}"
        ) from None


def _render_captured_stream(capture, prompt):
    discarded_tail, payload, truncated = capture.redaction_snapshot()
    value = payload.decode("utf-8", errors="replace")
    boundary_chars = _prompt_boundary_prefix_chars(
        discarded_tail,
        payload,
        str(prompt or "").encode("utf-8", errors="replace"),
    )
    value = _redact_prompt(value, prompt, boundary_chars=boundary_chars)
    if truncated:
        return f"{_OUTPUT_TRUNCATION_MARKER}\n{value}"
    return value


def _prompt_boundary_prefix_chars(discarded_tail, payload, transported):
    """Return retained leading characters proven to continue a discarded prompt echo."""
    if not discarded_tail or not payload or len(transported) < 2:
        return 0
    seed_bytes = min(_PROMPT_ECHO_SEED_CHARS, len(transported))
    max_left_size = min(seed_bytes - 1, len(discarded_tail))
    if max_left_size <= 0:
        return 0
    high = min(len(payload), _PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES)
    matched_bytes = 0
    for left_size in range(1, max_left_size + 1):
        left = discarded_tail[-left_size:]
        if left + payload[:1] not in transported:
            continue
        low = 1
        candidate_high = high
        while low < candidate_high:
            middle = (low + candidate_high + 1) // 2
            if left + payload[:middle] in transported:
                low = middle
            else:
                candidate_high = middle - 1
        matched_bytes = max(matched_bytes, low)
        if matched_bytes == high:
            break
    return len(payload[:matched_bytes].decode("utf-8", errors="replace"))


def _prompt_echo_spans(value, transported):
    """Return bounded prompt-derived spans, or None to request fail-closed redaction."""
    spans = []
    if not value or not transported:
        return spans
    if value in transported:
        return [(0, len(value))]
    for match in re.finditer(r"[^\r\n]+", value):
        line = match.group(0)
        stripped = line.strip()
        if stripped and stripped in transported:
            offset = line.find(stripped)
            spans.append((match.start() + offset, match.start() + offset + len(stripped)))
            if len(spans) > _PROMPT_ECHO_MAX_SPANS:
                return None

    seed_size = min(_PROMPT_ECHO_SEED_CHARS, len(transported))
    if seed_size <= 0 or len(value) < seed_size:
        return spans

    prompt_seeds = set()
    for index in range(len(transported) - seed_size + 1):
        prompt_seeds.add(transported[index : index + seed_size])
        if len(prompt_seeds) > _PROMPT_ECHO_MAX_UNIQUE_SEEDS:
            return None

    seed_spans = []
    last_seed_start = len(value) - seed_size
    for index in range(last_seed_start + 1):
        if value[index : index + seed_size] not in prompt_seeds:
            continue
        end = index + seed_size
        if seed_spans and index <= seed_spans[-1][1]:
            seed_spans[-1] = (
                seed_spans[-1][0],
                max(seed_spans[-1][1], end),
            )
        else:
            seed_spans.append((index, end))
            if len(spans) + len(seed_spans) > _PROMPT_ECHO_MAX_SPANS:
                return None
    spans.extend(seed_spans)
    return spans


def _redact_prompt(text, prompt, boundary_chars=0):
    value = text.decode("utf-8", errors="replace") if isinstance(text, bytes) else str(text or "")
    transported = str(prompt or "").encode("utf-8", errors="replace").decode("utf-8")
    spans = _prompt_echo_spans(value, transported)
    if spans is None:
        return _PROMPT_ECHO_MARKER if value else ""
    guard_end = min(len(value), max(0, int(boundary_chars)))
    if guard_end:
        if len(spans) >= _PROMPT_ECHO_MAX_SPANS:
            return _PROMPT_ECHO_MARKER
        spans.append((0, guard_end))
    if not spans:
        return value
    spans.sort()
    merged = []
    for start, end in spans:
        if merged and start <= merged[-1][1]:
            merged[-1] = (merged[-1][0], max(merged[-1][1], end))
        else:
            merged.append((start, end))
    parts = []
    cursor = 0
    for start, end in merged:
        parts.extend((value[cursor:start], _PROMPT_ECHO_MARKER))
        cursor = end
    parts.append(value[cursor:])
    return "".join(parts)


def _redact_timeout_stream(text):
    return _PROMPT_ECHO_MARKER if (
        text.decode("utf-8", errors="replace") if isinstance(text, bytes) else str(text or "")
    ) else ""


def _run_isolated_codex(command, prompt, cwd, env, timeout):
    process = subprocess.Popen(
        command,
        cwd=cwd,
        env=env,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        start_new_session=True,
    )
    process_io = _start_bounded_process_io(process, prompt)
    try:
        process.wait(timeout=timeout)
    except subprocess.TimeoutExpired:
        cleanup_failed = False
        try:
            _reap_process_group(process)
        except Exception:
            cleanup_failed = True
        try:
            process.wait(timeout=5)
        except subprocess.TimeoutExpired:
            cleanup_failed = True
        try:
            process_io.finish()
        except Exception:
            cleanup_failed = True
        finally:
            process_io.close_stopped_streams()
        try:
            cleanup_failed = (
                process.poll() is None
                or _process_group_alive(process.pid)
                or cleanup_failed
            )
        except Exception:
            cleanup_failed = True
        if cleanup_failed:
            raise _BoundedCaptureError(
                "bounded subprocess cleanup failed after timeout"
            ) from None
        error = subprocess.TimeoutExpired(command, timeout)
        error.stdout = (
            _PROMPT_ECHO_MARKER
            if process_io.stdout.total_bytes
            else ""
        )
        error.stderr = (
            _PROMPT_ECHO_MARKER
            if process_io.stderr.total_bytes
            else ""
        )
        raise error
    cleanup_failed = False
    try:
        _reap_process_group(process)
    except Exception:
        cleanup_failed = True
    try:
        group_alive = _process_group_alive(process.pid)
    except Exception:
        group_alive = True
    capture_error = None
    try:
        process_io.finish()
    except _BoundedCaptureError as error:
        capture_error = error
    finally:
        process_io.close_stopped_streams()
    if cleanup_failed or group_alive:
        raise _BoundedCaptureError("bounded subprocess cleanup failed") from None
    if capture_error is not None:
        raise capture_error
    return subprocess.CompletedProcess(
        command,
        process.returncode,
        _render_captured_stream(process_io.stdout, prompt),
        _render_captured_stream(process_io.stderr, prompt),
    )


def _short(text, limit=1000):
    value = str(text or "")
    return value if len(value) <= limit else f"{value[:limit]}..."


def _redact_sensitive(text):
    value = str(text or "")
    value = re.sub(r"/home/[A-Za-z0-9._-]+/", "~/", value)
    value = re.sub(r"/Users/[A-Za-z0-9._-]+/", "~/", value)
    value = re.sub(r"(?i)(https?://)[^/@\s]+:[^/@\s]+@", r"\1[redacted]@", value)
    value = re.sub(r"(?i)([?&](?:api[_-]?key|token|secret|password)=)[^&\s]+", r"\1[redacted]", value)
    value = re.sub(r"sk-[A-Za-z0-9_-]{20,}", "[redacted-token]", value)
    value = re.sub(
        r"\b(?:ghp_|github_pat_|xox[baprs]-|AKIA|ASIA)[A-Za-z0-9_/-]{12,}\b",
        "[redacted-token]",
        value,
    )
    return value


def _extract_json(text):
    value = (text or "").strip()
    if not value:
        raise ValueError("empty judge response")
    if value.startswith("```"):
        value = re.sub(r"^```(?:json)?\s*", "", value)
        value = re.sub(r"\s*```$", "", value)
    try:
        return json.loads(value)
    except json.JSONDecodeError:
        start = value.find("{")
        end = value.rfind("}")
        if start == -1 or end == -1 or end <= start:
            raise
        return json.loads(value[start : end + 1])


def _contains_expected_text(response, expected):
    text = (response or "").lower()
    return any(fragment.lower() in text for fragment in expected if isinstance(fragment, str))


def _bool(value):
    if isinstance(value, bool):
        return value
    if isinstance(value, str):
        return value.strip().lower() in {"true", "yes", "pass", "passed", "1"}
    return bool(value)


def _float_score(value):
    try:
        score = float(value)
    except (TypeError, ValueError):
        return 0.0
    return max(0.0, min(1.0, score))


def _match_assertion(response, assertion):
    text = response or ""
    raw = str(assertion or "").strip()
    if not raw:
        return None
    kind, _, value = raw.partition(":")
    kind = kind.strip().lower()
    value = value.strip()
    if not value:
        kind = "contains"
        value = raw

    haystack = text.lower()
    needle = value.lower()
    if kind in {"contains", "must_include", "include"}:
        passed = needle in haystack
        return {"text": raw, "passed": passed, "evidence": "substring present" if passed else "substring missing"}
    if kind in {"not_contains", "must_not_include", "exclude"}:
        passed = needle not in haystack
        return {"text": raw, "passed": passed, "evidence": "substring absent" if passed else "forbidden substring present"}
    if kind == "regex":
        try:
            passed = re.search(value, text, flags=re.IGNORECASE | re.MULTILINE) is not None
        except re.error as error:
            return {"text": raw, "passed": False, "evidence": f"invalid regex: {_short(error, 200)}"}
        return {"text": raw, "passed": passed, "evidence": "regex matched" if passed else "regex did not match"}
    if kind in {"section", "heading"}:
        pattern = rf"^#+\s+{re.escape(value)}\s*$"
        passed = re.search(pattern, text, flags=re.IGNORECASE | re.MULTILINE) is not None
        return {"text": raw, "passed": passed, "evidence": "heading present" if passed else "heading missing"}
    if kind in {"path", "artifact_path"}:
        passed = value in text
        return {"text": raw, "passed": passed, "evidence": "path referenced" if passed else "path not referenced"}

    passed = needle in haystack
    return {"text": raw, "passed": passed, "evidence": "substring present" if passed else "substring missing"}


def _artifact_matches(artifacts, pattern):
    matches = []
    for artifact in artifacts or []:
        artifact_path = str(artifact.get("path") or "")
        name = Path(artifact_path).name
        if fnmatch.fnmatch(artifact_path, pattern) or fnmatch.fnmatch(name, pattern):
            matches.append(artifact)
    return matches


def _decode_markdown_escapes(value):
    return re.sub(r"\\([\\`*_[\]{}()#+\-.!])", r"\1", value)


def _parse_visual_options(parts):
    options = {}
    for part in parts:
        match = re.match(r"^([A-Za-z_][A-Za-z0-9_]*)=(\d+)$", part)
        if not match:
            raise ValueError(f"invalid key=value token {part!r}")
        options[match.group(1)] = int(match.group(2))
    return options


def _parse_visual_assertion(assertion):
    raw = str(assertion or "").strip()
    if not raw:
        return None
    kind, separator, value = raw.partition(":")
    if not separator:
        raise ValueError("missing ':' separator")
    kind = kind.strip()
    value = value.strip()
    if not value:
        raise ValueError("missing assertion value")
    parts = value.split()
    glob = _decode_markdown_escapes(parts[0])
    if not glob:
        raise ValueError("missing artifact glob")
    return {"raw": raw, "kind": kind, "glob": glob, "parts": parts[1:]}


def _match_visual_assertion(artifacts, assertion):
    try:
        parsed = _parse_visual_assertion(assertion)
    except ValueError as error:
        return {"text": str(assertion), "passed": False, "evidence": f"invalid visual assertion: {error}"}
    if parsed is None:
        return None

    raw = parsed["raw"]
    kind = parsed["kind"]
    matches = _artifact_matches(artifacts, parsed["glob"])
    if not matches:
        return {"text": raw, "passed": False, "evidence": f"no artifact matched {parsed['glob']}"}

    if kind == "artifact_exists":
        return {"text": raw, "passed": True, "evidence": f"{len(matches)} matching artifact(s)"}

    if kind == "png_dimensions":
        try:
            options = _parse_visual_options(parsed["parts"])
        except ValueError as error:
            return {"text": raw, "passed": False, "evidence": str(error)}
        min_width = options.get("min_width")
        min_height = options.get("min_height")
        if not min_width or not min_height:
            return {
                "text": raw,
                "passed": False,
                "evidence": "png_dimensions requires min_width and min_height",
            }
        for artifact in matches:
            if (
                artifact.get("kind") == "png"
                and artifact.get("valid")
                and int(artifact.get("width") or 0) >= min_width
                and int(artifact.get("height") or 0) >= min_height
            ):
                return {
                    "text": raw,
                    "passed": True,
                    "evidence": f"{artifact.get('path')} is {artifact.get('width')}x{artifact.get('height')}",
                }
        return {"text": raw, "passed": False, "evidence": "no PNG met required dimensions"}

    if kind == "png_nonblank":
        try:
            options = _parse_visual_options(parsed["parts"])
        except ValueError as error:
            return {"text": raw, "passed": False, "evidence": str(error)}
        min_size = int(options.get("min_size") or 1024)
        for artifact in matches:
            if (
                artifact.get("kind") == "png"
                and artifact.get("valid")
                and int(artifact.get("width") or 0) > 0
                and int(artifact.get("height") or 0) > 0
                and int(artifact.get("size_bytes") or 0) >= min_size
                and bool(artifact.get("nonblank"))
            ):
                return {
                    "text": raw,
                    "passed": True,
                    "evidence": (
                        f"{artifact.get('path')} has {artifact.get('visible_pixel_count')} visible pixel(s) "
                        f"and {artifact.get('visible_distinct_pixels')} visible decoded pixel value(s)"
                    ),
                }
        return {"text": raw, "passed": False, "evidence": "no PNG passed decoded nonblank pixel check"}

    if kind == "svg_valid":
        invalid = [artifact for artifact in matches if artifact.get("kind") != "svg" or not artifact.get("valid")]
        return {
            "text": raw,
            "passed": not invalid,
            "evidence": "all matching SVG artifacts are valid" if not invalid else "one or more SVG artifacts are invalid",
        }

    if kind in {"svg_contains", "svg_not_contains"}:
        text = " ".join(parsed["parts"]).strip()
        if not text:
            return {"text": raw, "passed": False, "evidence": f"{kind} requires text"}
        inspectable = [
            artifact
            for artifact in matches
            if artifact.get("kind") == "svg"
            and artifact.get("valid")
            and "visible_text" in artifact
        ]
        if len(inspectable) != len(matches):
            return {
                "text": raw,
                "passed": False,
                "evidence": "one or more SVG artifacts lack validated visible-text metadata",
            }
        needle = " ".join(text.lower().split())
        containing = [
            artifact
            for artifact in inspectable
            if needle in " ".join(str(artifact.get("visible_text") or "").lower().split())
        ]
        if kind == "svg_contains":
            return {
                "text": raw,
                "passed": bool(containing),
                "evidence": (
                    f"{containing[0].get('path')} contains visible text"
                    if containing
                    else "no SVG contained visible text"
                ),
            }
        return {
            "text": raw,
            "passed": not containing,
            "evidence": (
                "forbidden visible text absent"
                if not containing
                else f"{containing[0].get('path')} contains forbidden visible text"
            ),
        }

    if kind == "svg_has_flow_animation":
        if parsed["parts"]:
            return {
                "text": raw,
                "passed": False,
                "evidence": "svg_has_flow_animation accepts only an artifact glob",
            }
        passing = [
            artifact
            for artifact in matches
            if artifact.get("kind") == "svg"
            and artifact.get("valid")
            and artifact.get("flow_animation")
            and int(artifact.get("external_references") or 0) == 0
            and int(artifact.get("unsupported_images") or 0) == 0
        ]
        return {
            "text": raw,
            "passed": bool(passing),
            "evidence": (
                f"{passing[0].get('path')} has active connector flow animation"
                if passing
                else "no SVG has active connector flow animation"
            ),
        }

    if kind == "svg_self_contained_images":
        passing = [
            artifact
            for artifact in matches
            if artifact.get("kind") == "svg"
            and artifact.get("valid")
            and int(artifact.get("embedded_svg_images") or 0) > 0
            and int(artifact.get("external_references") or 0) == 0
            and int(artifact.get("unsupported_images") or 0) == 0
        ]
        return {
            "text": raw,
            "passed": bool(passing),
            "evidence": (
                f"{passing[0].get('path')} has self-contained embedded SVG images"
                if passing
                else "no SVG has self-contained embedded SVG images"
            ),
        }

    if kind == "drawio_valid":
        try:
            options = _parse_visual_options(parsed["parts"])
        except ValueError as error:
            return {"text": raw, "passed": False, "evidence": str(error)}
        allowed = {
            "animation_on",
            "animation_off",
            "adaptive_colors",
            "min_pages",
            "min_native_stencils",
            "self_contained_svg",
            "uncompressed",
        }
        unsupported = sorted(set(options) - allowed)
        if unsupported:
            return {
                "text": raw,
                "passed": False,
                "evidence": f"drawio_valid does not support option {unsupported[0]}",
            }
        if options.get("animation_on") and options.get("animation_off"):
            return {
                "text": raw,
                "passed": False,
                "evidence": "drawio_valid cannot require animation_on and animation_off together",
            }
        for name in (
            "animation_on",
            "animation_off",
            "adaptive_colors",
            "self_contained_svg",
            "uncompressed",
        ):
            if name in options and options[name] != 1:
                return {
                    "text": raw,
                    "passed": False,
                    "evidence": f"drawio_valid option {name} must equal 1",
                }
        if "min_pages" in options and options["min_pages"] < 1:
            return {
                "text": raw,
                "passed": False,
                "evidence": "drawio_valid option min_pages must be positive",
            }
        if "min_native_stencils" in options and options["min_native_stencils"] < 1:
            return {
                "text": raw,
                "passed": False,
                "evidence": "drawio_valid option min_native_stencils must be positive",
            }
        if "min_native_stencils" in options and options.get("uncompressed") != 1:
            return {
                "text": raw,
                "passed": False,
                "evidence": "drawio_valid min_native_stencils requires uncompressed=1",
            }

        invalid = []
        for artifact in matches:
            if artifact.get("kind") != "drawio" or not artifact.get("valid"):
                invalid.append(artifact)
                continue
            if int(artifact.get("page_count") or 0) < int(options.get("min_pages") or 1):
                invalid.append(artifact)
                continue
            if int(artifact.get("native_stencil_count") or 0) < int(
                options.get("min_native_stencils") or 0
            ):
                invalid.append(artifact)
                continue
            if options.get("adaptive_colors") and (
                artifact.get("adaptive_colors") is not True
                or not artifact.get("page_graphs")
                or any(
                    page.get("adaptive_colors") is not True
                    for page in artifact.get("page_graphs") or []
                )
            ):
                invalid.append(artifact)
                continue
            if options.get("animation_on") and (
                not artifact.get("animation_on")
                or int(artifact.get("directed_flow_edges") or 0) < 1
                or int(artifact.get("animated_edges") or 0) < 1
            ):
                invalid.append(artifact)
                continue
            if options.get("animation_off") and not artifact.get("animation_off"):
                invalid.append(artifact)
                continue
            if options.get("self_contained_svg") and not artifact.get("self_contained_svg"):
                invalid.append(artifact)
                continue
            if options.get("uncompressed") and not artifact.get("uncompressed"):
                invalid.append(artifact)
        return {
            "text": raw,
            "passed": not invalid,
            "evidence": (
                "all matching draw.io artifacts passed bounded validation"
                if not invalid
                else "one or more draw.io artifacts failed validation or required options"
            ),
        }

    if kind == "drawio_embeds_svg_sha256":
        parts = parsed["parts"]
        if (
            len(parts) not in (1, 2)
            or re.fullmatch(r"[0-9a-f]{64}", parts[0]) is None
            or (
                len(parts) == 2
                and re.fullmatch(r"cell=[A-Za-z0-9_.:-]{1,128}", parts[1]) is None
            )
        ):
            return {
                "text": raw,
                "passed": False,
                "evidence": (
                    "drawio_embeds_svg_sha256 requires one lowercase SHA-256 digest "
                    "and optional cell=<stable-id>"
                ),
            }
        expected = parts[0]
        cell_id = parts[1][5:] if len(parts) == 2 else None
        expected_cell_digest = (
            hashlib.sha256(f"{cell_id}\0{expected}".encode("utf-8")).hexdigest()
            if cell_id
            else None
        )
        invalid = [
            artifact
            for artifact in matches
            if artifact.get("kind") != "drawio"
            or not artifact.get("valid")
            or not isinstance(artifact.get("embedded_svg_sha256s"), list)
            or expected not in artifact["embedded_svg_sha256s"]
            or (
                expected_cell_digest is not None
                and (
                    not isinstance(artifact.get("embedded_svg_cell_sha256s"), list)
                    or expected_cell_digest not in artifact["embedded_svg_cell_sha256s"]
                )
            )
        ]
        return {
            "text": raw,
            "passed": not invalid,
            "evidence": (
                "all matching draw.io artifacts embed the expected SVG digest"
                if not invalid
                else "one or more draw.io artifacts lack the expected embedded SVG digest"
            ),
        }

    if kind == "drawio_graph":
        options = {}
        for part in parsed["parts"]:
            match = re.fullmatch(
                r"(ids|native_ids|edges|not_edges|edge_roles|profile_styles|links|page)=(.+)",
                part,
            )
            if match is None or match.group(1) in options:
                return {
                    "text": raw,
                    "passed": False,
                    "evidence": f"drawio_graph has invalid or repeated option {part}",
                }
            options[match.group(1)] = match.group(2)
        if not options or not any(
            name in options
            for name in (
                "ids",
                "native_ids",
                "edges",
                "not_edges",
                "edge_roles",
                "profile_styles",
                "links",
            )
        ):
            return {
                "text": raw,
                "passed": False,
                "evidence": (
                    "drawio_graph requires ids=..., native_ids=..., edges=..., not_edges=..., "
                    "edge_roles=..., profile_styles=..., or links=..."
                ),
            }

        identifiers = options.get("ids", "").split(",") if "ids" in options else []
        native_identifiers = (
            options.get("native_ids", "").split(",") if "native_ids" in options else []
        )
        raw_edges = options.get("edges", "").split(",") if "edges" in options else []
        raw_not_edges = options.get("not_edges", "").split(",") if "not_edges" in options else []
        raw_edge_roles = (
            options.get("edge_roles", "").split(",") if "edge_roles" in options else []
        )
        raw_profile_styles = (
            options.get("profile_styles", "").split(",")
            if "profile_styles" in options
            else []
        )
        links = options.get("links", "").split(",") if "links" in options else []
        if any(
            len(values) > 128
            for values in (
                identifiers,
                native_identifiers,
                raw_edges,
                raw_not_edges,
                raw_edge_roles,
                raw_profile_styles,
                links,
            )
        ):
            return {"text": raw, "passed": False, "evidence": "drawio_graph exceeds 128 entries"}
        identifier_pattern = re.compile(r"[A-Za-z0-9_.:-]{1,128}")
        edges = []
        not_edges = []
        edge_roles = []
        profile_styles = []
        if any(
            identifier_pattern.fullmatch(identifier) is None
            for identifier in (*identifiers, *native_identifiers)
        ):
            return {"text": raw, "passed": False, "evidence": "drawio_graph has an invalid ID"}
        for raw_values, target in ((raw_edges, edges), (raw_not_edges, not_edges)):
            for edge in raw_values:
                match = re.fullmatch(r"([^>]+)>([^>]+)", edge)
                if match is None or any(
                    identifier_pattern.fullmatch(identifier) is None for identifier in match.groups()
                ):
                    return {
                        "text": raw,
                        "passed": False,
                        "evidence": "drawio_graph has an invalid directed edge pair",
                    }
                target.append(match.groups())
        for edge_role in raw_edge_roles:
            match = re.fullmatch(r"(.+):([a-z][a-z0-9_-]{0,63})", edge_role)
            if match is None or identifier_pattern.fullmatch(match.group(1)) is None:
                return {
                    "text": raw,
                    "passed": False,
                    "evidence": "drawio_graph has an invalid edge role mapping",
                }
            edge_roles.append(match.groups())
        encoded_component_pattern = re.compile(
            r"(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2})+"
        )
        style_key_pattern = re.compile(r"[A-Za-z][A-Za-z0-9_.-]{0,63}")
        for raw_profile_style in raw_profile_styles:
            parts = raw_profile_style.split(":")
            if (
                len(raw_profile_style) > 8192
                or len(parts) != 3
                or any(encoded_component_pattern.fullmatch(part) is None for part in parts)
            ):
                return {
                    "text": raw,
                    "passed": False,
                    "evidence": "drawio_graph has an invalid profile style mapping",
                }
            try:
                cell_id, style_key, style_value = (
                    urllib.parse.unquote(part, encoding="utf-8", errors="strict")
                    for part in parts
                )
            except UnicodeDecodeError:
                return {
                    "text": raw,
                    "passed": False,
                    "evidence": "drawio_graph has an invalid profile style mapping",
                }
            contains_control = any(
                ord(char) < 32 or 127 <= ord(char) <= 159
                for value in (cell_id, style_key, style_value)
                for char in value
            )
            if identifier_pattern.fullmatch(cell_id) is None:
                return {
                    "text": raw,
                    "passed": False,
                    "evidence": "drawio_graph has an invalid profile style ID",
                }
            if style_key_pattern.fullmatch(style_key) is None:
                return {
                    "text": raw,
                    "passed": False,
                    "evidence": "drawio_graph has an invalid profile style key",
                }
            if style_key not in DRAWIO_PROFILE_STYLE_KEYS:
                return {
                    "text": raw,
                    "passed": False,
                    "evidence": "drawio_graph has an unsupported profile style key",
                }
            if (
                not style_value
                or len(style_value) > 2048
                or contains_control
            ):
                return {
                    "text": raw,
                    "passed": False,
                    "evidence": "drawio_graph has an invalid profile style value",
                }
            profile_styles.append((cell_id, style_key, style_value))
        if any(len(link) > 2048 or re.fullmatch(r"https?://[^\s,]+", link) is None for link in links):
            return {"text": raw, "passed": False, "evidence": "drawio_graph has an invalid link"}
        page_name = None
        if "page" in options:
            encoded_page = options["page"]
            if (
                len(encoded_page) > 384
                or re.fullmatch(r"(?:[A-Za-z0-9_.:-]|%[0-9A-Fa-f]{2})+", encoded_page)
                is None
            ):
                return {"text": raw, "passed": False, "evidence": "drawio_graph has an invalid page"}
            page_name = urllib.parse.unquote(encoded_page)
            if not page_name or len(page_name) > 128 or any(ord(char) < 32 for char in page_name):
                return {"text": raw, "passed": False, "evidence": "drawio_graph has an invalid page"}

        expected_ids = {
            hashlib.sha256(identifier.encode("utf-8")).hexdigest()
            for identifier in identifiers
        }
        expected_native_ids = {
            hashlib.sha256(identifier.encode("utf-8")).hexdigest()
            for identifier in native_identifiers
        }
        expected_edges = {
            hashlib.sha256(f"{source}\0{target}".encode("utf-8")).hexdigest()
            for source, target in edges
        }
        forbidden_edges = {
            hashlib.sha256(f"{source}\0{target}".encode("utf-8")).hexdigest()
            for source, target in not_edges
        }
        expected_links = {hashlib.sha256(link.encode("utf-8")).hexdigest() for link in links}
        expected_edge_roles = {
            hashlib.sha256(f"{edge_id}\0{role}".encode("utf-8")).hexdigest()
            for edge_id, role in edge_roles
        }
        expected_profile_styles = {
            hashlib.sha256(f"{cell_id}\0{key}\0{value}".encode("utf-8")).hexdigest()
            for cell_id, key, value in profile_styles
        }
        expected_page = (
            hashlib.sha256(page_name.encode("utf-8")).hexdigest() if page_name is not None else None
        )
        invalid = []
        for artifact in matches:
            graph = artifact
            if expected_page is not None:
                graph = next(
                    (
                        page
                        for page in artifact.get("page_graphs") or []
                        if page.get("page_name_sha256") == expected_page
                    ),
                    None,
                )
            id_hashes = set((graph or {}).get("cell_id_sha256s") or [])
            native_id_hashes = set((graph or {}).get("native_stencil_cell_id_sha256s") or [])
            edge_hashes = set((graph or {}).get("directed_edge_sha256s") or [])
            edge_role_hashes = set((graph or {}).get("edge_role_sha256s") or [])
            if expected_page is not None:
                profile_styles_satisfied = expected_profile_styles.issubset(
                    set((graph or {}).get("profile_style_sha256s") or [])
                )
            else:
                profile_styles_satisfied = not expected_profile_styles or any(
                    expected_profile_styles.issubset(
                        set(page.get("profile_style_sha256s") or [])
                    )
                    for page in artifact.get("page_graphs") or []
                )
            link_hashes = set((graph or {}).get("link_sha256s") or [])
            if (
                artifact.get("kind") != "drawio"
                or not artifact.get("valid")
                or graph is None
                or not expected_ids.issubset(id_hashes)
                or not expected_native_ids.issubset(native_id_hashes)
                or not expected_edges.issubset(edge_hashes)
                or bool(forbidden_edges & edge_hashes)
                or not expected_edge_roles.issubset(edge_role_hashes)
                or not profile_styles_satisfied
                or not expected_links.issubset(link_hashes)
            ):
                invalid.append(artifact)
        return {
            "text": raw,
            "passed": not invalid,
            "evidence": (
                "all matching draw.io artifacts contain the required graph invariants"
                if not invalid
                else "one or more draw.io artifacts lack required graph invariants"
            ),
        }

    if kind == "drawio_self_contained_svg":
        invalid = [
            artifact
            for artifact in matches
            if artifact.get("kind") != "drawio"
            or not artifact.get("valid")
            or not artifact.get("uncompressed")
            or not artifact.get("self_contained_svg")
        ]
        return {
            "text": raw,
            "passed": not invalid,
            "evidence": (
                "all matching draw.io artifacts contain validated embedded SVG images"
                if not invalid
                else "one or more draw.io artifacts are invalid, compressed, or not self-contained"
            ),
        }

    return {"text": raw, "passed": False, "evidence": f"unsupported visual assertion kind {kind!r}"}


class AgentSkillsEvaluator:
    def __init__(self, config=None):
        self.config = dict(config or {})
        self.judge_backend = str(self.config.get("judge_backend") or "heuristic").strip().lower()
        self.optimizer_backend = str(
            self.config.get("optimizer_backend") or "openai_chat"
        ).strip().lower()
        self.codex_path = str(self.config.get("codex_exec_path") or "codex")
        self.codex_model = str(
            self.config.get("codex_cli_judge_model")
            or self.config.get("judge_model")
            or ""
        ).strip()
        self.codex_profile = str(self.config.get("codex_cli_judge_profile") or "").strip()
        self.codex_timeout = int(self.config.get("codex_cli_judge_timeout") or 240)
        self.codex_sandbox = str(self.config.get("codex_cli_judge_sandbox") or "read-only")

    def score(self, case, rollout):
        response = rollout.get("response", "")
        expected = case.get("expected_behavior", [])
        returncode = rollout.get("returncode", 0)
        if returncode not in (0, None):
            result = {
                "passed": False,
                "score": 0.0,
                "assertion_results": [
                    {
                        "text": "Target rollout command exits successfully",
                        "passed": False,
                        "evidence": f"returncode={returncode}; stderr={_short(rollout.get('stderr', ''), 500)}",
                    }
                ],
                "judge_backend": "runtime",
                "reason": f"target rollout failed with returncode={returncode}",
            }
        else:
            deterministic = self._score_deterministic(case, response)
            visual = self._score_visual(case, rollout)
            hard_assertions = self._combine_hard_assertions(deterministic, visual)
            if hard_assertions and not hard_assertions["passed"]:
                result = hard_assertions
            elif self.judge_backend == "codex_cli":
                result = self._score_with_codex_cli(case, response, expected, hard_assertions)
            elif self.judge_backend == "provider":
                result = self._score_with_provider(case, response, expected, hard_assertions)
            else:
                semantic = self._score_with_heuristic(response, expected)
                result = self._merge_passed_hard_assertions(semantic, hard_assertions)

        hard = 1 if result["passed"] else 0
        soft = _float_score(result.get("score", hard))
        task_type = str(case.get("task_type") or case.get("skill_name") or "agent_skill_quality")
        return {
            "id": case["id"],
            "hard": hard,
            "soft": soft,
            "prediction": response[:500],
            "assertion_results": result["assertion_results"],
            "judge_backend": result["judge_backend"],
            "judge_reason": result.get("reason", ""),
            "task_type": task_type,
        }

    def _score_with_heuristic(self, response, expected):
        assertion_results = []
        if expected:
            passed = _contains_expected_text(response, expected)
            assertion_results.append(
                {
                    "text": "Response semantically reflects expected behavior",
                    "passed": passed,
                    "evidence": "Matched expected behavior text" if passed else "No exact expected behavior text matched",
                }
            )
        else:
            passed = bool(response.strip())
            assertion_results.append(
                {
                    "text": "Response is non-empty",
                    "passed": passed,
                    "evidence": "Non-empty response" if passed else "Empty response",
                }
            )
        return {
            "passed": passed,
            "score": 1.0 if passed else 0.0,
            "assertion_results": assertion_results,
            "judge_backend": "heuristic",
            "reason": assertion_results[0]["evidence"],
        }

    def _score_deterministic(self, case, response):
        assertions = case.get("deterministic_assertions") or []
        results = [
            item
            for item in (_match_assertion(response, assertion) for assertion in assertions)
            if item is not None
        ]
        if not results:
            return None
        passed = all(item["passed"] for item in results)
        score = sum(1 for item in results if item["passed"]) / len(results)
        return {
            "passed": passed,
            "score": score,
            "assertion_results": results,
            "judge_backend": "deterministic",
            "reason": "deterministic assertions passed" if passed else "deterministic assertion failed",
        }

    def _score_visual(self, case, rollout):
        assertions = case.get("visual_assertions") or []
        results = [
            item
            for item in (_match_visual_assertion(rollout.get("artifacts") or [], assertion) for assertion in assertions)
            if item is not None
        ]
        if not results:
            return None
        passed = all(item["passed"] for item in results)
        score = sum(1 for item in results if item["passed"]) / len(results)
        return {
            "passed": passed,
            "score": score,
            "assertion_results": results,
            "judge_backend": "visual",
            "reason": "visual assertions passed" if passed else "visual assertion failed",
        }

    def _combine_hard_assertions(self, *groups):
        active = [group for group in groups if group]
        if not active:
            return None
        results = []
        for group in active:
            results.extend(group.get("assertion_results") or [])
        passed = all(item["passed"] for item in results)
        score = sum(1 for item in results if item["passed"]) / len(results) if results else 0.0
        backends = "+".join(group.get("judge_backend", "deterministic") for group in active)
        return {
            "passed": passed,
            "score": score,
            "assertion_results": results,
            "judge_backend": backends,
            "reason": "hard assertions passed" if passed else "hard assertion failed",
        }

    def _merge_passed_hard_assertions(self, semantic, hard_assertions):
        if not hard_assertions:
            return semantic
        results = list(hard_assertions.get("assertion_results") or [])
        results.extend(semantic.get("assertion_results") or [])
        passed = bool(hard_assertions.get("passed")) and bool(semantic.get("passed"))
        return {
            "passed": passed,
            "score": min(
                _float_score(hard_assertions.get("score", 0.0)),
                _float_score(semantic.get("score", 0.0)),
            ),
            "assertion_results": results,
            "judge_backend": (
                f"{hard_assertions.get('judge_backend', 'deterministic')}+"
                f"{semantic.get('judge_backend', 'heuristic')}"
            ),
            "reason": semantic.get("reason", "semantic evaluation completed"),
        }

    def _score_with_codex_cli(self, case, response, expected, deterministic=None):
        if not response.strip():
            return {
                "passed": False,
                "score": 0.0,
                "assertion_results": [
                    {
                        "text": "Response is non-empty",
                        "passed": False,
                        "evidence": "Empty response",
                    }
                ],
                "judge_backend": "codex_cli",
                "reason": "Empty response",
            }

        with tempfile.TemporaryDirectory(prefix="skillopt-codex-judge-") as tmp:
            out = Path(tmp) / "judge.json"
            process_env = _minimal_codex_environment()
            shell_env = _shell_environment(tmp, process_env)
            command = [
                self.codex_path,
                "exec",
                "--ignore-user-config",
                "--ignore-rules",
                "--skip-git-repo-check",
                "--ephemeral",
                "--color",
                "never",
                "--output-last-message",
                str(out),
                "--strict-config",
                "-c",
                f"default_permissions={_toml_string(JUDGE_PERMISSION_PROFILE)}",
                "-c",
                (
                    f"permissions.{JUDGE_PERMISSION_PROFILE}.filesystem="
                    f"{_text_filesystem_permissions(self.codex_path)}"
                ),
                "-c",
                f"permissions.{JUDGE_PERMISSION_PROFILE}.network.enabled=false",
                "-c",
                'shell_environment_policy.inherit="none"',
                "-c",
                f"shell_environment_policy.set={_toml_inline_string_map(shell_env)}",
            ]
            if self.codex_model:
                command.extend(["--model", self.codex_model])
            if self.codex_profile:
                command.extend(["--profile", self.codex_profile])
            prompt = self._judge_prompt(case, response, expected)
            command.append("-")
            try:
                completed = _run_isolated_codex(
                    command,
                    prompt,
                    tmp,
                    process_env,
                    self.codex_timeout,
                )
            except (OSError, subprocess.TimeoutExpired) as error:
                return self._judge_failure(f"Codex judge failed: {_short(error, 500)}")

            if completed.returncode != 0:
                return self._judge_failure(
                    f"Codex judge exited {completed.returncode}: {_short(completed.stderr, 500)}"
                )

            try:
                payload = _extract_json(out.read_text(encoding="utf-8") if out.exists() else completed.stdout)
                return self._normalise_judge_payload(
                    payload, expected, deterministic, backend="codex_cli"
                )
            except Exception as error:
                return self._judge_failure(f"Codex judge returned invalid JSON: {_short(error, 500)}")

    def _score_with_provider(self, case, response, expected, deterministic=None):
        if not response.strip():
            return self._judge_failure("Provider judge received an empty response", backend="provider")
        if self.optimizer_backend not in {"openai_chat", "qwen_chat"}:
            return self._judge_failure(
                f"Provider semantic judge does not support optimizer_backend={self.optimizer_backend}",
                backend="provider",
            )
        try:
            from skillopt.model import chat_optimizer

            payload_text, _usage = chat_optimizer(
                system=(
                    "You are a strict semantic evaluator for Agent Skill optimization. "
                    "Return only the requested JSON and do not execute tools."
                ),
                user=self._judge_prompt(case, response, expected),
                max_completion_tokens=2048,
                retries=2,
                stage="judge",
                timeout=self.codex_timeout,
            )
            payload = _extract_json(str(payload_text or ""))
            return self._normalise_judge_payload(
                payload, expected, deterministic, backend="provider"
            )
        except Exception as error:
            return self._judge_failure(
                f"Provider judge failed: {_short(error, 500)}", backend="provider"
            )

    def _judge_prompt(self, case, response, expected):
        payload = {
            "case_id": case.get("id", ""),
            "user_prompt": case.get("prompt", ""),
            "should_trigger": case.get("should_trigger", True),
            "expected_behavior": expected,
            "assistant_response": _short(response, 12000),
        }
        return (
            "You are a strict evaluator for Agent Skill optimization. Return JSON only.\n"
            "Judge whether the assistant response semantically satisfies the expected behavior. "
            "Do not require exact wording. Award partial score for materially correct but incomplete responses. "
            "Fail responses that invent repo facts, skip required verification/persistence gates, or only ask for context "
            "when the expected behavior requires a concrete provisional artifact/checkpoint.\n\n"
            "Return this exact shape:\n"
            "{\"passed\": boolean, \"score\": number, \"reason\": \"short\", "
            "\"assertions\": [{\"text\": \"criterion\", \"passed\": boolean, \"evidence\": \"short\"}]}\n\n"
            f"Evaluation input:\n{json.dumps(payload, ensure_ascii=False)}"
        )

    def _normalise_judge_payload(
        self, payload, expected, deterministic=None, backend="codex_cli"
    ):
        if not isinstance(payload, dict):
            raise ValueError("judge payload is not an object")
        passed = _bool(payload.get("passed"))
        score = _float_score(payload.get("score", 1.0 if passed else 0.0))
        assertions = payload.get("assertions")
        if not isinstance(assertions, list):
            assertions = []
        normalised = []
        for item in assertions[:12]:
            if not isinstance(item, dict):
                continue
            normalised.append(
                {
                    "text": _short(item.get("text") or "Judge assertion", 300),
                    "passed": _bool(item.get("passed")),
                    "evidence": _short(item.get("evidence") or item.get("reason") or "", 500),
                }
            )
        if not normalised:
            normalised.append(
                {
                    "text": "Response satisfies expected behavior",
                    "passed": passed,
                    "evidence": _short(payload.get("reason") or "Judge did not return assertions", 500),
                }
            )
        if expected and not any(not item["passed"] for item in normalised) and not passed:
            normalised.append(
                {
                    "text": "Overall expected behavior",
                    "passed": False,
                    "evidence": _short(payload.get("reason") or "Overall judge result failed", 500),
                }
            )
        if deterministic:
            normalised = deterministic["assertion_results"] + normalised
            passed = deterministic["passed"] and passed
            score = min(score, deterministic["score"])
        return {
            "passed": passed,
            "score": score,
            "assertion_results": normalised,
            "judge_backend": backend,
            "reason": _short(payload.get("reason") or "", 500),
        }

    def _judge_failure(self, evidence, backend="codex_cli"):
        evidence = _short(_redact_sensitive(evidence), 500)
        label = "Provider semantic judge completed" if backend == "provider" else "Codex CLI judge completed"
        return {
            "passed": False,
            "score": 0.0,
            "assertion_results": [
                {
                    "text": label,
                    "passed": False,
                    "evidence": evidence,
                }
            ],
            "judge_backend": backend,
            "reason": evidence,
        }
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall