Skip to content

stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup

skillopt-setup

Flagged

Set up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.

FileKindSize
SKILL.mdskill md12.9 KB
assets/agent-skills-benchmark/README.mdmarkdown1.7 KB
assets/agent-skills-benchmark/adapter.py.templateother14.4 KB
assets/agent-skills-benchmark/codex_cli_reflector.py.templateother27.5 KB
assets/agent-skills-benchmark/config.codex-cli-all.yamlconfig1.5 KB
assets/agent-skills-benchmark/config.hybrid-codex-target.yamlconfig1.7 KB
assets/agent-skills-benchmark/config.native-provider.yamlconfig1.3 KB
assets/agent-skills-benchmark/dataloader.py.templateother6.6 KB
assets/agent-skills-benchmark/evaluator.py.templateother62.5 KB
assets/agent-skills-benchmark/prompts/analyst_error.mdmarkdown193 B
assets/agent-skills-benchmark/prompts/analyst_error_full_rewrite.mdmarkdown215 B
assets/agent-skills-benchmark/prompts/analyst_success.mdmarkdown183 B
assets/agent-skills-benchmark/prompts/analyst_success_full_rewrite.mdmarkdown157 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_failure.mdmarkdown202 B
assets/agent-skills-benchmark/prompts/codex_cli_reflect_success.mdmarkdown186 B
assets/agent-skills-benchmark/rollout.py.templateother143.7 KB
assets/agent-skills-benchmark/schemas/codex_cli_patch.schema.jsonconfig1.2 KB
assets/agent-skills-benchmark/schemas/codex_cli_rollout.schema.jsonconfig854 B
references/adapter-contract.mdmarkdown5.8 KB
references/adoption-policy.mdmarkdown1.0 KB
references/codex-cli-runner.mdmarkdown3.3 KB
references/credential-modes.mdmarkdown5.8 KB
references/data-schema.mdmarkdown7.7 KB
references/local-artifact-audit.mdmarkdown1.9 KB
references/local-openai-gateway.mdmarkdown5.1 KB
references/official-best-practices.mdmarkdown4.1 KB
references/runbook.mdmarkdown20.0 KB
references/troubleshooting.mdmarkdown5.8 KB
scripts/apply-skillopt-best.mjsscript13.9 KB
scripts/audit-skillopt-local-artifacts.mjsscript22.0 KB
scripts/check-skillopt-readiness.mjsscript99.6 KB
scripts/codex-local-openai-chat-gateway.mjsscript51.6 KB
scripts/prepare-local-skillopt-adapter.mjsscript25.4 KB
scripts/prepare-skillopt-split.mjsscript22.9 KB
scripts/probe-codex-cli.mjsscript4.3 KB
scripts/probe-openai-compatible-endpoint.mjsscript7.9 KB
scripts/setup-skillopt-local.mjsscript40.9 KB
scripts/summarize-skillopt-run.mjsscript17.9 KB
scripts/verify-skillopt-run-artifacts.mjsscript7.4 KB

assets/agent-skills-benchmark/codex_cli_reflector.py.template

View on GitHub
from collections import deque
import errno
import json
import os
import re
import shutil
import signal
import subprocess
import tempfile
import time
from pathlib import Path
from threading import Lock, Thread


REFLECTOR_PERMISSION_PROFILE = "skillopt_reflector"


def _toml_string(value):
    return json.dumps(str(value))


def _path_is_within(path, root):
    try:
        path.relative_to(root)
        return True
    except ValueError:
        return False


def _codex_runtime_read_path(command):
    candidate = Path(str(command or "codex")).expanduser()
    if not candidate.is_absolute():
        resolved = shutil.which(str(candidate))
        if not resolved:
            return None
        candidate = Path(resolved)
    try:
        executable = candidate.resolve(strict=True)
    except OSError:
        return None
    runtime_path = executable
    for parent in executable.parents:
        package_json = parent / "package.json"
        if not package_json.is_file():
            continue
        try:
            package = json.loads(package_json.read_text(encoding="utf-8"))
        except (OSError, ValueError):
            continue
        package_name = str(package.get("name") or "")
        if package_name == "@openai/codex" or package_name.startswith("@openai/codex-"):
            runtime_path = parent
            break
    forbidden_roots = [(Path.home() / ".codex").resolve()]
    if os.environ.get("CODEX_HOME"):
        forbidden_roots.append(Path(os.environ["CODEX_HOME"]).expanduser().resolve())
    if runtime_path in {Path(runtime_path.anchor), Path.home()}:
        return None
    if any(_path_is_within(runtime_path, root) for root in forbidden_roots):
        return None
    return runtime_path


def _text_filesystem_permissions(command):
    entries = ['":minimal"="read"']
    runtime_path = _codex_runtime_read_path(command)
    if runtime_path is not None:
        entries.append(f"{_toml_string(runtime_path)}=\"read\"")
    return "{" + ",".join(entries) + "}"


def _minimal_codex_environment():
    allowed = (
        "HOME",
        "PATH",
        "USER",
        "LOGNAME",
        "SHELL",
        "TMPDIR",
        "TEMP",
        "TMP",
        "LANG",
        "LC_ALL",
        "LC_CTYPE",
        "TERM",
        "COLORTERM",
        "NO_COLOR",
        "CODEX_HOME",
        "SSL_CERT_FILE",
        "SSL_CERT_DIR",
    )
    return {key: os.environ[key] for key in allowed if os.environ.get(key)}


def _toml_inline_string_map(values):
    return "{" + ",".join(
        f"{_toml_string(key)}={_toml_string(value)}" for key, value in values.items()
    ) + "}"


def _shell_environment(workspace, process_env):
    temp_dir = Path(workspace) / ".tmp"
    temp_dir.mkdir(mode=0o700)
    values = {
        "HOME": str(workspace),
        "PATH": process_env.get("PATH", os.defpath),
        "TMPDIR": str(temp_dir),
        "TMP": str(temp_dir),
        "TEMP": str(temp_dir),
    }
    for key in ("LANG", "LC_ALL", "LC_CTYPE", "TERM", "COLORTERM", "NO_COLOR"):
        if process_env.get(key):
            values[key] = process_env[key]
    return values


def _process_group_alive(pgid):
    try:
        os.killpg(pgid, 0)
        return True
    except ProcessLookupError:
        return False
    except OSError:
        return True


def _reap_process_group(process, grace_seconds=1):
    if not _process_group_alive(process.pid):
        return
    try:
        os.killpg(process.pid, signal.SIGTERM)
    except ProcessLookupError:
        return
    deadline = time.monotonic() + grace_seconds
    while _process_group_alive(process.pid) and time.monotonic() < deadline:
        process.poll()
        time.sleep(0.05)
    if _process_group_alive(process.pid):
        try:
            os.killpg(process.pid, signal.SIGKILL)
        except ProcessLookupError:
            return
        deadline = time.monotonic() + 5
        while _process_group_alive(process.pid) and time.monotonic() < deadline:
            process.poll()
            time.sleep(0.05)


_PROMPT_ECHO_MARKER = "[redacted-prompt]"
_PROMPT_ECHO_SEED_CHARS = 16
_PROMPT_ECHO_MAX_UNIQUE_SEEDS = 65_536
_PROMPT_ECHO_MAX_SPANS = 65_536
_PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES = _PROMPT_ECHO_SEED_CHARS * 4
_MAX_STDOUT_CAPTURE_BYTES = 10 * 1024 * 1024
_MAX_STDERR_CAPTURE_BYTES = 64 * 1024
_CAPTURE_CHUNK_BYTES = 64 * 1024
_OUTPUT_TRUNCATION_MARKER = "[output truncated; retained tail]"


class _BoundedCaptureError(OSError):
    pass


class _BoundedByteCapture:
    def __init__(self, limit):
        self.limit = max(1, int(limit))
        self._chunks = deque()
        self._retained_bytes = 0
        self._total_bytes = 0
        self._discarded_tail = b""
        self._lock = Lock()

    def _remember_discarded(self, value):
        recent = value[-_PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES:]
        self._discarded_tail = (
            self._discarded_tail + recent
        )[-_PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES:]

    def append(self, chunk):
        value = bytes(chunk or b"")
        if not value:
            return
        with self._lock:
            self._total_bytes += len(value)
            self._chunks.append(value)
            self._retained_bytes += len(value)
            while self._retained_bytes > self.limit:
                excess = self._retained_bytes - self.limit
                first = self._chunks[0]
                if len(first) <= excess:
                    removed = self._chunks.popleft()
                    self._retained_bytes -= len(first)
                else:
                    removed = first[:excess]
                    self._chunks[0] = first[excess:]
                    self._retained_bytes -= excess
                self._remember_discarded(removed)

    @property
    def total_bytes(self):
        with self._lock:
            return self._total_bytes

    def snapshot(self):
        with self._lock:
            return b"".join(self._chunks), self._total_bytes > self.limit

    def redaction_snapshot(self):
        with self._lock:
            return (
                self._discarded_tail,
                b"".join(self._chunks),
                self._total_bytes > self.limit,
            )


class _BoundedProcessIO:
    def __init__(
        self,
        process,
        prompt,
        stdout_limit=_MAX_STDOUT_CAPTURE_BYTES,
        stderr_limit=_MAX_STDERR_CAPTURE_BYTES,
    ):
        self.process = process
        self.prompt = str(prompt or "").encode("utf-8", errors="replace")
        self.stdout = _BoundedByteCapture(stdout_limit)
        self.stderr = _BoundedByteCapture(stderr_limit)
        self._errors = []
        self._error_lock = Lock()
        self._readers = [
            Thread(
                target=self._read_stream,
                args=("stdout", process.stdout, self.stdout),
                name="skillopt-reflector-stdout-drain",
                daemon=True,
            ),
            Thread(
                target=self._read_stream,
                args=("stderr", process.stderr, self.stderr),
                name="skillopt-reflector-stderr-drain",
                daemon=True,
            ),
        ]
        self._writer = Thread(
            target=self._write_prompt,
            name="skillopt-reflector-stdin-writer",
            daemon=True,
        )
        self._threads = [*self._readers, self._writer]
        self._started_threads = []

    def _record_error(self, label, error):
        with self._error_lock:
            self._errors.append(f"{label}: {type(error).__name__}")

    def _read_stream(self, label, stream, capture):
        try:
            while True:
                chunk = stream.read(_CAPTURE_CHUNK_BYTES)
                if not chunk:
                    return
                capture.append(chunk)
        except Exception as error:
            self._record_error(label, error)
        finally:
            try:
                stream.close()
            except Exception:
                pass

    def _write_prompt(self):
        stream = self.process.stdin
        try:
            view = memoryview(self.prompt)
            while view:
                written = stream.write(view)
                if not written:
                    raise BrokenPipeError("stdin closed before prompt transport completed")
                view = view[written:]
            stream.flush()
        except BrokenPipeError:
            pass
        except OSError as error:
            if error.errno not in {errno.EINVAL, errno.EPIPE}:
                self._record_error("stdin", error)
        except Exception as error:
            self._record_error("stdin", error)
        finally:
            try:
                stream.close()
            except Exception:
                pass

    def start(self):
        for thread in self._threads:
            try:
                thread.start()
            finally:
                if thread.ident is not None and thread not in self._started_threads:
                    self._started_threads.append(thread)

    def close_unstarted_streams(self):
        thread_streams = (
            (self._readers[0], self.process.stdout),
            (self._readers[1], self.process.stderr),
            (self._writer, self.process.stdin),
        )
        for thread, stream in thread_streams:
            if thread in self._started_threads or thread.is_alive():
                continue
            try:
                stream.close()
            except Exception:
                pass

    def close_stopped_streams(self):
        thread_streams = (
            (self._readers[0], self.process.stdout),
            (self._readers[1], self.process.stderr),
            (self._writer, self.process.stdin),
        )
        for thread, stream in thread_streams:
            if thread.is_alive():
                continue
            try:
                stream.close()
            except Exception:
                pass

    def finish(self, timeout=7):
        deadline = time.monotonic() + max(0, timeout)
        for thread in self._started_threads:
            thread.join(timeout=max(0, deadline - time.monotonic()))
        alive = [thread.name for thread in self._started_threads if thread.is_alive()]
        if alive:
            raise _BoundedCaptureError("bounded subprocess I/O threads did not stop")
        with self._error_lock:
            errors = list(self._errors)
        if errors:
            raise _BoundedCaptureError(
                "bounded subprocess I/O failed: " + "; ".join(errors)
            )


def _close_process_pipes(process):
    for stream in (process.stdin, process.stdout, process.stderr):
        try:
            stream.close()
        except Exception:
            pass


def _cleanup_failed_process_io_start(process, process_io):
    cleanup_failed = False
    try:
        if process_io is not None:
            process_io.close_unstarted_streams()
    except Exception:
        cleanup_failed = True
    try:
        _reap_process_group(process, grace_seconds=0)
        try:
            process.wait(timeout=5)
        except subprocess.TimeoutExpired:
            cleanup_failed = True
    except Exception:
        cleanup_failed = True
    if process_io is not None:
        try:
            process_io.finish(timeout=1)
        except Exception:
            cleanup_failed = True
        finally:
            process_io.close_stopped_streams()
    else:
        _close_process_pipes(process)
    try:
        cleanup_failed = (
            process.poll() is None
            or _process_group_alive(process.pid)
            or cleanup_failed
        )
    except Exception:
        cleanup_failed = True
    return cleanup_failed


def _start_bounded_process_io(process, prompt, **kwargs):
    process_io = None
    try:
        process_io = _BoundedProcessIO(process, prompt, **kwargs)
        process_io.start()
        return process_io
    except BaseException as error:
        cleanup_failed = _cleanup_failed_process_io_start(process, process_io)
        if not isinstance(error, Exception):
            raise
        if cleanup_failed:
            raise _BoundedCaptureError(
                "bounded subprocess I/O startup cleanup failed"
            ) from None
        raise _BoundedCaptureError(
            f"bounded subprocess I/O failed to start: {type(error).__name__}"
        ) from None


def _render_captured_stream(capture, prompt):
    discarded_tail, payload, truncated = capture.redaction_snapshot()
    value = payload.decode("utf-8", errors="replace")
    boundary_chars = _prompt_boundary_prefix_chars(
        discarded_tail,
        payload,
        str(prompt or "").encode("utf-8", errors="replace"),
    )
    value = _redact_prompt(value, prompt, boundary_chars=boundary_chars)
    if truncated:
        return f"{_OUTPUT_TRUNCATION_MARKER}\n{value}"
    return value


def _prompt_boundary_prefix_chars(discarded_tail, payload, transported):
    """Return retained leading characters proven to continue a discarded prompt echo."""
    if not discarded_tail or not payload or len(transported) < 2:
        return 0
    seed_bytes = min(_PROMPT_ECHO_SEED_CHARS, len(transported))
    max_left_size = min(seed_bytes - 1, len(discarded_tail))
    if max_left_size <= 0:
        return 0
    high = min(len(payload), _PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES)
    matched_bytes = 0
    for left_size in range(1, max_left_size + 1):
        left = discarded_tail[-left_size:]
        if left + payload[:1] not in transported:
            continue
        low = 1
        candidate_high = high
        while low < candidate_high:
            middle = (low + candidate_high + 1) // 2
            if left + payload[:middle] in transported:
                low = middle
            else:
                candidate_high = middle - 1
        matched_bytes = max(matched_bytes, low)
        if matched_bytes == high:
            break
    return len(payload[:matched_bytes].decode("utf-8", errors="replace"))


def _prompt_echo_spans(value, transported):
    """Return bounded prompt-derived spans, or None to request fail-closed redaction."""
    spans = []
    if not value or not transported:
        return spans
    if value in transported:
        return [(0, len(value))]
    for match in re.finditer(r"[^\r\n]+", value):
        line = match.group(0)
        stripped = line.strip()
        if stripped and stripped in transported:
            offset = line.find(stripped)
            spans.append((match.start() + offset, match.start() + offset + len(stripped)))
            if len(spans) > _PROMPT_ECHO_MAX_SPANS:
                return None

    seed_size = min(_PROMPT_ECHO_SEED_CHARS, len(transported))
    if seed_size <= 0 or len(value) < seed_size:
        return spans

    prompt_seeds = set()
    for index in range(len(transported) - seed_size + 1):
        prompt_seeds.add(transported[index : index + seed_size])
        if len(prompt_seeds) > _PROMPT_ECHO_MAX_UNIQUE_SEEDS:
            return None

    seed_spans = []
    last_seed_start = len(value) - seed_size
    for index in range(last_seed_start + 1):
        if value[index : index + seed_size] not in prompt_seeds:
            continue
        end = index + seed_size
        if seed_spans and index <= seed_spans[-1][1]:
            seed_spans[-1] = (
                seed_spans[-1][0],
                max(seed_spans[-1][1], end),
            )
        else:
            seed_spans.append((index, end))
            if len(spans) + len(seed_spans) > _PROMPT_ECHO_MAX_SPANS:
                return None
    spans.extend(seed_spans)
    return spans


def _redact_prompt(text, prompt, boundary_chars=0):
    value = text.decode("utf-8", errors="replace") if isinstance(text, bytes) else str(text or "")
    transported = str(prompt or "").encode("utf-8", errors="replace").decode("utf-8")
    spans = _prompt_echo_spans(value, transported)
    if spans is None:
        return _PROMPT_ECHO_MARKER if value else ""
    guard_end = min(len(value), max(0, int(boundary_chars)))
    if guard_end:
        if len(spans) >= _PROMPT_ECHO_MAX_SPANS:
            return _PROMPT_ECHO_MARKER
        spans.append((0, guard_end))
    if not spans:
        return value
    spans.sort()
    merged = []
    for start, end in spans:
        if merged and start <= merged[-1][1]:
            merged[-1] = (merged[-1][0], max(merged[-1][1], end))
        else:
            merged.append((start, end))
    parts = []
    cursor = 0
    for start, end in merged:
        parts.extend((value[cursor:start], _PROMPT_ECHO_MARKER))
        cursor = end
    parts.append(value[cursor:])
    return "".join(parts)


def _redact_timeout_stream(text):
    return _PROMPT_ECHO_MARKER if (
        text.decode("utf-8", errors="replace") if isinstance(text, bytes) else str(text or "")
    ) else ""


def _run_isolated_codex(command, prompt, cwd, env, timeout):
    process = subprocess.Popen(
        command,
        cwd=cwd,
        env=env,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        start_new_session=True,
    )
    process_io = _start_bounded_process_io(process, prompt)
    try:
        process.wait(timeout=timeout)
    except subprocess.TimeoutExpired:
        cleanup_failed = False
        try:
            _reap_process_group(process)
        except Exception:
            cleanup_failed = True
        try:
            process.wait(timeout=5)
        except subprocess.TimeoutExpired:
            cleanup_failed = True
        try:
            process_io.finish()
        except Exception:
            cleanup_failed = True
        finally:
            process_io.close_stopped_streams()
        try:
            cleanup_failed = (
                process.poll() is None
                or _process_group_alive(process.pid)
                or cleanup_failed
            )
        except Exception:
            cleanup_failed = True
        if cleanup_failed:
            raise _BoundedCaptureError(
                "bounded subprocess cleanup failed after timeout"
            ) from None
        error = subprocess.TimeoutExpired(command, timeout)
        error.stdout = (
            _PROMPT_ECHO_MARKER
            if process_io.stdout.total_bytes
            else ""
        )
        error.stderr = (
            _PROMPT_ECHO_MARKER
            if process_io.stderr.total_bytes
            else ""
        )
        raise error
    cleanup_failed = False
    try:
        _reap_process_group(process)
    except Exception:
        cleanup_failed = True
    try:
        group_alive = _process_group_alive(process.pid)
    except Exception:
        group_alive = True
    capture_error = None
    try:
        process_io.finish()
    except _BoundedCaptureError as error:
        capture_error = error
    finally:
        process_io.close_stopped_streams()
    if cleanup_failed or group_alive:
        raise _BoundedCaptureError("bounded subprocess cleanup failed") from None
    if capture_error is not None:
        raise capture_error
    return subprocess.CompletedProcess(
        command,
        process.returncode,
        _render_captured_stream(process_io.stdout, prompt),
        _render_captured_stream(process_io.stderr, prompt),
    )


SECRET_PATTERN = re.compile(r"(sk-[A-Za-z0-9_-]{20,}|[A-Za-z0-9_-]{32,})")
FORBIDDEN_OPTIMIZER_PATTERN = re.compile(r"codex[-_\s]?opt", re.IGNORECASE)


def _extract_json_payload(text):
    value = (text or "").strip()
    if not value:
        return {}
    if value.startswith("```"):
        value = re.sub(r"^```(?:json)?\s*", "", value)
        value = re.sub(r"\s*```$", "", value)
    decoder = json.JSONDecoder()
    for index, char in enumerate(value):
        if char not in "{[":
            continue
        try:
            payload, _ = decoder.raw_decode(value[index:])
            return payload
        except json.JSONDecodeError:
            continue
    return {}


class CodexCliReflector:
    def __init__(self, env):
        self.env = env
        self.timeout = int(env.get("codex_cli_reflection_timeout", 240))
        self.codex_path = str(env.get("codex_exec_path") or "codex")
        self.model = str(env.get("codex_cli_reflection_model") or "").strip()
        self.profile = str(env.get("codex_exec_profile") or "").strip()
        self.edit_budget = max(1, int(env.get("edit_budget") or 2))

    def reflect(self, results, skill_content, out_dir, **kwargs):
        if not results:
            return []
        failures = sum(1 for item in results if not item.get("hard"))
        successes = len(results) - failures
        print(
            f"    [agent_skills reflection] codex_cli results={len(results)} failures={failures} successes={successes}",
            flush=True,
        )
        with tempfile.TemporaryDirectory(prefix="skillopt-codex-reflect-") as tmp:
            out = Path(tmp) / "patch.json"
            prompt = self._prompt(results, skill_content, kwargs)
            process_env = _minimal_codex_environment()
            shell_env = _shell_environment(tmp, process_env)
            command = [
                self.codex_path,
                "exec",
                "--ignore-user-config",
                "--ignore-rules",
                "--skip-git-repo-check",
                "--ephemeral",
                "--color",
                "never",
                "--output-last-message",
                str(out),
                "--strict-config",
                "-c",
                f"default_permissions={_toml_string(REFLECTOR_PERMISSION_PROFILE)}",
                "-c",
                (
                    f"permissions.{REFLECTOR_PERMISSION_PROFILE}.filesystem="
                    f"{_text_filesystem_permissions(self.codex_path)}"
                ),
                "-c",
                f"permissions.{REFLECTOR_PERMISSION_PROFILE}.network.enabled=false",
                "-c",
                'shell_environment_policy.inherit="none"',
                "-c",
                f"shell_environment_policy.set={_toml_inline_string_map(shell_env)}",
            ]
            if self.model:
                command.extend(["--model", self.model])
            if self.profile:
                command.extend(["--profile", self.profile])
            command.append("-")
            try:
                completed = _run_isolated_codex(
                    command,
                    prompt,
                    tmp,
                    process_env,
                    self.timeout,
                )
            except (OSError, subprocess.TimeoutExpired) as error:
                print(f"    [agent_skills reflection skipped] codex_cli failed: {error}", flush=True)
                return []
            if completed.returncode != 0:
                print(
                    f"    [agent_skills reflection skipped] codex_cli exited {completed.returncode}",
                    flush=True,
                )
                return []
            payload_text = out.read_text(encoding="utf-8") if out.exists() else completed.stdout
            payload = _extract_json_payload(payload_text)
            source_type = "failure" if failures else "success"
            patches = self._normalise_payload(
                payload,
                batch_size=len(results),
                source_type=source_type,
            )
            patches_dir = Path(out_dir) / "patches"
            patches_dir.mkdir(parents=True, exist_ok=True)
            valid_patches = []
            for index, patch in enumerate(patches):
                try:
                    self._validate_patch(patch)
                except ValueError as error:
                    print(
                        f"    [agent_skills reflection skipped patch] {error}",
                        flush=True,
                    )
                    continue
                valid_patches.append(patch)
                (patches_dir / f"codex_cli_reflection_{index:03d}.json").write_text(
                    json.dumps(patch, ensure_ascii=False, indent=2),
                    encoding="utf-8",
                )
            print(
                f"    [agent_skills reflection done] patches={len(valid_patches)}",
                flush=True,
            )
            return valid_patches

    def _prompt(self, results, skill_content, kwargs):
        failures = [item for item in results if not item.get("hard")]
        successes = [item for item in results if item.get("hard")]
        update_mode = kwargs.get("update_mode", "patch")
        return (
            "Return exactly one JSON object only, with no markdown and no second object. "
            "Produce bounded SkillOpt RawPatch objects for an Agent Skill body. "
            "Use this shape: {\"raw_patches\":[{\"source_type\":\"failure\","
            "\"patch\":{\"reasoning\":\"...\",\"edits\":[{\"op\":\"replace\","
            "\"target\":\"exact text\",\"content\":\"replacement text\"}]}}]}. "
            f"Return at most {self.edit_budget} edits total. "
            "Return an empty raw_patches list when no useful edit is justified. "
            "Do not include frontmatter, secrets, local auth paths, or whole-skill rewrites.\n\n"
            f"Update mode: {update_mode}\n\n"
            f"Current skill:\n{skill_content[:8000]}\n\n"
            f"Failures:\n{json.dumps(failures, ensure_ascii=False)[:8000]}\n\n"
            f"Successes:\n{json.dumps(successes, ensure_ascii=False)[:5000]}"
        )

    def _normalise_payload(self, payload, batch_size, source_type):
        if isinstance(payload, dict) and isinstance(payload.get("raw_patches"), list):
            candidates = payload["raw_patches"]
        elif isinstance(payload, list):
            candidates = payload
        elif isinstance(payload, dict) and "patch" in payload:
            candidates = [payload]
        elif isinstance(payload, dict) and "edits" in payload:
            candidates = [{"source_type": "failure", "patch": payload}]
        else:
            return []

        edits = []
        reasoning = []
        seen = set()
        for candidate in candidates:
            if not isinstance(candidate, dict):
                continue
            patch = candidate.get("patch", candidate)
            if not isinstance(patch, dict):
                continue
            if patch.get("reasoning"):
                reasoning.append(str(patch.get("reasoning")))
            for edit in patch.get("edits", []):
                if not isinstance(edit, dict):
                    continue
                key = json.dumps(edit, sort_keys=True, ensure_ascii=False)
                if key in seen:
                    continue
                seen.add(key)
                edit = dict(edit)
                edit.setdefault("source_type", candidate.get("source_type", source_type))
                edits.append(edit)

        if not edits:
            return []

        selected = edits[: self.edit_budget]
        merged_reasoning = " ".join(reasoning).strip() or "codex_cli local merge"
        if len(edits) > len(selected):
            merged_reasoning += f" [codex-cli local truncation {len(edits)}->{len(selected)} edits]"
        return [
            {
                "source_type": source_type,
                "batch_size": batch_size,
                "patch": {
                    "reasoning": merged_reasoning,
                    "edits": selected,
                },
            }
        ]

    def _validate_patch(self, payload):
        text = json.dumps(payload, ensure_ascii=False)
        if "---" in text:
            raise ValueError("patch must not include frontmatter")
        if SECRET_PATTERN.search(text):
            raise ValueError("patch contains a secret-like string")
        if FORBIDDEN_OPTIMIZER_PATTERN.search(text):
            raise ValueError("patch references an unsupported optimizer source")
        edits = payload.get("patch", {}).get("edits", [])
        if len(edits) > 8:
            raise ValueError("patch exceeds edit budget")
        for edit in edits:
            if edit.get("op") == "replace_all":
                raise ValueError("whole-skill replacement is disabled in patch mode")
finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the workStars: 322kUpdated Claude CodeCodexVerifiedInstall
systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixesStars: 322kUpdated Claude CodeCodexVerifiedInstall
test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation codeStars: 322kUpdated Claude CodeCodexVerifiedInstall
skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.Stars: 198kUpdated Claude CodeCodexVerifiedInstall