stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup
skillopt-setup
FlaggedSet up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.
assets/agent-skills-benchmark/codex_cli_reflector.py.template
View on GitHubfrom collections import deque
import errno
import json
import os
import re
import shutil
import signal
import subprocess
import tempfile
import time
from pathlib import Path
from threading import Lock, Thread
REFLECTOR_PERMISSION_PROFILE = "skillopt_reflector"
def _toml_string(value):
return json.dumps(str(value))
def _path_is_within(path, root):
try:
path.relative_to(root)
return True
except ValueError:
return False
def _codex_runtime_read_path(command):
candidate = Path(str(command or "codex")).expanduser()
if not candidate.is_absolute():
resolved = shutil.which(str(candidate))
if not resolved:
return None
candidate = Path(resolved)
try:
executable = candidate.resolve(strict=True)
except OSError:
return None
runtime_path = executable
for parent in executable.parents:
package_json = parent / "package.json"
if not package_json.is_file():
continue
try:
package = json.loads(package_json.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
package_name = str(package.get("name") or "")
if package_name == "@openai/codex" or package_name.startswith("@openai/codex-"):
runtime_path = parent
break
forbidden_roots = [(Path.home() / ".codex").resolve()]
if os.environ.get("CODEX_HOME"):
forbidden_roots.append(Path(os.environ["CODEX_HOME"]).expanduser().resolve())
if runtime_path in {Path(runtime_path.anchor), Path.home()}:
return None
if any(_path_is_within(runtime_path, root) for root in forbidden_roots):
return None
return runtime_path
def _text_filesystem_permissions(command):
entries = ['":minimal"="read"']
runtime_path = _codex_runtime_read_path(command)
if runtime_path is not None:
entries.append(f"{_toml_string(runtime_path)}=\"read\"")
return "{" + ",".join(entries) + "}"
def _minimal_codex_environment():
allowed = (
"HOME",
"PATH",
"USER",
"LOGNAME",
"SHELL",
"TMPDIR",
"TEMP",
"TMP",
"LANG",
"LC_ALL",
"LC_CTYPE",
"TERM",
"COLORTERM",
"NO_COLOR",
"CODEX_HOME",
"SSL_CERT_FILE",
"SSL_CERT_DIR",
)
return {key: os.environ[key] for key in allowed if os.environ.get(key)}
def _toml_inline_string_map(values):
return "{" + ",".join(
f"{_toml_string(key)}={_toml_string(value)}" for key, value in values.items()
) + "}"
def _shell_environment(workspace, process_env):
temp_dir = Path(workspace) / ".tmp"
temp_dir.mkdir(mode=0o700)
values = {
"HOME": str(workspace),
"PATH": process_env.get("PATH", os.defpath),
"TMPDIR": str(temp_dir),
"TMP": str(temp_dir),
"TEMP": str(temp_dir),
}
for key in ("LANG", "LC_ALL", "LC_CTYPE", "TERM", "COLORTERM", "NO_COLOR"):
if process_env.get(key):
values[key] = process_env[key]
return values
def _process_group_alive(pgid):
try:
os.killpg(pgid, 0)
return True
except ProcessLookupError:
return False
except OSError:
return True
def _reap_process_group(process, grace_seconds=1):
if not _process_group_alive(process.pid):
return
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
deadline = time.monotonic() + grace_seconds
while _process_group_alive(process.pid) and time.monotonic() < deadline:
process.poll()
time.sleep(0.05)
if _process_group_alive(process.pid):
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
return
deadline = time.monotonic() + 5
while _process_group_alive(process.pid) and time.monotonic() < deadline:
process.poll()
time.sleep(0.05)
_PROMPT_ECHO_MARKER = "[redacted-prompt]"
_PROMPT_ECHO_SEED_CHARS = 16
_PROMPT_ECHO_MAX_UNIQUE_SEEDS = 65_536
_PROMPT_ECHO_MAX_SPANS = 65_536
_PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES = _PROMPT_ECHO_SEED_CHARS * 4
_MAX_STDOUT_CAPTURE_BYTES = 10 * 1024 * 1024
_MAX_STDERR_CAPTURE_BYTES = 64 * 1024
_CAPTURE_CHUNK_BYTES = 64 * 1024
_OUTPUT_TRUNCATION_MARKER = "[output truncated; retained tail]"
class _BoundedCaptureError(OSError):
pass
class _BoundedByteCapture:
def __init__(self, limit):
self.limit = max(1, int(limit))
self._chunks = deque()
self._retained_bytes = 0
self._total_bytes = 0
self._discarded_tail = b""
self._lock = Lock()
def _remember_discarded(self, value):
recent = value[-_PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES:]
self._discarded_tail = (
self._discarded_tail + recent
)[-_PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES:]
def append(self, chunk):
value = bytes(chunk or b"")
if not value:
return
with self._lock:
self._total_bytes += len(value)
self._chunks.append(value)
self._retained_bytes += len(value)
while self._retained_bytes > self.limit:
excess = self._retained_bytes - self.limit
first = self._chunks[0]
if len(first) <= excess:
removed = self._chunks.popleft()
self._retained_bytes -= len(first)
else:
removed = first[:excess]
self._chunks[0] = first[excess:]
self._retained_bytes -= excess
self._remember_discarded(removed)
@property
def total_bytes(self):
with self._lock:
return self._total_bytes
def snapshot(self):
with self._lock:
return b"".join(self._chunks), self._total_bytes > self.limit
def redaction_snapshot(self):
with self._lock:
return (
self._discarded_tail,
b"".join(self._chunks),
self._total_bytes > self.limit,
)
class _BoundedProcessIO:
def __init__(
self,
process,
prompt,
stdout_limit=_MAX_STDOUT_CAPTURE_BYTES,
stderr_limit=_MAX_STDERR_CAPTURE_BYTES,
):
self.process = process
self.prompt = str(prompt or "").encode("utf-8", errors="replace")
self.stdout = _BoundedByteCapture(stdout_limit)
self.stderr = _BoundedByteCapture(stderr_limit)
self._errors = []
self._error_lock = Lock()
self._readers = [
Thread(
target=self._read_stream,
args=("stdout", process.stdout, self.stdout),
name="skillopt-reflector-stdout-drain",
daemon=True,
),
Thread(
target=self._read_stream,
args=("stderr", process.stderr, self.stderr),
name="skillopt-reflector-stderr-drain",
daemon=True,
),
]
self._writer = Thread(
target=self._write_prompt,
name="skillopt-reflector-stdin-writer",
daemon=True,
)
self._threads = [*self._readers, self._writer]
self._started_threads = []
def _record_error(self, label, error):
with self._error_lock:
self._errors.append(f"{label}: {type(error).__name__}")
def _read_stream(self, label, stream, capture):
try:
while True:
chunk = stream.read(_CAPTURE_CHUNK_BYTES)
if not chunk:
return
capture.append(chunk)
except Exception as error:
self._record_error(label, error)
finally:
try:
stream.close()
except Exception:
pass
def _write_prompt(self):
stream = self.process.stdin
try:
view = memoryview(self.prompt)
while view:
written = stream.write(view)
if not written:
raise BrokenPipeError("stdin closed before prompt transport completed")
view = view[written:]
stream.flush()
except BrokenPipeError:
pass
except OSError as error:
if error.errno not in {errno.EINVAL, errno.EPIPE}:
self._record_error("stdin", error)
except Exception as error:
self._record_error("stdin", error)
finally:
try:
stream.close()
except Exception:
pass
def start(self):
for thread in self._threads:
try:
thread.start()
finally:
if thread.ident is not None and thread not in self._started_threads:
self._started_threads.append(thread)
def close_unstarted_streams(self):
thread_streams = (
(self._readers[0], self.process.stdout),
(self._readers[1], self.process.stderr),
(self._writer, self.process.stdin),
)
for thread, stream in thread_streams:
if thread in self._started_threads or thread.is_alive():
continue
try:
stream.close()
except Exception:
pass
def close_stopped_streams(self):
thread_streams = (
(self._readers[0], self.process.stdout),
(self._readers[1], self.process.stderr),
(self._writer, self.process.stdin),
)
for thread, stream in thread_streams:
if thread.is_alive():
continue
try:
stream.close()
except Exception:
pass
def finish(self, timeout=7):
deadline = time.monotonic() + max(0, timeout)
for thread in self._started_threads:
thread.join(timeout=max(0, deadline - time.monotonic()))
alive = [thread.name for thread in self._started_threads if thread.is_alive()]
if alive:
raise _BoundedCaptureError("bounded subprocess I/O threads did not stop")
with self._error_lock:
errors = list(self._errors)
if errors:
raise _BoundedCaptureError(
"bounded subprocess I/O failed: " + "; ".join(errors)
)
def _close_process_pipes(process):
for stream in (process.stdin, process.stdout, process.stderr):
try:
stream.close()
except Exception:
pass
def _cleanup_failed_process_io_start(process, process_io):
cleanup_failed = False
try:
if process_io is not None:
process_io.close_unstarted_streams()
except Exception:
cleanup_failed = True
try:
_reap_process_group(process, grace_seconds=0)
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
cleanup_failed = True
except Exception:
cleanup_failed = True
if process_io is not None:
try:
process_io.finish(timeout=1)
except Exception:
cleanup_failed = True
finally:
process_io.close_stopped_streams()
else:
_close_process_pipes(process)
try:
cleanup_failed = (
process.poll() is None
or _process_group_alive(process.pid)
or cleanup_failed
)
except Exception:
cleanup_failed = True
return cleanup_failed
def _start_bounded_process_io(process, prompt, **kwargs):
process_io = None
try:
process_io = _BoundedProcessIO(process, prompt, **kwargs)
process_io.start()
return process_io
except BaseException as error:
cleanup_failed = _cleanup_failed_process_io_start(process, process_io)
if not isinstance(error, Exception):
raise
if cleanup_failed:
raise _BoundedCaptureError(
"bounded subprocess I/O startup cleanup failed"
) from None
raise _BoundedCaptureError(
f"bounded subprocess I/O failed to start: {type(error).__name__}"
) from None
def _render_captured_stream(capture, prompt):
discarded_tail, payload, truncated = capture.redaction_snapshot()
value = payload.decode("utf-8", errors="replace")
boundary_chars = _prompt_boundary_prefix_chars(
discarded_tail,
payload,
str(prompt or "").encode("utf-8", errors="replace"),
)
value = _redact_prompt(value, prompt, boundary_chars=boundary_chars)
if truncated:
return f"{_OUTPUT_TRUNCATION_MARKER}\n{value}"
return value
def _prompt_boundary_prefix_chars(discarded_tail, payload, transported):
"""Return retained leading characters proven to continue a discarded prompt echo."""
if not discarded_tail or not payload or len(transported) < 2:
return 0
seed_bytes = min(_PROMPT_ECHO_SEED_CHARS, len(transported))
max_left_size = min(seed_bytes - 1, len(discarded_tail))
if max_left_size <= 0:
return 0
high = min(len(payload), _PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES)
matched_bytes = 0
for left_size in range(1, max_left_size + 1):
left = discarded_tail[-left_size:]
if left + payload[:1] not in transported:
continue
low = 1
candidate_high = high
while low < candidate_high:
middle = (low + candidate_high + 1) // 2
if left + payload[:middle] in transported:
low = middle
else:
candidate_high = middle - 1
matched_bytes = max(matched_bytes, low)
if matched_bytes == high:
break
return len(payload[:matched_bytes].decode("utf-8", errors="replace"))
def _prompt_echo_spans(value, transported):
"""Return bounded prompt-derived spans, or None to request fail-closed redaction."""
spans = []
if not value or not transported:
return spans
if value in transported:
return [(0, len(value))]
for match in re.finditer(r"[^\r\n]+", value):
line = match.group(0)
stripped = line.strip()
if stripped and stripped in transported:
offset = line.find(stripped)
spans.append((match.start() + offset, match.start() + offset + len(stripped)))
if len(spans) > _PROMPT_ECHO_MAX_SPANS:
return None
seed_size = min(_PROMPT_ECHO_SEED_CHARS, len(transported))
if seed_size <= 0 or len(value) < seed_size:
return spans
prompt_seeds = set()
for index in range(len(transported) - seed_size + 1):
prompt_seeds.add(transported[index : index + seed_size])
if len(prompt_seeds) > _PROMPT_ECHO_MAX_UNIQUE_SEEDS:
return None
seed_spans = []
last_seed_start = len(value) - seed_size
for index in range(last_seed_start + 1):
if value[index : index + seed_size] not in prompt_seeds:
continue
end = index + seed_size
if seed_spans and index <= seed_spans[-1][1]:
seed_spans[-1] = (
seed_spans[-1][0],
max(seed_spans[-1][1], end),
)
else:
seed_spans.append((index, end))
if len(spans) + len(seed_spans) > _PROMPT_ECHO_MAX_SPANS:
return None
spans.extend(seed_spans)
return spans
def _redact_prompt(text, prompt, boundary_chars=0):
value = text.decode("utf-8", errors="replace") if isinstance(text, bytes) else str(text or "")
transported = str(prompt or "").encode("utf-8", errors="replace").decode("utf-8")
spans = _prompt_echo_spans(value, transported)
if spans is None:
return _PROMPT_ECHO_MARKER if value else ""
guard_end = min(len(value), max(0, int(boundary_chars)))
if guard_end:
if len(spans) >= _PROMPT_ECHO_MAX_SPANS:
return _PROMPT_ECHO_MARKER
spans.append((0, guard_end))
if not spans:
return value
spans.sort()
merged = []
for start, end in spans:
if merged and start <= merged[-1][1]:
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
else:
merged.append((start, end))
parts = []
cursor = 0
for start, end in merged:
parts.extend((value[cursor:start], _PROMPT_ECHO_MARKER))
cursor = end
parts.append(value[cursor:])
return "".join(parts)
def _redact_timeout_stream(text):
return _PROMPT_ECHO_MARKER if (
text.decode("utf-8", errors="replace") if isinstance(text, bytes) else str(text or "")
) else ""
def _run_isolated_codex(command, prompt, cwd, env, timeout):
process = subprocess.Popen(
command,
cwd=cwd,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
start_new_session=True,
)
process_io = _start_bounded_process_io(process, prompt)
try:
process.wait(timeout=timeout)
except subprocess.TimeoutExpired:
cleanup_failed = False
try:
_reap_process_group(process)
except Exception:
cleanup_failed = True
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
cleanup_failed = True
try:
process_io.finish()
except Exception:
cleanup_failed = True
finally:
process_io.close_stopped_streams()
try:
cleanup_failed = (
process.poll() is None
or _process_group_alive(process.pid)
or cleanup_failed
)
except Exception:
cleanup_failed = True
if cleanup_failed:
raise _BoundedCaptureError(
"bounded subprocess cleanup failed after timeout"
) from None
error = subprocess.TimeoutExpired(command, timeout)
error.stdout = (
_PROMPT_ECHO_MARKER
if process_io.stdout.total_bytes
else ""
)
error.stderr = (
_PROMPT_ECHO_MARKER
if process_io.stderr.total_bytes
else ""
)
raise error
cleanup_failed = False
try:
_reap_process_group(process)
except Exception:
cleanup_failed = True
try:
group_alive = _process_group_alive(process.pid)
except Exception:
group_alive = True
capture_error = None
try:
process_io.finish()
except _BoundedCaptureError as error:
capture_error = error
finally:
process_io.close_stopped_streams()
if cleanup_failed or group_alive:
raise _BoundedCaptureError("bounded subprocess cleanup failed") from None
if capture_error is not None:
raise capture_error
return subprocess.CompletedProcess(
command,
process.returncode,
_render_captured_stream(process_io.stdout, prompt),
_render_captured_stream(process_io.stderr, prompt),
)
SECRET_PATTERN = re.compile(r"(sk-[A-Za-z0-9_-]{20,}|[A-Za-z0-9_-]{32,})")
FORBIDDEN_OPTIMIZER_PATTERN = re.compile(r"codex[-_\s]?opt", re.IGNORECASE)
def _extract_json_payload(text):
value = (text or "").strip()
if not value:
return {}
if value.startswith("```"):
value = re.sub(r"^```(?:json)?\s*", "", value)
value = re.sub(r"\s*```$", "", value)
decoder = json.JSONDecoder()
for index, char in enumerate(value):
if char not in "{[":
continue
try:
payload, _ = decoder.raw_decode(value[index:])
return payload
except json.JSONDecodeError:
continue
return {}
class CodexCliReflector:
def __init__(self, env):
self.env = env
self.timeout = int(env.get("codex_cli_reflection_timeout", 240))
self.codex_path = str(env.get("codex_exec_path") or "codex")
self.model = str(env.get("codex_cli_reflection_model") or "").strip()
self.profile = str(env.get("codex_exec_profile") or "").strip()
self.edit_budget = max(1, int(env.get("edit_budget") or 2))
def reflect(self, results, skill_content, out_dir, **kwargs):
if not results:
return []
failures = sum(1 for item in results if not item.get("hard"))
successes = len(results) - failures
print(
f" [agent_skills reflection] codex_cli results={len(results)} failures={failures} successes={successes}",
flush=True,
)
with tempfile.TemporaryDirectory(prefix="skillopt-codex-reflect-") as tmp:
out = Path(tmp) / "patch.json"
prompt = self._prompt(results, skill_content, kwargs)
process_env = _minimal_codex_environment()
shell_env = _shell_environment(tmp, process_env)
command = [
self.codex_path,
"exec",
"--ignore-user-config",
"--ignore-rules",
"--skip-git-repo-check",
"--ephemeral",
"--color",
"never",
"--output-last-message",
str(out),
"--strict-config",
"-c",
f"default_permissions={_toml_string(REFLECTOR_PERMISSION_PROFILE)}",
"-c",
(
f"permissions.{REFLECTOR_PERMISSION_PROFILE}.filesystem="
f"{_text_filesystem_permissions(self.codex_path)}"
),
"-c",
f"permissions.{REFLECTOR_PERMISSION_PROFILE}.network.enabled=false",
"-c",
'shell_environment_policy.inherit="none"',
"-c",
f"shell_environment_policy.set={_toml_inline_string_map(shell_env)}",
]
if self.model:
command.extend(["--model", self.model])
if self.profile:
command.extend(["--profile", self.profile])
command.append("-")
try:
completed = _run_isolated_codex(
command,
prompt,
tmp,
process_env,
self.timeout,
)
except (OSError, subprocess.TimeoutExpired) as error:
print(f" [agent_skills reflection skipped] codex_cli failed: {error}", flush=True)
return []
if completed.returncode != 0:
print(
f" [agent_skills reflection skipped] codex_cli exited {completed.returncode}",
flush=True,
)
return []
payload_text = out.read_text(encoding="utf-8") if out.exists() else completed.stdout
payload = _extract_json_payload(payload_text)
source_type = "failure" if failures else "success"
patches = self._normalise_payload(
payload,
batch_size=len(results),
source_type=source_type,
)
patches_dir = Path(out_dir) / "patches"
patches_dir.mkdir(parents=True, exist_ok=True)
valid_patches = []
for index, patch in enumerate(patches):
try:
self._validate_patch(patch)
except ValueError as error:
print(
f" [agent_skills reflection skipped patch] {error}",
flush=True,
)
continue
valid_patches.append(patch)
(patches_dir / f"codex_cli_reflection_{index:03d}.json").write_text(
json.dumps(patch, ensure_ascii=False, indent=2),
encoding="utf-8",
)
print(
f" [agent_skills reflection done] patches={len(valid_patches)}",
flush=True,
)
return valid_patches
def _prompt(self, results, skill_content, kwargs):
failures = [item for item in results if not item.get("hard")]
successes = [item for item in results if item.get("hard")]
update_mode = kwargs.get("update_mode", "patch")
return (
"Return exactly one JSON object only, with no markdown and no second object. "
"Produce bounded SkillOpt RawPatch objects for an Agent Skill body. "
"Use this shape: {\"raw_patches\":[{\"source_type\":\"failure\","
"\"patch\":{\"reasoning\":\"...\",\"edits\":[{\"op\":\"replace\","
"\"target\":\"exact text\",\"content\":\"replacement text\"}]}}]}. "
f"Return at most {self.edit_budget} edits total. "
"Return an empty raw_patches list when no useful edit is justified. "
"Do not include frontmatter, secrets, local auth paths, or whole-skill rewrites.\n\n"
f"Update mode: {update_mode}\n\n"
f"Current skill:\n{skill_content[:8000]}\n\n"
f"Failures:\n{json.dumps(failures, ensure_ascii=False)[:8000]}\n\n"
f"Successes:\n{json.dumps(successes, ensure_ascii=False)[:5000]}"
)
def _normalise_payload(self, payload, batch_size, source_type):
if isinstance(payload, dict) and isinstance(payload.get("raw_patches"), list):
candidates = payload["raw_patches"]
elif isinstance(payload, list):
candidates = payload
elif isinstance(payload, dict) and "patch" in payload:
candidates = [payload]
elif isinstance(payload, dict) and "edits" in payload:
candidates = [{"source_type": "failure", "patch": payload}]
else:
return []
edits = []
reasoning = []
seen = set()
for candidate in candidates:
if not isinstance(candidate, dict):
continue
patch = candidate.get("patch", candidate)
if not isinstance(patch, dict):
continue
if patch.get("reasoning"):
reasoning.append(str(patch.get("reasoning")))
for edit in patch.get("edits", []):
if not isinstance(edit, dict):
continue
key = json.dumps(edit, sort_keys=True, ensure_ascii=False)
if key in seen:
continue
seen.add(key)
edit = dict(edit)
edit.setdefault("source_type", candidate.get("source_type", source_type))
edits.append(edit)
if not edits:
return []
selected = edits[: self.edit_budget]
merged_reasoning = " ".join(reasoning).strip() or "codex_cli local merge"
if len(edits) > len(selected):
merged_reasoning += f" [codex-cli local truncation {len(edits)}->{len(selected)} edits]"
return [
{
"source_type": source_type,
"batch_size": batch_size,
"patch": {
"reasoning": merged_reasoning,
"edits": selected,
},
}
]
def _validate_patch(self, payload):
text = json.dumps(payload, ensure_ascii=False)
if "---" in text:
raise ValueError("patch must not include frontmatter")
if SECRET_PATTERN.search(text):
raise ValueError("patch contains a secret-like string")
if FORBIDDEN_OPTIMIZER_PATTERN.search(text):
raise ValueError("patch references an unsupported optimizer source")
edits = payload.get("patch", {}).get("edits", [])
if len(edits) > 8:
raise ValueError("patch exceeds edit budget")
for edit in edits:
if edit.get("op") == "replace_all":
raise ValueError("whole-skill replacement is disabled in patch mode")
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |