stark-ai-de/agent-skills / incubator/skills/skill-maintenance/skillopt-setup
skillopt-setup
FlaggedSet up Microsoft SkillOpt for Agent Skills improvement. Use when installing SkillOpt, preparing per-skill train/val/test eval splits, configuring Codex CLI or provider-backed execution, running SkillOpt against SKILL.md bodies, reviewing best_skill.md, or importing optimized skill changes. Do not use for ordinary skill authoring or manual repo validation.
assets/agent-skills-benchmark/rollout.py.template
View on GitHubimport base64
import binascii
from collections import deque
import copy
import errno
import html
import json
import hashlib
import math
import os
import re
import shutil
import signal
import stat
import subprocess
import struct
import tempfile
import time
import urllib.parse
import xml.etree.ElementTree as ET
import zlib
from html.parser import HTMLParser
from pathlib import Path
from threading import Lock, Thread
def redact(text):
return str(text or "").replace(str(Path.home()), "~")
def short(text, limit=8000):
value = redact(text)
return value if len(value) <= limit else f"{value[:limit]}..."
def as_text(value):
if value is None:
return ""
if isinstance(value, bytes):
return value.decode("utf-8", errors="replace")
return str(value)
PROMPT_ECHO_MARKER = "[redacted-prompt]"
PROMPT_ECHO_SEED_CHARS = 16
PROMPT_ECHO_MAX_UNIQUE_SEEDS = 65_536
PROMPT_ECHO_MAX_SPANS = 65_536
PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES = PROMPT_ECHO_SEED_CHARS * 4
MAX_STDOUT_CAPTURE_BYTES = 10 * 1024 * 1024
MAX_STDERR_CAPTURE_BYTES = 64 * 1024
CAPTURE_CHUNK_BYTES = 64 * 1024
OUTPUT_TRUNCATION_MARKER = "[output truncated; retained tail]"
class BoundedCaptureError(OSError):
pass
class BoundedByteCapture:
def __init__(self, limit):
self.limit = max(1, int(limit))
self._chunks = deque()
self._retained_bytes = 0
self._total_bytes = 0
self._discarded_tail = b""
self._lock = Lock()
def _remember_discarded(self, value):
recent = value[-PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES:]
self._discarded_tail = (
self._discarded_tail + recent
)[-PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES:]
def append(self, chunk):
value = bytes(chunk or b"")
if not value:
return
with self._lock:
self._total_bytes += len(value)
self._chunks.append(value)
self._retained_bytes += len(value)
while self._retained_bytes > self.limit:
excess = self._retained_bytes - self.limit
first = self._chunks[0]
if len(first) <= excess:
removed = self._chunks.popleft()
self._retained_bytes -= len(first)
else:
removed = first[:excess]
self._chunks[0] = first[excess:]
self._retained_bytes -= excess
self._remember_discarded(removed)
@property
def total_bytes(self):
with self._lock:
return self._total_bytes
def snapshot(self):
with self._lock:
return b"".join(self._chunks), self._total_bytes > self.limit
def redaction_snapshot(self):
with self._lock:
return (
self._discarded_tail,
b"".join(self._chunks),
self._total_bytes > self.limit,
)
class BoundedProcessIO:
def __init__(
self,
process,
prompt,
stdout_limit=MAX_STDOUT_CAPTURE_BYTES,
stderr_limit=MAX_STDERR_CAPTURE_BYTES,
):
self.process = process
self.prompt = str(prompt or "").encode("utf-8", errors="replace")
self.stdout = BoundedByteCapture(stdout_limit)
self.stderr = BoundedByteCapture(stderr_limit)
self._errors = []
self._error_lock = Lock()
self._readers = [
Thread(
target=self._read_stream,
args=("stdout", process.stdout, self.stdout),
name="skillopt-stdout-drain",
daemon=True,
),
Thread(
target=self._read_stream,
args=("stderr", process.stderr, self.stderr),
name="skillopt-stderr-drain",
daemon=True,
),
]
self._writer = Thread(
target=self._write_prompt,
name="skillopt-stdin-writer",
daemon=True,
)
self._threads = [*self._readers, self._writer]
self._started_threads = []
def _record_error(self, label, error):
with self._error_lock:
self._errors.append(f"{label}: {type(error).__name__}")
def _read_stream(self, label, stream, capture):
try:
while True:
chunk = stream.read(CAPTURE_CHUNK_BYTES)
if not chunk:
return
capture.append(chunk)
except Exception as error:
self._record_error(label, error)
finally:
try:
stream.close()
except Exception:
pass
def _write_prompt(self):
stream = self.process.stdin
try:
view = memoryview(self.prompt)
while view:
written = stream.write(view)
if not written:
raise BrokenPipeError("stdin closed before prompt transport completed")
view = view[written:]
stream.flush()
except BrokenPipeError:
pass
except OSError as error:
if error.errno not in {errno.EINVAL, errno.EPIPE}:
self._record_error("stdin", error)
except Exception as error:
self._record_error("stdin", error)
finally:
try:
stream.close()
except Exception:
pass
def start(self):
for thread in self._threads:
try:
thread.start()
finally:
if thread.ident is not None and thread not in self._started_threads:
self._started_threads.append(thread)
def close_unstarted_streams(self):
thread_streams = (
(self._readers[0], self.process.stdout),
(self._readers[1], self.process.stderr),
(self._writer, self.process.stdin),
)
for thread, stream in thread_streams:
if thread in self._started_threads or thread.is_alive():
continue
try:
stream.close()
except Exception:
pass
def close_stopped_streams(self):
thread_streams = (
(self._readers[0], self.process.stdout),
(self._readers[1], self.process.stderr),
(self._writer, self.process.stdin),
)
for thread, stream in thread_streams:
if thread.is_alive():
continue
try:
stream.close()
except Exception:
pass
def finish(self, timeout=7):
deadline = time.monotonic() + max(0, timeout)
for thread in self._started_threads:
thread.join(timeout=max(0, deadline - time.monotonic()))
alive = [thread.name for thread in self._started_threads if thread.is_alive()]
if alive:
raise BoundedCaptureError("bounded subprocess I/O threads did not stop")
with self._error_lock:
errors = list(self._errors)
if errors:
raise BoundedCaptureError("bounded subprocess I/O failed: " + "; ".join(errors))
def render_captured_stream(capture, prompt):
discarded_tail, payload, truncated = capture.redaction_snapshot()
value = payload.decode("utf-8", errors="replace")
boundary_chars = prompt_boundary_prefix_chars(
discarded_tail,
payload,
str(prompt or "").encode("utf-8", errors="replace"),
)
value = redact_prompt(value, prompt, boundary_chars=boundary_chars)
if truncated:
return f"{OUTPUT_TRUNCATION_MARKER}\n{value}"
return value
def prompt_boundary_prefix_chars(discarded_tail, payload, transported):
"""Return retained leading characters proven to continue a discarded prompt echo."""
if not discarded_tail or not payload or len(transported) < 2:
return 0
seed_bytes = min(PROMPT_ECHO_SEED_CHARS, len(transported))
max_left_size = min(seed_bytes - 1, len(discarded_tail))
if max_left_size <= 0:
return 0
high = min(len(payload), PROMPT_ECHO_BOUNDARY_CONTEXT_BYTES)
matched_bytes = 0
for left_size in range(1, max_left_size + 1):
left = discarded_tail[-left_size:]
if left + payload[:1] not in transported:
continue
low = 1
candidate_high = high
while low < candidate_high:
middle = (low + candidate_high + 1) // 2
if left + payload[:middle] in transported:
low = middle
else:
candidate_high = middle - 1
matched_bytes = max(matched_bytes, low)
if matched_bytes == high:
break
return len(payload[:matched_bytes].decode("utf-8", errors="replace"))
def prompt_echo_spans(value, transported):
"""Return bounded prompt-derived spans, or None to request fail-closed redaction."""
spans = []
if not value or not transported:
return spans
# A short captured stream or an individual line can be a truncated stdin echo.
# Treat it as sensitive even when it is shorter than the rolling-match seed.
if value in transported:
return [(0, len(value))]
for match in re.finditer(r"[^\r\n]+", value):
line = match.group(0)
stripped = line.strip()
if stripped and stripped in transported:
offset = line.find(stripped)
spans.append((match.start() + offset, match.start() + offset + len(stripped)))
if len(spans) > PROMPT_ECHO_MAX_SPANS:
return None
seed_size = min(PROMPT_ECHO_SEED_CHARS, len(transported))
if seed_size <= 0 or len(value) < seed_size:
return spans
prompt_seeds = set()
for index in range(len(transported) - seed_size + 1):
prompt_seeds.add(transported[index : index + seed_size])
if len(prompt_seeds) > PROMPT_ECHO_MAX_UNIQUE_SEEDS:
return None
seed_spans = []
last_seed_start = len(value) - seed_size
for index in range(last_seed_start + 1):
if value[index : index + seed_size] not in prompt_seeds:
continue
end = index + seed_size
if seed_spans and index <= seed_spans[-1][1]:
seed_spans[-1] = (
seed_spans[-1][0],
max(seed_spans[-1][1], end),
)
else:
seed_spans.append((index, end))
if len(spans) + len(seed_spans) > PROMPT_ECHO_MAX_SPANS:
return None
spans.extend(seed_spans)
return spans
def redact_prompt(text, prompt, boundary_chars=0):
value = as_text(text)
transported = str(prompt or "").encode("utf-8", errors="replace").decode("utf-8")
spans = prompt_echo_spans(value, transported)
if spans is None:
return PROMPT_ECHO_MARKER if value else ""
guard_end = min(len(value), max(0, int(boundary_chars)))
if guard_end:
if len(spans) >= PROMPT_ECHO_MAX_SPANS:
return PROMPT_ECHO_MARKER
spans.append((0, guard_end))
if not spans:
return value
spans.sort()
merged = []
for start, end in spans:
if merged and start <= merged[-1][1]:
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
else:
merged.append((start, end))
parts = []
cursor = 0
for start, end in merged:
parts.extend((value[cursor:start], PROMPT_ECHO_MARKER))
cursor = end
parts.append(value[cursor:])
return "".join(parts)
def redact_timeout_stream(text):
# Once transport is interrupted, even a one-character captured fragment can
# be prompt-derived. Its ownership cannot be distinguished from diagnostics.
return PROMPT_ECHO_MARKER if as_text(text) else ""
def normalize_task_prompt(prompt, skill_name="drawio-diagrams"):
text = str(prompt or "")
return (
text.replace(f"${skill_name}", "the provided Agent Skill body")
.replace(f"@{skill_name}", "the provided Agent Skill body")
)
def config_bool(value):
if isinstance(value, bool):
return value
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
def has_visual_assertions(case):
assertions = case.get("visual_assertions") or []
if not isinstance(assertions, list):
return False
return any(str(assertion or "").strip() for assertion in assertions)
def find_drawio_cli():
for command in ("drawio", "diagrams.net"):
executable = shutil.which(command)
if executable:
try:
resolved = Path(executable).resolve(strict=True)
except OSError:
continue
if resolved.is_file():
return {"command": command, "path": str(resolved)}
return None
def toml_string(value):
return json.dumps(str(value))
def safe_rmtree(path):
try:
shutil.rmtree(path)
return ""
except OSError as error:
return short(error, 500)
def process_group_alive(pgid):
try:
os.killpg(pgid, 0)
return True
except ProcessLookupError:
return False
except OSError:
return True
def signal_process_group(process, sig):
try:
os.killpg(process.pid, sig)
return ""
except ProcessLookupError:
return "already exited"
except OSError as error:
return short(error, 500)
def wait_after_signal(process, timeout):
try:
process.wait(timeout=timeout)
return False
except subprocess.TimeoutExpired:
return True
def wait_for_process_group_exit(pgid, timeout):
deadline = time.monotonic() + max(0, timeout)
while process_group_alive(pgid) and time.monotonic() < deadline:
time.sleep(0.05)
return not process_group_alive(pgid)
def reap_process_group(process, grace_seconds=1):
notes = []
pgid = process.pid
had_remaining_processes = process_group_alive(pgid)
if not had_remaining_processes:
return notes, False, False
term_error = signal_process_group(process, signal.SIGTERM)
if term_error and term_error != "already exited":
notes.append(f"SIGTERM: {term_error}")
if not wait_for_process_group_exit(pgid, grace_seconds):
kill_error = signal_process_group(process, signal.SIGKILL)
if kill_error and kill_error != "already exited":
notes.append(f"SIGKILL: {kill_error}")
wait_for_process_group_exit(pgid, 5)
still_alive = process_group_alive(pgid)
if still_alive:
notes.append("process group remained alive after SIGKILL")
return notes, still_alive, had_remaining_processes
def close_process_pipes(process):
for stream in (process.stdin, process.stdout, process.stderr):
try:
stream.close()
except Exception:
pass
def cleanup_failed_process_io_start(process, process_io):
cleanup_failed = False
try:
if process_io is not None:
process_io.close_unstarted_streams()
except Exception:
cleanup_failed = True
try:
signal_process_group(process, signal.SIGTERM)
still_running = wait_after_signal(process, 0)
if still_running:
signal_process_group(process, signal.SIGKILL)
cleanup_failed = wait_after_signal(process, 5) or cleanup_failed
reap_process_group(process, grace_seconds=0)
except Exception:
cleanup_failed = True
if process_io is not None:
try:
process_io.finish(timeout=1)
except Exception:
cleanup_failed = True
finally:
process_io.close_stopped_streams()
else:
close_process_pipes(process)
try:
cleanup_failed = (
process.poll() is None
or process_group_alive(process.pid)
or cleanup_failed
)
except Exception:
cleanup_failed = True
return cleanup_failed
def start_bounded_process_io(process, prompt, **kwargs):
process_io = None
try:
process_io = BoundedProcessIO(process, prompt, **kwargs)
process_io.start()
return process_io
except BaseException as error:
cleanup_failed = cleanup_failed_process_io_start(process, process_io)
if not isinstance(error, Exception):
raise
if cleanup_failed:
raise BoundedCaptureError(
"bounded subprocess I/O startup cleanup failed"
) from None
raise BoundedCaptureError(
f"bounded subprocess I/O failed to start: {type(error).__name__}"
) from None
PACKAGE_DIRS_TO_SEED = ("scripts", "references", "assets", "agents")
ROLLOUT_CONTROL_DIR = ".skillopt-control"
ROLLOUT_BIN_DIR = ".skillopt-bin"
ARTIFACT_EXCLUDED_TOP_LEVEL = frozenset(
(*PACKAGE_DIRS_TO_SEED, "fixtures", ROLLOUT_CONTROL_DIR, ROLLOUT_BIN_DIR)
)
VISUAL_ROLLOUT_PERMISSION_PROFILE = "skillopt_visual_rollout"
TEXT_ROLLOUT_PERMISSION_PROFILE = "skillopt_text_rollout"
MAX_WORKSPACE_ENTRIES = 4096
MAX_ARTIFACT_COUNT = 32
MAX_ARTIFACT_FILE_BYTES = 20 * 1024 * 1024
MAX_ARTIFACT_TOTAL_BYTES = 64 * 1024 * 1024
MAX_SVG_FILE_BYTES = 5 * 1024 * 1024
MAX_EMBEDDED_IMAGE_BYTES = 2 * 1024 * 1024
MAX_SVG_ELEMENTS = 20000
MAX_SVG_DEPTH = 128
MAX_DRAWIO_ELEMENTS = 50000
MAX_DRAWIO_DEPTH = 256
MAX_INFLATED_DRAWIO_BYTES = 32 * 1024 * 1024
MAX_PNG_DIMENSION = 16384
MAX_PNG_PIXELS = 2_000_000
MAX_PNG_INFLATED_BYTES = 16 * 1024 * 1024
MAX_DISTINCT_PIXEL_SAMPLES = 1024
MAX_PROVIDER_CONTEXT_FILES = 64
MAX_PROVIDER_CONTEXT_FILE_BYTES = 128 * 1024
MAX_PROVIDER_FIXTURE_BYTES = 64 * 1024
MAX_PROVIDER_CONTEXT_TOTAL_BYTES = 768 * 1024
class ArtifactPolicyError(ValueError):
pass
class ProviderContextError(ValueError):
pass
def path_is_within(path, root):
try:
path.relative_to(root)
return True
except ValueError:
return False
def codex_runtime_read_path(command):
candidate = Path(str(command or "codex")).expanduser()
if not candidate.is_absolute():
resolved_command = shutil.which(str(candidate))
if not resolved_command:
return None
candidate = Path(resolved_command)
try:
executable = candidate.resolve(strict=True)
except OSError:
return None
runtime_path = executable
for parent in executable.parents:
package_json = parent / "package.json"
if not package_json.is_file():
continue
try:
package = json.loads(package_json.read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
package_name = str(package.get("name") or "")
if package_name == "@openai/codex" or package_name.startswith("@openai/codex-"):
runtime_path = parent
break
forbidden_roots = [(Path.home() / ".codex").resolve()]
if os.environ.get("CODEX_HOME"):
forbidden_roots.append(Path(os.environ["CODEX_HOME"]).expanduser().resolve())
if runtime_path == Path(runtime_path.anchor) or runtime_path == Path.home():
return None
if any(path_is_within(runtime_path, root) for root in forbidden_roots):
return None
return runtime_path
def drawio_runtime_read_path(command):
executable = Path(command).resolve(strict=True)
for parent in executable.parents:
name = parent.name.lower()
if "drawio" in name or "draw.io" in name or "diagrams" in name:
return parent
return executable
def visual_rollout_filesystem_permissions(codex_command, drawio_cli=None):
entries = [
'":minimal"="read"',
(
'":workspace_roots"={"."="write",'
f'"{ROLLOUT_CONTROL_DIR}"="deny"'
"}"
),
]
runtime_path = codex_runtime_read_path(codex_command)
if runtime_path is not None:
entries.append(f"{toml_string(runtime_path)}=\"read\"")
if drawio_cli:
entries.append(f"{toml_string(drawio_runtime_read_path(drawio_cli))}=\"read\"")
return "{" + ",".join(entries) + "}"
def text_rollout_filesystem_permissions(codex_command):
entries = [
'":minimal"="read"',
(
'":workspace_roots"={"."="deny",'
f'"{ROLLOUT_CONTROL_DIR}"="deny"'
"}"
),
]
runtime_path = codex_runtime_read_path(codex_command)
if runtime_path is not None:
entries.append(f"{toml_string(runtime_path)}=\"read\"")
return "{" + ",".join(entries) + "}"
def prepare_drawio_command_alias(workspace, drawio_cli):
if not drawio_cli or drawio_cli.get("command") == "drawio":
return None
alias_dir = workspace / ROLLOUT_BIN_DIR
alias_dir.mkdir(mode=0o700)
alias = alias_dir / "drawio"
alias.symlink_to(Path(drawio_cli["path"]))
return alias_dir
def minimal_rollout_environment(path_prefix=None):
allowed = (
"HOME",
"PATH",
"USER",
"LOGNAME",
"SHELL",
"TMPDIR",
"TEMP",
"TMP",
"LANG",
"LC_ALL",
"LC_CTYPE",
"TERM",
"COLORTERM",
"NO_COLOR",
"CODEX_HOME",
"SSL_CERT_FILE",
"SSL_CERT_DIR",
)
env = {key: os.environ[key] for key in allowed if os.environ.get(key)}
if path_prefix is not None:
current_path = env.get("PATH", os.defpath)
env["PATH"] = f"{path_prefix}{os.pathsep}{current_path}"
return env
def shell_environment_for_workspace(workspace, process_env):
temp_dir = workspace / ".tmp"
temp_dir.mkdir(mode=0o700)
values = {
"HOME": str(workspace),
"PATH": process_env.get("PATH", os.defpath),
"TMPDIR": str(temp_dir),
"TMP": str(temp_dir),
"TEMP": str(temp_dir),
"XDG_CACHE_HOME": str(workspace / ".cache"),
"XDG_CONFIG_HOME": str(workspace / ".config"),
"XDG_DATA_HOME": str(workspace / ".local" / "share"),
}
for key in ("LANG", "LC_ALL", "LC_CTYPE", "TERM", "COLORTERM", "NO_COLOR"):
if process_env.get(key):
values[key] = process_env[key]
return values
def toml_inline_string_map(values):
return "{" + ",".join(
f"{toml_string(key)}={toml_string(value)}" for key, value in sorted(values.items())
) + "}"
def find_repo_root():
for candidate in [Path.cwd(), *Path.cwd().parents]:
has_skill_tree = (candidate / "skills").is_dir() or (
candidate / "incubator" / "skills"
).is_dir()
has_repo_sentinel = any(
(candidate / name).exists() for name in ("AGENTS.md", "package.json", "skill-evals")
)
if has_skill_tree and has_repo_sentinel:
return candidate
return None
def find_skill_dir(repo_root, skill_name):
matches = []
for search_root in (repo_root / "skills", repo_root / "incubator" / "skills"):
if not search_root.is_dir():
continue
for skill_file in search_root.rglob("SKILL.md"):
if skill_file.parent.name == skill_name:
matches.append(skill_file.parent)
matches = sorted(set(matches), key=lambda item: item.as_posix())
if len(matches) == 1:
return matches[0], ""
if not matches:
return (
None,
f"No target skill package named {skill_name!r} was found under skills/ or incubator/skills.",
)
relative_matches = ", ".join(str(match.relative_to(repo_root)) for match in matches)
return (
None,
f"Multiple target skill packages named {skill_name!r} were found: {relative_matches}. "
"Cannot seed rollout helpers safely.",
)
def copy_tree(src, dst):
for root, directories, files in os.walk(src, followlinks=False):
for name in [*directories, *files]:
candidate = Path(root) / name
try:
mode = candidate.stat(follow_symlinks=False).st_mode
except OSError as error:
raise ValueError(
f"skill package resource cannot be inspected safely: {candidate}"
) from error
if stat.S_ISLNK(mode):
raise ValueError(f"skill package resources may not contain symlinks: {candidate}")
if not (stat.S_ISDIR(mode) or stat.S_ISREG(mode)):
raise ValueError(
f"skill package resources must be regular files or directories: {candidate}"
)
shutil.copytree(
src,
dst,
dirs_exist_ok=True,
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
)
def fixture_source(repo_root, fixture):
if not isinstance(fixture, str) or not fixture.strip():
raise ValueError("fixture paths must be non-empty strings")
relative = Path(fixture)
if relative.is_absolute() or ".." in relative.parts:
raise ValueError(f"fixture path must stay relative to the repository: {fixture!r}")
candidate = repo_root / relative
current = repo_root
for part in relative.parts:
current = current / part
if current.is_symlink():
raise ValueError(f"fixture path may not contain symlinks: {fixture!r}")
try:
resolved = candidate.resolve(strict=True)
mode = candidate.stat(follow_symlinks=False).st_mode
except OSError as error:
raise ValueError(f"fixture does not exist or cannot be inspected: {fixture!r}") from error
if not path_is_within(resolved, repo_root) or not stat.S_ISREG(mode):
raise ValueError(f"fixture must be a regular repository file: {fixture!r}")
return resolved
def seed_workspace(workspace, case, skill_name):
repo_root = find_repo_root()
if repo_root is None:
return "No repository helper files were found for this isolated rollout."
repo_root = repo_root.resolve(strict=True)
notes = []
skill_dir, skill_error = find_skill_dir(repo_root, skill_name)
if skill_error:
notes.append(skill_error)
if skill_dir is not None:
for dirname in PACKAGE_DIRS_TO_SEED:
src = skill_dir / dirname
if src.is_dir():
copy_tree(src, workspace / dirname)
notes.append(f"- ./{dirname}/ from the target skill package")
fixture_notes = []
fixture_dir = workspace / "fixtures"
fixtures = case.get("fixtures") or []
if not isinstance(fixtures, list):
raise ValueError("case fixtures must be a list of repository-relative paths")
for fixture in fixtures:
src = fixture_source(repo_root, fixture)
fixture_dir.mkdir(parents=True, exist_ok=True)
dst = fixture_dir / src.name
if dst.exists():
raise ValueError(
f"fixture basename collision would overwrite {dst.name!r}; use unique names"
)
shutil.copy2(src, dst)
fixture_notes.append(f"- ./fixtures/{dst.name} copied from {fixture}")
if fixture_notes:
notes.append("Fixtures:")
notes.extend(fixture_notes)
if not notes:
return "No case-specific helper files were required for this isolated rollout."
return "\n".join(notes)
def sanitize_provider_context(text):
value = redact(text)
value = re.sub(r"/(?:home|Users)/[A-Za-z0-9._-]+/", "~/", value)
value = re.sub(
r"(?i)(https?://)[^/@\s]+:[^/@\s]+@",
r"\1[redacted-credentials]@",
value,
)
value = re.sub(r"(?i)\b(Bearer\s+)[^\s,;]+", r"\1[redacted-token]", value)
value = re.sub(
r"(?i)([?&](?:api[_-]?key|access[_-]?token|token|secret|password|signature)=)[^&\s]+",
r"\1[redacted-token]",
value,
)
value = re.sub(r"sk-[A-Za-z0-9_-]{20,}", "[redacted-token]", value)
value = re.sub(
r"\b(?:ghp_|github_pat_|xox[baprs]-|AKIA|ASIA)[A-Za-z0-9_/-]{12,}\b",
"[redacted-token]",
value,
)
return value
def provider_resource_snapshot(workspace):
roots = [workspace / name for name in (*PACKAGE_DIRS_TO_SEED, "fixtures")]
files = []
for root in roots:
if not root.is_dir():
continue
for candidate in root.rglob("*"):
if candidate.is_symlink() or not candidate.is_file():
continue
files.append(candidate)
files = sorted(set(files), key=lambda item: item.relative_to(workspace).as_posix())
if len(files) > MAX_PROVIDER_CONTEXT_FILES:
raise ProviderContextError(
f"provider resource snapshot exceeds {MAX_PROVIDER_CONTEXT_FILES} files"
)
sections = []
total_bytes = 0
for candidate in files:
relative = candidate.relative_to(workspace).as_posix()
file_size = candidate.stat(follow_symlinks=False).st_size
is_fixture = relative.startswith("fixtures/")
file_limit = (
MAX_PROVIDER_FIXTURE_BYTES if is_fixture else MAX_PROVIDER_CONTEXT_FILE_BYTES
)
if file_size > file_limit:
raise ProviderContextError(
f"provider resource {relative} exceeds {file_limit} bytes"
)
content = candidate.read_bytes()
if b"\x00" in content:
if is_fixture:
raise ProviderContextError(
f"provider target cannot inline binary fixture {relative}"
)
sections.append(f"<resource path={json.dumps(relative)}>\n[non-text resource omitted]\n</resource>")
continue
try:
decoded = content.decode("utf-8", errors="strict")
except UnicodeDecodeError as error:
if is_fixture:
raise ProviderContextError(
f"provider target cannot inline non-UTF-8 fixture {relative}"
) from error
sections.append(f"<resource path={json.dumps(relative)}>\n[non-text resource omitted]\n</resource>")
continue
sanitized = sanitize_provider_context(decoded)
encoded_size = len(sanitized.encode("utf-8"))
total_bytes += encoded_size
if total_bytes > MAX_PROVIDER_CONTEXT_TOTAL_BYTES:
raise ProviderContextError(
f"provider resource snapshot exceeds {MAX_PROVIDER_CONTEXT_TOTAL_BYTES} total bytes"
)
sections.append(
f"<resource path={json.dumps(relative)}>\n{sanitized}\n</resource>"
)
if not sections:
return "No textual helper resources or fixtures were required."
return "\n\n".join(sections)
def paeth_predictor(left, up, upper_left):
estimate = left + up - upper_left
left_distance = abs(estimate - left)
up_distance = abs(estimate - up)
upper_left_distance = abs(estimate - upper_left)
if left_distance <= up_distance and left_distance <= upper_left_distance:
return left
if up_distance <= upper_left_distance:
return up
return upper_left
def png_channels(color_type):
if color_type in {0, 3}:
return 1
if color_type == 2:
return 3
if color_type == 4:
return 2
if color_type == 6:
return 4
raise ValueError(f"unsupported PNG color type {color_type}")
def png_pixel_sample(row, x, bit_depth, pixel_stride):
if bit_depth >= 8:
start = x * pixel_stride
return bytes(row[start : start + pixel_stride]).hex()
bit_offset = x * bit_depth
byte = row[bit_offset // 8]
shift = 8 - bit_depth - (bit_offset % 8)
mask = (1 << bit_depth) - 1
return str((byte >> shift) & mask)
def sample_has_value(data):
return any(data)
def sample_equals(data, expected):
return expected is not None and bytes(data) == bytes(expected)
def png_visible_pixel_sample(
row, x, bit_depth, color_type, pixel_stride, transparency, palette
):
if bit_depth >= 8:
start = x * pixel_stride
bytes_per_sample = (bit_depth + 7) // 8
if color_type == 0:
sample = row[start : start + bytes_per_sample]
if sample_equals(sample, transparency.get("grayscale")):
return None
return bytes(sample).hex()
if color_type == 2:
sample = row[start : start + 3 * bytes_per_sample]
if sample_equals(sample, transparency.get("rgb")):
return None
return bytes(sample).hex()
if color_type == 3:
palette_index = row[start]
alpha = transparency.get("palette", {}).get(palette_index, 255)
if alpha == 0:
return None
return (palette[palette_index] + bytes([alpha])).hex()
if color_type == 4:
alpha_start = start + bytes_per_sample
alpha = row[alpha_start : alpha_start + bytes_per_sample]
if not sample_has_value(alpha):
return None
return bytes(row[start:alpha_start]).hex()
if color_type == 6:
alpha_start = start + 3 * bytes_per_sample
alpha = row[alpha_start : alpha_start + bytes_per_sample]
if not sample_has_value(alpha):
return None
return bytes(row[start:alpha_start]).hex()
return bytes(row[start : start + pixel_stride]).hex()
if color_type == 0:
sample = png_pixel_sample(row, x, bit_depth, pixel_stride)
if transparency.get("grayscale_value") is not None and sample == str(transparency["grayscale_value"]):
return None
return sample
if color_type == 3:
palette_index = int(png_pixel_sample(row, x, bit_depth, pixel_stride))
alpha = transparency.get("palette", {}).get(palette_index, 255)
if alpha == 0:
return None
return (palette[palette_index] + bytes([alpha])).hex()
return png_pixel_sample(row, x, bit_depth, pixel_stride)
def png_transparency_sample_bytes(data, bit_depth, sample_index):
bytes_per_sample = (bit_depth + 7) // 8
sample_start = sample_index * 2
return bytes(data[sample_start + 2 - bytes_per_sample : sample_start + 2])
def decompress_png_rows(idat_chunks, expected_bytes):
if expected_bytes > MAX_PNG_INFLATED_BYTES:
raise ValueError(
f"PNG decoded rows exceed {MAX_PNG_INFLATED_BYTES} byte safety limit"
)
compressed = b"".join(idat_chunks)
decompressor = zlib.decompressobj()
inflated = bytearray()
pending = compressed
while pending:
remaining = expected_bytes + 1 - len(inflated)
if remaining <= 0:
raise ValueError("PNG IDAT data expands beyond expected image rows")
before = len(pending)
inflated.extend(decompressor.decompress(pending, remaining))
if len(inflated) > expected_bytes:
raise ValueError("PNG IDAT data expands beyond expected image rows")
pending = decompressor.unconsumed_tail
if pending and len(pending) == before and not decompressor.eof:
raise ValueError("PNG IDAT decompressor made no progress")
if not decompressor.eof:
raise ValueError("PNG IDAT stream is truncated")
if decompressor.unused_data:
raise ValueError("PNG IDAT stream contains trailing compressed data")
if len(inflated) != expected_bytes:
raise ValueError(
f"PNG IDAT decoded {len(inflated)} bytes; expected {expected_bytes}"
)
return bytes(inflated)
ADAM7_PASSES = (
(0, 0, 8, 8),
(4, 0, 8, 8),
(0, 4, 4, 8),
(2, 0, 4, 4),
(0, 2, 2, 4),
(1, 0, 2, 2),
(0, 1, 1, 2),
)
def adam7_extent(size, start, step):
return 0 if size <= start else (size - start + step - 1) // step
def png_passes(width, height, interlace_method):
if interlace_method == 0:
return ((0, 0, 1, 1, width, height),)
return tuple(
(
x_start,
y_start,
x_step,
y_step,
adam7_extent(width, x_start, x_step),
adam7_extent(height, y_start, y_step),
)
for x_start, y_start, x_step, y_step in ADAM7_PASSES
)
def png_expected_inflated_bytes(width, height, bits_per_pixel, interlace_method):
total = 0
for _, _, _, _, pass_width, pass_height in png_passes(
width, height, interlace_method
):
if pass_width and pass_height:
pass_row_bytes = (pass_width * bits_per_pixel + 7) // 8
total += pass_height * (pass_row_bytes + 1)
return total
def unfilter_png_row(encoded, previous, filter_stride, row_filter):
row = bytearray(len(encoded))
for index, value in enumerate(encoded):
left = row[index - filter_stride] if index >= filter_stride else 0
up = previous[index] if index < len(previous) else 0
upper_left = previous[index - filter_stride] if index >= filter_stride else 0
if row_filter == 0:
predictor = 0
elif row_filter == 1:
predictor = left
elif row_filter == 2:
predictor = up
elif row_filter == 3:
predictor = (left + up) // 2
elif row_filter == 4:
predictor = paeth_predictor(left, up, upper_left)
else:
raise ValueError(f"unsupported PNG row filter {row_filter}")
row[index] = (value + predictor) & 0xFF
return row
def packed_png_sample(row, x, bit_depth):
bit_offset = x * bit_depth
byte_index = bit_offset // 8
shift = 8 - bit_depth - (bit_offset % 8)
return (row[byte_index] >> shift) & ((1 << bit_depth) - 1)
def set_packed_png_sample(row, x, bit_depth, value):
bit_offset = x * bit_depth
byte_index = bit_offset // 8
shift = 8 - bit_depth - (bit_offset % 8)
mask = ((1 << bit_depth) - 1) << shift
row[byte_index] = (row[byte_index] & ~mask) | ((value << shift) & mask)
def decode_png_rows(inflated, width, height, bits_per_pixel, filter_stride, interlace_method):
full_row_bytes = (width * bits_per_pixel + 7) // 8
if interlace_method == 0:
rows = []
previous = bytearray(full_row_bytes)
offset = 0
for _ in range(height):
row_filter = inflated[offset]
encoded = inflated[offset + 1 : offset + 1 + full_row_bytes]
offset += full_row_bytes + 1
decoded = unfilter_png_row(encoded, previous, filter_stride, row_filter)
rows.append(decoded)
previous = decoded
if offset != len(inflated):
raise ValueError("PNG decoded row data has unexpected trailing bytes")
return rows
rows = [bytearray(full_row_bytes) for _ in range(height)]
offset = 0
for x_start, y_start, x_step, y_step, pass_width, pass_height in png_passes(
width, height, interlace_method
):
if not pass_width or not pass_height:
continue
pass_row_bytes = (pass_width * bits_per_pixel + 7) // 8
previous = bytearray(pass_row_bytes)
for pass_y in range(pass_height):
row_filter = inflated[offset]
encoded = inflated[offset + 1 : offset + 1 + pass_row_bytes]
offset += pass_row_bytes + 1
decoded = unfilter_png_row(encoded, previous, filter_stride, row_filter)
target = rows[y_start + pass_y * y_step]
if bits_per_pixel >= 8:
pixel_bytes = bits_per_pixel // 8
for pass_x in range(pass_width):
source_start = pass_x * pixel_bytes
target_start = (x_start + pass_x * x_step) * pixel_bytes
target[target_start : target_start + pixel_bytes] = decoded[
source_start : source_start + pixel_bytes
]
else:
for pass_x in range(pass_width):
sample = packed_png_sample(decoded, pass_x, bits_per_pixel)
set_packed_png_sample(
target, x_start + pass_x * x_step, bits_per_pixel, sample
)
previous = decoded
if offset != len(inflated):
raise ValueError("PNG decoded row data has unexpected trailing bytes")
return rows
def png_metadata(path, data=None):
try:
if data is None:
data = path.read_bytes()
if len(data) > MAX_ARTIFACT_FILE_BYTES:
raise ValueError(f"PNG exceeds {MAX_ARTIFACT_FILE_BYTES} byte safety limit")
if len(data) < 33 or data[:8] != b"\x89PNG\r\n\x1a\n":
return {"valid": False, "error": "invalid PNG signature"}
offset = 8
width = None
height = None
bit_depth = None
color_type = None
interlace_method = None
transparency = {}
idat_chunks = []
palette = []
palette_entries = 0
seen_ihdr = False
seen_plte = False
seen_trns = False
seen_idat = False
idat_ended = False
seen_iend = False
chunk_index = 0
while offset + 12 <= len(data):
length = struct.unpack(">I", data[offset : offset + 4])[0]
chunk_type_bytes = data[offset + 4 : offset + 8]
try:
chunk_type = chunk_type_bytes.decode("ascii")
except UnicodeDecodeError as error:
raise ValueError("PNG chunk type is not ASCII") from error
if len(chunk_type) != 4 or not chunk_type.isalpha():
raise ValueError(f"invalid PNG chunk type {chunk_type!r}")
if chunk_type_bytes[2] & 0x20:
raise ValueError(f"PNG chunk {chunk_type} uses a lowercase reserved bit")
data_start = offset + 8
data_end = data_start + length
if data_end + 4 > len(data):
raise ValueError("truncated PNG chunk")
chunk = data[data_start:data_end]
stored_crc = struct.unpack(">I", data[data_end : data_end + 4])[0]
calculated_crc = zlib.crc32(chunk_type_bytes)
calculated_crc = zlib.crc32(chunk, calculated_crc) & 0xFFFFFFFF
if stored_crc != calculated_crc:
raise ValueError(f"PNG {chunk_type} chunk CRC mismatch")
if chunk_index == 0 and chunk_type != "IHDR":
raise ValueError("PNG IHDR must be the first chunk")
if chunk_type != "IDAT" and seen_idat:
idat_ended = True
if chunk_type == "IHDR":
if seen_ihdr or chunk_index != 0:
raise ValueError("PNG must contain exactly one leading IHDR")
if length != 13:
raise ValueError("invalid PNG IHDR length")
width, height = struct.unpack(">II", chunk[:8])
bit_depth = chunk[8]
color_type = chunk[9]
if chunk[10] != 0 or chunk[11] != 0 or chunk[12] not in {0, 1}:
raise ValueError("unsupported PNG compression, filter, or interlace method")
interlace_method = chunk[12]
legal_depths = {
0: {1, 2, 4, 8, 16},
2: {8, 16},
3: {1, 2, 4, 8},
4: {8, 16},
6: {8, 16},
}
if color_type not in legal_depths or bit_depth not in legal_depths[color_type]:
raise ValueError(
f"invalid PNG bit depth {bit_depth} for color type {color_type}"
)
if width <= 0 or height <= 0:
raise ValueError("PNG dimensions must be positive")
if width > MAX_PNG_DIMENSION or height > MAX_PNG_DIMENSION:
raise ValueError(
f"PNG dimensions exceed {MAX_PNG_DIMENSION} pixel safety limit"
)
if width * height > MAX_PNG_PIXELS:
raise ValueError(f"PNG exceeds {MAX_PNG_PIXELS} pixel safety limit")
seen_ihdr = True
elif chunk_type == "PLTE":
if not seen_ihdr or seen_idat or seen_plte or seen_trns:
raise ValueError("PNG PLTE chunk is duplicated or out of order")
if color_type in {0, 4}:
raise ValueError(f"PNG color type {color_type} may not contain PLTE")
if length == 0 or length > 768 or length % 3 != 0:
raise ValueError("invalid PNG PLTE length")
palette_entries = length // 3
palette = [chunk[index : index + 3] for index in range(0, length, 3)]
if color_type == 3 and palette_entries > 2**bit_depth:
raise ValueError("PNG PLTE has more entries than indexed bit depth permits")
seen_plte = True
elif chunk_type == "IDAT":
if not seen_ihdr or seen_iend or idat_ended:
raise ValueError("PNG IDAT chunks must be consecutive and precede IEND")
if color_type == 3 and not seen_plte:
raise ValueError("indexed PNG is missing PLTE before IDAT")
idat_chunks.append(chunk)
seen_idat = True
elif chunk_type == "tRNS":
if not seen_ihdr or seen_idat or seen_trns:
raise ValueError("PNG tRNS chunk is duplicated or does not precede IDAT")
if color_type == 0:
if length != 2:
raise ValueError("invalid PNG tRNS length for grayscale image")
transparency["grayscale"] = png_transparency_sample_bytes(chunk, bit_depth, 0)
transparency["grayscale_value"] = struct.unpack(">H", chunk)[0]
elif color_type == 2:
if length != 6:
raise ValueError("invalid PNG tRNS length for truecolor image")
transparency["rgb"] = b"".join(
png_transparency_sample_bytes(chunk, bit_depth, sample_index)
for sample_index in range(3)
)
elif color_type == 3:
if not seen_plte or length == 0 or length > palette_entries:
raise ValueError("invalid PNG tRNS length for indexed image")
transparency["palette"] = {index: alpha for index, alpha in enumerate(chunk)}
else:
raise ValueError(f"PNG color type {color_type} may not contain tRNS")
seen_trns = True
elif chunk_type == "IEND":
if length != 0 or not seen_idat or seen_iend:
raise ValueError("invalid or duplicate PNG IEND chunk")
seen_iend = True
offset = data_end + 4
break
elif chunk_type_bytes[0] & 0x20 == 0:
raise ValueError(f"unsupported critical PNG chunk {chunk_type}")
offset = data_end + 4
chunk_index += 1
if not seen_ihdr or width is None or height is None:
raise ValueError("PNG missing IHDR")
if not seen_idat or not idat_chunks:
raise ValueError("PNG missing IDAT data")
if not seen_iend:
raise ValueError("PNG missing IEND")
if offset != len(data):
raise ValueError("PNG contains trailing data after IEND")
channels = png_channels(color_type)
bits_per_pixel = channels * bit_depth
filter_stride = max(1, (bits_per_pixel + 7) // 8)
pixel_stride = filter_stride if bit_depth >= 8 else 1
expected_bytes = png_expected_inflated_bytes(
width, height, bits_per_pixel, interlace_method
)
inflated = decompress_png_rows(idat_chunks, expected_bytes)
rows = decode_png_rows(
inflated, width, height, bits_per_pixel, filter_stride, interlace_method
)
distinct_pixels = set()
visible_distinct_pixels = set()
visible_pixel_count = 0
transparent_pixel_count = 0
nonblank = False
for row in rows:
if color_type == 3:
for x in range(width):
if int(png_pixel_sample(row, x, bit_depth, pixel_stride)) >= palette_entries:
raise ValueError("indexed PNG references a missing palette entry")
if not nonblank and bit_depth == 8 and color_type == 6:
alpha_values = row[3::4]
transparent_in_row = alpha_values.count(0)
visible_in_row = width - transparent_in_row
transparent_pixel_count += transparent_in_row
visible_pixel_count += visible_in_row
if visible_pixel_count > 0 and transparent_pixel_count > 0:
if visible_in_row and not visible_distinct_pixels:
first_visible = next(
index for index, alpha in enumerate(alpha_values) if alpha != 0
)
start = first_visible * 4
visible_distinct_pixels.add(bytes(row[start : start + 3]).hex())
nonblank = True
elif visible_in_row:
for x in range(width):
start = x * 4
pixel_sample = bytes(row[start : start + 4]).hex()
visible_sample = bytes(row[start : start + 3]).hex()
if len(distinct_pixels) < MAX_DISTINCT_PIXEL_SAMPLES:
distinct_pixels.add(pixel_sample)
if len(visible_distinct_pixels) < MAX_DISTINCT_PIXEL_SAMPLES:
visible_distinct_pixels.add(visible_sample)
if len(visible_distinct_pixels) > 1:
nonblank = True
break
elif not nonblank:
for x in range(width):
pixel_sample = png_pixel_sample(row, x, bit_depth, pixel_stride)
if len(distinct_pixels) < MAX_DISTINCT_PIXEL_SAMPLES:
distinct_pixels.add(pixel_sample)
visible_sample = png_visible_pixel_sample(
row,
x,
bit_depth,
color_type,
pixel_stride,
transparency,
palette,
)
if visible_sample is None:
transparent_pixel_count += 1
else:
visible_pixel_count += 1
if len(visible_distinct_pixels) < MAX_DISTINCT_PIXEL_SAMPLES:
visible_distinct_pixels.add(visible_sample)
if (
visible_pixel_count > 0 and transparent_pixel_count > 0
) or len(visible_distinct_pixels) > 1:
nonblank = True
break
return {
"valid": True,
"width": width,
"height": height,
"bit_depth": bit_depth,
"color_type": color_type,
"interlace_method": interlace_method,
"distinct_pixels": len(distinct_pixels),
"visible_distinct_pixels": len(visible_distinct_pixels),
"distinct_pixel_samples_capped": (
len(distinct_pixels) == MAX_DISTINCT_PIXEL_SAMPLES
or len(visible_distinct_pixels) == MAX_DISTINCT_PIXEL_SAMPLES
),
"visible_pixel_count": visible_pixel_count,
"transparent_pixel_count": transparent_pixel_count,
"nonblank": nonblank,
}
except Exception as error:
return {"valid": False, "error": short(error, 300)}
NON_RENDERED_SVG_ELEMENTS = {
"clipPath",
"defs",
"desc",
"marker",
"mask",
"metadata",
"pattern",
"script",
"style",
"symbol",
"title",
}
SVG_TEXT_CONTAINERS = {"foreignObject", "text", "textPath", "tspan"}
SVG_11_DOCTYPE = re.compile(
r'''<!DOCTYPE\s+svg\s+PUBLIC\s+["']-//W3C//DTD\s+SVG\s+1\.1//EN["']\s+'''
r'''["']http://www\.w3\.org/Graphics/SVG/1\.1/DTD/svg11\.dtd["']\s*>''',
flags=re.IGNORECASE,
)
SVG_NAMESPACE = "http://www.w3.org/2000/svg"
SVG_XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace"
SVG_URL_RE = re.compile(r'''url\(\s*(['"]?)(.*?)\1\s*\)''', flags=re.IGNORECASE)
SVG_KEYFRAMES_START_RE = re.compile(
r"@(?:-webkit-)?keyframes\s+[_A-Za-z][_A-Za-z0-9-]*\s*\{",
flags=re.IGNORECASE,
)
SVG_EXTERNAL_REFERENCE_PREFIXES = ("http://", "https://", "//")
SVG_ANIMATION_ELEMENTS = {
"animate",
"animatecolor",
"animatemotion",
"animatetransform",
"discard",
"mpath",
"set",
}
SVG_UNSAFE_RESOURCE_ELEMENTS = SVG_ANIMATION_ELEMENTS | {
"embed",
"frame",
"iframe",
"object",
"script",
}
def svg_local_name(value):
if not isinstance(value, str):
return ""
return value.rsplit("}", 1)[-1].split(":")[-1]
def svg_namespace(value):
if not isinstance(value, str) or not value.startswith("{") or "}" not in value:
return ""
return value[1:].split("}", 1)[0]
def svg_attribute(element, name):
for raw_name, value in element.attrib.items():
if svg_local_name(raw_name).lower() == name:
return str(value).strip()
return ""
def svg_css_reference_counts(value, identifiers):
literal = str(value or "")
external = 1 if re.search(r"@import\b", literal, flags=re.IGNORECASE) or "\\" in literal else 0
unsupported = 0
for match in SVG_URL_RE.finditer(literal):
reference = match.group(2).strip()
if not reference or reference.lower().startswith("data:"):
continue
if reference.startswith("#"):
if reference[1:] not in identifiers:
unsupported += 1
else:
external += 1
return external, unsupported
def xml_tree_shape(root, *, max_elements, max_depth, label):
count = 0
deepest = 0
stack = [(root, 1)]
while stack:
element, depth = stack.pop()
count += 1
deepest = max(deepest, depth)
if count > max_elements or deepest > max_depth:
raise ValueError(f"{label} exceeds element or depth safety limit")
stack.extend((child, depth + 1) for child in element)
return count, deepest
def svg_tree_shape(root):
return xml_tree_shape(
root,
max_elements=MAX_SVG_ELEMENTS,
max_depth=MAX_SVG_DEPTH,
label="SVG",
)
def drawio_tree_shape(root, max_elements=None):
return xml_tree_shape(
root,
max_elements=MAX_DRAWIO_ELEMENTS if max_elements is None else max_elements,
max_depth=MAX_DRAWIO_DEPTH,
label="draw.io XML",
)
def positive_svg_length(value):
match = re.fullmatch(
r"\s*([+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[Ee][+-]?\d+)?)(?:[A-Za-z%]+)?\s*",
str(value or ""),
)
if not match:
return False
try:
number = float(match.group(1))
return math.isfinite(number) and number > 0
except ValueError:
return False
def svg_has_bounds(root):
view_box = root.attrib.get("viewBox") or root.attrib.get("viewbox")
if view_box:
try:
values = [float(value) for value in re.split(r"[\s,]+", view_box.strip()) if value]
except ValueError:
values = []
if (
len(values) == 4
and all(math.isfinite(value) for value in values)
and values[2] > 0
and values[3] > 0
):
return True
return positive_svg_length(root.attrib.get("width")) and positive_svg_length(
root.attrib.get("height")
)
def svg_identifier_map(root):
result = {}
for element in root.iter():
for raw_name, raw_value in element.attrib.items():
if svg_local_name(raw_name).lower() != "id":
continue
identifier = str(raw_value).strip()
if not identifier:
raise ValueError("SVG contains an empty id")
if identifier in result:
raise ValueError(f"SVG contains duplicate id {identifier!r}")
result[identifier] = element
return result
def svg_stylesheet_has_only_keyframes(css):
literal = str(css or "")
if not literal.strip():
return True
if "\\" in literal or "/*" in literal or "*/" in literal:
return False
position = 0
while position < len(literal):
while position < len(literal) and literal[position].isspace():
position += 1
if position == len(literal):
return True
match = SVG_KEYFRAMES_START_RE.match(literal, position)
if match is None:
return False
depth = 1
quote = None
position = match.end()
while position < len(literal) and depth:
character = literal[position]
if quote:
if character == quote:
quote = None
elif character in {'"', "'"}:
quote = character
elif character == "{":
depth += 1
elif character == "}":
depth -= 1
position += 1
if depth or quote:
return False
return True
def validate_svg_stylesheets(root):
for element in root.iter():
if svg_local_name(element.tag).lower() != "style":
continue
if not svg_stylesheet_has_only_keyframes("".join(element.itertext())):
raise ValueError("SVG contains a non-keyframe stylesheet rule")
def svg_numeric_length(value, default=0):
match = re.fullmatch(
r"\s*([+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[Ee][+-]?\d+)?)(?:[A-Za-z%]+)?\s*",
str(value or ""),
)
if not match:
return default
try:
number = float(match.group(1))
except ValueError:
return default
return number if math.isfinite(number) else default
def svg_points(value):
try:
numbers = [
float(item)
for item in re.findall(
r"[-+]?(?:\d*\.\d+|\d+\.?\d*)(?:[Ee][-+]?\d+)?",
str(value or ""),
)
]
except ValueError:
return []
if len(numbers) % 2 or not all(math.isfinite(number) for number in numbers):
return []
return list(zip(numbers[::2], numbers[1::2]))
def svg_points_have_area(points):
if not points:
return False
origin = points[0]
direction = next((point for point in points[1:] if point != origin), None)
if direction is None:
return False
first = (direction[0] - origin[0], direction[1] - origin[1])
return any(
first[0] * (point[1] - origin[1]) != first[1] * (point[0] - origin[0])
for point in points[1:]
)
def svg_path_geometry(value):
data = str(value or "").strip()
if not data:
return False, False, False
arity = {"m": 2, "l": 2, "h": 1, "v": 1, "c": 6, "s": 4, "q": 4, "t": 2, "a": 7}
commands = frozenset("MmZzLlHhVvCcSsQqTtAa")
number_re = re.compile(r"[-+]?(?:\d*\.\d+|\d+\.?\d*)(?:[Ee][-+]?\d+)?")
index = 0
command = None
relative = False
saw_command = False
drew_segment = False
drew_zero_length_segment = False
fillable = False
current = (0.0, 0.0)
subpath_start = None
fill_direction = None
def record_fill_point(point):
nonlocal fill_direction, fillable
if subpath_start is None or point == subpath_start:
return
if fill_direction is None:
fill_direction = point
return
first = (fill_direction[0] - subpath_start[0], fill_direction[1] - subpath_start[1])
candidate = (point[0] - subpath_start[0], point[1] - subpath_start[1])
if first[0] * candidate[1] != first[1] * candidate[0]:
fillable = True
def skip_whitespace(position):
while position < len(data) and data[position].isspace():
position += 1
return position
def consume_separator(position, allow_comma):
position = skip_whitespace(position)
if position < len(data) and data[position] == ",":
if not allow_comma:
return None
position = skip_whitespace(position + 1)
return position
while index < len(data):
index = skip_whitespace(index)
if index >= len(data):
break
if data[index] in commands:
raw_command = data[index]
command = raw_command.lower()
relative = raw_command.islower()
if not saw_command and command != "m":
return False, False, False
saw_command = True
index += 1
if command == "z":
if subpath_start is not None:
drew_segment = drew_segment or current != subpath_start
current = subpath_start
command = None
continue
elif command is None:
return False, False, False
parameter_count = arity.get(command)
if not parameter_count:
return False, False, False
group_count = 0
while True:
values = []
for parameter in range(parameter_count):
separated = consume_separator(index, parameter > 0 or group_count > 0)
if separated is None:
return False, False, False
index = separated
if index >= len(data) or data[index] in commands:
return False, False, False
if command == "a" and parameter in {3, 4}:
if data[index] not in {"0", "1"}:
return False, False, False
values.append(data[index])
index += 1
else:
match = number_re.match(data, index)
if match is None:
return False, False, False
values.append(match.group(0))
index = match.end()
numbers = [float(item) for item in values]
if not all(math.isfinite(number) for number in numbers):
return False, False, False
if command == "a" and (numbers[0] < 0 or numbers[1] < 0):
return False, False, False
origin = current
def point(x, y):
return (origin[0] + x, origin[1] + y) if relative else (x, y)
if command == "m":
endpoint = point(numbers[0], numbers[1])
if group_count == 0:
subpath_start = endpoint
fill_direction = None
else:
drew_segment = drew_segment or endpoint != current
drew_zero_length_segment = drew_zero_length_segment or endpoint == current
record_fill_point(endpoint)
current = endpoint
elif command in {"l", "t"}:
endpoint = point(numbers[0], numbers[1])
drew_segment = drew_segment or endpoint != current
drew_zero_length_segment = drew_zero_length_segment or endpoint == current
record_fill_point(endpoint)
current = endpoint
elif command == "h":
endpoint = (origin[0] + numbers[0], origin[1]) if relative else (numbers[0], origin[1])
drew_segment = drew_segment or endpoint != current
drew_zero_length_segment = drew_zero_length_segment or endpoint == current
record_fill_point(endpoint)
current = endpoint
elif command == "v":
endpoint = (origin[0], origin[1] + numbers[0]) if relative else (origin[0], numbers[0])
drew_segment = drew_segment or endpoint != current
drew_zero_length_segment = drew_zero_length_segment or endpoint == current
record_fill_point(endpoint)
current = endpoint
elif command == "c":
control_1 = point(numbers[0], numbers[1])
control_2 = point(numbers[2], numbers[3])
endpoint = point(numbers[4], numbers[5])
has_extent = any(item != current for item in (control_1, control_2, endpoint))
drew_segment = drew_segment or has_extent
drew_zero_length_segment = drew_zero_length_segment or not has_extent
for item in (control_1, control_2, endpoint):
record_fill_point(item)
current = endpoint
elif command in {"s", "q"}:
control = point(numbers[0], numbers[1])
endpoint = point(numbers[2], numbers[3])
has_extent = control != current or endpoint != current
drew_segment = drew_segment or has_extent
drew_zero_length_segment = drew_zero_length_segment or not has_extent
for item in (control, endpoint):
record_fill_point(item)
current = endpoint
elif command == "a":
endpoint = point(numbers[5], numbers[6])
drew_segment = drew_segment or endpoint != current
drew_zero_length_segment = drew_zero_length_segment or endpoint == current
if numbers[0] > 0 and numbers[1] > 0 and endpoint != current:
fillable = True
record_fill_point(endpoint)
current = endpoint
group_count += 1
index = skip_whitespace(index)
if index >= len(data):
return drew_segment, fillable, drew_zero_length_segment
if data[index] in commands:
break
return (
saw_command and drew_segment,
saw_command and fillable,
saw_command and drew_zero_length_segment,
)
def svg_color_visible(value, identifiers):
literal = re.sub(r"\s+", "", str(value or "")).lower()
local_paint = re.fullmatch(r'''url\((?:['"])?#([^)'"]+)(?:['"])?\)''', literal)
if local_paint:
return urllib.parse.unquote(local_paint.group(1)) in identifiers
if literal.startswith("url("):
return False
if literal in {"", "none", "transparent"}:
return False
if re.fullmatch(r"#[0-9a-f]{4}", literal):
return literal[-1] != "0"
if re.fullmatch(r"#[0-9a-f]{8}", literal):
return literal[-2:] != "00"
alpha = re.fullmatch(r"(?:rgba|hsla)\(.*,([^,\)]*)\)", literal)
if alpha is None:
alpha = re.fullmatch(r"(?:rgba?|hsla?)\(.*\/([^\)]*)\)", literal)
if alpha:
value = alpha.group(1)
try:
divisor = 100 if value.endswith("%") else 1
return float(value.rstrip("%")) / divisor > 0
except ValueError:
return True
return True
def svg_paint_visible(state, identifiers, *, fill=True, stroke=True):
fill_color = state["color"] if state["fill"] == "currentcolor" else state["fill"]
stroke_color = state["color"] if state["stroke"] == "currentcolor" else state["stroke"]
fill_visible = (
fill
and svg_color_visible(fill_color, identifiers)
and state["fill_opacity"] > 0
)
stroke_visible = (
stroke
and svg_color_visible(stroke_color, identifiers)
and state["stroke_opacity"] > 0
and state["stroke_width"] > 0
)
return fill_visible or stroke_visible
def svg_renderable_graphics(root):
identifiers = svg_identifier_map(root)
validate_svg_stylesheets(root)
initial_state = {
"color": "black",
"displayed": True,
"visibility": "visible",
"opacity": 1.0,
"fill": "black",
"fill_opacity": 1.0,
"stroke": "none",
"stroke_opacity": 1.0,
"stroke_width": 1.0,
"stroke_linecap": "butt",
"hidden_attribute": False,
}
non_rendering = {
"clippath",
"defs",
"desc",
"marker",
"mask",
"metadata",
"pattern",
"style",
"symbol",
"title",
}
def visible_geometry(element, name, state):
if name == "image":
return positive_svg_length(element.attrib.get("width")) and positive_svg_length(
element.attrib.get("height")
)
if name == "rect":
geometry = positive_svg_length(element.attrib.get("width")) and positive_svg_length(
element.attrib.get("height")
)
return geometry and svg_paint_visible(state, identifiers)
elif name == "circle":
geometry = positive_svg_length(element.attrib.get("r"))
return geometry and svg_paint_visible(state, identifiers)
elif name == "ellipse":
geometry = positive_svg_length(element.attrib.get("rx")) and positive_svg_length(
element.attrib.get("ry")
)
return geometry and svg_paint_visible(state, identifiers)
elif name == "line":
geometry = (
svg_numeric_length(element.attrib.get("x1")),
svg_numeric_length(element.attrib.get("y1")),
) != (
svg_numeric_length(element.attrib.get("x2")),
svg_numeric_length(element.attrib.get("y2")),
)
return geometry and svg_paint_visible(
state, identifiers, fill=False, stroke=True
)
elif name in {"polygon", "polyline"}:
points = svg_points(element.attrib.get("points"))
geometry = len(set(points)) >= 2
return geometry and svg_paint_visible(
state,
identifiers,
fill=svg_points_have_area(points),
stroke=True,
)
elif name == "path":
has_segment, has_fill_area, has_zero_length_segment = svg_path_geometry(
element.attrib.get("d")
)
if has_segment:
return svg_paint_visible(
state, identifiers, fill=has_fill_area, stroke=True
)
if has_zero_length_segment and state["stroke_linecap"] in {"round", "square"}:
return svg_paint_visible(state, identifiers, fill=False, stroke=True)
return False
elif name == "text":
geometry = bool("".join(element.itertext()).strip()) and svg_numeric_length(
svg_properties(element).get("font-size"), 16
) > 0
return geometry and svg_paint_visible(state, identifiers)
else:
return False
def visit(element, parent_state, resolving=frozenset(), referenced=False):
name = svg_local_name(element.tag).lower()
namespace = svg_namespace(element.tag)
if namespace not in {"", SVG_NAMESPACE}:
return 0
state = svg_render_state(element, parent_state)
hidden = (
not state["displayed"]
or state["hidden_attribute"]
or state["visibility"] in {"hidden", "collapse"}
or state["opacity"] <= 0
)
if hidden:
return 0
if name == "use":
href = svg_attribute(element, "href")
identifier = urllib.parse.unquote(href[1:]) if href.startswith("#") else ""
if not identifier or identifier in resolving or identifier not in identifiers:
return 0
return visit(
identifiers[identifier],
state,
resolving | {identifier},
referenced=True,
)
if name in non_rendering and not (referenced and name == "symbol"):
return 0
own = 1 if visible_geometry(element, name, state) else 0
return own + sum(visit(child, state, resolving) for child in element)
return visit(root, initial_state)
def decode_data_image(source):
header, separator, payload = str(source or "").partition(",")
if not separator or not header.lower().startswith("data:image/"):
raise ValueError("image source is not a data URI")
parts = header[5:].split(";")
media_type = parts[0].lower()
parameters = {part.lower() for part in parts[1:] if part}
if "base64" not in parameters and re.search(r"%(?![0-9A-Fa-f]{2})", payload):
raise ValueError("embedded image contains malformed percent encoding")
try:
if "base64" in parameters:
raw = base64.b64decode(payload, validate=True)
elif media_type == "image/svg+xml" and payload.lstrip().startswith("<"):
raw = payload.encode("utf-8")
elif re.search(r"%[0-9A-Fa-f]{2}", payload):
raw = urllib.parse.unquote_to_bytes(payload)
else:
raw = base64.b64decode(payload, validate=True)
except (binascii.Error, ValueError) as error:
raise ValueError("invalid embedded image encoding") from error
if not raw or len(raw) > MAX_EMBEDDED_IMAGE_BYTES:
raise ValueError("embedded image is empty or exceeds the safety limit")
return media_type, raw
def validate_embedded_image(source):
media_type, raw = decode_data_image(source)
if media_type == "image/png":
if not png_metadata(Path("embedded.png"), raw).get("valid"):
raise ValueError("invalid embedded PNG")
return "png"
if media_type != "image/svg+xml":
raise ValueError("unsupported embedded image media type")
text = raw.decode("utf-8", errors="strict")
lowered = text.lower()
if "\x00" in text or "<!doctype" in lowered or "<!entity" in lowered:
raise ValueError("embedded SVG contains a forbidden declaration")
without_xml_declaration = re.sub(
r"^\ufeff?\s*<\?xml\s+[^?]*\?>", "", text, count=1, flags=re.IGNORECASE
)
if "<?" in without_xml_declaration:
raise ValueError("embedded SVG processing instructions are not allowed")
root = ET.fromstring(text)
if svg_local_name(root.tag).lower() != "svg" or svg_namespace(root.tag) not in {
"",
SVG_NAMESPACE,
}:
raise ValueError("embedded image root is not SVG")
svg_tree_shape(root)
if not svg_has_bounds(root) or svg_renderable_graphics(root) <= 0:
raise ValueError("embedded SVG lacks bounds or renderable graphics")
for element in root.iter():
name = svg_local_name(element.tag).lower()
if name == "image":
raise ValueError("nested embedded SVG images are not allowed")
if name == "foreignobject":
raise ValueError("embedded SVG foreignObject content is not allowed")
metadata = svg_resource_metadata(root, None)
if metadata["external_references"] or metadata["unsupported_images"]:
raise ValueError("embedded SVG contains unsafe resource references")
return "svg+xml"
def svg_resource_metadata(root, embedded_image_validator):
identifiers = set(svg_identifier_map(root))
embedded_svg_images = 0
embedded_raster_images = 0
external_references = 0
unsupported_images = 0
for element in root.iter():
name = svg_local_name(element.tag).lower()
namespace = svg_namespace(element.tag)
href = svg_attribute(element, "href")
image_sources = []
if name == "image" and namespace in {"", SVG_NAMESPACE}:
image_sources = [href] if href else []
elif name in {"img", "source"} and namespace != SVG_NAMESPACE:
image_sources = [
value
for value in (svg_attribute(element, "src"), svg_attribute(element, "srcset"))
if value
]
if name in {"image", "img", "source"}:
if not image_sources:
unsupported_images += 1
for source in image_sources:
if source.startswith("#"):
if source[1:] not in identifiers:
unsupported_images += 1
continue
if source.lower().startswith("data:image/"):
if embedded_image_validator is None:
unsupported_images += 1
continue
try:
media_type = embedded_image_validator(source)
except Exception:
unsupported_images += 1
else:
if media_type == "svg+xml":
embedded_svg_images += 1
else:
embedded_raster_images += 1
continue
if source.lower().startswith(SVG_EXTERNAL_REFERENCE_PREFIXES):
external_references += 1
else:
unsupported_images += 1
elif href and name != "a":
if href.startswith("#"):
if href[1:] not in identifiers:
unsupported_images += 1
elif href.lower().startswith("data:"):
unsupported_images += 1
else:
external_references += 1
elif name == "a" and href:
navigation = normalize_url_for_scheme(href).lower()
if not navigation.startswith(("#", "http://", "https://", "mailto:")):
external_references += 1
if name in SVG_UNSAFE_RESOURCE_ELEMENTS:
external_references += 1
if name == "style":
external, unsupported = svg_css_reference_counts("".join(element.itertext()), identifiers)
external_references += external
unsupported_images += unsupported
for raw_name, value in element.attrib.items():
key = svg_local_name(raw_name).lower()
literal = normalize_url_for_scheme(value).lower()
if raw_name == f"{{{SVG_XML_NAMESPACE}}}base" or str(raw_name).lower() == "xml:base":
external_references += 1
if key.startswith("on") or literal.startswith(("javascript:", "vbscript:", "data:text/html")):
external_references += 1
external, unsupported = svg_css_reference_counts(value, identifiers)
external_references += external
unsupported_images += unsupported
return {
"embedded_svg_images": embedded_svg_images,
"embedded_raster_images": embedded_raster_images,
"external_references": external_references,
"unsupported_images": unsupported_images,
}
def svg_style_properties(element):
properties = {}
for item in str(element.attrib.get("style") or "").split(";"):
name, separator, value = item.partition(":")
if separator:
properties[name.strip().lower()] = value.strip().lower().split("!important", 1)[0].strip()
return properties
def svg_properties(element):
attributes = {
svg_local_name(name).lower(): str(value).strip().lower()
for name, value in element.attrib.items()
}
attributes.update(svg_style_properties(element))
return attributes
def svg_flow_keyframe_offsets(root):
offsets = {}
for element in root.iter():
if svg_local_name(element.tag).lower() != "style":
continue
css = "".join(element.itertext())
for match in re.finditer(
r"@keyframes\s+([_A-Za-z][_A-Za-z0-9-]*)\s*\{", css, re.IGNORECASE
):
start = match.end() - 1
depth = 0
end = None
for index in range(start, len(css)):
if css[index] == "{":
depth += 1
elif css[index] == "}":
depth -= 1
if depth == 0:
end = index
break
if end is None:
continue
body = css[start + 1 : end]
values = set()
has_from = has_to = False
for rule in re.finditer(r"([^{}]+)\{([^{}]*)\}", body):
declaration = re.search(
r"stroke-dashoffset\s*:\s*([+-]?(?:\d+(?:\.\d*)?|\.\d+))(?:px)?(?:\s*[;}])",
rule.group(2),
re.IGNORECASE,
)
if declaration is None or not math.isfinite(float(declaration.group(1))):
continue
values.add(float(declaration.group(1)))
selectors = {selector.strip().lower() for selector in rule.group(1).split(",")}
has_from = has_from or bool(selectors & {"from", "0%", "0.0%"})
has_to = has_to or bool(selectors & {"to", "100%", "100.0%"})
if values:
offsets[match.group(1).lower()] = {
"values": values,
"has_from": has_from,
"has_to": has_to,
}
return offsets
def svg_has_flow_animation(root):
keyframes = svg_flow_keyframe_offsets(root)
if not keyframes:
return False
identifiers = svg_identifier_map(root)
initial_state = {
"color": "black",
"displayed": True,
"visibility": "visible",
"opacity": 1.0,
"fill": "black",
"fill_opacity": 1.0,
"stroke": "none",
"stroke_opacity": 1.0,
"stroke_width": 1.0,
"stroke_linecap": "butt",
"hidden_attribute": False,
}
def visit(element, parent_state):
state = svg_render_state(element, parent_state)
properties = svg_properties(element)
animation = properties.get("animation", "")
tokens = set(re.split(r"\s+", animation))
times = [
float(value) * (0.001 if unit.lower() == "ms" else 1)
for value, unit in re.findall(
r"(?<![-\w.])(\d+(?:\.\d+)?)(ms|s)(?![-\w])", animation, re.IGNORECASE
)
]
dash_array = properties.get("stroke-dasharray", "")
dash_offset = properties.get("stroke-dashoffset", "")
animation_name = next((name for name in keyframes if name in tokens), None)
inline_offsets = {
float(value)
for value in re.findall(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)", dash_offset)
if math.isfinite(float(value))
} or {0.0}
keyframe = keyframes.get(animation_name, {})
keyframe_offsets = keyframe.get("values", set())
motion = len(keyframe_offsets) > 1 or (
not (keyframe.get("has_from") and keyframe.get("has_to"))
and len(inline_offsets | keyframe_offsets) > 1
)
stroke_color = state["color"] if state["stroke"] == "currentcolor" else state["stroke"]
animated = (
svg_local_name(element.tag).lower() in {"path", "line", "polyline"}
and animation_name
and "running" in tokens
and "infinite" in tokens
and times
and times[0] > 0
and any(
float(value) > 0
for value in re.findall(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)", dash_array)
)
and motion
and state["displayed"]
and not state["hidden_attribute"]
and state["visibility"] not in {"hidden", "collapse"}
and state["opacity"] > 0
and state["stroke_opacity"] > 0
and state["stroke_width"] > 0
and svg_color_visible(stroke_color, identifiers)
)
return animated or any(visit(child, state) for child in element)
return visit(root, initial_state)
def svg_number(value, fallback):
text = str(value or "").strip().lower()
if not text:
return fallback
try:
if text.endswith("%"):
return float(text[:-1]) / 100
match = re.match(r"^[+-]?(?:\d+(?:\.\d*)?|\.\d+)", text)
return float(match.group(0)) if match else fallback
except ValueError:
return fallback
def svg_render_state(element, parent):
properties = svg_properties(element)
display = properties.get("display", "")
visibility = properties.get("visibility", parent["visibility"])
if visibility == "inherit":
visibility = parent["visibility"]
own_opacity = max(0.0, svg_number(properties.get("opacity"), 1.0))
def inherited(name):
value = properties.get(name)
return parent[name.replace("-", "_")] if not value or value == "inherit" else value
return {
"displayed": parent["displayed"] and display != "none",
"visibility": visibility,
"opacity": parent["opacity"] * own_opacity,
"color": inherited("color"),
"fill": inherited("fill"),
"fill_opacity": max(
0.0,
svg_number(inherited("fill-opacity"), parent["fill_opacity"]),
),
"stroke": inherited("stroke"),
"stroke_opacity": max(
0.0,
svg_number(inherited("stroke-opacity"), parent["stroke_opacity"]),
),
"stroke_width": max(
0.0,
svg_number(inherited("stroke-width"), parent["stroke_width"]),
),
"stroke_linecap": inherited("stroke-linecap"),
"hidden_attribute": parent["hidden_attribute"]
or "hidden" in properties
and properties.get("hidden") in {"", "hidden", "true"}
or properties.get("aria-hidden") == "true",
}
def svg_state_is_visible(state, identifiers, foreign_text=False):
if (
not state["displayed"]
or state["hidden_attribute"]
or state["visibility"] in {"hidden", "collapse"}
or state["opacity"] <= 0
):
return False
if foreign_text:
return True
return svg_paint_visible(state, identifiers)
def svg_visible_text(root):
parts = []
identifiers = svg_identifier_map(root)
initial_state = {
"color": "black",
"displayed": True,
"visibility": "visible",
"opacity": 1.0,
"fill": "black",
"fill_opacity": 1.0,
"stroke": "none",
"stroke_opacity": 1.0,
"stroke_width": 1.0,
"stroke_linecap": "butt",
"hidden_attribute": False,
}
def visit(element, parent_state, text_context=""):
local_name = svg_local_name(element.tag)
state = svg_render_state(element, parent_state)
non_rendered = local_name in NON_RENDERED_SVG_ELEMENTS
child_text_context = text_context
if local_name == "foreignObject":
child_text_context = "foreign"
elif local_name in SVG_TEXT_CONTAINERS and not child_text_context:
child_text_context = "svg"
visible = not non_rendered and svg_state_is_visible(
state, identifiers, foreign_text=child_text_context == "foreign"
)
if visible and child_text_context and element.text:
parts.append(element.text)
for child in element:
visit(
child,
state if not non_rendered else {**state, "displayed": False},
child_text_context,
)
if visible and child_text_context and child.tail:
parts.append(child.tail)
visit(root, initial_state)
return " ".join(" ".join(parts).split())
def validate_svg_doctype(text):
if re.search(r"<!ENTITY\b", text, flags=re.IGNORECASE):
raise ValueError("SVG entity declarations are not allowed")
doctypes = list(re.finditer(r"<!DOCTYPE\b", text, flags=re.IGNORECASE))
if not doctypes:
return
if len(doctypes) != 1:
raise ValueError("SVG may contain at most one doctype")
match = SVG_11_DOCTYPE.search(text)
if match is None or match.start() != doctypes[0].start():
raise ValueError("unsupported SVG doctype or internal subset")
if "[" in match.group(0) or "]" in match.group(0):
raise ValueError("SVG doctype internal subsets are not allowed")
def svg_metadata(path, data=None):
try:
if data is None:
data = path.read_bytes()
if len(data) > MAX_SVG_FILE_BYTES:
raise ValueError(f"SVG exceeds {MAX_SVG_FILE_BYTES} byte safety limit")
text = data.decode("utf-8", errors="strict")
validate_svg_doctype(text)
without_xml_declaration = re.sub(
r"^\ufeff?\s*<\?xml\s+[^?]*\?>", "", text, count=1, flags=re.IGNORECASE
)
if "<?" in without_xml_declaration:
raise ValueError("SVG processing instructions are not allowed")
root = ET.fromstring(text)
local_name = svg_local_name(root.tag)
namespace = svg_namespace(root.tag)
svg_tree_shape(root)
visible_text = svg_visible_text(root)
renderable_graphics = svg_renderable_graphics(root)
resource_metadata = svg_resource_metadata(root, validate_embedded_image)
return {
"valid": (
local_name == "svg"
and namespace in {"", SVG_NAMESPACE}
and svg_has_bounds(root)
and renderable_graphics > 0
),
"visible_text": visible_text,
"renderable_graphics": renderable_graphics,
"flow_animation": svg_has_flow_animation(root),
**resource_metadata,
}
except Exception as error:
return {"valid": False, "error": short(error, 300)}
def drawio_style_map(style):
values = {}
for token in str(style or "").split(";"):
key, separator, value = token.partition("=")
if separator:
values[key.strip()] = value.strip()
return values
def drawio_is_image_style(style):
literal = str(style or "")
parsed = drawio_style_map(literal)
flags = {
token.strip().lower()
for token in literal.split(";")
if token.strip() and "=" not in token
}
return parsed.get("shape", "").strip().lower() == "image" or "image" in flags
DRAWIO_FLOW_ANIMATION_KEYS = {
"flowAnimation",
"flowAnimationDirection",
"flowAnimationDuration",
"flowAnimationTimingFunction",
}
DRAWIO_FLOW_ANIMATION_DIRECTIONS = {"normal", "reverse", "alternate", "alternate-reverse"}
DRAWIO_FLOW_ANIMATION_TIMINGS = {"linear", "ease", "ease-in", "ease-out", "ease-in-out"}
DRAWIO_EXPLICIT_FLOW_ROLES = {
"flow",
"runtime-flow",
"data-flow",
"process-flow",
"request",
"event",
}
DRAWIO_STATIC_EDGE_ROLES = {
"association",
"dependency",
"containment",
"ownership",
"annotation",
"decorative",
"legend",
"structural",
}
DRAWIO_PROFILE_STYLE_HASH_KEYS = frozenset(
{
"designProfile",
"shape",
"dataRole",
"strokeColor",
"fillColor",
"gradientColor",
"gradientDirection",
"shadow",
"glass",
"arcSize",
"strokeWidth",
"fontColor",
"fontSize",
"profileRole",
}
)
def drawio_image_source(style):
literal = str(style or "")
match = None
for candidate in re.finditer(r"(?:^|;)image=", literal):
match = candidate
if match is None:
return ""
tail = literal[match.end():]
boundary = re.search(r";(?=[A-Za-z_][A-Za-z0-9_.-]*=)", tail)
source = (tail[:boundary.start()] if boundary else tail).strip()
if boundary is None and source.endswith(";"):
source = source[:-1].rstrip()
return source
def drawio_srcset_sources(value):
literal = str(value or "")
sources = []
position = 0
whitespace = "\t\n\f\r "
while position < len(literal):
while position < len(literal) and literal[position] in whitespace + ",":
position += 1
if position >= len(literal):
break
start = position
is_data_uri = literal[position : position + 5].lower() == "data:"
while position < len(literal):
character = literal[position]
if character in whitespace or (character == "," and not is_data_uri):
break
position += 1
source = literal[start:position]
if position < len(literal) and literal[position] == ",":
position += 1
if source:
sources.append(source)
continue
if source.endswith(","):
source = source.rstrip(",")
if source:
sources.append(source)
continue
if source:
sources.append(source)
parentheses = 0
while position < len(literal):
character = literal[position]
if character == "(":
parentheses += 1
elif character == ")" and parentheses:
parentheses -= 1
elif character == "," and not parentheses:
position += 1
break
position += 1
return sources
def normalize_url_for_scheme(value):
without_tab_or_newline = re.sub(r"[\t\n\r]", "", str(value or ""))
return re.sub(r"^[\x00-\x20]+|[\x00-\x20]+$", "", without_tab_or_newline)
class DrawioImageSourceParser(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.sources = []
self.style_depth = 0
def collect_css_sources(self, value):
literal = str(value or "")
if "\\" in literal:
self.sources.append("unsafe-css:escape")
if re.search(r"@import\b", literal, flags=re.IGNORECASE):
self.sources.append("unsafe-css:import")
if re.search(
r"(?<![-_A-Za-z0-9])(?:-webkit-)?image-set\s*\(",
literal,
flags=re.IGNORECASE,
):
self.sources.append("unsupported-css:image-set")
for match in SVG_URL_RE.finditer(literal):
self.sources.append(match.group(2).strip())
def handle_starttag(self, tag, attrs):
tag_name = str(tag or "").lower()
values = {}
for name, value in attrs:
values.setdefault(str(name or "").lower(), []).append(str(value or ""))
if tag_name == "style":
self.style_depth += 1
if tag_name in {"img", "image", "source"}:
image_sources = []
for name in ("src", "href", "xlink:href"):
image_sources.extend(
source.strip() for source in values.get(name, ()) if source.strip()
)
for srcset in values.get("srcset", ()):
image_sources.extend(drawio_srcset_sources(srcset))
self.sources.extend(image_sources)
if not image_sources:
self.sources.append(f"missing-image-source:{tag_name}")
resource_attributes = {
"animate": ("href", "xlink:href"),
"animatemotion": ("href", "xlink:href"),
"animatetransform": ("href", "xlink:href"),
"audio": ("src",),
"base": ("href",),
"discard": ("href", "xlink:href"),
"embed": ("src",),
"feimage": ("href", "xlink:href"),
"filter": ("href", "xlink:href"),
"frame": ("src",),
"iframe": ("src",),
"input": ("src",),
"lineargradient": ("href", "xlink:href"),
"link": ("href",),
"mpath": ("href", "xlink:href"),
"object": ("data",),
"pattern": ("href", "xlink:href"),
"radialgradient": ("href", "xlink:href"),
"script": ("src", "href"),
"set": ("href", "xlink:href"),
"textpath": ("href", "xlink:href"),
"track": ("src",),
"use": ("href", "xlink:href"),
"video": ("src", "poster"),
}
for name in resource_attributes.get(tag_name, ()):
self.sources.extend(
source.strip() for source in values.get(name, ()) if source.strip()
)
if tag_name in SVG_UNSAFE_RESOURCE_ELEMENTS:
self.sources.append(f"active-content:{tag_name}")
for name, attribute_values in values.items():
for value in attribute_values:
literal = normalize_url_for_scheme(value).lower()
if name == "xml:base":
self.sources.append("active-content:xml:base")
elif name.startswith("on") or literal.startswith(
("javascript:", "vbscript:", "data:text/html")
):
self.sources.append(value.strip() or f"active-content:{name}")
if name == "style":
self.collect_css_sources(value)
def handle_endtag(self, tag):
if str(tag or "").lower() == "style" and self.style_depth:
self.style_depth -= 1
def handle_data(self, data):
if self.style_depth:
self.collect_css_sources(data)
def handle_startendtag(self, tag, attrs):
self.handle_starttag(tag, attrs)
self.handle_endtag(tag)
def handle_pi(self, data):
self.sources.append("active-content:processing-instruction")
def drawio_cell_resource_sources(cell):
sources = []
style = str(cell.attrib.get("style") or "")
style_source = drawio_image_source(style)
if style_source:
sources.append(style_source)
value = html.unescape(str(cell.attrib.get("value") or ""))
parser = DrawioImageSourceParser()
try:
parser.feed(value)
parser.close()
except (TypeError, ValueError):
parser.sources.append("")
sources.extend(parser.sources)
for text in (style, value):
for match in SVG_URL_RE.finditer(text):
sources.append(match.group(2).strip())
return list(dict.fromkeys(str(source or "").strip() for source in sources))
def drawio_root_cells(root):
cells = []
for child in root:
if child.tag == "mxCell":
cells.append(child)
continue
if child.tag.lower() not in {"object", "userobject"}:
continue
nested = [element for element in child if element.tag == "mxCell"]
if len(nested) != 1:
raise ValueError("draw.io object wrapper must contain exactly one mxCell")
normalized = copy.deepcopy(nested[0])
wrapper_id = child.attrib.get("id")
inner_id = normalized.attrib.get("id")
if wrapper_id and inner_id and wrapper_id != inner_id:
raise ValueError("draw.io object wrapper and mxCell IDs disagree")
if wrapper_id and not inner_id:
normalized.attrib["id"] = wrapper_id
if "value" not in normalized.attrib:
wrapper_value = child.attrib.get("label") or child.attrib.get("value")
if wrapper_value is not None:
normalized.attrib["value"] = wrapper_value
if "link" not in normalized.attrib and child.attrib.get("link") is not None:
normalized.attrib["link"] = child.attrib["link"]
cells.append(normalized)
return cells
def validate_drawio_animation_style(cell, style):
is_edge = cell.attrib.get("edge") == "1"
animation_keys = {key for key in style if key.startswith("flowAnimation")}
if animation_keys - DRAWIO_FLOW_ANIMATION_KEYS:
raise ValueError("draw.io cell contains an unknown flow animation style")
if animation_keys and not is_edge:
raise ValueError("draw.io flow animation styles are only valid on edges")
enabled = style.get("flowAnimation")
if enabled is not None and enabled not in {"0", "1"}:
raise ValueError("draw.io flowAnimation must be 0 or 1")
duration = style.get("flowAnimationDuration")
if duration is not None and (not duration.isdigit() or int(duration) <= 0):
raise ValueError("draw.io flowAnimationDuration must be a positive integer")
if (
style.get("flowAnimationTimingFunction") is not None
and style["flowAnimationTimingFunction"] not in DRAWIO_FLOW_ANIMATION_TIMINGS
):
raise ValueError("draw.io flowAnimationTimingFunction is invalid")
if (
style.get("flowAnimationDirection") is not None
and style["flowAnimationDirection"] not in DRAWIO_FLOW_ANIMATION_DIRECTIONS
):
raise ValueError("draw.io flowAnimationDirection is invalid")
def drawio_directed_semantic_flow(style):
role = str(style.get("dataRole") or "").strip().lower()
if role in DRAWIO_EXPLICIT_FLOW_ROLES:
return True
if role in DRAWIO_STATIC_EDGE_ROLES:
return False
return style.get("endArrow") != "none" or style.get("startArrow") not in {None, "none"}
def drawio_has_static_direction(style):
return style.get("endArrow") != "none" or style.get("startArrow") not in {None, "none"}
def drawio_visible_profile_vertex(cell, cells_by_id):
if cell.attrib.get("vertex") != "1":
return False
geometries = [child for child in cell if child.tag == "mxGeometry"]
if len(geometries) != 1 or geometries[0].attrib.get("as") != "geometry":
return False
try:
width = float(geometries[0].attrib.get("width", ""))
height = float(geometries[0].attrib.get("height", ""))
except (TypeError, ValueError):
return False
if not math.isfinite(width) or not math.isfinite(height) or width <= 0 or height <= 0:
return False
current = cell
seen = set()
while current is not None:
if current.attrib.get("visible") == "0":
return False
current_id = str(current.attrib.get("id") or "")
if not current_id or current_id in seen:
return False
seen.add(current_id)
parent = current.attrib.get("parent")
current = cells_by_id.get(parent) if parent is not None else None
return True
def drawio_finite_number(value, label):
try:
number = float(value)
except (TypeError, ValueError) as error:
raise ValueError(f"draw.io {label} must be numeric") from error
if not math.isfinite(number):
raise ValueError(f"draw.io {label} must be finite")
return number
def drawio_cell_geometry(cell, *, edge=False):
geometries = [child for child in cell if child.tag == "mxGeometry"]
if len(geometries) != 1:
raise ValueError("draw.io vertex or edge must contain exactly one mxGeometry")
geometry = geometries[0]
if geometry.attrib.get("as") != "geometry":
raise ValueError("draw.io mxGeometry must set as=geometry")
if edge and geometry.attrib.get("relative") != "1":
raise ValueError("draw.io edge geometry must set relative=1")
for name in ("x", "y", "width", "height"):
if name in geometry.attrib:
drawio_finite_number(geometry.attrib[name], f"geometry {name}")
for point in geometry.iter("mxPoint"):
for name in ("x", "y"):
if name in point.attrib:
drawio_finite_number(point.attrib[name], f"point {name}")
return geometry
def validate_drawio_model_cells(cells, cells_by_id):
content = [cell for cell in cells if cell.attrib.get("id") not in {"0", "1"}]
semantic_cells = 0
for cell in content:
cell_id = str(cell.attrib["id"])
parent = cell.attrib.get("parent")
if not parent or parent not in cells_by_id:
raise ValueError("draw.io content cell must reference an existing parent")
is_vertex = cell.attrib.get("vertex") == "1"
is_edge = cell.attrib.get("edge") == "1"
if is_vertex and is_edge:
raise ValueError("draw.io cell cannot be both a vertex and an edge")
if not is_vertex and not is_edge:
continue
semantic_cells += 1
geometry = drawio_cell_geometry(cell, edge=is_edge)
style = drawio_style_map(cell.attrib.get("style"))
if is_vertex:
required = ("x", "y", "width", "height")
if any(name not in geometry.attrib for name in required):
raise ValueError("draw.io vertex requires x, y, width, and height geometry")
width = drawio_finite_number(geometry.attrib["width"], "vertex width")
height = drawio_finite_number(geometry.attrib["height"], "vertex height")
if width < 0 or height < 0:
raise ValueError("draw.io vertex geometry cannot be negative")
if (
drawio_image_source(cell.attrib.get("style"))
or drawio_is_image_style(cell.attrib.get("style"))
) and (width <= 0 or height <= 0):
raise ValueError("draw.io image vertex dimensions must be positive")
continue
for endpoint_name in ("source", "target"):
endpoint_id = cell.attrib.get(endpoint_name)
if endpoint_id:
endpoint = cells_by_id.get(endpoint_id)
if endpoint is None or endpoint.attrib.get("vertex") != "1":
raise ValueError("draw.io edge endpoint must reference a vertex")
else:
point_name = f"{endpoint_name}Point"
points = [
child
for child in geometry
if child.tag == "mxPoint" and child.attrib.get("as") == point_name
]
if len(points) != 1:
raise ValueError("draw.io edge requires a vertex endpoint or terminal point")
if "x" not in points[0].attrib or "y" not in points[0].attrib:
raise ValueError("draw.io terminal point must contain x and y")
for port_name in ("sourcePort", "targetPort"):
port_id = style.get(port_name)
if port_id:
port = cells_by_id.get(port_id)
if port is None or port.attrib.get("vertex") != "1":
raise ValueError("draw.io edge port must reference a vertex")
if semantic_cells == 0:
raise ValueError("draw.io page has no vertices or edges")
for cell in content:
seen = {str(cell.attrib["id"])}
parent = cell.attrib.get("parent")
while parent is not None:
if parent in seen:
raise ValueError("draw.io cell parent chain contains a cycle")
seen.add(parent)
parent_cell = cells_by_id.get(parent)
if parent_cell is None:
raise ValueError("draw.io cell parent chain is incomplete")
parent = parent_cell.attrib.get("parent")
def decode_compressed_drawio(
payload,
max_inflated_bytes=MAX_INFLATED_DRAWIO_BYTES,
max_xml_elements=MAX_DRAWIO_ELEMENTS,
):
if max_inflated_bytes <= 0:
raise ValueError("compressed draw.io pages exceed the aggregate inflated-size limit")
if max_xml_elements <= 0:
raise ValueError("draw.io pages exceed the aggregate XML element limit")
compact = re.sub(r"\s+", "", str(payload or ""))
if not compact:
raise ValueError("compressed draw.io page is empty")
try:
compressed = base64.b64decode(compact, validate=True)
except (binascii.Error, ValueError) as error:
raise ValueError("compressed draw.io page is not valid base64") from error
try:
decompressor = zlib.decompressobj(-15)
inflated = decompressor.decompress(compressed, max_inflated_bytes + 1)
if len(inflated) > max_inflated_bytes or decompressor.unconsumed_tail:
raise ValueError("compressed draw.io page exceeds the inflated-size limit")
inflated += decompressor.flush(max_inflated_bytes + 1 - len(inflated))
except zlib.error as error:
raise ValueError("compressed draw.io page has an invalid deflate stream") from error
if not decompressor.eof or decompressor.unused_data:
raise ValueError("compressed draw.io page has an incomplete or trailing deflate stream")
try:
encoded_xml = inflated.decode("utf-8", errors="strict")
if re.search(r"%(?![0-9A-Fa-f]{2})", encoded_xml):
raise ValueError("compressed draw.io page contains malformed percent encoding")
xml = urllib.parse.unquote_to_bytes(encoded_xml).decode("utf-8", errors="strict")
lowered = xml.lower()
without_xml_declaration = re.sub(
r"^\ufeff?\s*<\?xml\s+[^?]*\?>", "", xml, count=1, flags=re.IGNORECASE
)
if "\x00" in xml or "<!doctype" in lowered or "<!entity" in lowered:
raise ValueError("compressed draw.io page contains a forbidden declaration")
if "<?" in without_xml_declaration:
raise ValueError("compressed draw.io page contains a processing instruction")
model = ET.fromstring(xml)
except (UnicodeDecodeError, ET.ParseError) as error:
raise ValueError("compressed draw.io page does not contain valid UTF-8 XML") from error
if model.tag != "mxGraphModel":
raise ValueError("compressed draw.io page is not an mxGraphModel")
element_count, _ = drawio_tree_shape(model, max_elements=max_xml_elements)
return model, len(inflated), element_count
def drawio_metadata(path, data=None):
try:
if data is None:
data = path.read_bytes()
text = data.decode("utf-8", errors="strict")
lowered = text.lower()
without_xml_declaration = re.sub(
r"^\ufeff?\s*<\?xml\s+[^?]*\?>", "", text, count=1, flags=re.IGNORECASE
)
if "\x00" in text or "<!doctype" in lowered or "<!entity" in lowered:
raise ValueError("draw.io source contains a forbidden declaration")
if "<?" in without_xml_declaration:
raise ValueError("draw.io processing instructions are not allowed")
root = ET.fromstring(text)
outer_element_count, _ = drawio_tree_shape(root)
if svg_namespace(root.tag):
raise ValueError("draw.io root must not use an XML namespace")
page_models = []
page_count = 0
uncompressed = True
animation_inspectable = True
if root.tag == "mxGraphModel":
page_models.append((path.name, root))
page_count = 1
elif root.tag == "mxfile":
diagrams = [child for child in root if child.tag == "diagram"]
if not diagrams:
raise ValueError("draw.io source has no diagram pages")
page_count = len(diagrams)
inflated_drawio_bytes = 0
decoded_element_count = 0
for page_index, diagram in enumerate(diagrams, start=1):
page_name = str(diagram.attrib.get("name") or f"Page-{page_index}")
models = [child for child in diagram if child.tag == "mxGraphModel"]
if len(models) == 1:
page_models.append((page_name, models[0]))
elif not models and str(diagram.text or "").strip():
uncompressed = False
model, inflated_bytes, element_count = decode_compressed_drawio(
diagram.text,
MAX_INFLATED_DRAWIO_BYTES - inflated_drawio_bytes,
MAX_DRAWIO_ELEMENTS - outer_element_count - decoded_element_count,
)
inflated_drawio_bytes += inflated_bytes
decoded_element_count += element_count
page_models.append((page_name, model))
else:
raise ValueError("draw.io page has an invalid model payload")
else:
raise ValueError("draw.io source is missing an mxfile or mxGraphModel root")
embedded_svg_images = 0
embedded_raster_images = 0
embedded_svg_sha256s = []
embedded_svg_cell_sha256s = set()
external_images = 0
unsupported_images = 0
animation_on = animation_inspectable
animation_off = animation_inspectable
directed_flow_edges = 0
animated_edges = 0
animated_static_edges = 0
native_stencil_count = 0
cell_id_sha256s = set()
native_stencil_cell_id_sha256s = set()
directed_edge_sha256s = set()
edge_role_sha256s = set()
profile_style_sha256s = set()
link_sha256s = set()
adaptive_colors = bool(page_models)
page_graphs = []
for page_name, model in page_models:
if model.tag != "mxGraphModel":
raise ValueError("draw.io page model uses an unexpected namespace")
page_adaptive_colors = model.attrib.get("adaptiveColors") == "auto"
adaptive_colors = adaptive_colors and page_adaptive_colors
graph_roots = [child for child in model if child.tag == "root"]
if len(graph_roots) != 1:
raise ValueError("draw.io page is missing its graph root")
cells = drawio_root_cells(graph_roots[0])
identifiers = [str(cell.attrib.get("id") or "") for cell in cells]
if not all(identifiers) or len(identifiers) != len(set(identifiers)):
raise ValueError("draw.io page contains missing or duplicate cell IDs")
by_id = set(identifiers)
cells_by_id = {str(cell.attrib["id"]): cell for cell in cells}
page_cell_id_sha256s = {
hashlib.sha256(identifier.encode("utf-8")).hexdigest()
for identifier in identifiers
}
cell_id_sha256s.update(page_cell_id_sha256s)
page_link_sha256s = {
hashlib.sha256(link.encode("utf-8")).hexdigest()
for cell in cells
if (link := str(cell.attrib.get("link") or "").strip())
}
link_sha256s.update(page_link_sha256s)
page_native_stencil_cell_id_sha256s = set()
page_directed_edge_sha256s = set()
page_edge_role_sha256s = set()
page_profile_style_sha256s = set()
if "0" not in by_id or "1" not in by_id:
raise ValueError("draw.io page is missing base cells")
if cells_by_id["0"].attrib.get("parent") is not None:
raise ValueError("draw.io root base cell must not have a parent")
if cells_by_id["1"].attrib.get("parent") != "0":
raise ValueError("draw.io layer base cell must have parent 0")
validate_drawio_model_cells(cells, cells_by_id)
for cell in cells:
style = drawio_style_map(cell.attrib.get("style"))
validate_drawio_animation_style(cell, style)
if str(style.get("designProfile") or "").strip() and drawio_visible_profile_vertex(
cell, cells_by_id
):
for key, value in style.items():
if key not in DRAWIO_PROFILE_STYLE_HASH_KEYS:
continue
profile_digest = hashlib.sha256(
f"{cell.attrib['id']}\0{key}\0{value}".encode("utf-8")
).hexdigest()
profile_style_sha256s.add(profile_digest)
page_profile_style_sha256s.add(profile_digest)
if cell.attrib.get("vertex") == "1" and str(
style.get("shape") or ""
).lower().startswith("mxgraph."):
native_stencil_count += 1
native_digest = hashlib.sha256(
str(cell.attrib["id"]).encode("utf-8")
).hexdigest()
native_stencil_cell_id_sha256s.add(native_digest)
page_native_stencil_cell_id_sha256s.add(native_digest)
if cell.attrib.get("edge") == "1":
animated = style.get("flowAnimation") == "1"
directed = drawio_directed_semantic_flow(style)
static_direction = drawio_has_static_direction(style)
role = str(style.get("dataRole") or "").strip().lower()
if role:
role_digest = hashlib.sha256(
f"{cell.attrib['id']}\0{role}".encode("utf-8")
).hexdigest()
edge_role_sha256s.add(role_digest)
page_edge_role_sha256s.add(role_digest)
animated_static = animated and role in DRAWIO_STATIC_EDGE_ROLES
animated_edges += int(animated)
animated_static_edges += int(animated_static)
directed_flow_edges += int(directed)
if animated:
animation_off = False
if directed and (not animated or not static_direction) or animated_static:
animation_on = False
source = str(cell.attrib.get("source") or "")
target = str(cell.attrib.get("target") or "")
if source and target:
edge_digest = hashlib.sha256(
f"{source}\0{target}".encode("utf-8")
).hexdigest()
directed_edge_sha256s.add(edge_digest)
page_directed_edge_sha256s.add(edge_digest)
source = drawio_image_source(cell.attrib.get("style"))
image_declared = re.search(
r"(?:^|;)image=", str(cell.attrib.get("style") or "")
) is not None
image_shape = drawio_is_image_style(cell.attrib.get("style"))
if (image_declared or image_shape) and not source:
raise ValueError("draw.io image cell has an empty or truncated image source")
for resource_source in drawio_cell_resource_sources(cell):
if resource_source.lower().startswith("data:image/"):
try:
media_type = validate_embedded_image(resource_source)
except Exception as error:
raise ValueError(
"draw.io source contains an invalid embedded image"
) from error
else:
if media_type == "svg+xml":
embedded_svg_images += 1
_, raw_image = decode_data_image(resource_source)
svg_digest = hashlib.sha256(raw_image).hexdigest()
embedded_svg_sha256s.append(svg_digest)
embedded_svg_cell_sha256s.add(
hashlib.sha256(
f"{cell.attrib['id']}\0{svg_digest}".encode("utf-8")
).hexdigest()
)
else:
embedded_raster_images += 1
elif resource_source.lower().startswith(
SVG_EXTERNAL_REFERENCE_PREFIXES
):
external_images += 1
else:
unsupported_images += 1
page_graphs.append(
{
"page_name_sha256": hashlib.sha256(
page_name.encode("utf-8")
).hexdigest(),
"adaptive_colors": page_adaptive_colors,
"cell_id_sha256s": sorted(page_cell_id_sha256s),
"native_stencil_cell_id_sha256s": sorted(
page_native_stencil_cell_id_sha256s
),
"directed_edge_sha256s": sorted(page_directed_edge_sha256s),
"edge_role_sha256s": sorted(page_edge_role_sha256s),
"profile_style_sha256s": sorted(page_profile_style_sha256s),
"link_sha256s": sorted(page_link_sha256s),
}
)
animation_on = (
animation_on
and directed_flow_edges > 0
and animated_edges > 0
and animated_static_edges == 0
)
return {
"valid": True,
"page_count": page_count,
"adaptive_colors": adaptive_colors,
"uncompressed": uncompressed,
"embedded_svg_images": embedded_svg_images,
"embedded_raster_images": embedded_raster_images,
"embedded_svg_sha256s": sorted(set(embedded_svg_sha256s)),
"embedded_svg_cell_sha256s": sorted(embedded_svg_cell_sha256s),
"external_references": external_images,
"unsupported_images": unsupported_images,
"animation_on": animation_on,
"animation_off": animation_off,
"animation_inspectable": animation_inspectable,
"directed_flow_edges": directed_flow_edges,
"animated_edges": animated_edges,
"animated_static_edges": animated_static_edges,
"native_stencil_count": native_stencil_count,
"cell_id_sha256s": sorted(cell_id_sha256s),
"native_stencil_cell_id_sha256s": sorted(native_stencil_cell_id_sha256s),
"directed_edge_sha256s": sorted(directed_edge_sha256s),
"edge_role_sha256s": sorted(edge_role_sha256s),
"profile_style_sha256s": sorted(profile_style_sha256s),
"link_sha256s": sorted(link_sha256s),
"page_graphs": page_graphs,
"self_contained_svg": (
embedded_svg_images > 0
and external_images == 0
and unsupported_images == 0
),
}
except Exception as error:
return {
"valid": False,
"self_contained_svg": False,
"error": short(error, 300),
}
def read_artifact_snapshot(path):
if not hasattr(os, "O_NOFOLLOW") or not hasattr(os, "O_NONBLOCK"):
raise ArtifactPolicyError("artifact inspection requires O_NOFOLLOW and O_NONBLOCK support")
flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK
if hasattr(os, "O_CLOEXEC"):
flags |= os.O_CLOEXEC
try:
descriptor = os.open(path, flags)
except OSError as error:
raise ArtifactPolicyError(f"could not open artifact {path.name}: {error}") from error
try:
before = os.fstat(descriptor)
if not stat.S_ISREG(before.st_mode):
raise ArtifactPolicyError(f"artifact is not a regular file: {path.name}")
if before.st_size > MAX_ARTIFACT_FILE_BYTES:
raise ArtifactPolicyError(
f"artifact {path.name} exceeds {MAX_ARTIFACT_FILE_BYTES} byte limit"
)
chunks = []
remaining = MAX_ARTIFACT_FILE_BYTES + 1
while remaining > 0:
chunk = os.read(descriptor, min(1024 * 1024, remaining))
if not chunk:
break
chunks.append(chunk)
remaining -= len(chunk)
content = b"".join(chunks)
after = os.fstat(descriptor)
identity_before = (
before.st_dev,
before.st_ino,
before.st_size,
before.st_mtime_ns,
before.st_ctime_ns,
)
identity_after = (
after.st_dev,
after.st_ino,
after.st_size,
after.st_mtime_ns,
after.st_ctime_ns,
)
if identity_before != identity_after or len(content) != before.st_size:
raise ArtifactPolicyError(f"artifact changed while being inspected: {path.name}")
return content
finally:
os.close(descriptor)
def collect_artifacts(workspace):
artifacts = []
artifact_paths = []
scanned_entries = 0
for root, dirs, files in os.walk(workspace, topdown=True, followlinks=False):
root_path = Path(root)
raw_dirs = sorted(dirs)
dirs[:] = [
name
for name in raw_dirs
if not (root_path / name).is_symlink()
and not (
root_path == workspace and name in ARTIFACT_EXCLUDED_TOP_LEVEL
)
]
scanned_entries += len(dirs)
if scanned_entries > MAX_WORKSPACE_ENTRIES:
raise ArtifactPolicyError(
f"rollout workspace exceeds {MAX_WORKSPACE_ENTRIES} inspected entries"
)
for name in sorted(files):
scanned_entries += 1
if scanned_entries > MAX_WORKSPACE_ENTRIES:
raise ArtifactPolicyError(
f"rollout workspace exceeds {MAX_WORKSPACE_ENTRIES} inspected entries"
)
path = root_path / name
suffix = path.suffix.lower()
if suffix not in {".drawio", ".png", ".svg"}:
continue
if path.is_symlink():
continue
relative_parts = path.relative_to(workspace).parts
if relative_parts and relative_parts[0] in ARTIFACT_EXCLUDED_TOP_LEVEL:
continue
artifact_paths.append(path)
if len(artifact_paths) > MAX_ARTIFACT_COUNT:
raise ArtifactPolicyError(
f"rollout produced more than {MAX_ARTIFACT_COUNT} visual artifacts"
)
total_bytes = 0
for path in sorted(artifact_paths):
content = read_artifact_snapshot(path)
total_bytes += len(content)
if total_bytes > MAX_ARTIFACT_TOTAL_BYTES:
raise ArtifactPolicyError(
f"visual artifacts exceed {MAX_ARTIFACT_TOTAL_BYTES} total byte limit"
)
suffix = path.suffix.lower()
item = {
"path": path.relative_to(workspace).as_posix(),
"kind": suffix[1:],
"size_bytes": len(content),
"sha256": hashlib.sha256(content).hexdigest(),
}
if suffix == ".png":
item.update(png_metadata(path, content))
elif suffix == ".svg":
item.update(svg_metadata(path, content))
elif suffix == ".drawio":
item.update(drawio_metadata(path, content))
artifacts.append(item)
return artifacts
class AgentSkillsRollout:
def __init__(self, config):
self.config = config
self.timeout = int(config.get("exec_timeout", 240))
self.visual_exec_timeout = int(config.get("visual_exec_timeout") or min(self.timeout, 120))
self.tool_rollout_for_visual_assertions = config_bool(
config.get("tool_rollout_for_visual_assertions", True)
)
self.require_drawio_cli_for_visual_rollouts = config_bool(
config.get("require_drawio_cli_for_visual_rollouts", True)
)
self.target_backend = str(config.get("target_backend") or "codex_exec").strip().lower()
self.target_max_completion_tokens = max(
1, int(config.get("target_max_completion_tokens") or 16384)
)
self.target_retries = max(0, int(config.get("target_retries") or 5))
self.model = str(config.get("target_model") or config.get("target") or "").strip()
self.profile = str(config.get("codex_exec_profile") or "").strip()
self.reasoning_effort = str(config.get("codex_exec_reasoning_effort") or "").strip()
self.approval_policy = str(config.get("codex_exec_approval_policy") or "never").strip()
self.web_search = config_bool(config.get("codex_exec_web_search"))
self.network_access = config_bool(config.get("codex_exec_network_access"))
def allow_workspace_tools(self, case):
return self.tool_rollout_for_visual_assertions and has_visual_assertions(case)
def rollout_bounds(self, case):
if self.allow_workspace_tools(case):
return {
"timeout": self.visual_exec_timeout,
"tool_rollout_enabled": True,
"policy": (
"This eval case declares visual assertions, so the nested rollout may create "
"and edit files only inside the current working directory and may run local "
"shell commands only for copied helper scripts, draw.io XML files, deterministic "
"XML/rule validation, and requested PNG/SVG artifact export. Browser tools, "
"package managers, network access, installs, MCP tools, and reads outside the "
"current working directory remain forbidden. Run at most two validation/export "
"attempts before reporting skipped or unavailable capabilities honestly."
),
}
return {
"timeout": self.timeout,
"tool_rollout_enabled": False,
"policy": (
"Do not call shell commands, file-editing tools, browser tools, or external tools "
"in this nested rollout environment; provide the final response text directly, "
"describing the minimal files, commands, validations, and skipped capabilities "
"that the skill would use for the task."
),
}
def uses_chat_target(self):
return self.target_backend in {
"openai_chat",
"qwen_chat",
"minimax_chat",
}
def run_chat_target(self, case, system_prompt, user_prompt, timeout):
try:
from skillopt.model import chat_target
response, usage = chat_target(
system=system_prompt,
user=user_prompt,
max_completion_tokens=self.target_max_completion_tokens,
retries=self.target_retries,
stage="target",
reasoning_effort=self.reasoning_effort or None,
timeout=timeout,
)
normalized_usage = {}
if isinstance(usage, dict):
for key, value in usage.items():
if isinstance(value, (int, float)) and not isinstance(value, bool):
normalized_usage[str(key)] = int(value)
return {
"id": case["id"],
"case_id": case["id"],
"response": redact(as_text(response)),
"returncode": 0,
"stderr": "",
"stdout": "",
"artifacts": [],
"tool_rollout_enabled": False,
"target_backend": self.target_backend,
"usage": normalized_usage,
}
except Exception as error:
return {
"id": case["id"],
"case_id": case["id"],
"response": "",
"returncode": 1,
"stderr": short(
sanitize_provider_context(f"provider target failed: {error}"), 1000
),
"stdout": "",
"artifacts": [],
"tool_rollout_enabled": False,
"target_backend": self.target_backend,
}
def run(self, case, skill_body):
workspace = Path(tempfile.mkdtemp(prefix="skillopt-agent-skill-"))
keep_workspace = False
cleanup_error = ""
outcome = None
try:
skill_name = str(case.get("skill_name") or "drawio-diagrams")
task_prompt = normalize_task_prompt(case["prompt"], skill_name)
workspace_note = seed_workspace(workspace, case, skill_name)
bounds = self.rollout_bounds(case)
if self.uses_chat_target() and has_visual_assertions(case):
outcome = {
"id": case["id"],
"case_id": case["id"],
"response": (
"Provider-backed chat targets cannot create or inspect local visual "
"artifacts. Use the generated data-text-only split or select the "
"read-isolated codex_exec target for visual assertions."
),
"returncode": 126,
"stderr": "visual assertions require a local artifact-capable target",
"stdout": "",
"artifacts": [],
"tool_rollout_enabled": False,
"target_backend": self.target_backend,
"visual_rollout_blocker": "provider chat target cannot produce local artifacts",
}
return outcome
if self.target_backend != "codex_exec" and not self.uses_chat_target():
outcome = {
"id": case["id"],
"case_id": case["id"],
"response": "",
"returncode": 2,
"stderr": f"unsupported target_backend: {self.target_backend}",
"stdout": "",
"artifacts": [],
"tool_rollout_enabled": False,
"target_backend": self.target_backend,
}
return outcome
resource_snapshot = ""
if self.uses_chat_target() or not bounds["tool_rollout_enabled"]:
try:
resource_snapshot = provider_resource_snapshot(workspace)
except ProviderContextError as error:
outcome = {
"id": case["id"],
"case_id": case["id"],
"response": "",
"returncode": 127,
"stderr": short(error, 1000),
"stdout": "",
"artifacts": [],
"tool_rollout_enabled": False,
"target_backend": self.target_backend,
"resource_snapshot_blocker": short(error, 1000),
}
return outcome
drawio_cli = None
drawio_alias_dir = None
if bounds["tool_rollout_enabled"]:
drawio_cli = find_drawio_cli()
if drawio_cli is None and self.require_drawio_cli_for_visual_rollouts:
response = (
"Visual artifact rollout could not run because draw.io Desktop CLI was "
"not found on PATH. Install or expose `drawio`/`diagrams.net`, or run an "
"explicitly text-only split before scaling SkillOpt."
)
outcome = {
"id": case["id"],
"case_id": case["id"],
"response": response,
"returncode": 0,
"stderr": "",
"stdout": "",
"artifacts": [],
"tool_rollout_enabled": False,
"visual_rollout_blocker": "draw.io Desktop CLI not found",
}
return outcome
if drawio_cli is not None:
drawio_alias_dir = prepare_drawio_command_alias(workspace, drawio_cli)
system_prompt = (
"Use only the following Agent Skill body as the active operating procedure. "
"Do not read or load installed skill files from ~/.agents/skills, ~/.codex/skills, "
"the workspace, or any other skill registry. If the task prompt names a skill, "
"interpret it as referring only to the skill body below. Local helper scripts, "
"references, and fixtures for this isolated rollout are available only inside "
"the current working directory; use those local copies instead of searching "
"outside the workspace.\n\n"
"This is a bounded SkillOpt rollout. Keep the implementation minimal and finish "
"with a concise delivery summary. Do not perform open-ended environment searches, "
"do not wait for unavailable optional tools, and follow the validation/export "
"bounds below before reporting skipped or unavailable capabilities honestly. "
f"{bounds['policy']}\n\n"
f"{skill_body}"
)
user_parts = ["Rollout workspace files:\n", f"{workspace_note}\n\n"]
if resource_snapshot:
user_parts.extend([
"Bounded textual helper resources and fixtures:\n",
f"{resource_snapshot}\n\n",
])
user_parts.extend(["Task prompt:\n", f"{task_prompt}\n"])
user_prompt = "".join(user_parts)
if self.uses_chat_target():
outcome = self.run_chat_target(
case, system_prompt, user_prompt, bounds["timeout"]
)
return outcome
prompt = f"{system_prompt}\n\n{user_prompt}"
control_dir = workspace / ROLLOUT_CONTROL_DIR
control_dir.mkdir(mode=0o700)
final = control_dir / "final.md"
codex_command = self.config.get("codex_exec_path", "codex")
process_env = minimal_rollout_environment(drawio_alias_dir)
shell_env = shell_environment_for_workspace(workspace, process_env)
command = [
codex_command,
"exec",
"--ignore-user-config",
"--ignore-rules",
"--skip-git-repo-check",
"--ephemeral",
"--color",
"never",
"--output-last-message",
str(final),
]
# Legacy Codex sandboxes limit writes but still permit host-wide reads. Every rollout
# therefore uses a strict permission profile. Text-only cases receive only minimal
# runtime reads; visual cases additionally receive scoped rollout-workspace writes.
permission_profile = (
VISUAL_ROLLOUT_PERMISSION_PROFILE
if bounds["tool_rollout_enabled"]
else TEXT_ROLLOUT_PERMISSION_PROFILE
)
filesystem_permissions = (
visual_rollout_filesystem_permissions(
codex_command, drawio_cli["path"] if drawio_cli else None
)
if bounds["tool_rollout_enabled"]
else text_rollout_filesystem_permissions(codex_command)
)
command.extend([
"--strict-config",
"-c",
f"default_permissions={toml_string(permission_profile)}",
"-c",
f"permissions.{permission_profile}.filesystem={filesystem_permissions}",
"-c",
f"permissions.{permission_profile}.network.enabled=false",
])
command.extend([
"-c",
'shell_environment_policy.inherit="none"',
"-c",
f"shell_environment_policy.set={toml_inline_string_map(shell_env)}",
])
if self.model:
command.extend(["--model", self.model])
if self.reasoning_effort:
command.extend(["-c", f'model_reasoning_effort="{self.reasoning_effort}"'])
if self.approval_policy:
command.extend(["-c", f"approval_policy={toml_string(self.approval_policy)}"])
if self.profile:
command.extend(["--profile", self.profile])
command.append("-")
try:
process = subprocess.Popen(
command,
cwd=workspace,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
start_new_session=True,
env=process_env,
)
process_io = start_bounded_process_io(process, prompt)
process.wait(timeout=bounds["timeout"])
process_notes, process_group_still_alive, reaped_descendants = reap_process_group(
process,
grace_seconds=0,
)
capture_error = ""
try:
process_io.finish()
except BoundedCaptureError as error:
capture_error = str(error)
finally:
process_io.close_stopped_streams()
if process_group_still_alive or capture_error:
keep_workspace = True
outcome = {
"id": case["id"],
"case_id": case["id"],
"response": "",
"returncode": 125,
"stderr": short(
(
"rollout process group could not be terminated: "
+ "; ".join(process_notes)
)
if process_group_still_alive
else capture_error
),
"stdout": "",
"artifacts": [],
"tool_rollout_enabled": bounds["tool_rollout_enabled"],
}
if process_group_still_alive:
outcome["process_group_alive"] = True
if capture_error:
outcome["capture_error"] = True
return outcome
stdout = render_captured_stream(process_io.stdout, prompt)
stderr = render_captured_stream(process_io.stderr, prompt)
response = final.read_text(encoding="utf-8") if final.exists() else stdout
artifacts = []
artifact_error = ""
if process.returncode == 0:
try:
artifacts = collect_artifacts(workspace)
except ArtifactPolicyError as error:
artifact_error = short(error, 1000)
outcome = {
"id": case["id"],
"case_id": case["id"],
"response": redact(response),
"returncode": 125 if artifact_error else process.returncode,
"stderr": short(
f"{stderr}\nartifact policy error: {artifact_error}"
if artifact_error
else stderr
),
"stdout": short(stdout),
"artifacts": artifacts,
"tool_rollout_enabled": bounds["tool_rollout_enabled"],
}
if reaped_descendants:
outcome["reaped_descendants"] = True
if process_notes:
outcome["process_cleanup_notes"] = process_notes
if artifact_error:
outcome["artifact_policy_error"] = artifact_error
except BoundedCaptureError as error:
keep_workspace = True
outcome = {
"id": case["id"],
"case_id": case["id"],
"response": "",
"returncode": 125,
"stderr": short(str(error)),
"stdout": "",
"artifacts": [],
"tool_rollout_enabled": bounds["tool_rollout_enabled"],
"capture_error": True,
}
except subprocess.TimeoutExpired:
termination_notes = []
process_alive_after_kill = False
term_error = signal_process_group(process, signal.SIGTERM)
if term_error:
termination_notes.append(f"SIGTERM: {term_error}")
still_running = wait_after_signal(process, 5)
if still_running:
kill_error = signal_process_group(process, signal.SIGKILL)
if kill_error:
termination_notes.append(f"SIGKILL: {kill_error}")
process_alive_after_kill = wait_after_signal(process, 5)
process_notes, group_alive, _ = reap_process_group(process)
termination_notes.extend(process_notes)
process_alive_after_kill = process_alive_after_kill or group_alive
try:
process_io.finish()
except BoundedCaptureError:
termination_notes.append("bounded subprocess I/O cleanup failed")
finally:
process_io.close_stopped_streams()
stdout = (
PROMPT_ECHO_MARKER if process_io.stdout.total_bytes else ""
)
stderr = (
PROMPT_ECHO_MARKER if process_io.stderr.total_bytes else ""
)
# On timeout, avoid inspecting or deleting the workspace. Some
# failed Codex/tool subprocesses can leave cwd/file handles in a
# kernel-blocked state; walking the tree or rmtree can wedge the
# trainer even after the direct child has exited.
keep_workspace = True
response = stdout
stderr_text = (
f"codex exec timed out after {bounds['timeout']} seconds\n"
f"process_alive_after_kill={process_alive_after_kill}\n"
f"termination_notes={termination_notes}\n"
f"stdout before timeout:\n{stdout}\n"
f"stderr before timeout:\n{stderr}"
)
outcome = {
"id": case["id"],
"case_id": case["id"],
"response": redact(response),
"returncode": 124,
"stderr": short(stderr_text),
"stdout": short(stdout),
"artifacts": [],
"timeout": True,
"tool_rollout_enabled": bounds["tool_rollout_enabled"],
}
if keep_workspace:
outcome["workspace_preserved"] = redact(str(workspace))
return outcome
finally:
if keep_workspace:
if outcome is not None:
outcome["workspace_preserved"] = redact(str(workspace))
else:
cleanup_error = safe_rmtree(workspace)
if cleanup_error and outcome is not None:
outcome["cleanup_error"] = cleanup_error
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| finishing-a-development-branchobra/superpowersUse when implementation is complete, all tests pass, and you need to decide how to integrate the work | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| systematic-debuggingobra/superpowersUse when encountering any bug, test failure, or unexpected behavior, before proposing fixes | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| test-driven-developmentobra/superpowersUse when implementing any feature or bugfix, before writing implementation code | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| skill-creatoranthropics/skillsCreate new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |