Skip to content

ericrisco/rsc-harness / skills/roast-me

roast-me

Verified

Use when someone wants an honest, comedic audit of their own prompting — reads their past agent transcripts, scores prompt quality and compute efficiency, names their worst habits, builds a model-selection cheat sheet. NOT reviewing code an agent produced (that is `code-review`), NOT prompting technique in general (that is `prompt-engineering`).

FileKindSize
SKILL.mdskill md5.4 KB
evals/README.mdmarkdown3.8 KB
evals/cases.yamlconfig5.7 KB
prompts/analyze.mdmarkdown5.5 KB
prompts/compute.mdmarkdown4.8 KB
prompts/roast.mdmarkdown7.6 KB
tools/adapters/__init__.pyscript36 B
tools/adapters/base.pyscript2.4 KB
tools/adapters/claude.pyscript4.2 KB
tools/adapters/codex.pyscript8.0 KB
tools/adapters/gemini.pyscript5.4 KB
tools/adapters/registry.pyscript2.0 KB
tools/extract_prompts.pyscript20.2 KB

tools/adapters/codex.py

View on GitHub
"""Codex CLI adapter (@openai/codex).

Session files: ~/.codex/sessions/YYYY/MM/DD/rollout-<timestamp>.jsonl
Format: JSON Lines. Each line is {"timestamp", "type", "payload"}.

CONFIRMED (2026-06, against live ~/.codex/sessions on macOS):
  - Base path / layout / naming: ~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl
  - Line envelope: {"timestamp": ISO8601, "type": str, "payload": {...}}
  - The genuine user prompt is an `event_msg` line whose payload.type is
    "user_message", carrying the typed text in payload.message. (Injected
    AGENTS.md / developer preambles appear only as response_item/message lines,
    so keying on user_message excludes them.)
  - Assistant turns: event_msg/agent_message (payload.message) and
    response_item/message role=assistant.
  - Tool calls: response_item/function_call (payload.name).
  - Tool results: response_item/function_call_output (payload.output) — shell
    execs include a "Process exited with code N" line; N != 0 (or common error
    markers) means the command failed.
  - Model: turn_context.model / session_meta payload.

Degradation: if ~/.codex/ does not exist, discover() returns []; any unexpected
line shape is skipped, so parse() never raises.
"""

from __future__ import annotations

import datetime
import json
import re
from pathlib import Path
from typing import Any

from .base import CORRECTION_RE, BaseAdapter

# A shell tool result failed when it reports a non-zero exit code, or carries a
# common error marker with no explicit success code. Conservative on purpose —
# better to miss a soft error than to invent one.
_EXIT_CODE_RE = re.compile(r"exited with code\s+(\d+)", re.IGNORECASE)
_ERROR_MARKER_RE = re.compile(
    r"\b(command not found|no such file|permission denied|traceback \(most recent|"
    r"fatal:|panic:|segmentation fault|unhandled exception)\b",
    re.IGNORECASE,
)


def _looks_like_error(output: str) -> bool:
    if not output:
        return False
    m = _EXIT_CODE_RE.search(output)
    if m:
        return m.group(1) != "0"
    return bool(_ERROR_MARKER_RE.search(output))


def _ts(value: Any) -> float | None:
    if isinstance(value, (int, float)):
        return float(value)
    if isinstance(value, str):
        try:
            return datetime.datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
        except (ValueError, AttributeError):
            return None
    return None


class CodexAdapter(BaseAdapter):
    RUNTIME_ID = "codex"

    SESSIONS_DIR = Path.home() / ".codex" / "sessions"

    def discover(self) -> list[Path]:
        """Find all rollout-*.jsonl files under ~/.codex/sessions/."""
        if not self.SESSIONS_DIR.exists():
            return []
        files: list[Path] = []
        try:
            for jsonl in self.SESSIONS_DIR.rglob("rollout-*.jsonl"):
                try:
                    if jsonl.is_file():
                        files.append(jsonl)
                except OSError:
                    continue
        except OSError:
            return []
        return sorted(files, key=lambda f: f.stat().st_mtime, reverse=True)

    def parse(self, path: Path) -> list[dict[str, Any]]:
        """Parse one Codex session JSONL file into user-prompt records with
        session-aware error/correction context."""
        try:
            lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
        except OSError:
            return []

        # 1) Flatten the file into an ordered timeline of turns.
        timeline: list[dict[str, Any]] = []
        model = ""
        pending_tool = ""
        for line in lines:
            line = line.strip()
            if not line:
                continue
            try:
                obj = json.loads(line)
            except json.JSONDecodeError:
                continue
            if not isinstance(obj, dict):
                continue

            top_type = obj.get("type")
            payload = obj.get("payload")
            if not isinstance(payload, dict):
                payload = {}
            ts = _ts(obj.get("timestamp"))

            # Track the active model from session metadata / turn context.
            if top_type in ("session_meta", "turn_context"):
                m = payload.get("model")
                if isinstance(m, str) and m:
                    model = m
                continue

            if top_type == "event_msg":
                p_type = payload.get("type")
                if p_type == "user_message":
                    text = payload.get("message", "")
                    if isinstance(text, str) and text.strip():
                        timeline.append({"kind": "user", "text": text, "ts": ts})
                elif p_type == "agent_message":
                    text = payload.get("message", "")
                    if isinstance(text, str) and text.strip():
                        timeline.append({"kind": "assistant", "text": text, "ts": ts})
                continue

            if top_type == "response_item":
                p_type = payload.get("type")
                if p_type == "function_call":
                    name = payload.get("name", "")
                    pending_tool = name if isinstance(name, str) else ""
                elif p_type == "function_call_output":
                    output = payload.get("output", "")
                    output = output if isinstance(output, str) else json.dumps(output)
                    timeline.append({
                        "kind": "tool_output",
                        "tool": pending_tool,
                        "is_error": _looks_like_error(output),
                        "text": output,
                        "ts": ts,
                    })
                    pending_tool = ""
                # response_item/message (assistant/user/developer) is intentionally
                # ignored: genuine prompts come from event_msg/user_message, and the
                # assistant context comes from event_msg/agent_message.
                continue

        # 2) Walk the timeline; enrich each user prompt with what follows it.
        user_indices = [i for i, t in enumerate(timeline) if t["kind"] == "user"]
        records: list[dict[str, Any]] = []
        for n, idx in enumerate(user_indices):
            turn = timeline[idx]
            next_user = user_indices[n + 1] if n + 1 < len(user_indices) else len(timeline)

            followed_by_error = False
            error_tool = ""
            error_text = ""
            for j in range(idx + 1, next_user):
                t = timeline[j]
                if t["kind"] == "tool_output" and t.get("is_error"):
                    followed_by_error = True
                    error_tool = t.get("tool", "")
                    error_text = t.get("text", "")
                    break

            followed_by_correction = False
            correction_text = ""
            if next_user < len(timeline):
                nxt = timeline[next_user]["text"]
                if CORRECTION_RE.search(nxt):
                    followed_by_correction = True
                    correction_text = nxt

            # context_before: last assistant text strictly before this prompt.
            context_before = ""
            for j in range(idx - 1, -1, -1):
                if timeline[j]["kind"] == "assistant":
                    context_before = timeline[j]["text"]
                    break

            error_was_recovered = followed_by_error and not followed_by_correction

            records.append({
                "runtime": self.RUNTIME_ID,
                "session_file": str(path),
                "prompt_text": turn["text"],
                "timestamp": turn.get("ts"),
                "model": model,
                "context_before": context_before,
                "followed_by_error": followed_by_error,
                "error_was_recovered": error_was_recovered,
                "followed_by_correction": followed_by_correction,
                "correction_text": correction_text,
                "error_tool": error_tool,
                "error_text": error_text,
            })

        return records
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 471kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall