Skip to content

borghei/Claude-Skills / engineering/computer-use-automation

computer-use-automation

Verified

Decides when a GUI-driving (computer-use) agent beats an API and lints planned screen actions for missing confirmation, verification and grounding. Use when designing a computer-use agent or reviewing its action plan.

scripts/action_safety_linter.py

View on GitHub
#!/usr/bin/env python3
"""action_safety_linter.py — Lint a planned computer-use action sequence.

Reads a list of planned agent actions (JSON or simple text) and flags
reliability and safety gaps before the agent ever touches a real target:

  - destructive / irreversible verbs (delete, send, pay, submit, purchase...)
    with no confirmation gate
  - state-changing actions with no verification step afterward
  - patterns that trigger blocking native/modal dialogs the agent cannot see
    (file pickers, OS print/save dialogs, downloads)
  - navigation/typing without a preceding fresh-screenshot grounding step

Input formats
-------------
JSON: a list of objects, e.g.
  [{"type": "click", "target": "Delete", "confirmed": false, "verified": false}]

Text: one action per line, "<type> <target>", e.g.
  click Delete
  type  search box
  submit Confirm

Reads from --file or stdin. Standard library only. Supports --json and
human-readable / markdown output.
"""

import argparse
import json
import re
import sys

# Typed text that would be destructive if it lands in a terminal or query box.
DESTRUCTIVE_INPUT_RE = re.compile(
    r"rm\s+-rf|drop\s+(table|database)|truncate\s+table|mkfs|format\s+[a-z]:|shutdown|"
    r"del\s+/[sq]|git\s+push\s+.*--force|sudo\s", re.IGNORECASE)
# Targets that suggest the agent is typing a secret or payment detail.
SENSITIVE_TARGET_RE = re.compile(r"password|passcode|card\s*number|cvv|ssn|social security|iban|api[_ -]?key",
                                 re.IGNORECASE)

DESTRUCTIVE_VERBS = {
    "delete", "remove", "send", "pay", "purchase", "buy", "submit", "confirm",
    "transfer", "delete_all", "drop", "wipe", "erase", "deactivate",
    "unsubscribe", "post", "publish", "approve", "cancel", "checkout",
}

# Action types that change state and therefore need a verification observation.
STATE_CHANGING = {
    "click", "type", "submit", "navigate", "drag", "select", "upload",
    "keypress", "press", "scroll_submit", "send",
}

# Verbs / targets that tend to open blocking native dialogs.
DIALOG_TRIGGERS = {
    "upload", "download", "print", "save_as", "saveas", "open_file",
    "file_picker", "choose_file", "browse",
}

# Actions that read screen state (count as verification / grounding).
VERIFY_TYPES = {"screenshot", "observe", "verify", "read", "assert", "wait_for", "check"}

SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2}


def load_actions(raw):
    raw = raw.strip()
    if not raw:
        return []
    # Try JSON first.
    if raw[0] in "[{":
        parsed = json.loads(raw)
        if isinstance(parsed, dict):
            parsed = [parsed]
        actions = []
        for item in parsed:
            if isinstance(item, str):
                actions.append(parse_text_line(item))
            elif isinstance(item, dict):
                atype = item.get("type", item.get("action"))  # "action" accepted as an alias
                if not atype:
                    raise ValueError("action %r has no 'type' (or 'action') field" % (item,))
                actions.append({
                    "type": str(atype).strip().lower(),
                    "target": str(item.get("target", "")).strip(),
                    "text": str(item.get("text", item.get("value", ""))),
                    "confirmed": bool(item.get("confirmed", False)),
                    "verified": bool(item.get("verified", False)),
                })
            else:
                raise ValueError("action %r is neither an object nor a text line" % (item,))
        return actions
    # Otherwise treat as text lines.
    return [parse_text_line(line) for line in raw.splitlines() if line.strip()]


def parse_text_line(line):
    parts = line.strip().split(None, 1)
    atype = parts[0].strip().lower() if parts else ""
    target = parts[1].strip() if len(parts) > 1 else ""
    return {"type": atype, "target": target, "text": "", "confirmed": False, "verified": False}


def is_destructive(action):
    blob = (action["type"] + " " + action["target"]).lower()
    for verb in DESTRUCTIVE_VERBS:
        if verb in blob.split() or verb in action["type"]:
            return True
        if verb in blob:
            return True
    return False


def is_dialog_trigger(action):
    blob = (action["type"] + " " + action["target"]).lower()
    return any(t in blob for t in DIALOG_TRIGGERS)


def has_following_verification(actions, idx):
    """True if the action itself is marked verified or a later action reads state
    before the next state-changing action."""
    if actions[idx].get("verified"):
        return True
    for j in range(idx + 1, len(actions)):
        if actions[j]["type"] in VERIFY_TYPES:
            return True
        if actions[j]["type"] in STATE_CHANGING:
            return False  # another state change happened with no read in between
    return False


def has_preceding_grounding(actions, idx):
    """True if a fresh screenshot/observe precedes this action (grounding)."""
    for j in range(idx - 1, -1, -1):
        if actions[j]["type"] in VERIFY_TYPES:
            return True
        if actions[j]["type"] in STATE_CHANGING:
            return False
    return False


def lint(actions):
    findings = []

    def add(idx, sev, code, msg):
        if idx is not None:
            label = ("%s %s" % (actions[idx]["type"], actions[idx]["target"])).strip()
        else:
            label = "(sequence)"
        findings.append({
            "index": idx,
            "action": label,
            "severity": sev,
            "code": code,
            "message": msg,
        })

    grounded_at_start = False
    if actions and actions[0]["type"] in VERIFY_TYPES:
        grounded_at_start = True

    for i, a in enumerate(actions):
        destructive = is_destructive(a)

        if destructive and not a.get("confirmed"):
            add(i, "high", "no-confirmation-gate",
                "Destructive/irreversible action has no confirmation gate "
                "(set confirmed=true once a human/explicit gate approves it).")

        if a["type"] in STATE_CHANGING and not has_following_verification(actions, i):
            sev = "high" if destructive else "medium"
            add(i, sev, "missing-verification",
                "State-changing action is not verified — add a screenshot/observe "
                "step afterward to confirm the expected change actually happened.")

        typed = a.get("text", "") + " " + (a["target"] if a["type"] in {"type", "keypress"} else "")
        if a["type"] in {"type", "keypress", "press"} and DESTRUCTIVE_INPUT_RE.search(typed):
            add(i, "high", "destructive-input",
                "Typed text looks like a destructive command or query; require a human gate "
                "and run it in a sandbox, never on a live system.")
        if a["type"] == "type" and SENSITIVE_TARGET_RE.search(a["target"]):
            add(i, "medium", "sensitive-input",
                "Agent types into a secret or payment field; inject the value from a vault "
                "at run time and keep it out of screenshots and logs.")

        if is_dialog_trigger(a):
            add(i, "medium", "blocking-dialog",
                "Action likely opens a blocking native/modal dialog (file picker, "
                "print/save, download) the agent cannot see or dismiss — prefer a "
                "flow that keeps state on the page.")

        if a["type"] in {"click", "type", "drag", "submit"} and not has_preceding_grounding(actions, i):
            add(i, "low", "no-grounding",
                "Action is not grounded on a fresh screenshot/observe — re-capture "
                "the current screen before acting to avoid stale-layout misclicks.")

    if actions and not grounded_at_start:
        add(0, "low", "no-initial-screenshot",
            "Sequence does not start with a screenshot/observe — ground the first "
            "action in the current screen state.")

    findings.sort(key=lambda f: (SEVERITY_ORDER[f["severity"]],
                                 f["index"] if f["index"] is not None else -1))
    return findings


def summarize(findings):
    counts = {"high": 0, "medium": 0, "low": 0}
    for f in findings:
        counts[f["severity"]] += 1
    risk = "high" if counts["high"] else "medium" if counts["medium"] else \
        "low" if counts["low"] else "clean"
    return counts, risk


def render_human(actions, findings, counts, risk, markdown=False):
    h1 = "## " if markdown else ""
    bullet = "- " if markdown else "  - "
    lines = []
    lines.append("%sAction Safety Lint Report" % h1)
    lines.append("")
    lines.append("Actions analyzed : %d" % len(actions))
    lines.append("Overall risk     : %s" % risk.upper())
    lines.append("Findings         : %d high, %d medium, %d low"
                 % (counts["high"], counts["medium"], counts["low"]))
    lines.append("")
    if not findings:
        lines.append("No issues found. Still dry-run in a sandbox before any real target.")
        return "\n".join(lines)
    lines.append("%sFindings" % ("### " if markdown else ""))
    for f in findings:
        loc = "#%d" % f["index"] if f["index"] is not None else "seq"
        lines.append("%s[%s] %s (%s): %s"
                     % (bullet, f["severity"].upper(), loc + " " + f["action"],
                        f["code"], f["message"]))
    return "\n".join(lines)


def main():
    p = argparse.ArgumentParser(
        description="Lint a planned computer-use action sequence for safety/reliability gaps."
    )
    p.add_argument("--file", help="Path to JSON or text action list (default: stdin)")
    p.add_argument("--format", choices=["text", "markdown"], default="text",
                   help="Human-readable output format")
    p.add_argument("--json", action="store_true", help="Emit JSON instead of text")
    args = p.parse_args()

    if args.file:
        try:
            with open(args.file, "r", encoding="utf-8") as fh:
                raw = fh.read()
        except OSError as exc:
            sys.stderr.write("error: cannot read %s: %s\n" % (args.file, exc.strerror))
            sys.exit(2)
    else:
        raw = sys.stdin.read()

    try:
        actions = load_actions(raw)
    except (ValueError, json.JSONDecodeError) as exc:
        sys.stderr.write("error: could not parse actions: %s\n" % exc)
        sys.exit(2)

    findings = lint(actions)
    counts, risk = summarize(findings)

    if args.json:
        print(json.dumps({
            "actions_analyzed": len(actions),
            "overall_risk": risk,
            "counts": counts,
            "findings": findings,
        }, indent=2))
    else:
        print(render_human(actions, findings, counts, risk,
                           markdown=(args.format == "markdown")))

    sys.exit(1 if counts["high"] else 0)


if __name__ == "__main__":
    main()
brainstormingobra/superpowersYou MUST use this before any creative work - creating features, building components, adding functionality, modifying behavior, or planning anything new, in software or out of it (a talk, a business, a renovation).Stars: 322kUpdated Claude CodeCodexVerifiedInstall
brand-guidelinesanthropics/skillsApplies Anthropic's official brand colors and typography to any sort of artifact that may benefit from having Anthropic's look-and-feel. Use it when brand colors or style guidelines, visual formatting, or company design standards apply.Stars: 198kUpdated Claude CodeCodexVerifiedInstall
frontend-designanthropics/skillsGuidance for distinctive, intentional visual design when building new UI or reshaping an existing one. Helps with aesthetic direction, typography, and making choices that don't read as templated defaults.Stars: 198kUpdated Claude CodeCodexVerifiedInstall
mcp-builderanthropics/skillsGuide for creating high-quality MCP (Model Context Protocol) servers that enable LLMs to interact with external services through well-designed tools. Use when building MCP servers to integrate external APIs or services, whether in Python (FastMCP) or Node/TypeScript (MCP SDK).Stars: 198kUpdated Claude CodeCodexVerifiedInstall