Skip to content

datarobot-oss/datarobot-agent-skills / skills/datarobot-gap-analysis

datarobot-gap-analysis

Flagged

Use when the user wants to assess whether an AI agent repository (DataRobot-built or not) is enterprise-ready, run a gap analysis / readiness scan against security, governance, reliability, or regulatory standards, score an agent against EU AI Act requirements, or find and fix gaps (secrets, unpinned models, missing CI, missing guardrails, over-permissioned identity, etc.) before deploying an agent to production. Works on any repository, not only ones built with DataRobot.

FileKindSize
SKILL.mdskill md17.2 KB
references/pillars-and-layers.mdmarkdown7.8 KB
references/policy-authoring.mdmarkdown6.3 KB
references/remediation-paths.mdmarkdown3.0 KB
scripts/gap_analysis/__init__.pyscript534 B
scripts/gap_analysis/_bootstrap.pyscript2.7 KB
scripts/gap_analysis/agent_prompt.pyscript2.5 KB
scripts/gap_analysis/cli.pyscript19.9 KB
scripts/gap_analysis/conformance.pyscript9.6 KB
scripts/gap_analysis/detect.pyscript24.4 KB
scripts/gap_analysis/docs.pyscript2.9 KB
scripts/gap_analysis/engine.pyscript9.3 KB
scripts/gap_analysis/ingest.pyscript3.4 KB
scripts/gap_analysis/inventory.pyscript28.0 KB
scripts/gap_analysis/llm.pyscript2.6 KB
scripts/gap_analysis/migrate.pyscript11.7 KB
scripts/gap_analysis/models.pyscript5.9 KB
scripts/gap_analysis/opencode.pyscript2.7 KB
scripts/gap_analysis/paths.pyscript1.1 KB
scripts/gap_analysis/policy.pyscript1.3 KB
scripts/gap_analysis/posture.pyscript9.6 KB
scripts/gap_analysis/remediate.pyscript20.1 KB
scripts/gap_analysis/report.pyscript20.8 KB
scripts/gap_analysis/report_html.pyscript41.0 KB
scripts/gap_analysis/risk_management.pyscript36.3 KB
scripts/gap_analysis/scanners.pyscript52.0 KB
scripts/gap_analysis/settings.pyscript2.2 KB
scripts/gap_analysis/taxonomy.pyscript5.3 KB
scripts/gap_analysis/variants.pyscript1.5 KB
scripts/policy/defaults.yamlconfig4.6 KB
scripts/prompts/_contract.mdmarkdown3.0 KB
scripts/prompts/_deployment_datarobot_app.mdmarkdown1.2 KB
scripts/prompts/_fix_contract.mdmarkdown1.3 KB
scripts/prompts/_migrate_contract.mdmarkdown1.6 KB
scripts/prompts/_verify.mdmarkdown2.2 KB
scripts/prompts/aig-001-guardrails.mdmarkdown475 B
scripts/prompts/aig-002-model-pinning.mdmarkdown446 B
scripts/prompts/aig-004-evals.mdmarkdown357 B
scripts/prompts/aig-005-human-in-loop.mdmarkdown370 B
scripts/prompts/aig-006-cost-controls.mdmarkdown368 B
scripts/prompts/aig-007-grounding.mdmarkdown406 B
scripts/prompts/aig-008-prompt-versioning.mdmarkdown363 B
scripts/prompts/fix-aig-001-guardrails.mdmarkdown441 B
scripts/prompts/fix-aig-002-model-pinning.mdmarkdown527 B
scripts/prompts/fix-aig-003-approved-model.mdmarkdown351 B
scripts/prompts/fix-aig-006-cost-controls.mdmarkdown354 B
scripts/prompts/fix-idn-003-narrow-scope.mdmarkdown415 B
scripts/prompts/fix-ita-005-base-image.mdmarkdown305 B
scripts/prompts/fix-ops-001-logging.mdmarkdown298 B
scripts/prompts/fix-ops-002-tracing.mdmarkdown375 B
scripts/prompts/fix-pol-pulumi.mdmarkdown3.1 KB
scripts/prompts/fix-rel-003-resilience.mdmarkdown361 B
scripts/prompts/fix-sec-004-redact-log.mdmarkdown418 B
scripts/prompts/fix-sec-011-injection.mdmarkdown598 B
scripts/prompts/fix-sec-012-validation.mdmarkdown374 B
scripts/prompts/fix-sec-013-encryption.mdmarkdown370 B
scripts/prompts/idn-001-human-identity.mdmarkdown548 B
scripts/prompts/idn-002-rotation.mdmarkdown410 B
scripts/prompts/idn-003-over-permissioned.mdmarkdown711 B
scripts/prompts/idn-004-rbac.mdmarkdown433 B
scripts/prompts/migrate-extract-spec.mdmarkdown1.9 KB
scripts/prompts/ops-001-logging.mdmarkdown396 B
scripts/prompts/ops-002-tracing.mdmarkdown351 B
scripts/prompts/ops-003-health.mdmarkdown363 B
scripts/prompts/rel-003-resilience.mdmarkdown401 B
scripts/prompts/rel-004-fallback.mdmarkdown410 B
scripts/prompts/risk-management-mitigation.mdmarkdown2.7 KB
scripts/prompts/sec-001-secret-in-context.mdmarkdown701 B
scripts/prompts/sec-011-injection.mdmarkdown711 B
scripts/prompts/sec-012-input-validation.mdmarkdown999 B
scripts/prompts/sec-013-encryption.mdmarkdown410 B
scripts/risk_management_mitigations.yamlconfig32.0 KB
scripts/run_gap_analysis.pyexecutablescript1.5 KB
scripts/taxonomy.yamlconfig28.8 KB
scripts/validate_risk_management_mapping.pyexecutablescript2.0 KB

scripts/gap_analysis/conformance.py

View on GitHub
# Copyright (c) 2026 DataRobot, Inc. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Layer 3 — conformance of the repo against the merged policy."""

from __future__ import annotations

import fnmatch
import json
import shutil
import subprocess
import threading
from typing import Any

from .models import Finding
from .taxonomy import Condition, Taxonomy


def _ver_tuple(v: str, width: int = 3) -> tuple[int, ...]:
    """A comparable version tuple, zero-padded: "3.11" and "3.11.0" are equal,
    where the raw split tuples would order the shorter one first."""
    parts = tuple(int(x) for x in v.split(".") if x.isdigit())
    return parts + (0,) * max(0, width - len(parts))


def _glob_any(value: str, patterns: list[str]) -> bool:
    """True when `value` matches any of the patterns (allow/deny lists)."""
    return any(fnmatch.fnmatch(value, p) for p in patterns)


def _matched_by(pattern: str, values: set[str] | list[str]) -> bool:
    """True when any value matches `pattern` (a require entry is the pattern)."""
    return any(fnmatch.fnmatch(v, pattern) for v in values)


_GATEWAY_MODELS: list[str] | None = None
_GATEWAY_LOCK = threading.Lock()


def llm_gateway_models(offline: bool = False) -> list[str]:
    """Model ids served by the org's DataRobot LLM Gateway, via `dr llm-gateway list`.

    Empty when the CLI is missing, unauthenticated, slow, or `offline`. Models
    the gateway serves are governed by the platform, so they count as approved
    alongside the policy allowlist.

    Layer 2 workers and Layer 3 call this from different threads, so the cache
    is only published once the fetch has finished: an empty list means "asked
    and got nothing", never "asking right now".
    """
    global _GATEWAY_MODELS
    if _GATEWAY_MODELS is not None:
        return _GATEWAY_MODELS
    with _GATEWAY_LOCK:
        if _GATEWAY_MODELS is not None:
            return _GATEWAY_MODELS
        models: list[str] = []
        dr = shutil.which("dr")
        if dr and not offline:
            try:
                proc = subprocess.run(
                    [dr, "llm-gateway", "list", "--output-format", "json"],
                    capture_output=True,
                    text=True,
                    timeout=30,
                    check=False,
                )
                data = json.loads(proc.stdout or "{}")
                models = sorted(
                    {m.get("model") for m in data.get("llms", []) if m.get("model")}
                )
            except (OSError, subprocess.TimeoutExpired, ValueError, AttributeError):
                models = []
        _GATEWAY_MODELS = models
        return _GATEWAY_MODELS


def _gateway_serves(model_id: str, catalog: list[str]) -> bool:
    return model_id in catalog or any(g.startswith(model_id + "-") for g in catalog)


def check_conformance(
    inventory: dict[str, Any],
    policy: dict[str, Any],
    taxonomy: Taxonomy,
    offline: bool = False,
) -> tuple[list[Finding], list[str]]:
    findings: list[Finding] = []
    notes: list[str] = []
    it = policy.get("it_admin", {})

    # ITA-001 — Python minimum version
    cond = taxonomy.get("ITA-001")
    min_v = (it.get("python", {}) or {}).get("min_version")
    repo_v = inventory.get("python_version")
    if cond and min_v:
        if repo_v is None:
            notes.append(
                "ITA-001: no declared Python version found — cannot confirm minimum."
            )
        elif _ver_tuple(repo_v) < _ver_tuple(str(min_v)):
            findings.append(
                _mk(
                    cond,
                    _py_source(inventory),
                    None,
                    f"declared Python {repo_v} < required {min_v}",
                    f"Project targets Python {repo_v}; policy requires >= {min_v}.",
                )
            )

    # ITA-002 — library allow/deny/require
    cond = taxonomy.get("ITA-002")
    libs = it.get("libraries", {}) or {}
    deps = set(inventory.get("dependencies", []))
    if cond:
        allow = [a.lower() for a in libs.get("allow", []) or []]
        deny = [d.lower() for d in libs.get("deny", []) or []]
        require = [r.lower() for r in libs.get("require", []) or []]
        for dep in sorted(deps):
            if dep in deny:
                findings.append(
                    _mk(
                        cond,
                        _manifest(inventory),
                        None,
                        dep,
                        f"Dependency '{dep}' is on the policy deny list.",
                    )
                )
            elif allow and not _glob_any(dep, allow):
                findings.append(
                    _mk(
                        cond,
                        _manifest(inventory),
                        None,
                        dep,
                        f"Dependency '{dep}' is not on the policy allow list.",
                    )
                )
        for req in require:
            if req not in deps and not _matched_by(req, deps):
                findings.append(
                    _mk(
                        cond,
                        _manifest(inventory),
                        None,
                        req,
                        f"Required library '{req}' is missing from dependencies.",
                    )
                )

    # AIG-003 / ITA-003 — approved models
    allow_models = (it.get("models", {}) or {}).get("allow", []) or []
    catalog = (
        llm_gateway_models(offline)
        if allow_models and inventory.get("model_ids")
        else []
    )
    if catalog:
        notes.append(
            f"AIG-003/ITA-003: {len(catalog)} model id(s) served by the DataRobot LLM "
            "Gateway are treated as approved."
        )
    for cid in ("AIG-003", "ITA-003"):
        cond = taxonomy.get(cid)
        if not cond or not allow_models:
            continue
        seen: set[str] = set()
        for mid in inventory.get("model_ids", []):
            if mid in seen:
                continue
            seen.add(mid)
            if not _glob_any(mid, allow_models) and not _gateway_serves(mid, catalog):
                findings.append(
                    _mk(
                        cond,
                        None,
                        None,
                        mid,
                        f"Model '{mid}' is not on the approved-model allowlist.",
                    )
                )

    # ITA-005 — approved base images
    cond = taxonomy.get("ITA-005")
    allow_imgs = (it.get("base_images", {}) or {}).get("allow", []) or []
    if cond and allow_imgs:
        image_files = inventory.get("base_image_files") or {}
        for img in inventory.get("base_images", []):
            if not _glob_any(img, allow_imgs):
                findings.append(
                    _mk(
                        cond,
                        (image_files.get(img) or [_dockerfile(inventory)])[0],
                        None,
                        img,
                        f"Base image '{img}' is not on the approved-image allowlist.",
                    )
                )

    # ITA-004 — offline coverage is limited to the repo's own declared license
    # (package.json / pyproject.toml); dependency licenses need registry or
    # installed-package metadata.
    cond = taxonomy.get("ITA-004")
    deny_licenses = (it.get("licenses", {}) or {}).get("deny") or []
    if cond and deny_licenses:
        for rel, lic in inventory.get("declared_licenses", []) or []:
            if lic in deny_licenses:
                findings.append(
                    _mk(
                        cond,
                        rel,
                        None,
                        lic,
                        f"Manifest declares license '{lic}', which is on the "
                        "org's denied-license list.",
                    )
                )
        notes.append(
            "ITA-004: dependency licenses require installed package metadata; "
            "only the repo's own declared license was checked offline."
        )

    return findings, notes


_PY_FLOOR_FILES = (
    ".python-version",
    "runtime.txt",
    "pyproject.toml",
    "setup.cfg",
    "setup.py",
)


def _py_source(inv: dict[str, Any]) -> str | None:
    """The file that declares the lowest Python floor: the manifest in the
    component directory the inventory attributed that floor to."""
    versions = inv.get("python_versions") or {}
    floor = inv.get("python_version")
    files = set(inv.get("files", []))
    dirs = sorted(d for d, v in versions.items() if v == floor) or sorted(versions)
    for d in dirs:
        for name in _PY_FLOOR_FILES:
            rel = name if d in (".", "") else f"{d}/{name}"
            if rel in files:
                return rel
    man = inv.get("key_files", {}).get("manifests", [])
    return str(man[0]) if man else None


def _manifest(inv: dict[str, Any]) -> str | None:
    man = inv.get("key_files", {}).get("manifests")
    return str(man[0]) if man else None


def _dockerfile(inv: dict[str, Any]) -> str | None:
    files = inv.get("key_files", {}).get("dockerfiles")
    return str(files[0]) if files else None


def _mk(
    cond: Condition,
    file: str | None,
    line: int | None,
    evidence: str,
    explanation: str,
) -> Finding:
    return Finding(
        condition_id=cond.id,
        pillar=cond.pillar,
        severity=cond.severity,
        title=cond.title,
        file=file,
        line=line,
        evidence=evidence,
        explanation=explanation,
        remediation=cond.remediation,
        fix_type=cond.fix_type,
        fix_strategy=cond.fix_strategy,
        fix_risk=cond.fix_risk,
        layer=cond.layer,
        detector=cond.detector,
    )
rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first.Stars: 194kUpdated Claude CodeCodexVerifiedInstall