datarobot-oss/datarobot-agent-skills / skills/datarobot-gap-analysis
datarobot-gap-analysis
FlaggedUse when the user wants to assess whether an AI agent repository (DataRobot-built or not) is enterprise-ready, run a gap analysis / readiness scan against security, governance, reliability, or regulatory standards, score an agent against EU AI Act requirements, or find and fix gaps (secrets, unpinned models, missing CI, missing guardrails, over-permissioned identity, etc.) before deploying an agent to production. Works on any repository, not only ones built with DataRobot.
scripts/gap_analysis/posture.py
View on GitHub# Copyright (c) 2026 DataRobot, Inc. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Remediation-posture decision gate.
Turns a flat gap list into a recommendation: PATCH, HYBRID, or RE-PLATFORM.
The premise: fix-in-place vs. rebuild-from-scratch is a false binary. The unit of
decision is the *gap*, not the agent. Plumbing gaps (secrets, pins, scaffolding) patch
safely; STRUCTURAL gaps (no observability, no guardrails, human identity, no resilience)
can't be surgically fixed — "fixing" them means restructuring someone else's business
logic. When structural gaps dominate, lifting the business logic into a fresh, conformant
af-components base (see migrate.py) is safer than in-place surgery.
`structural` is a per-condition flag in taxonomy.yaml for Layers 1-3 (any advisory
high/critical also counts); Layer 4 findings carry it directly on the Finding instead,
since they're generated dynamically from a DataRobot risk-management policy rather
than a taxonomy.yaml entry (almost always true there, since satisfying a mitigation
nearly always means adopting a DataRobot platform feature, not patching code in
place). Thresholds live under `posture:` in the policy.
"""
from __future__ import annotations
from typing import Any
from .models import AnalysisResult, Finding, Severity
from .taxonomy import Taxonomy
from .variants import _select_variant_text, shipped_deployment_variant
# Severity weights — structural density is weighted, so a couple of critical structural
# gaps outweigh a long tail of low-severity plumbing.
_WEIGHT = {
Severity.CRITICAL: 4.0,
Severity.HIGH: 3.0,
Severity.MEDIUM: 2.0,
Severity.LOW: 1.0,
}
PATCH = "PATCH"
HYBRID = "HYBRID"
REPLATFORM = "RE-PLATFORM"
_DEFAULTS = {
# score <= patch_max -> PATCH
# patch_max < score < replatform_min -> HYBRID
# score >= replatform_min -> RE-PLATFORM
"patch_max": 0.25,
"replatform_min": 0.50,
# Absolute override: this many high/critical structural gaps forces RE-PLATFORM
# regardless of density (a small repo can be all-structural at low total weight).
"replatform_structural_count": 4,
"max_drivers": 8,
}
def assess_posture(
result: AnalysisResult,
policy: dict[str, Any] | None = None,
taxonomy: Taxonomy | None = None,
) -> dict[str, Any]:
"""Return {recommendation, score, structural_count, total, drivers, rationale}."""
cfg = {**_DEFAULTS, **(policy or {}).get("posture", {})}
tax = taxonomy or Taxonomy.load()
findings = result.findings
total = len(findings)
if not total:
return {
"recommendation": PATCH,
"score": 0.0,
"structural_count": 0,
"total": 0,
"drivers": [],
"rationale": "No gaps detected — nothing to remediate or re-platform.",
}
def is_structural(f: Finding) -> bool:
# Findings with a taxonomy.yaml entry (Layers 1-3) defer to the
# condition's own flag. Layer 4 findings are dynamically generated (no
# taxonomy entry, driven by the org's live DataRobot risk-management
# policy (see risk_management.py) and carry the flag on the Finding
# itself instead.
# A verification pass that saw the actual remediation shape wins over
# the condition template.
if f.shape in ("patch", "structural"):
return f.shape == "structural"
c = tax.get(f.condition_id)
if c is not None:
return bool(c.structural)
return bool(f.structural)
total_weight = sum(_WEIGHT[f.severity] for f in findings)
structural = [f for f in findings if is_structural(f)]
structural_weight = sum(_WEIGHT[f.severity] for f in structural)
score = round(structural_weight / total_weight, 3) if total_weight else 0.0
high_structural = [
f for f in structural if f.severity in (Severity.CRITICAL, Severity.HIGH)
]
# Decide.
if (
len(high_structural) >= cfg["replatform_structural_count"]
or score >= cfg["replatform_min"]
):
rec = REPLATFORM
elif score <= cfg["patch_max"]:
rec = PATCH
else:
rec = HYBRID
drivers = _drivers(structural, int(cfg["max_drivers"]))
advice = migration_advice(result.inventory, result.iac)
return {
"recommendation": rec,
"score": score,
"structural_count": len(structural),
"total": total,
"drivers": drivers,
"advice": advice,
"rationale": _rationale(
rec, score, len(structural), total, high_structural, advice
),
}
def migration_advice(
inventory: dict[str, Any] | None, iac: dict[str, Any] | None = None
) -> str:
"""What "re-platform" means for this repo.
A repo already generated from af-components keeps its application; the
structural fix is to put the model or LLM path behind a DataRobot
deployment of the right target type so guards and monitoring can attach.
When the Pulumi program already ships a variant that declares one, the fix
is to select it. The agent framework is never changed unless the user asks:
frameworks without a native DataRobot template deploy through the generic
Base flavor.
"""
inv = inventory or {}
iac = iac or {}
sources = [s for s in inv.get("template_sources", []) if "af-component" in s]
frameworks = inv.get("agent_frameworks") or []
llm = inv.get("llm_usage") or {}
target = iac.get("target_type") or inv.get("deploy_target")
variant = shipped_deployment_variant(iac)
if variant:
text = (
"The active Pulumi configuration deploys no datarobot.Deployment, but the "
"repo already ships one. " + _select_variant_text(iac, variant)
)
elif sources:
text = (
f"This repo already builds on af-components ({', '.join(sources)}). Keep the "
f"application and put the {_path_name(target)} behind {_deployment_shape(target)}, "
"so guards, tracing and monitoring attach."
)
if llm.get("gateway"):
text += (
" The LLM calls can keep going through the DataRobot LLM Gateway from "
"inside that model; the gateway is the provider, the deployment is the "
"governed entity."
)
else:
text = (
"Re-platform onto af-components with the datarobot-agent-assist skill: extract "
"the business logic into a fresh af-components base rather than restructuring "
"in place."
)
if frameworks:
native = [f["name"] for f in frameworks if f["native"]]
other = [f["name"] for f in frameworks if not f["native"]]
if other:
choices = inv.get("agent_template_choices") or {}
natives = [label for label, value in choices.items() if value != "base"]
offered = ", ".join(natives) if natives else "a fixed set of frameworks"
text += (
f" The agent uses {', '.join(other)}; the DataRobot agent template offers "
f"{offered} natively, so deploy it through its generic Base flavor, which "
"wraps any Python agent, rather than rewriting it onto another framework."
)
elif native:
text += f" The agent uses {', '.join(native)}, which has a native DataRobot agent template."
return text
def _path_name(target: str | None) -> str:
if target == "TextGeneration":
return "LLM path"
if target and target not in ("AgenticWorkflow",):
return "model"
return "agent or LLM path"
def _deployment_shape(target: str | None) -> str:
if target == "TextGeneration":
return (
"an LLM deployment: an LLM blueprint or a datarobot.CustomModel with target "
"type TextGeneration behind a datarobot.Deployment"
)
if target and target not in ("AgenticWorkflow",):
return "a registered model behind a datarobot.Deployment"
return (
"an agentic deployment: a datarobot.CustomModel with target type "
"AgenticWorkflow behind a datarobot.Deployment"
)
def _drivers(structural: list[Finding], limit: int) -> list[dict[str, str]]:
"""One row per structural condition (deduped), worst severity first."""
seen: dict[str, dict[str, str]] = {}
for f in structural:
cur = seen.get(f.condition_id)
if cur is None or f.severity.rank < Severity(cur["severity"]).rank:
seen[f.condition_id] = {
"condition_id": f.condition_id,
"severity": f.severity.value,
"title": f.title,
}
ordered = sorted(seen.values(), key=lambda d: Severity(d["severity"]).rank)
return ordered[:limit]
def _rationale(
rec: str,
score: float,
structural_count: int,
total: int,
high_structural: list[Finding],
advice: str = "",
) -> str:
pct = round(score * 100)
if rec == PATCH:
return (
f"Only {structural_count} of {total} gaps are structural "
f"({pct}% of weighted risk). Patch in place — the fixes are surgical and "
f"low-risk to existing business logic."
)
if rec == REPLATFORM:
return (
f"{len(high_structural)} high/critical structural gaps and {pct}% of "
f"weighted risk is architectural: these gaps are closed by changing how the "
f"system is deployed and governed, not by editing individual files. {advice}"
)
return (
f"Mixed profile — {structural_count} of {total} gaps are structural "
f"({pct}% of weighted risk). Patch the plumbing now, and plan the structural "
f"core separately. {advice}"
)
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| rea-analysis-optimizationmorluto/reaOptimize REA providers and shared analysis workflows for agent context, latency, CPU, and memory while preserving correctness and Evidence. Use for analysis bottlenecks and cache or query design; CI/test-suite optimization belongs to rea-ci-tests-audit. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-ci-tests-auditmorluto/reaAudit REA CI and test suites for correctness, fixture drift, false passes, and measured performance; use for suite or workflow investigations and efficiency changes, rather than ordinary feature testing or a single test failure. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| rea-tool-designmorluto/reaAudit, investigate, design, or change REA investigation tools, CLI/MCP contracts, provider capabilities, and Evidence semantics. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-auditDietrichGebert/ponytailQuality audit of the whole repo: bugs, security, real load, missing tests, speed, and what to delete. Most important first. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |