#!/usr/bin/env python3
"""
analyze_repo.py — Repository analysis script for the repo-estimator skill.
Produces a structured JSON summary of a repository's composition, complexity
signals, and measurable characteristics. This is raw data for Claude to
interpret — not a final estimate.
Usage:
python scripts/analyze_repo.py <repo_path> [--output <output.json>] [--verbose]
Output:
JSON to stdout (or file if --output specified)
"""
import argparse
import json
import os
import re
import subprocess
import sys
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
# ---------------------------------------------------------------------------
# Language detection
# ---------------------------------------------------------------------------
EXTENSION_MAP = {
# Languages
".py": "Python",
".js": "JavaScript",
".jsx": "JavaScript (React)",
".ts": "TypeScript",
".tsx": "TypeScript (React)",
".rb": "Ruby",
".go": "Go",
".rs": "Rust",
".java": "Java",
".kt": "Kotlin",
".swift": "Swift",
".cs": "C#",
".cpp": "C++",
".cc": "C++",
".c": "C",
".h": "C/C++ Header",
".hpp": "C++ Header",
".php": "PHP",
".scala": "Scala",
".ex": "Elixir",
".exs": "Elixir Script",
".erl": "Erlang",
".clj": "Clojure",
".hs": "Haskell",
".lua": "Lua",
".r": "R",
".R": "R",
".dart": "Dart",
".vue": "Vue",
".svelte": "Svelte",
# Web
".html": "HTML",
".htm": "HTML",
".css": "CSS",
".scss": "SCSS",
".sass": "Sass",
".less": "Less",
# Data / Config
".json": "JSON",
".yaml": "YAML",
".yml": "YAML",
".toml": "TOML",
".xml": "XML",
".env": "Environment",
# Infra / DevOps
".tf": "Terraform",
".hcl": "HCL",
".dockerfile": "Dockerfile",
# Queries
".sql": "SQL",
".graphql": "GraphQL",
".gql": "GraphQL",
".proto": "Protobuf",
# Docs
".md": "Markdown",
".rst": "reStructuredText",
".txt": "Text",
# Shell
".sh": "Shell",
".bash": "Bash",
".zsh": "Zsh",
".fish": "Fish",
# Mobile
".m": "Objective-C",
".xib": "Xcode Interface",
".storyboard": "Xcode Storyboard",
}
# Files to skip (generated, vendored, or irrelevant)
SKIP_DIRS = {
"node_modules", ".git", "vendor", "venv", ".venv", "env", ".env",
"__pycache__", ".pytest_cache", ".mypy_cache", ".tox", "dist",
"build", ".build", "out", ".next", ".nuxt", "coverage", ".coverage",
"htmlcoverage", ".eggs", "*.egg-info", ".gradle", ".idea", ".vscode",
"target", "Pods", "DerivedData", ".dart_tool", ".pub-cache",
"generated", "gen", "pb_generated", "*.generated",
"migrations", # Will be counted separately
}
SKIP_EXTENSIONS = {
".lock", ".sum", ".snap", ".min.js", ".min.css", ".map",
".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg", ".webp",
".woff", ".woff2", ".ttf", ".eot", ".otf",
".mp4", ".mp3", ".avi", ".mov", ".webm",
".zip", ".tar", ".gz", ".bz2", ".7z",
".pdf", ".doc", ".docx", ".xls", ".xlsx",
".pyc", ".pyo", ".class", ".o", ".a", ".so", ".dylib", ".dll",
".exe", ".bin",
}
# Patterns suggesting generated files
GENERATED_PATTERNS = [
r"DO NOT EDIT",
r"Code generated",
r"This file was automatically generated",
r"@generated",
r"Auto-generated",
r"Generated by",
r"GENERATED CODE",
]
# ---------------------------------------------------------------------------
# Complexity signal patterns (regex applied to file contents)
# ---------------------------------------------------------------------------
COMPLEXITY_SIGNALS = {
# Authentication & authorization
"auth_oauth": [r"oauth", r"openid", r"oidc", r"jwt", r"access_token", r"refresh_token"],
"auth_mfa": [r"totp", r"mfa", r"two.factor", r"authenticator", r"otp"],
"auth_rbac": [r"role.based", r"rbac", r"permission", r"policy", r"acl", r"casbin", r"pundit"],
# Real-time
"realtime_websocket": [r"websocket", r"ws://", r"wss://", r"socket\.io", r"ActionCable", r"pusher", r"ably"],
"realtime_sse": [r"server.sent", r"EventSource", r"text/event-stream"],
# Data
"database_orm": [r"prisma", r"sequelize", r"typeorm", r"sqlalchemy", r"activerecord", r"gorm", r"ecto"],
"database_migrations": [r"migrate", r"migration", r"schema.*version", r"alembic", r"flyway"],
"database_redis": [r"redis", r"ioredis", r"bull", r"sidekiq", r"celery", r"rqueue"],
"database_elasticsearch": [r"elasticsearch", r"opensearch", r"solr", r"meilisearch", r"typesense"],
"database_graphql": [r"graphql", r"apollo", r"relay", r"gql"],
# Payments
"payments_stripe": [r"stripe", r"payment_intent", r"subscription", r"webhook.*stripe"],
"payments_other": [r"paypal", r"braintree", r"square", r"adyen", r"checkout", r"plaid"],
# Cloud / Infra
"infra_kubernetes": [r"kubernetes", r"k8s", r"kubectl", r"helm", r"kustomize", r"pod", r"deployment.*yaml"],
"infra_terraform": [r"terraform", r"\.tf$"],
"infra_docker": [r"dockerfile", r"docker-compose", r"container"],
"infra_aws": [r"amazonaws\.com", r"boto3", r"aws-sdk", r"s3://", r"arn:aws"],
"infra_gcp": [r"googleapis\.com", r"google-cloud", r"gcloud", r"pubsub"],
"infra_azure": [r"azure\.com", r"azure-sdk", r"microsoft\.azure"],
# ML / AI
"ml_training": [r"torch", r"tensorflow", r"keras", r"model\.fit", r"train_step", r"optimizer\.step"],
"ml_inference": [r"model\.predict", r"inference", r"onnx", r"tensorrt", r"triton"],
"ml_llm": [r"openai", r"anthropic", r"langchain", r"llamaindex", r"huggingface", r"transformers"],
"ml_embeddings": [r"embedding", r"vector.*store", r"faiss", r"pinecone", r"weaviate", r"chroma"],
# Testing
"testing_unit": [r"describe\(", r"it\(", r"test\(", r"def test_", r"@Test", r"func Test"],
"testing_integration": [r"integration.test", r"supertest", r"testcontainers", r"factory_bot"],
"testing_e2e": [r"playwright", r"cypress", r"selenium", r"puppeteer", r"nightwatch"],
"testing_performance": [r"k6", r"locust", r"jmeter", r"gatling", r"artillery"],
# Multi-tenancy
"multitenancy": [r"tenant", r"organization", r"workspace", r"schema.*switching", r"row.level.security"],
# File handling
"file_uploads": [r"multipart", r"file.upload", r"s3.*upload", r"multer", r"carrierwave", r"shrine", r"active.storage"],
# Email
"email": [r"sendgrid", r"ses", r"postmark", r"mailgun", r"resend", r"nodemailer", r"action.mailer"],
# Mobile specific
"mobile_push": [r"fcm", r"apns", r"push.notification", r"firebase.messaging"],
"mobile_offline": [r"offline", r"sync.*conflict", r"crdt", r"local.first"],
# Cryptography
"crypto": [r"encrypt", r"decrypt", r"cipher", r"aes", r"rsa", r"hmac", r"bcrypt", r"argon2"],
# Queues / async
"queues": [r"queue", r"worker", r"job", r"task", r"celery", r"sidekiq", r"bullmq", r"temporal", r"inngest"],
# Search
"search": [r"search", r"index", r"facet", r"full.text", r"elasticsearch", r"meilisearch"],
# Observability
"observability": [r"opentelemetry", r"datadog", r"sentry", r"newrelic", r"prometheus", r"grafana", r"honeycomb"],
# Analytics
"analytics": [r"segment", r"mixpanel", r"amplitude", r"posthog", r"analytics", r"event.tracking"],
# Webhooks
"webhooks": [r"webhook", r"event.*callback", r"ngrok", r"svix"],
}
# ---------------------------------------------------------------------------
# Rebuild Difficulty Detection
# Five dimensions: domain, infra, data model, integrations, operational maturity
# ---------------------------------------------------------------------------
# Dimension 1: Specialized knowledge domains
# Each key = domain name, value = (patterns, difficulty_score)
# difficulty_score: 1=moderate, 2=high/very_high, 3=extreme
DOMAIN_SIGNALS = {
"payments_fintech": (["pci", "payment_intent", "settlement", "reconcil", "acquirer",
"merchant", "chargeback", "card_network", "issuer", "fx_rate",
"plaid", "open_banking", "wire_transfer", "ach", "swift", "sepa"], 2),
"healthcare": (["hl7", "fhir", "dicom", "hipaa", "ehr", "emr", "icd.10",
"cpt_code", "npi", "patient_record", "clinical", "pharmacy"], 2),
"cryptography_custom": (["cipher_suite", "key_derivation", "hmac", "zero.knowledge",
"homomorphic", "elliptic.curve", "schnorr", "zk.proof",
"merkle", "threshold.signature", "hsm", "pkcs"], 3),
"distributed_consensus": (["raft", "paxos", "viewstamped", "pbft", "consensus",
"leader_election", "quorum", "log.replication",
"crdt", "vector.clock", "lamport", "hlc"], 3),
"realtime_trading": (["order_book", "matching_engine", "fix_protocol", "latency",
"market_data", "tick_data", "hft", "dark_pool",
"risk_limit", "margin_call", "position_limit"], 3),
"signal_processing": (["fft", "dsp", "filter.*kernel", "convolution", "spectrogram",
"wavelet", "sample_rate", "nyquist", "frequency_domain",
"audio_codec", "video_codec", "h264", "h265"], 2),
"compilers_runtimes": (["ast", "lexer", "parser", "bytecode", "ir_gen",
"register_alloc", "jit", "garbage.collect",
"llvm", "cranelift", "wasm", "runtime.*vm"], 2),
"robotics_embedded": (["ros", "pid_controller", "kalman", "slam", "odometry",
"actuator", "servo", "pwm", "interrupt", "gpio",
"motor_driver", "sensor_fusion", "imu"], 2),
"blockchain": (["solidity", "smart_contract", "evm", "web3", "ethers",
"transaction.*hash", "gas_limit", "nonce", "wallet",
"defi", "liquidity_pool", "amm"], 2),
"tax_compliance": (["tax_rate", "jurisdiction", "nexus", "vat", "gst",
"sales_tax", "withholding", "1099", "w2",
"tax_code", "avalara", "taxjar"], 2),
"ml_research": (["training_loop", "backward.*pass", "gradient", "loss_fn",
"optimizer", "epoch", "batch_size", "learning_rate",
"attention_head", "transformer", "embedding_dim"], 2),
"geospatial": (["postgis", "geojson", "latitude", "longitude", "bbox",
"spatial_index", "geodesic", "projection", "wgs84",
"routing", "isochrone", "h3", "s2geometry"], 1),
}
# Dimension 2: Infrastructure coupling
INFRA_COUPLING_SIGNALS = {
"kubernetes_advanced": ["kubernetes", "helm", "kustomize", "kubectl", "operator.*sdk",
"custom.*resource.*definition", "crd", "argo", "flux", "gitops"],
"kubernetes_basic": ["k8s", "pod", "deployment.*yaml", "service.*yaml", "ingress"],
"terraform": ["terraform", r"\.tf", "hcl", "cloudformation", "pulumi", "cdk"],
"multi_cloud": ["multi.cloud", "cloud.agnostic", "vendor.agnostic"],
"service_mesh": ["istio", "linkerd", "envoy", "consul", "service.mesh"],
"docker_compose": ["docker-compose", "docker_compose", "compose.yaml"],
"ci_cd_advanced": ["matrix.*strategy", "reusable.*workflow", "workflow_call",
"parallel.*jobs", "cache.*action", "artifact.*upload"],
"chaos_engineering": ["chaos", "fault.inject", "kill.*pod", "toxiproxy",
"chaos.monkey", "litmus", "gremlin"],
"data_platform": ["airflow", "prefect", "dagster", "spark", "flink",
"dbt", "snowflake", "bigquery", "redshift", "databricks"],
}
# Dimension 3: Integration surface area - external APIs and services
INTEGRATION_SIGNALS = {
# Payment & financial
"stripe": ["stripe", "payment_intent", "stripe_webhook"],
"paypal": ["paypal", "braintree"],
"plaid": ["plaid", "link_token", "access_token.*plaid"],
"adyen": ["adyen"],
"square": ["square", "squareup"],
# Communication
"twilio": ["twilio", "sms.*send", "voice.*call.*api"],
"sendgrid": ["sendgrid"],
"mailgun": ["mailgun"],
"postmark": ["postmark"],
"resend": ["resend"],
"vonage": ["vonage", "nexmo"],
# Auth
"auth0": ["auth0"],
"okta": ["okta", "okta.*sdk"],
"clerk": ["clerk", "clerkjs"],
"firebase_auth": ["firebase.auth", "firebase_auth"],
# Cloud storage
"aws_s3": ["s3_bucket", "s3\\.upload", "putobject", "getobject.*s3", "aws.*s3"],
"gcs": ["google.*storage", "gcs.*bucket", "storage\\.bucket"],
"cloudinary": ["cloudinary"],
# Infra / DevOps APIs
"github_api": ["octokit", "github.*api", "github.*token", "repos.*github"],
"datadog_api": ["datadog.*api", "dd_api_key"],
"pagerduty": ["pagerduty"],
# CRM / Sales
"salesforce": ["salesforce", "sfdc", "soql", "apex.*salesforce"],
"hubspot": ["hubspot", "hapi.*key"],
"intercom": ["intercom"],
# Analytics
"segment": ["segment\\.io", "analytics\\.track", "writekey"],
"mixpanel": ["mixpanel"],
"amplitude": ["amplitude"],
"posthog": ["posthog"],
# Maps / geo
"google_maps": ["maps\\.googleapis", "places.*api", "geocod"],
"mapbox": ["mapbox", "mapbox.*token"],
# ML / AI APIs
"openai_api": ["openai\\.api_key", "openai\\.chat", "gpt.4", "gpt.3"],
"anthropic_api": ["anthropic", "claude.*api"],
# Misc platform
"shopify": ["shopify", "storefront.*api", "admin.*api.*shopify"],
"zendesk": ["zendesk"],
"slack_api": ["slack.*token", "bolt.*slack", "webclient.*slack"],
"linear_api": ["linear.*client", "linearapp"],
"notion_api": ["notion.*api", "notion.*client"],
}
# Dimension 5: Operational maturity signals (content-based)
OPERATIONAL_MATURITY_SIGNALS = {
"slo_definitions": ["error_budget", "slo", "sla.*target", "availability.*target",
"p99", "p95", "latency.*slo", "sli"],
"runbooks": ["runbook", "playbook.*incident", "on.call", "escalation.*policy",
"postmortem", "incident.*response"],
"load_testing": ["k6", "locust", "jmeter", "artillery", "gatling",
"loadtest", "stress.*test", "soak.*test"],
"chaos_testing": ["chaos", "fault.inject", "toxiproxy", "chaos.monkey",
"kill.*service", "network.*partition"],
"feature_flags": ["feature.*flag", "feature.*toggle", "launchdarkly",
"unleash", "flagsmith", "split\\.io", "growthbook"],
"ab_testing": ["a.b.*test", "experiment", "variant", "control.*group",
"treatment.*group", "statistical.*significance"],
"distributed_tracing":["opentelemetry", "jaeger", "zipkin", "x.ray",
"trace.*id", "span.*id", "propagat.*context"],
"advanced_alerting": ["pagerduty", "alertmanager", "alert.*rule",
"threshold.*alert", "anomaly.*detect"],
"canary_deployment": ["canary", "blue.green", "traffic.*split",
"progressive.*delivery", "rollout.*strategy"],
}
def count_orm_models(repo_path: Path) -> int:
"""Estimate number of database tables/models by scanning ORM model definitions."""
model_count = 0
model_patterns = [
re.compile(r"class\s+\w+\s*\(.*Model.*\)", re.IGNORECASE), # Django/SQLAlchemy
re.compile(r"class\s+\w+\s*\(.*Base.*\)", re.IGNORECASE), # SQLAlchemy Base
re.compile(r"model\s+\w+\s*\{", re.IGNORECASE), # Prisma
re.compile(r"create_table\s+['\"]?\w+", re.IGNORECASE), # Rails migration
re.compile(r"@Entity\b", re.IGNORECASE), # JPA/TypeORM
re.compile(r"@Table\(", re.IGNORECASE), # JPA
re.compile(r"Schema\(\s*\{", re.IGNORECASE), # Mongoose
re.compile(r"defmodule\s+\w+\s+do\s*\n.*use.*Ecto\.Schema", re.IGNORECASE | re.MULTILINE), # Ecto
]
skip_dirs = {"node_modules", ".git", "vendor", "venv", ".venv", "__pycache__", "dist", "build"}
try:
for root, dirs, files in os.walk(repo_path):
dirs[:] = [d for d in dirs if d not in skip_dirs]
for fname in files:
if not any(fname.endswith(ext) for ext in [".py", ".rb", ".ts", ".js", ".java", ".kt", ".ex", ".scala"]):
if fname != "schema.prisma":
continue
fpath = Path(root) / fname
try:
content = fpath.read_text(encoding="utf-8", errors="replace")
for pattern in model_patterns:
model_count += len(pattern.findall(content))
except Exception:
pass
except Exception:
pass
return model_count
def count_migrations(repo_path: Path) -> int:
"""Count migration files across common migration directory patterns."""
migration_dirs = [
"migrations", "db/migrate", "alembic/versions",
"flyway/sql", "database/migrations", "prisma/migrations",
"db/migrations", "src/migrations",
]
migration_extensions = {".sql", ".py", ".rb", ".ts", ".js", ".java", ".scala"}
count = 0
for mdir in migration_dirs:
full_path = repo_path / mdir
if full_path.exists() and full_path.is_dir():
for f in full_path.rglob("*"):
if f.is_file() and f.suffix.lower() in migration_extensions:
count += 1
return count
def detect_integrations(signals: dict) -> list:
"""From accumulated signals, return list of detected external integrations."""
found = []
for integration_key in INTEGRATION_SIGNALS:
if integration_key in signals and signals[integration_key] > 0:
found.append(integration_key)
return found
def score_rebuild_difficulty(
domain_hits: dict,
infra_signals: dict,
model_count: int,
migration_count: int,
integration_count: int,
enterprise_integrations: int,
operational_signals: dict,
) -> dict:
"""
Compute the 5-dimension rebuild difficulty score.
Returns a dict with per-dimension scores, total, rating, and multiplier.
"""
# --- Dimension 1: Domain Knowledge ---
domain_score = 0.0
detected_domains = []
for domain, (_, diff_score) in DOMAIN_SIGNALS.items():
if domain in domain_hits and domain_hits[domain] > 0:
detected_domains.append(domain)
domain_score = max(domain_score, diff_score) # take highest single domain
# Stack bonus for multiple high domains
high_domains = [d for d in detected_domains if DOMAIN_SIGNALS[d][1] >= 2]
if len(high_domains) >= 2:
domain_score = min(domain_score + 1, 4)
# --- Dimension 2: Infrastructure Coupling ---
infra_score = 0.0
detected_infra = []
if "kubernetes_advanced" in infra_signals and infra_signals["kubernetes_advanced"] > 0:
infra_score = 3.0
detected_infra.append("kubernetes_advanced")
elif "service_mesh" in infra_signals and infra_signals["service_mesh"] > 0:
infra_score = max(infra_score, 2.5)
detected_infra.append("service_mesh")
elif "kubernetes_basic" in infra_signals and infra_signals["kubernetes_basic"] > 0:
infra_score = max(infra_score, 2.0)
detected_infra.append("kubernetes_basic")
if "terraform" in infra_signals and infra_signals["terraform"] > 0:
infra_score = max(infra_score, infra_score + 1.0)
detected_infra.append("terraform")
if "data_platform" in infra_signals and infra_signals["data_platform"] > 0:
infra_score = max(infra_score, infra_score + 1.0)
detected_infra.append("data_platform")
if "chaos_engineering" in infra_signals and infra_signals["chaos_engineering"] > 0:
infra_score += 0.5
detected_infra.append("chaos_engineering")
if "docker_compose" in infra_signals and infra_signals["docker_compose"] > 0 and infra_score == 0:
infra_score = 0.5
detected_infra.append("docker_compose")
infra_score = min(infra_score, 4.0)
# --- Dimension 3: Data Model ---
data_score = 0.0
# Tables/models
if model_count >= 100:
data_score = 3.0
elif model_count >= 60:
data_score = 2.5
elif model_count >= 30:
data_score = 2.0
elif model_count >= 10:
data_score = 1.0
# Migrations
if migration_count >= 300:
data_score = max(data_score, 3.0)
elif migration_count >= 100:
data_score = max(data_score, 2.0)
elif migration_count >= 20:
data_score = max(data_score, 1.0)
# --- Dimension 4: Integration Surface ---
integration_score = 0.0
if integration_count >= 16:
integration_score = 3.0
elif integration_count >= 11:
integration_score = 2.5
elif integration_count >= 6:
integration_score = 2.0
elif integration_count >= 3:
integration_score = 1.0
integration_score += min(enterprise_integrations, 2) # cap bonus at +2
# --- Dimension 5: Operational Maturity ---
ops_score = 0.0
ops_signal_count = sum(1 for v in operational_signals.values() if v > 0)
if "chaos_testing" in operational_signals and operational_signals["chaos_testing"] > 0:
ops_score = 3.0
elif "canary_deployment" in operational_signals and operational_signals["canary_deployment"] > 0:
ops_score = max(ops_score, 2.5)
if ops_signal_count >= 5:
ops_score = max(ops_score, 2.5)
elif ops_signal_count >= 3:
ops_score = max(ops_score, 2.0)
elif ops_signal_count >= 2:
ops_score = max(ops_score, 1.0)
elif ops_signal_count == 1:
ops_score = max(ops_score, 0.5)
total = domain_score + infra_score + data_score + integration_score + ops_score
# Rating
if total <= 2:
rating = "LOW"
multiplier_low, multiplier_high = 1.0, 1.0
elif total <= 4:
rating = "MODERATE"
multiplier_low, multiplier_high = 1.1, 1.2
elif total <= 7:
rating = "HIGH"
multiplier_low, multiplier_high = 1.25, 1.45
elif total <= 10:
rating = "VERY HIGH"
multiplier_low, multiplier_high = 1.5, 1.7
elif total <= 14:
rating = "EXTREME"
multiplier_low, multiplier_high = 1.9, 2.5
else:
rating = "EXCEPTIONAL"
multiplier_low, multiplier_high = 2.5, 4.0
return {
"rating": rating,
"total_score": round(total, 1),
"multiplier_range": f"{multiplier_low}–{multiplier_high}×",
"dimensions": {
"domain_knowledge": {
"score": round(domain_score, 1),
"detected_domains": detected_domains,
},
"infrastructure_coupling": {
"score": round(infra_score, 1),
"detected_signals": detected_infra,
},
"data_model": {
"score": round(data_score, 1),
"estimated_models": model_count,
"migration_count": migration_count,
},
"integration_surface": {
"score": round(integration_score, 1),
"integration_count": integration_count,
"enterprise_integrations": enterprise_integrations,
},
"operational_maturity": {
"score": round(ops_score, 1),
"signals_detected": ops_signal_count,
},
},
"note": "See references/rebuild-difficulty.md for scoring methodology",
}
def is_generated_file(content_head: str) -> bool:
for pattern in GENERATED_PATTERNS:
if re.search(pattern, content_head, re.IGNORECASE):
return True
return False
def count_lines(filepath: Path) -> tuple[int, bool]:
"""Return (line_count, is_generated). Returns (0, False) on binary/error."""
try:
with open(filepath, "r", encoding="utf-8", errors="replace") as f:
content = f.read()
lines = content.split("\n")
generated = is_generated_file(content[:500])
return len(lines), generated
except Exception:
return 0, False
def scan_file_for_signals(filepath: Path, signals: dict) -> set[str]:
"""Check a file for complexity signals. Returns set of matching signal keys."""
found = set()
try:
with open(filepath, "r", encoding="utf-8", errors="replace") as f:
content = f.read().lower()
for signal_key, patterns in signals.items():
for pattern in patterns:
if re.search(pattern, content, re.IGNORECASE):
found.add(signal_key)
break
except Exception:
pass
return found
def get_git_stats(repo_path: Path) -> dict:
"""Extract git history statistics."""
stats = {
"has_git": False,
"total_commits": None,
"contributors": None,
"first_commit_date": None,
"last_commit_date": None,
"active_months": None,
}
git_dir = repo_path / ".git"
if not git_dir.exists():
return stats
stats["has_git"] = True
try:
result = subprocess.run(
["git", "-C", str(repo_path), "log", "--oneline"],
capture_output=True, text=True, timeout=15
)
if result.returncode == 0:
lines = [l for l in result.stdout.strip().split("\n") if l]
stats["total_commits"] = len(lines)
result = subprocess.run(
["git", "-C", str(repo_path), "shortlog", "-sn", "--no-merges"],
capture_output=True, text=True, timeout=15
)
if result.returncode == 0:
contributors = [l.strip() for l in result.stdout.strip().split("\n") if l.strip()]
stats["contributors"] = len(contributors)
result = subprocess.run(
["git", "-C", str(repo_path), "log", "--format=%ai", "--reverse"],
capture_output=True, text=True, timeout=15
)
if result.returncode == 0:
dates = [l.strip() for l in result.stdout.strip().split("\n") if l.strip()]
if dates:
stats["first_commit_date"] = dates[0][:10]
stats["last_commit_date"] = dates[-1][:10]
try:
first = datetime.fromisoformat(dates[0][:10])
last = datetime.fromisoformat(dates[-1][:10])
delta = last - first
stats["active_months"] = max(1, round(delta.days / 30))
except Exception:
pass
except Exception as e:
stats["git_error"] = str(e)
return stats
def find_key_files(repo_path: Path) -> dict:
"""Detect important configuration and project files."""
key_files = {
"package_json": False,
"requirements_txt": False,
"pyproject_toml": False,
"gemfile": False,
"go_mod": False,
"cargo_toml": False,
"pubspec_yaml": False,
"build_gradle": False,
"pom_xml": False,
"dockerfile": False,
"docker_compose": False,
"kubernetes_manifests": False,
"terraform_files": False,
"github_actions": False,
"circleci": False,
"makefile": False,
"readme": False,
"openapi_spec": False,
"prisma_schema": False,
"graphql_schema": False,
"env_example": False,
"migrations_dir": False,
}
for name in os.listdir(repo_path):
lower = name.lower()
if lower == "package.json": key_files["package_json"] = True
if lower == "requirements.txt": key_files["requirements_txt"] = True
if lower == "pyproject.toml": key_files["pyproject_toml"] = True
if lower == "gemfile": key_files["gemfile"] = True
if lower == "go.mod": key_files["go_mod"] = True
if lower == "cargo.toml": key_files["cargo_toml"] = True
if lower == "pubspec.yaml": key_files["pubspec_yaml"] = True
if "build.gradle" in lower: key_files["build_gradle"] = True
if lower == "pom.xml": key_files["pom_xml"] = True
if lower == "dockerfile": key_files["dockerfile"] = True
if "docker-compose" in lower: key_files["docker_compose"] = True
if lower == "makefile": key_files["makefile"] = True
if "readme" in lower: key_files["readme"] = True
if lower in (".env.example", ".env.sample", ".env.template"): key_files["env_example"] = True
if lower in ("migrations", "db/migrate", "alembic"): key_files["migrations_dir"] = True
if lower == ".github": key_files["github_actions"] = True
if lower == ".circleci": key_files["circleci"] = True
# Walk for deeper files
for root, dirs, files in os.walk(repo_path):
# Prune skipped dirs
dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
for f in files:
lower = f.lower()
rel = os.path.relpath(os.path.join(root, f), repo_path)
if "openapi" in lower or "swagger" in lower: key_files["openapi_spec"] = True
if lower == "schema.prisma": key_files["prisma_schema"] = True
if lower.endswith(".graphql") or lower.endswith(".gql"): key_files["graphql_schema"] = True
if "terraform" in lower or lower.endswith(".tf"): key_files["terraform_files"] = True
if "kubernetes" in rel or "/k8s/" in rel or "/manifests/" in rel: key_files["kubernetes_manifests"] = True
return key_files
def analyze_repository(repo_path: Path, verbose: bool = False) -> dict:
"""Main analysis function. Returns structured summary dict."""
repo_path = repo_path.resolve()
if not repo_path.exists():
return {"error": f"Path does not exist: {repo_path}"}
if verbose:
print(f"Analyzing: {repo_path}", file=sys.stderr)
# Counters
lang_lines = defaultdict(int) # language -> total LOC
lang_files = defaultdict(int) # language -> file count
generated_lines = 0
generated_files = 0
skipped_files = 0
total_files = 0
test_files = 0
test_lines = 0
config_lines = 0
all_signals = defaultdict(int) # signal_key -> file count
domain_hits = defaultdict(int) # domain_key -> file count
infra_hits = defaultdict(int) # infra_key -> file count
integration_hits = defaultdict(int) # integration_key -> file count
ops_hits = defaultdict(int) # ops_key -> file count
dir_summary = defaultdict(int) # top-level dir -> file count
file_size_buckets = {"tiny": 0, "small": 0, "medium": 0, "large": 0, "huge": 0}
largest_files = []
# Build flat signal dicts for scanning (patterns only, no metadata)
domain_patterns = {k: v[0] for k, v in DOMAIN_SIGNALS.items()}
infra_patterns = {k: v for k, v in INFRA_COUPLING_SIGNALS.items()}
integration_patterns = {k: v for k, v in INTEGRATION_SIGNALS.items()}
ops_patterns = {k: v for k, v in OPERATIONAL_MATURITY_SIGNALS.items()}
for root, dirs, files in os.walk(repo_path):
rel_root = Path(root).relative_to(repo_path)
parts = rel_root.parts
# Track top-level directory
top_dir = parts[0] if parts else "(root)"
# Prune skipped directories
dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".")]
for filename in files:
filepath = Path(root) / filename
ext = filepath.suffix.lower()
rel_path = str(filepath.relative_to(repo_path))
total_files += 1
# Skip binary/asset extensions
if ext in SKIP_EXTENSIONS:
skipped_files += 1
continue
# Skip lock files and other auto-generated
if filename in ("package-lock.json", "yarn.lock", "pnpm-lock.yaml",
"Gemfile.lock", "poetry.lock", "Pipfile.lock",
"cargo.lock", "go.sum", "composer.lock"):
skipped_files += 1
continue
lang = EXTENSION_MAP.get(ext, f"Other ({ext})" if ext else "No extension")
lines, is_generated = count_lines(filepath)
if is_generated:
generated_lines += lines
generated_files += 1
skipped_files += 1
continue
# Classify test files
is_test = (
"test" in rel_path.lower() or
"spec" in rel_path.lower() or
filename.startswith("test_") or
filename.endswith("_test.py") or
filename.endswith(".test.js") or
filename.endswith(".test.ts") or
filename.endswith(".spec.js") or
filename.endswith(".spec.ts") or
filename.endswith("_spec.rb")
)
# Classify config files
is_config = ext in (".json", ".yaml", ".yml", ".toml", ".env", ".ini", ".cfg", ".hcl")
if is_test:
test_files += 1
test_lines += lines
elif is_config:
config_lines += lines
lang_lines[lang] += lines
lang_files[lang] += 1
dir_summary[top_dir] += 1
# File size classification
if lines < 50:
file_size_buckets["tiny"] += 1
elif lines < 200:
file_size_buckets["small"] += 1
elif lines < 500:
file_size_buckets["medium"] += 1
elif lines < 1000:
file_size_buckets["large"] += 1
else:
file_size_buckets["huge"] += 1
largest_files.append({"path": rel_path, "lines": lines, "lang": lang})
# Scan for all signal types (skip tiny config files for speed)
if lines > 10 and ext not in (".json", ".yaml", ".yml", ".md", ".txt", ".env"):
signals = scan_file_for_signals(filepath, COMPLEXITY_SIGNALS)
for s in signals:
all_signals[s] += 1
# Rebuild difficulty dimensions
for s in scan_file_for_signals(filepath, domain_patterns):
domain_hits[s] += 1
for s in scan_file_for_signals(filepath, infra_patterns):
infra_hits[s] += 1
for s in scan_file_for_signals(filepath, integration_patterns):
integration_hits[s] += 1
for s in scan_file_for_signals(filepath, ops_patterns):
ops_hits[s] += 1
elif lines > 5 and ext in (".yaml", ".yml"):
# Infra and ops signals commonly live in YAML (k8s, CI, monitoring)
for s in scan_file_for_signals(filepath, infra_patterns):
infra_hits[s] += 1
for s in scan_file_for_signals(filepath, ops_patterns):
ops_hits[s] += 1
if verbose and total_files % 500 == 0:
print(f" Scanned {total_files} files...", file=sys.stderr)
# Sort and trim
largest_files.sort(key=lambda x: x["lines"], reverse=True)
largest_files = largest_files[:20]
top_langs = sorted(lang_lines.items(), key=lambda x: x[1], reverse=True)
total_loc = sum(lang_lines.values())
total_analyzed = total_files - skipped_files
# Test ratio
test_ratio = round(test_lines / total_loc, 3) if total_loc > 0 else 0
# Coverage signal
if test_ratio == 0:
test_coverage_signal = "none"
elif test_ratio < 0.10:
test_coverage_signal = "minimal"
elif test_ratio < 0.25:
test_coverage_signal = "moderate"
elif test_ratio < 0.50:
test_coverage_signal = "thorough"
else:
test_coverage_signal = "comprehensive"
git_stats = get_git_stats(repo_path)
key_files = find_key_files(repo_path)
# Top-level directory breakdown (top 15)
top_dirs = sorted(dir_summary.items(), key=lambda x: x[1], reverse=True)[:15]
# Signals summary — only include those with >= 1 file hit
significant_signals = {k: v for k, v in all_signals.items() if v >= 1}
significant_signals = dict(sorted(significant_signals.items(), key=lambda x: x[1], reverse=True))
# --- Rebuild Difficulty Analysis ---
model_count = count_orm_models(repo_path)
migration_count = count_migrations(repo_path)
detected_integrations = detect_integrations(integration_hits)
# Flag enterprise/complex integrations for bonus scoring
enterprise_integration_keys = {"salesforce", "hubspot", "okta", "zendesk", "shopify"}
enterprise_count = sum(1 for i in detected_integrations if i in enterprise_integration_keys)
rebuild_difficulty = score_rebuild_difficulty(
domain_hits=dict(domain_hits),
infra_signals=dict(infra_hits),
model_count=model_count,
migration_count=migration_count,
integration_count=len(detected_integrations),
enterprise_integrations=enterprise_count,
operational_signals=dict(ops_hits),
)
rebuild_difficulty["detected_integrations"] = detected_integrations
result = {
"meta": {
"repo_path": str(repo_path),
"repo_name": repo_path.name,
"analyzed_at": datetime.now(timezone.utc).isoformat(),
},
"file_counts": {
"total_files": total_files,
"analyzed_files": total_analyzed,
"skipped_files": skipped_files,
"generated_files": generated_files,
"test_files": test_files,
},
"line_counts": {
"total_loc": total_loc,
"test_lines": test_lines,
"config_lines": config_lines,
"generated_lines": generated_lines,
"non_test_loc": total_loc - test_lines - config_lines,
},
"test_coverage": {
"signal": test_coverage_signal,
"test_to_code_ratio": test_ratio,
"test_files": test_files,
"test_lines": test_lines,
},
"languages": {
"primary": top_langs[0][0] if top_langs else "Unknown",
"breakdown": [
{
"language": lang,
"lines": lines,
"files": lang_files[lang],
"pct": round(lines / total_loc * 100, 1) if total_loc > 0 else 0,
}
for lang, lines in top_langs[:15]
],
},
"file_complexity": {
"size_distribution": file_size_buckets,
"largest_files": largest_files,
},
"top_directories": [{"dir": d, "files": c} for d, c in top_dirs],
"complexity_signals": significant_signals,
"key_files": key_files,
"git_stats": git_stats,
"rebuild_difficulty": rebuild_difficulty,
}
return result
def main():
parser = argparse.ArgumentParser(
description="Analyze a repository and output structured JSON for estimation."
)
parser.add_argument("repo_path", help="Path to the repository to analyze")
parser.add_argument("--output", "-o", help="Output file path (default: stdout)")
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose progress output")
args = parser.parse_args()
result = analyze_repository(Path(args.repo_path), verbose=args.verbose)
output = json.dumps(result, indent=2)
if args.output:
with open(args.output, "w") as f:
f.write(output)
print(f"Analysis written to {args.output}", file=sys.stderr)
else:
print(output)
if __name__ == "__main__":
main()