#!/usr/bin/env python3
"""
AST-based index extractor for the wiki-doc protocol (v1).
Extracts function definitions, module-level / class-level / closure-local variables,
and class definitions from Python source files. Powers Level 2a / Level 2b line-number
generation, the salvage protocol's pre-documentation audit (Phase S-3), and the
Reference Refresh diff mode (R-1).
Usage:
python ast_index.py <source.py> # functions only
python ast_index.py <source.py> --vars # functions + variables
python ast_index.py <source.py> --all # functions + vars + classes
python ast_index.py <source.py> --audit # pre-S-3 audit report
python ast_index.py <source.py> --diff <snapshot.tsv> # drift report vs. snapshot
Output format (default, --vars, --all):
KIND<TAB>NAME<TAB>LINE<TAB>SCOPE
where KIND is DEF | VAR | CLASS, and SCOPE is auto-detected.
Output format (--audit):
Structured Markdown report on stdout.
Output format (--diff):
Structured drift report on stdout (DRIFT/ADDED/DELETED/UNCHANGED rows).
Scope tags:
top — module-level
cls:ClassName — class method (immediate member)
cls:ClassName (in <enclosing>) — nested def inside a class method
cl:<closure_name> — closure-local def
cl:<outer> (in <inner>) — sub-closure-local def
Note: Python-specific. For other languages, author an equivalent tool that produces the
same output format. See SKILL.md Section 20.
"""
import ast
import argparse
import sys
from collections import defaultdict
# ---------------------------------------------------------------------------
# Scope tracking
# ---------------------------------------------------------------------------
def _scope_tag(stack):
"""
Compute the scope tag for a node whose enclosing context is `stack`.
`stack` is a list of (kind, name) tuples representing the chain of enclosing
scopes from outermost to innermost — but NOT including the current node itself.
Rules:
empty stack → "top"
stack ends in ClassDef (immediate) → "cls:ClassName"
stack ends in FunctionDef (immediate) → "cl:FunctionName"
deeper nesting:
cls then fn → "cls:ClassName (in fn_name)"
fn then fn → "cl:outer_fn (in inner_fn)"
cls then fn then fn → "cls:ClassName (in inner_fn)"
The convention prioritizes the most recent class context for class-method-nested
defs, with the immediately enclosing function name in the parenthetical.
"""
if not stack:
return "top"
# Find the deepest class in the stack (if any) and the deepest function.
deepest_class_idx = None
for i in range(len(stack) - 1, -1, -1):
if stack[i][0] == "ClassDef":
deepest_class_idx = i
break
immediate = stack[-1]
if deepest_class_idx is not None:
class_name = stack[deepest_class_idx][1]
# If the immediate enclosing scope is the class itself, no (in ...) needed.
if deepest_class_idx == len(stack) - 1:
return f"cls:{class_name}"
# Otherwise the immediate enclosing scope is a function inside the class
# (or nested deeper). Use the immediate function name in the parenthetical.
if immediate[0] in ("FunctionDef", "AsyncFunctionDef"):
return f"cls:{class_name} (in {immediate[1]})"
# Edge case: shouldn't happen with sane Python, but fall back gracefully.
return f"cls:{class_name}"
else:
# Pure closure nesting (no class in the chain).
if len(stack) == 1:
return f"cl:{stack[0][1]}"
# Outermost function name + immediate enclosing function name.
outer = stack[0][1]
immediate_name = immediate[1]
if outer == immediate_name:
return f"cl:{outer}"
return f"cl:{outer} (in {immediate_name})"
def walk_with_scope(tree):
"""
Walk the AST yielding (node, scope_tag) tuples. Scope is computed for each
node based on its enclosing chain. Stack tracks (kind, name) of enclosing
scopes from outermost to innermost.
"""
def _walk(node, stack):
# Yield this node first with current scope.
yield node, _scope_tag(stack)
# Decide whether to push this node onto the stack for its children.
push = None
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
push = (type(node).__name__, node.name)
elif isinstance(node, ast.ClassDef):
push = ("ClassDef", node.name)
new_stack = stack + [push] if push else stack
for child in ast.iter_child_nodes(node):
yield from _walk(child, new_stack)
yield from _walk(tree, [])
# ---------------------------------------------------------------------------
# Extraction
# ---------------------------------------------------------------------------
def extract_functions(tree):
"""All function definitions with name, line number, scope."""
results = []
for node, scope in walk_with_scope(tree):
# Skip the module node itself; we want children only.
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
# `scope` was computed for the node based on its enclosing chain
# — that's exactly the scope tag for this def.
results.append(("DEF", node.name, node.lineno, scope))
return results
def extract_classes(tree):
"""All class definitions with name, line number, scope."""
results = []
for node, scope in walk_with_scope(tree):
if isinstance(node, ast.ClassDef):
results.append(("CLASS", node.name, node.lineno, scope))
return results
def extract_module_vars(tree):
"""
Module-level + closure-local-introduced + class-level variable assignments.
Captures:
- Top-level Assign / AnnAssign with a Name target.
- Inside FunctionDef bodies: variables introduced by Assign / AnnAssign /
For / With targets at the function's top level (not inside nested defs).
Scope tag is the enclosing closure (cl:<fn>).
- Inside ClassDef bodies: class-level Assign / AnnAssign with a Name target.
Scope tag is cls:<Class>.
Skips: nested defs' locals (those are covered by their own L2b extension if needed).
"""
results = []
for node, scope in walk_with_scope(tree):
# Top-level assignments
if isinstance(node, ast.Module):
for child in ast.iter_child_nodes(node):
if isinstance(child, ast.Assign):
for target in child.targets:
if isinstance(target, ast.Name):
results.append(("VAR", target.id, child.lineno, "top"))
elif isinstance(child, ast.AnnAssign) and isinstance(child.target, ast.Name):
results.append(("VAR", child.target.id, child.lineno, "top"))
# Class-level assignments
if isinstance(node, ast.ClassDef):
class_scope = f"cls:{node.name}"
for child in node.body:
if isinstance(child, ast.Assign):
for target in child.targets:
if isinstance(target, ast.Name):
results.append(("VAR", target.id, child.lineno, class_scope))
elif isinstance(child, ast.AnnAssign) and isinstance(child.target, ast.Name):
results.append(("VAR", child.target.id, child.lineno, class_scope))
return results
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def emit_rows(rows, out=sys.stdout):
"""Sort by line number then emit TSV rows: KIND\\tNAME\\tLINE\\tSCOPE."""
rows.sort(key=lambda r: r[2])
for kind, name, lineno, scope in rows:
out.write(f"{kind}\t{name}\t{lineno}\t{scope}\n")
# ---------------------------------------------------------------------------
# Audit mode (Phase S-3)
# ---------------------------------------------------------------------------
def run_audit(tree, source_path):
"""
Pre-documentation audit. Surfaces:
1. Duplicate-name candidates (same def name in multiple scopes).
2. Zero-caller candidates (defs with no observed call sites in this file).
3. Shape anomalies (defs inside `if False:` or `if 0:` branches).
"""
# 1. Collect all defs with their scopes.
defs = []
for node, scope in walk_with_scope(tree):
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
defs.append((node.name, node.lineno, scope))
# 2. Duplicates: group by name across different scopes.
by_name = defaultdict(list)
for name, line, scope in defs:
by_name[name].append((line, scope))
duplicates = {n: locs for n, locs in by_name.items() if len(locs) > 1}
# 3. Zero-callers: collect all called names within the file.
called_names = set()
for node in ast.walk(tree):
if isinstance(node, ast.Call):
func = node.func
if isinstance(func, ast.Name):
called_names.add(func.id)
elif isinstance(func, ast.Attribute):
called_names.add(func.attr)
elif isinstance(node, ast.Attribute):
# Conservative: an attribute access like `self._method` may or may not
# be a call site — count it as a potential reference.
called_names.add(node.attr)
# Also count names referenced in command= / .bind() etc. — these appear as
# Name nodes inside Attribute or kwarg contexts, not as direct Call.func.
for node in ast.walk(tree):
if isinstance(node, ast.Name):
called_names.add(node.id)
zero_callers = []
for name, line, scope in defs:
# A def is zero-caller if its name is NEVER referenced anywhere in the file
# except in its own definition. The Name walk above includes the def's own
# FunctionDef.name implicitly via the AST? Actually no — FunctionDef has
# node.name as a string, not an ast.Name. So if the function's name appears
# anywhere as an ast.Name, it's a reference (call site or assignment).
if name not in called_names:
zero_callers.append((name, line, scope))
# 4. Shape anomalies: defs inside `if False:` / `if 0:` / `if __debug__ is False:` etc.
anomalies = []
for node in ast.walk(tree):
if isinstance(node, ast.If):
test = node.test
unreachable = False
if isinstance(test, ast.Constant) and test.value in (False, 0, None):
unreachable = True
elif isinstance(test, ast.NameConstant) and test.value is False: # py<3.8
unreachable = True
if unreachable:
# Walk node's body for any defs.
for child in ast.walk(node):
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
anomalies.append((child.name, child.lineno, "inside-unreachable-if"))
# Emit Markdown audit report.
print(f"# Pre-S-3 Audit Report — {source_path}")
print()
print(f"**Total defs scanned:** {len(defs)}")
print()
print("---")
print()
print("## 1. Duplicate-name candidates")
print()
if not duplicates:
print("None found.")
else:
print(f"Found {len(duplicates)} name(s) with multiple scope occurrences:")
print()
for name in sorted(duplicates.keys()):
print(f"### `{name}`")
for line, scope in sorted(duplicates[name]):
print(f"- line {line}, scope `{scope}`")
print()
print()
print("---")
print()
print("## 2. Zero-caller candidates (intra-file)")
print()
if not zero_callers:
print("None found.")
else:
print(f"Found {len(zero_callers)} def(s) with no observed in-file call sites.")
print("Note: this is intra-file only. Cross-file callers are not detected.")
print()
print("| Name | Line | Scope |")
print("|------|------|-------|")
for name, line, scope in sorted(zero_callers, key=lambda r: r[1]):
print(f"| `{name}` | {line} | `{scope}` |")
print()
print()
print("---")
print()
print("## 3. Shape anomalies")
print()
if not anomalies:
print("None found.")
else:
print(f"Found {len(anomalies)} def(s) inside potentially unreachable branches.")
print()
print("| Name | Line | Anomaly |")
print("|------|------|---------|")
for name, line, kind in sorted(anomalies, key=lambda r: r[1]):
print(f"| `{name}` | {line} | `{kind}` |")
# ---------------------------------------------------------------------------
# Diff mode (Reference Refresh, R-1)
# ---------------------------------------------------------------------------
def run_diff(tree, snapshot_path):
"""
Compare current AST against a saved snapshot TSV. Outputs drift report.
Snapshot format: KIND\\tNAME\\tLINE\\tSCOPE (one entry per line).
"""
# Load snapshot.
snapshot = {}
try:
with open(snapshot_path, encoding="utf-8") as f:
for raw in f:
line = raw.rstrip("\n")
if not line:
continue
parts = line.split("\t")
if len(parts) == 3:
# v0.2 format (no scope) — treat scope as empty string.
kind, name, lineno = parts
scope = ""
elif len(parts) == 4:
kind, name, lineno, scope = parts
else:
continue
key = (kind, name, scope)
snapshot[key] = int(lineno)
except FileNotFoundError:
print(f"ERROR: snapshot file not found: {snapshot_path}", file=sys.stderr)
sys.exit(2)
# Compute current.
current_rows = []
current_rows.extend(extract_functions(tree))
current_rows.extend(extract_module_vars(tree))
current_rows.extend(extract_classes(tree))
current = {(k, n, s): l for k, n, l, s in current_rows}
# Compare.
snapshot_keys = set(snapshot.keys())
current_keys = set(current.keys())
added = current_keys - snapshot_keys
deleted = snapshot_keys - current_keys
common = current_keys & snapshot_keys
shifted = []
unchanged = []
for key in common:
if snapshot[key] != current[key]:
shifted.append((key, snapshot[key], current[key]))
else:
unchanged.append((key, current[key]))
# Emit report.
print(f"# Drift Report")
print()
print(f"Snapshot: `{snapshot_path}`")
print()
print(f"- Added: {len(added)}")
print(f"- Deleted: {len(deleted)}")
print(f"- Shifted: {len(shifted)}")
print(f"- Unchanged: {len(unchanged)}")
print()
if shifted:
print("## SHIFTED")
print()
print("| Kind | Name | Scope | Old line | New line | Δ |")
print("|------|------|-------|----------|----------|---|")
for (kind, name, scope), old, new in sorted(shifted, key=lambda r: r[2]):
delta = new - old
print(f"| {kind} | `{name}` | `{scope}` | {old} | {new} | {delta:+d} |")
print()
if added:
print("## ADDED")
print()
print("| Kind | Name | Scope | Line |")
print("|------|------|-------|------|")
for kind, name, scope in sorted(added, key=lambda r: current[r]):
print(f"| {kind} | `{name}` | `{scope}` | {current[(kind, name, scope)]} |")
print()
if deleted:
print("## DELETED")
print()
print("| Kind | Name | Scope | Old line |")
print("|------|------|-------|----------|")
for kind, name, scope in sorted(deleted, key=lambda r: snapshot[r]):
print(f"| {kind} | `{name}` | `{scope}` | {snapshot[(kind, name, scope)]} |")
print()
# ---------------------------------------------------------------------------
# Entrypoint
# ---------------------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(
description="AST index extractor for wiki-doc protocol (v1).")
parser.add_argument("source", help="Python source file to analyze")
parser.add_argument("--vars", action="store_true",
help="Include module-level and class-level variables")
parser.add_argument("--all", action="store_true",
help="Include functions, variables, and classes")
parser.add_argument("--audit", action="store_true",
help="Run pre-documentation audit (Phase S-3): duplicates, "
"zero-callers, shape anomalies. Output is Markdown report.")
parser.add_argument("--diff", metavar="SNAPSHOT",
help="Compare current AST to a saved snapshot TSV; emit drift "
"report. Powers Reference Refresh (R-1).")
args = parser.parse_args()
with open(args.source, encoding="utf-8") as f:
source = f.read()
tree = ast.parse(source)
if args.audit:
run_audit(tree, args.source)
return
if args.diff:
run_diff(tree, args.diff)
return
rows = extract_functions(tree)
if args.vars or args.all:
rows.extend(extract_module_vars(tree))
if args.all:
rows.extend(extract_classes(tree))
emit_rows(rows)
if __name__ == "__main__":
main()