LeoYeAI/agent-master-skills / skills/agent-mode-upgrades
agentic-loop-upgrade
FlaggedEnhanced agentic loop with planning, parallel execution, confidence gates, semantic error recovery, and observable state machine. Includes Mode dashboard UI for easy configuration.
src/state/step-tracker.ts
View on GitHub/**
* Automatic Step Completion Detection
*
* Analyzes tool results to determine if plan steps are complete.
* Uses heuristics and optional LLM verification.
*/
import type { PlanStep, ToolCall, ToolResult, LLMCaller } from "../types.js";
import type { PlanStateManager, PlanState } from "./persistence.js";
// ============================================================================
// Types
// ============================================================================
export interface StepTrackerConfig {
/** Use LLM to verify completion (more accurate but costs tokens) */
useLLMVerification: boolean;
/** Confidence threshold for heuristic-only detection (0-1) */
heuristicThreshold: number;
/** Auto-advance to next step on completion */
autoAdvance: boolean;
/** Keywords that indicate failure */
failureKeywords: string[];
/** Keywords that indicate success */
successKeywords: string[];
}
export interface CompletionAnalysis {
isComplete: boolean;
confidence: number;
reason: string;
suggestedResult?: string;
}
const DEFAULT_CONFIG: StepTrackerConfig = {
useLLMVerification: false,
heuristicThreshold: 0.7,
autoAdvance: true,
failureKeywords: [
"error", "failed", "failure", "exception", "cannot", "unable",
"denied", "permission", "not found", "timeout", "refused"
],
successKeywords: [
"success", "complete", "created", "done", "finished", "wrote",
"saved", "built", "deployed", "installed", "configured"
],
};
// ============================================================================
// Step Tracker
// ============================================================================
export class StepTracker {
private config: StepTrackerConfig;
private stateManager: PlanStateManager;
private llmCaller?: LLMCaller;
constructor(
stateManager: PlanStateManager,
config: Partial<StepTrackerConfig> = {},
llmCaller?: LLMCaller
) {
this.stateManager = stateManager;
this.config = { ...DEFAULT_CONFIG, ...config };
this.llmCaller = llmCaller;
}
/**
* Analyze a tool result to determine if current step is complete
*/
async analyzeToolResult(
tool: ToolCall,
result: ToolResult
): Promise<CompletionAnalysis> {
const state = this.stateManager.getOrNull();
const activeStep = state ? this.stateManager.getActiveStep() : null;
if (!activeStep) {
return {
isComplete: false,
confidence: 1.0,
reason: "No active step to track",
};
}
// First, do heuristic analysis
const heuristic = this.heuristicAnalysis(activeStep, tool, result);
// If confident enough or LLM verification disabled, return heuristic
if (heuristic.confidence >= this.config.heuristicThreshold || !this.config.useLLMVerification) {
if (heuristic.isComplete && this.config.autoAdvance) {
this.stateManager.completeStep(activeStep.id, heuristic.suggestedResult);
}
return heuristic;
}
// Use LLM for verification if available
if (this.llmCaller) {
const llmAnalysis = await this.llmVerification(activeStep, tool, result, heuristic);
if (llmAnalysis.isComplete && this.config.autoAdvance) {
this.stateManager.completeStep(activeStep.id, llmAnalysis.suggestedResult);
}
return llmAnalysis;
}
return heuristic;
}
/**
* Manually mark step complete (for explicit completion)
*/
markComplete(stepId?: string, result?: string): void {
const state = this.stateManager.getOrNull();
if (!state?.plan) return;
const id = stepId ?? state.activeStepId;
if (id) {
this.stateManager.completeStep(id, result);
}
}
/**
* Manually mark step failed
*/
markFailed(stepId?: string, error?: string): void {
const state = this.stateManager.getOrNull();
if (!state?.plan) return;
const id = stepId ?? state.activeStepId;
if (id) {
this.stateManager.failStep(id, error ?? "Step failed");
}
}
/**
* Check if plan is complete
*/
isPlanComplete(): boolean {
const state = this.stateManager.getOrNull();
if (!state?.plan) return false;
return state.plan.steps.every(
s => s.status === "complete" || s.status === "skipped"
);
}
/**
* Get completion status summary
*/
getStatus(): {
hasActivePlan: boolean;
activeStep: PlanStep | null;
progress: { completed: number; total: number; percent: number };
isComplete: boolean;
} {
const state = this.stateManager.getOrNull();
return {
hasActivePlan: !!state?.plan,
activeStep: this.stateManager.getActiveStep(),
progress: this.stateManager.getProgress(),
isComplete: this.isPlanComplete(),
};
}
// ──────────────────────────────────────────────────────────────────────────
// Private Methods
// ──────────────────────────────────────────────────────────────────────────
private heuristicAnalysis(
step: PlanStep,
tool: ToolCall,
result: ToolResult
): CompletionAnalysis {
const resultText = this.resultToText(result);
const lowerResult = resultText.toLowerCase();
const lowerAction = step.action.toLowerCase();
const lowerCriteria = step.successCriteria.toLowerCase();
// Check for explicit failure
const hasFailure = this.config.failureKeywords.some(kw =>
lowerResult.includes(kw) && !lowerResult.includes(`no ${kw}`)
);
if (hasFailure && result.error) {
return {
isComplete: false,
confidence: 0.9,
reason: `Tool returned error: ${result.error.slice(0, 100)}`,
};
}
// Check for explicit success keywords
const successCount = this.config.successKeywords.filter(kw =>
lowerResult.includes(kw)
).length;
// Check if tool matches expected action
const toolMatchesAction = this.toolMatchesStepAction(tool.name, lowerAction);
// Check if result mentions success criteria keywords
const criteriaWords = lowerCriteria.split(/\s+/).filter(w => w.length > 3);
const criteriaMatches = criteriaWords.filter(w => lowerResult.includes(w)).length;
const criteriaMatchRatio = criteriaWords.length > 0
? criteriaMatches / criteriaWords.length
: 0;
// Calculate confidence
let confidence = 0;
let reasons: string[] = [];
if (result.success) {
confidence += 0.4;
reasons.push("Tool succeeded");
}
if (successCount > 0) {
confidence += Math.min(0.2, successCount * 0.1);
reasons.push(`Found ${successCount} success indicators`);
}
if (toolMatchesAction) {
confidence += 0.2;
reasons.push("Tool matches expected action");
}
if (criteriaMatchRatio > 0.3) {
confidence += 0.2 * criteriaMatchRatio;
reasons.push(`Success criteria ${Math.round(criteriaMatchRatio * 100)}% matched`);
}
const isComplete = confidence >= this.config.heuristicThreshold && result.success;
return {
isComplete,
confidence,
reason: reasons.join("; ") || "Insufficient indicators",
suggestedResult: isComplete ? this.summarizeResult(resultText) : undefined,
};
}
private async llmVerification(
step: PlanStep,
tool: ToolCall,
result: ToolResult,
heuristic: CompletionAnalysis
): Promise<CompletionAnalysis> {
if (!this.llmCaller) return heuristic;
const prompt = `Analyze if this plan step is complete based on the tool result.
Step: ${step.title}
Action: ${step.action}
Success Criteria: ${step.successCriteria}
Tool Used: ${tool.name}
Tool Arguments: ${JSON.stringify(tool.arguments)}
Tool Result: ${this.resultToText(result).slice(0, 1000)}
Heuristic Analysis: ${heuristic.reason} (confidence: ${heuristic.confidence})
Answer in JSON:
{
"isComplete": true/false,
"confidence": 0.0-1.0,
"reason": "Brief explanation",
"suggestedResult": "One-line summary if complete"
}`;
try {
const response = await this.llmCaller({
messages: [
{ role: "system", content: "You analyze task completion. Output valid JSON only." },
{ role: "user", content: prompt },
],
maxTokens: 200,
});
const parsed = JSON.parse(response.content);
return {
isComplete: parsed.isComplete ?? heuristic.isComplete,
confidence: parsed.confidence ?? heuristic.confidence,
reason: parsed.reason ?? heuristic.reason,
suggestedResult: parsed.suggestedResult,
};
} catch {
return heuristic;
}
}
private toolMatchesStepAction(toolName: string, action: string): boolean {
const toolActionMap: Record<string, string[]> = {
exec: ["run", "execute", "install", "build", "deploy", "create", "setup", "configure"],
Write: ["write", "create", "save", "add", "generate"],
Edit: ["edit", "modify", "update", "change", "fix"],
Read: ["read", "check", "verify", "inspect", "look"],
web_search: ["search", "find", "research", "look up"],
web_fetch: ["fetch", "get", "download", "retrieve"],
browser: ["browse", "navigate", "open", "click", "interact"],
message: ["send", "message", "notify", "email", "communicate"],
};
const keywords = toolActionMap[toolName] ?? [];
return keywords.some(kw => action.includes(kw));
}
private resultToText(result: ToolResult): string {
if (result.error) return `Error: ${result.error}`;
if (typeof result.output === "string") return result.output;
if (result.output) return JSON.stringify(result.output);
return result.success ? "Success (no output)" : "Failed (no details)";
}
private summarizeResult(text: string): string {
// Take first meaningful line or truncate
const lines = text.split("\n").filter(l => l.trim().length > 0);
const firstLine = lines[0] ?? "Completed";
return firstLine.length > 100 ? firstLine.slice(0, 97) + "..." : firstLine;
}
}
// ============================================================================
// Factory
// ============================================================================
export function createStepTracker(
stateManager: PlanStateManager,
config?: Partial<StepTrackerConfig>,
llmCaller?: LLMCaller
): StepTracker {
return new StepTracker(stateManager, config, llmCaller);
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| brainstormingobra/superpowersYou MUST use this before any creative work - creating features, building components, adding functionality, modifying behavior, or planning anything new, in software or out of it (a talk, a business, a renovation). | Stars: 322k | Updated | Claude CodeCodex | Verified | Install |
| brand-guidelinesanthropics/skillsApplies Anthropic's official brand colors and typography to any sort of artifact that may benefit from having Anthropic's look-and-feel. Use it when brand colors or style guidelines, visual formatting, or company design standards apply. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| frontend-designanthropics/skillsGuidance for distinctive, intentional visual design when building new UI or reshaping an existing one. Helps with aesthetic direction, typography, and making choices that don't read as templated defaults. | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| mcp-builderanthropics/skillsGuide for creating high-quality MCP (Model Context Protocol) servers that enable LLMs to interact with external services through well-designed tools. Use when building MCP servers to integrate external APIs or services, whether in Python (FastMCP) or Node/TypeScript (MCP SDK). | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |