Skip to content

LeoYeAI/agent-master-skills / skills/agent-mode-upgrades

agentic-loop-upgrade

Flagged

Enhanced agentic loop with planning, parallel execution, confidence gates, semantic error recovery, and observable state machine. Includes Mode dashboard UI for easy configuration.

src/state/step-tracker.ts

View on GitHub
/**
 * Automatic Step Completion Detection
 * 
 * Analyzes tool results to determine if plan steps are complete.
 * Uses heuristics and optional LLM verification.
 */

import type { PlanStep, ToolCall, ToolResult, LLMCaller } from "../types.js";
import type { PlanStateManager, PlanState } from "./persistence.js";

// ============================================================================
// Types
// ============================================================================

export interface StepTrackerConfig {
  /** Use LLM to verify completion (more accurate but costs tokens) */
  useLLMVerification: boolean;
  /** Confidence threshold for heuristic-only detection (0-1) */
  heuristicThreshold: number;
  /** Auto-advance to next step on completion */
  autoAdvance: boolean;
  /** Keywords that indicate failure */
  failureKeywords: string[];
  /** Keywords that indicate success */
  successKeywords: string[];
}

export interface CompletionAnalysis {
  isComplete: boolean;
  confidence: number;
  reason: string;
  suggestedResult?: string;
}

const DEFAULT_CONFIG: StepTrackerConfig = {
  useLLMVerification: false,
  heuristicThreshold: 0.7,
  autoAdvance: true,
  failureKeywords: [
    "error", "failed", "failure", "exception", "cannot", "unable",
    "denied", "permission", "not found", "timeout", "refused"
  ],
  successKeywords: [
    "success", "complete", "created", "done", "finished", "wrote",
    "saved", "built", "deployed", "installed", "configured"
  ],
};

// ============================================================================
// Step Tracker
// ============================================================================

export class StepTracker {
  private config: StepTrackerConfig;
  private stateManager: PlanStateManager;
  private llmCaller?: LLMCaller;

  constructor(
    stateManager: PlanStateManager,
    config: Partial<StepTrackerConfig> = {},
    llmCaller?: LLMCaller
  ) {
    this.stateManager = stateManager;
    this.config = { ...DEFAULT_CONFIG, ...config };
    this.llmCaller = llmCaller;
  }

  /**
   * Analyze a tool result to determine if current step is complete
   */
  async analyzeToolResult(
    tool: ToolCall,
    result: ToolResult
  ): Promise<CompletionAnalysis> {
    const state = this.stateManager.getOrNull();
    const activeStep = state ? this.stateManager.getActiveStep() : null;

    if (!activeStep) {
      return {
        isComplete: false,
        confidence: 1.0,
        reason: "No active step to track",
      };
    }

    // First, do heuristic analysis
    const heuristic = this.heuristicAnalysis(activeStep, tool, result);

    // If confident enough or LLM verification disabled, return heuristic
    if (heuristic.confidence >= this.config.heuristicThreshold || !this.config.useLLMVerification) {
      if (heuristic.isComplete && this.config.autoAdvance) {
        this.stateManager.completeStep(activeStep.id, heuristic.suggestedResult);
      }
      return heuristic;
    }

    // Use LLM for verification if available
    if (this.llmCaller) {
      const llmAnalysis = await this.llmVerification(activeStep, tool, result, heuristic);
      if (llmAnalysis.isComplete && this.config.autoAdvance) {
        this.stateManager.completeStep(activeStep.id, llmAnalysis.suggestedResult);
      }
      return llmAnalysis;
    }

    return heuristic;
  }

  /**
   * Manually mark step complete (for explicit completion)
   */
  markComplete(stepId?: string, result?: string): void {
    const state = this.stateManager.getOrNull();
    if (!state?.plan) return;

    const id = stepId ?? state.activeStepId;
    if (id) {
      this.stateManager.completeStep(id, result);
    }
  }

  /**
   * Manually mark step failed
   */
  markFailed(stepId?: string, error?: string): void {
    const state = this.stateManager.getOrNull();
    if (!state?.plan) return;

    const id = stepId ?? state.activeStepId;
    if (id) {
      this.stateManager.failStep(id, error ?? "Step failed");
    }
  }

  /**
   * Check if plan is complete
   */
  isPlanComplete(): boolean {
    const state = this.stateManager.getOrNull();
    if (!state?.plan) return false;

    return state.plan.steps.every(
      s => s.status === "complete" || s.status === "skipped"
    );
  }

  /**
   * Get completion status summary
   */
  getStatus(): {
    hasActivePlan: boolean;
    activeStep: PlanStep | null;
    progress: { completed: number; total: number; percent: number };
    isComplete: boolean;
  } {
    const state = this.stateManager.getOrNull();
    
    return {
      hasActivePlan: !!state?.plan,
      activeStep: this.stateManager.getActiveStep(),
      progress: this.stateManager.getProgress(),
      isComplete: this.isPlanComplete(),
    };
  }

  // ──────────────────────────────────────────────────────────────────────────
  // Private Methods
  // ──────────────────────────────────────────────────────────────────────────

  private heuristicAnalysis(
    step: PlanStep,
    tool: ToolCall,
    result: ToolResult
  ): CompletionAnalysis {
    const resultText = this.resultToText(result);
    const lowerResult = resultText.toLowerCase();
    const lowerAction = step.action.toLowerCase();
    const lowerCriteria = step.successCriteria.toLowerCase();

    // Check for explicit failure
    const hasFailure = this.config.failureKeywords.some(kw => 
      lowerResult.includes(kw) && !lowerResult.includes(`no ${kw}`)
    );
    
    if (hasFailure && result.error) {
      return {
        isComplete: false,
        confidence: 0.9,
        reason: `Tool returned error: ${result.error.slice(0, 100)}`,
      };
    }

    // Check for explicit success keywords
    const successCount = this.config.successKeywords.filter(kw =>
      lowerResult.includes(kw)
    ).length;

    // Check if tool matches expected action
    const toolMatchesAction = this.toolMatchesStepAction(tool.name, lowerAction);

    // Check if result mentions success criteria keywords
    const criteriaWords = lowerCriteria.split(/\s+/).filter(w => w.length > 3);
    const criteriaMatches = criteriaWords.filter(w => lowerResult.includes(w)).length;
    const criteriaMatchRatio = criteriaWords.length > 0 
      ? criteriaMatches / criteriaWords.length 
      : 0;

    // Calculate confidence
    let confidence = 0;
    let reasons: string[] = [];

    if (result.success) {
      confidence += 0.4;
      reasons.push("Tool succeeded");
    }

    if (successCount > 0) {
      confidence += Math.min(0.2, successCount * 0.1);
      reasons.push(`Found ${successCount} success indicators`);
    }

    if (toolMatchesAction) {
      confidence += 0.2;
      reasons.push("Tool matches expected action");
    }

    if (criteriaMatchRatio > 0.3) {
      confidence += 0.2 * criteriaMatchRatio;
      reasons.push(`Success criteria ${Math.round(criteriaMatchRatio * 100)}% matched`);
    }

    const isComplete = confidence >= this.config.heuristicThreshold && result.success;

    return {
      isComplete,
      confidence,
      reason: reasons.join("; ") || "Insufficient indicators",
      suggestedResult: isComplete ? this.summarizeResult(resultText) : undefined,
    };
  }

  private async llmVerification(
    step: PlanStep,
    tool: ToolCall,
    result: ToolResult,
    heuristic: CompletionAnalysis
  ): Promise<CompletionAnalysis> {
    if (!this.llmCaller) return heuristic;

    const prompt = `Analyze if this plan step is complete based on the tool result.

Step: ${step.title}
Action: ${step.action}
Success Criteria: ${step.successCriteria}

Tool Used: ${tool.name}
Tool Arguments: ${JSON.stringify(tool.arguments)}
Tool Result: ${this.resultToText(result).slice(0, 1000)}

Heuristic Analysis: ${heuristic.reason} (confidence: ${heuristic.confidence})

Answer in JSON:
{
  "isComplete": true/false,
  "confidence": 0.0-1.0,
  "reason": "Brief explanation",
  "suggestedResult": "One-line summary if complete"
}`;

    try {
      const response = await this.llmCaller({
        messages: [
          { role: "system", content: "You analyze task completion. Output valid JSON only." },
          { role: "user", content: prompt },
        ],
        maxTokens: 200,
      });

      const parsed = JSON.parse(response.content);
      return {
        isComplete: parsed.isComplete ?? heuristic.isComplete,
        confidence: parsed.confidence ?? heuristic.confidence,
        reason: parsed.reason ?? heuristic.reason,
        suggestedResult: parsed.suggestedResult,
      };
    } catch {
      return heuristic;
    }
  }

  private toolMatchesStepAction(toolName: string, action: string): boolean {
    const toolActionMap: Record<string, string[]> = {
      exec: ["run", "execute", "install", "build", "deploy", "create", "setup", "configure"],
      Write: ["write", "create", "save", "add", "generate"],
      Edit: ["edit", "modify", "update", "change", "fix"],
      Read: ["read", "check", "verify", "inspect", "look"],
      web_search: ["search", "find", "research", "look up"],
      web_fetch: ["fetch", "get", "download", "retrieve"],
      browser: ["browse", "navigate", "open", "click", "interact"],
      message: ["send", "message", "notify", "email", "communicate"],
    };

    const keywords = toolActionMap[toolName] ?? [];
    return keywords.some(kw => action.includes(kw));
  }

  private resultToText(result: ToolResult): string {
    if (result.error) return `Error: ${result.error}`;
    if (typeof result.output === "string") return result.output;
    if (result.output) return JSON.stringify(result.output);
    return result.success ? "Success (no output)" : "Failed (no details)";
  }

  private summarizeResult(text: string): string {
    // Take first meaningful line or truncate
    const lines = text.split("\n").filter(l => l.trim().length > 0);
    const firstLine = lines[0] ?? "Completed";
    return firstLine.length > 100 ? firstLine.slice(0, 97) + "..." : firstLine;
  }
}

// ============================================================================
// Factory
// ============================================================================

export function createStepTracker(
  stateManager: PlanStateManager,
  config?: Partial<StepTrackerConfig>,
  llmCaller?: LLMCaller
): StepTracker {
  return new StepTracker(stateManager, config, llmCaller);
}
brainstormingobra/superpowersYou MUST use this before any creative work - creating features, building components, adding functionality, modifying behavior, or planning anything new, in software or out of it (a talk, a business, a renovation).Stars: 322kUpdated Claude CodeCodexVerifiedInstall
brand-guidelinesanthropics/skillsApplies Anthropic's official brand colors and typography to any sort of artifact that may benefit from having Anthropic's look-and-feel. Use it when brand colors or style guidelines, visual formatting, or company design standards apply.Stars: 198kUpdated Claude CodeCodexVerifiedInstall
frontend-designanthropics/skillsGuidance for distinctive, intentional visual design when building new UI or reshaping an existing one. Helps with aesthetic direction, typography, and making choices that don't read as templated defaults.Stars: 198kUpdated Claude CodeCodexVerifiedInstall
mcp-builderanthropics/skillsGuide for creating high-quality MCP (Model Context Protocol) servers that enable LLMs to interact with external services through well-designed tools. Use when building MCP servers to integrate external APIs or services, whether in Python (FastMCP) or Node/TypeScript (MCP SDK).Stars: 198kUpdated Claude CodeCodexVerifiedInstall