Skip to content

Hello-QM/catgo-LRG / server/catgo/workflow

catgo-workflow

Verified

Create and manage computational chemistry workflows with CatGo. Supports VASP, CP2K, ORCA, MLP, LAMMPS. Build OER/HER/CO2RR workflows, geometry optimization, frequency analysis, Gibbs energy calculations.

FileKindSize
SKILL.mdskill md3.8 KB
__init__.pyscript1.1 KB
builtins.pyscript4.7 KB
builtins_impl.pyscript21.2 KB
config.pyscript6.5 KB
db.pyscript25.3 KB
engine/__init__.pyscript615 B
engine/advancer.pyscript9.3 KB
engine/ai_diagnosis.pyscript3.8 KB
engine/batch_submitter.pyscript9.7 KB
engine/broadcast.pyscript1.8 KB
engine/collector.pyscript8.3 KB
engine/control_flow.pyscript5.7 KB
engine/engine_builtins.pyscript19.6 KB
engine/engine_registry.pyscript1.5 KB
engine/error_handler.pyscript3.6 KB
engine/hpc_utils.pyscript4.0 KB
engine/job_script.pyscript11.6 KB
engine/lifecycle.pyscript10.8 KB
engine/orca_progress.pyscript3.4 KB
engine/poller.pyscript11.7 KB
engine/resolver.pyscript2.7 KB
engine/result_collector.pyscript56.3 KB
engine/result_handler.pyscript5.6 KB
engine/scanner.pyscript52.0 KB
engine/smart_recovery.pyscript3.6 KB
engine/submitter.pyscript26.2 KB
engine/v1_monitor.pyscript2.7 KB
engine/vasp_submission.pyscript29.0 KB
graph_converter.pyscript6.5 KB
mcp_tools.pyscript7.5 KB
provenance.pyscript5.8 KB
reference.pyscript1.1 KB
service.pyscript4.0 KB
state_map.pyscript1.5 KB
states.pyscript7.5 KB
task_decorator.pyscript2.5 KB
task_ids.pyscript986 B
v1_compat.pyscript2.8 KB
workflow.pyscript12.0 KB

engine/smart_recovery.py

View on GitHub
"""Three-tier smart error recovery for failed HPC tasks.

Tier 1: Custodian (handles VASP errors at runtime -- already integrated)
Tier 2: Rule-based diagnosis (parse error message, apply known fixes)
Tier 3: Escalate to user (PAUSED with diagnosis message)
"""

from __future__ import annotations

import json
import logging
from typing import Any

from catgo.workflow.db import WorkflowDB

logger = logging.getLogger(__name__)

# Known error patterns -> parameter fixes
_VASP_FIXES: list[dict] = [
    {
        "pattern": "ZBRENT",
        "diagnosis": "Ionic step optimizer (ZBRENT) failed to bracket minimum",
        "fixes": {"IBRION": 1, "POTIM": 0.1},
    },
    {
        "pattern": "BRMIX",
        "diagnosis": "Charge density mixing instability",
        "fixes": {"AMIX": 0.1, "BMIX": 0.001, "AMIX_MAG": 0.2, "BMIX_MAG": 0.001},
    },
    {
        "pattern": "EDDDAV",
        "diagnosis": "Sub-space rotation failed in electronic minimization",
        "fixes": {"ALGO": "All"},
    },
    {
        "pattern": "VERY BAD NEWS",
        "diagnosis": "Internal VASP error -- often caused by bad geometry or pseudopotential",
        "fixes": {"SYMPREC": 1e-4},
    },
    {
        "pattern": "RSPHER",
        "diagnosis": "Atoms too close together -- augmentation sphere overlap",
        "fixes": {},  # Can't auto-fix, escalate
    },
    {
        "pattern": "exceeded",  # NSW exceeded
        "diagnosis": "Ionic relaxation did not converge within NSW steps",
        "fixes": {"NSW": 500},
    },
    {
        "pattern": "PRICEL",
        "diagnosis": "Primitive cell determination failed",
        "fixes": {"SYMPREC": 1e-4, "ISYM": 0},
    },
    {
        "pattern": "not converge",  # SCF not converged
        "diagnosis": "Electronic SCF did not converge",
        "fixes": {"ALGO": "All", "NELM": 300, "AMIX": 0.1, "BMIX": 0.001},
    },
]

_ORCA_FIXES: list[dict] = [
    {
        "pattern": "SCF NOT CONVERGED",
        "diagnosis": "ORCA SCF did not converge",
        "fixes": {"scf_max_iter": 500, "scf_conv": "SlowConv"},
    },
    {
        "pattern": "ORCA TERMINATED ABNORMALLY",
        "diagnosis": "ORCA crashed -- often memory or input error",
        "fixes": {},  # Escalate
    },
]


def diagnose_and_fix(
    db: WorkflowDB,
    task: dict,
    config: dict,
) -> dict | None:
    """Analyze error message and return a fix dict, or None to escalate.

    Returns: {"diagnosis": str, "fixes": dict, "tier": int} or None
    """
    error_msg = task.get("error_message", "") or ""
    software = task.get("software", "")

    # Select fix database based on software
    if software == "vasp":
        fixes_db = _VASP_FIXES
    elif software == "orca":
        fixes_db = _ORCA_FIXES
    else:
        fixes_db = []

    for entry in fixes_db:
        if entry["pattern"].lower() in error_msg.lower():
            if entry["fixes"]:
                return {
                    "diagnosis": entry["diagnosis"],
                    "fixes": entry["fixes"],
                    "tier": 2,
                }
            else:
                # Known error but no auto-fix -- escalate
                return {
                    "diagnosis": entry["diagnosis"],
                    "fixes": {},
                    "tier": 3,
                }

    return None  # Unknown error -- escalate


def apply_fix(db: WorkflowDB, task_id: str, fixes: dict) -> None:
    """Merge fixes into task params and reset to READY."""
    task = db.get_task(task_id)
    params = json.loads(task.get("params_json", "{}") or "{}")
    params.update(fixes)
    db.update_task(task_id, params_json=json.dumps(params))
    logger.info("Applied fix to task %s: %s", task_id, fixes)
planning-and-task-breakdownaddyosmani/agent-skillsBreaks work into ordered tasks. Use when you have a spec or clear requirements and need to break work into implementable tasks. Use when a task feels too large to start, when you need to estimate scope, or when parallel work is possible.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
screenpipethedotmack/claude-memSearch screen and audio history recorded by Screenpipe and optionally save selected context into claude-mem. Use when the user asks to recall desktop activity, a meeting, or something seen in another app through Screenpipe.Stars: 118kUpdated Claude CodeCodexVerifiedInstall
contact-widgetnexu-io/open-designSelf-contained floating chat widget with welcome screen, social links, meeting button, and message input. Single HTML file, zero dependencies.Stars: 108kUpdated Claude CodeCodexVerifiedInstall
meeting-notesnexu-io/open-designMeeting notes page — title bar with attendees, agenda checklist, decisions block, action items table with owners + dates, and a "next meeting" footer. Use when the brief mentions "meeting notes", "minutes", "1:1 notes", "all-hands recap", or "会议纪要".Stars: 108kUpdated Claude CodeCodexVerifiedInstall