Skip to content

Hello-QM/catgo-LRG / server/catgo/workflow

catgo-workflow

Verified

Create and manage computational chemistry workflows with CatGo. Supports VASP, CP2K, ORCA, MLP, LAMMPS. Build OER/HER/CO2RR workflows, geometry optimization, frequency analysis, Gibbs energy calculations.

FileKindSize
SKILL.mdskill md3.8 KB
__init__.pyscript1.1 KB
builtins.pyscript4.7 KB
builtins_impl.pyscript21.2 KB
config.pyscript6.5 KB
db.pyscript25.3 KB
engine/__init__.pyscript615 B
engine/advancer.pyscript9.3 KB
engine/ai_diagnosis.pyscript3.8 KB
engine/batch_submitter.pyscript9.7 KB
engine/broadcast.pyscript1.8 KB
engine/collector.pyscript8.3 KB
engine/control_flow.pyscript5.7 KB
engine/engine_builtins.pyscript19.6 KB
engine/engine_registry.pyscript1.5 KB
engine/error_handler.pyscript3.6 KB
engine/hpc_utils.pyscript4.0 KB
engine/job_script.pyscript11.6 KB
engine/lifecycle.pyscript10.8 KB
engine/orca_progress.pyscript3.4 KB
engine/poller.pyscript11.7 KB
engine/resolver.pyscript2.7 KB
engine/result_collector.pyscript56.3 KB
engine/result_handler.pyscript5.6 KB
engine/scanner.pyscript52.0 KB
engine/smart_recovery.pyscript3.6 KB
engine/submitter.pyscript26.2 KB
engine/v1_monitor.pyscript2.7 KB
engine/vasp_submission.pyscript29.0 KB
graph_converter.pyscript6.5 KB
mcp_tools.pyscript7.5 KB
provenance.pyscript5.8 KB
reference.pyscript1.1 KB
service.pyscript4.0 KB
state_map.pyscript1.5 KB
states.pyscript7.5 KB
task_decorator.pyscript2.5 KB
task_ids.pyscript986 B
v1_compat.pyscript2.8 KB
workflow.pyscript12.0 KB

engine/lifecycle.py

View on GitHub
"""Workflow engine lifecycle — start, pause, resume, reset, cancel HPC jobs."""

from __future__ import annotations
import asyncio
import logging
import threading
from typing import Any, Optional

from catgo.workflow.db import WorkflowDB
from catgo.workflow.states import TaskState
from catgo.workflow.engine.scanner import WorkflowEngine

logger = logging.getLogger(__name__)

# The engine runs in a dedicated daemon thread with its own asyncio loop.
# This isolates slow SQLite / SSH / subprocess work in scan_cycle from
# FastAPI's request-handling loop, so HTTP endpoints and MCP tool calls
# never stall on an engine operation.
_engine: WorkflowEngine | None = None
_engine_thread: threading.Thread | None = None
_engine_loop: asyncio.AbstractEventLoop | None = None
_engine_task: asyncio.Task | None = None
_engine_ready = threading.Event()


def get_engine() -> WorkflowEngine | None:
    """Get the global engine instance."""
    return _engine


def get_engine_loop() -> asyncio.AbstractEventLoop | None:
    """Return the engine's asyncio loop (running on a dedicated thread)."""
    return _engine_loop


def assert_engine_loop() -> None:
    """Sanity check — must be called from the engine loop.

    Triggers immediately (AssertionError) if a caller accidentally invokes
    engine-side SSH helpers from the FastAPI loop. The enforcement exists
    to make the one-way ownership invariant (FastAPI -> engine via
    call_soon_threadsafe; engine -> FastAPI via run_on_owner) impossible
    to violate silently.
    """
    if _engine_loop is None:
        return  # Engine not started yet (tests, cold start)
    try:
        current = asyncio.get_running_loop()
    except RuntimeError:
        return  # Not inside an async context; nothing to assert
    if current is not _engine_loop:
        raise AssertionError(
            f"assert_engine_loop: called on loop {id(current)} but engine "
            f"loop is {id(_engine_loop)}. Engine-only helpers must not be "
            f"invoked from FastAPI-side code."
        )


async def start_engine(db: WorkflowDB, config: dict[str, Any]) -> WorkflowEngine:
    """Start the global workflow engine in a dedicated thread.

    The engine gets its own asyncio event loop so its scan cycles never
    block the FastAPI request loop. The call returns once the engine loop
    is up and ready to accept cross-thread scheduling.
    """
    global _engine, _engine_thread, _engine_loop, _engine_task

    if _engine is not None:
        return _engine

    _engine = WorkflowEngine(db=db, config=config)
    _engine_ready.clear()

    def runner() -> None:
        global _engine_loop, _engine_task
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        _engine_loop = loop
        _engine._loop = loop  # expose to cancel_inflight for thread-safe cancel

        task = loop.create_task(_engine.run_forever())
        _engine_task = task
        _engine_ready.set()
        try:
            loop.run_until_complete(task)
        except asyncio.CancelledError:
            pass
        except Exception:
            logger.exception("Workflow engine thread crashed")
        finally:
            try:
                loop.close()
            except Exception:
                pass

    _engine_thread = threading.Thread(
        target=runner, daemon=True, name="workflow-engine",
    )
    _engine_thread.start()

    # Wait until the loop is set so subsequent call_soon_threadsafe users are safe.
    if not _engine_ready.wait(timeout=5.0):
        logger.error("Workflow engine thread failed to start within 5s")

    logger.info("Workflow engine started in dedicated thread")
    return _engine


async def stop_engine() -> None:
    """Stop the global engine scanner (thread-safe)."""
    global _engine, _engine_thread, _engine_loop, _engine_task

    if _engine_loop is not None and _engine_task is not None and not _engine_task.done():
        _engine_loop.call_soon_threadsafe(_engine_task.cancel)

    if _engine_thread is not None:
        _engine_thread.join(timeout=5.0)

    _engine = None
    _engine_thread = None
    _engine_loop = None
    _engine_task = None
    logger.info("Workflow engine stopped")


def submit_workflow(db: WorkflowDB, workflow_id: str) -> None:
    """Mark a workflow as running so the engine picks it up."""
    db.update_workflow(workflow_id, status="running")
    logger.info("Workflow %s submitted for execution", workflow_id)


def pause_workflow(db: WorkflowDB, workflow_id: str) -> None:
    """Pause a workflow — engine will skip it on next cycle."""
    db.update_workflow(workflow_id, status="paused")
    # Mark active tasks as paused
    for status in (TaskState.READY, TaskState.WAITING):
        tasks = db.get_tasks_by_status(workflow_id, status.value)
        for t in tasks:
            db.update_task(t["id"], status=TaskState.PAUSED.value)
    logger.info("Workflow %s paused", workflow_id)


def resume_workflow(db: WorkflowDB, workflow_id: str) -> None:
    """Resume a paused workflow."""
    # Unpause paused tasks back to WAITING
    tasks = db.get_tasks_by_status(workflow_id, TaskState.PAUSED.value)
    for t in tasks:
        db.update_task(t["id"], status=TaskState.WAITING.value)
    db.update_workflow(workflow_id, status="running")
    logger.info("Workflow %s resumed", workflow_id)


def reset_workflow(db: WorkflowDB, workflow_id: str) -> None:
    """Reset all tasks to WAITING and clear their HPC job binding.

    Clears hpc_job_id and work_dir so a re-run submits FRESH jobs. Previously a
    reset left the old hpc_job_id in place: on the next run the engine polled
    that (now-finished or cancelled) job, saw it gone, and marked the task
    FAILED instead of resubmitting — the whole workflow then failed without ever
    launching a new calculation.

    Sets workflow status to 'resetting' so the scanner skips it,
    then resets all task states with retry logic for DB lock contention.
    Always ensures the workflow ends in 'draft' status (never stuck in 'resetting').
    """
    # 1. Cancel any in-flight async tasks (MLP subprocess, analysis, etc.)
    engine = get_engine()
    if engine:
        engine.cancel_inflight(workflow_id)

    # 2. Tell the scanner to stop processing this workflow
    try:
        db.update_workflow(workflow_id, status="resetting")
    except Exception:
        pass

    # 2. Reset all tasks with retry, always restoring status on failure
    try:
        tasks = db.get_all_tasks(workflow_id)
        max_retries = 5
        for attempt in range(max_retries):
            try:
                for t in tasks:
                    db.update_task(t["id"],
                        status=TaskState.WAITING.value,
                        error_message=None,
                        error_type=None,
                        retry_count=0,
                        hpc_job_id=None,
                        work_dir=None,
                    )
                db.update_workflow(workflow_id, status="draft")
                logger.info("Workflow %s reset (%d tasks)", workflow_id, len(tasks))
                return
            except Exception as e:
                if "locked" in str(e).lower() and attempt < max_retries - 1:
                    logger.warning("Reset retry %d/%d for %s: %s", attempt + 1, max_retries, workflow_id, e)
                    # Brief non-blocking pause — acceptable since reset is synchronous
                    import time
                    time.sleep(0.2)
                else:
                    raise
    except Exception:
        # Always ensure we don't leave workflow stuck in "resetting"
        try:
            db.update_workflow(workflow_id, status="draft")
        except Exception:
            pass
        raise


# ---------------------------------------------------------------------------
# HPC job cancellation
# ---------------------------------------------------------------------------

_CANCELABLE_STATES = {
    TaskState.RUNNING.value,
    TaskState.SUBMITTED.value,
    TaskState.QUEUED.value,
}


async def cancel_workflow_jobs(
    db: WorkflowDB,
    workflow_id: str,
    only_task_ids: list[str] | None = None,
) -> list[dict]:
    """Cancel running/queued HPC jobs for a workflow (V2 DB).

    Args:
        db: V2 WorkflowDB instance.
        workflow_id: The workflow to cancel jobs for.
        only_task_ids: If provided, only cancel jobs for these task IDs.
                       If None, cancel all running/queued jobs.

    Returns list of {step_id, job_id, success, message} dicts.
    Best-effort: individual failures are logged but don't raise.
    """
    tasks = db.get_all_tasks(workflow_id)
    results = []

    for task in tasks:
        status = task.get("status", "")
        job_id = task.get("hpc_job_id")
        session_id = task.get("hpc_session_id")

        if not job_id or status not in _CANCELABLE_STATES:
            continue

        if only_task_ids is not None and task["id"] not in only_task_ids:
            continue

        success, message = await _cancel_single_job(session_id, job_id)
        results.append({
            "step_id": task["id"],
            "job_id": job_id,
            "success": success,
            "message": message,
        })

        if success:
            db.update_task(task["id"], status=TaskState.CANCELLED.value)

        level = logging.INFO if success else logging.WARNING
        logger.log(
            level,
            "Workflow %s: cancel job %s for task %s — %s",
            workflow_id, job_id, task["id"],
            "OK" if success else f"FAILED: {message}",
        )

    return results


async def _cancel_single_job(
    session_id: Optional[str], job_id: str,
) -> tuple[bool, str]:
    """Cancel a single HPC job via its scheduler.

    Returns (success, message). Never raises — errors are caught and
    returned as (False, error_message).
    """
    if not session_id:
        return False, "No session_id available for this task"

    try:
        from catgo.utils.hpc_client import pool

        hpc = pool.get_connection(session_id)
        if not hpc:
            # Try any available session as fallback
            for sid, conn in list(pool.connections.items()):
                if conn and conn.conn:
                    logger.info(
                        "Session %s unavailable, trying fallback session %s for job %s",
                        session_id, sid, job_id,
                    )
                    hpc = conn
                    break
            if not hpc:
                return False, f"HPC session '{session_id}' not connected and no fallback available"

        success, message = await hpc.run_on_owner(
            lambda: hpc.scheduler.cancel_job(hpc.conn, job_id)
        )
        if not success:
            logger.error("scancel failed for job %s: %s", job_id, message)
        return success, message
    except Exception as e:
        logger.error("Cancel job %s failed with exception: %s", job_id, e, exc_info=True)
        return False, f"Cancel failed: {e}"
planning-and-task-breakdownaddyosmani/agent-skillsBreaks work into ordered tasks. Use when you have a spec or clear requirements and need to break work into implementable tasks. Use when a task feels too large to start, when you need to estimate scope, or when parallel work is possible.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
screenpipethedotmack/claude-memSearch screen and audio history recorded by Screenpipe and optionally save selected context into claude-mem. Use when the user asks to recall desktop activity, a meeting, or something seen in another app through Screenpipe.Stars: 118kUpdated Claude CodeCodexVerifiedInstall
contact-widgetnexu-io/open-designSelf-contained floating chat widget with welcome screen, social links, meeting button, and message input. Single HTML file, zero dependencies.Stars: 108kUpdated Claude CodeCodexVerifiedInstall
meeting-notesnexu-io/open-designMeeting notes page — title bar with attendees, agenda checklist, decisions block, action items table with owners + dates, and a "next meeting" footer. Use when the brief mentions "meeting notes", "minutes", "1:1 notes", "all-hands recap", or "会议纪要".Stars: 108kUpdated Claude CodeCodexVerifiedInstall