Skip to content

Hello-QM/catgo-LRG / server/catgo/workflow

catgo-workflow

Verified

Create and manage computational chemistry workflows with CatGo. Supports VASP, CP2K, ORCA, MLP, LAMMPS. Build OER/HER/CO2RR workflows, geometry optimization, frequency analysis, Gibbs energy calculations.

FileKindSize
SKILL.mdskill md3.8 KB
__init__.pyscript1.1 KB
builtins.pyscript4.7 KB
builtins_impl.pyscript21.2 KB
config.pyscript6.5 KB
db.pyscript25.3 KB
engine/__init__.pyscript615 B
engine/advancer.pyscript9.3 KB
engine/ai_diagnosis.pyscript3.8 KB
engine/batch_submitter.pyscript9.7 KB
engine/broadcast.pyscript1.8 KB
engine/collector.pyscript8.3 KB
engine/control_flow.pyscript5.7 KB
engine/engine_builtins.pyscript19.6 KB
engine/engine_registry.pyscript1.5 KB
engine/error_handler.pyscript3.6 KB
engine/hpc_utils.pyscript4.0 KB
engine/job_script.pyscript11.6 KB
engine/lifecycle.pyscript10.8 KB
engine/orca_progress.pyscript3.4 KB
engine/poller.pyscript11.7 KB
engine/resolver.pyscript2.7 KB
engine/result_collector.pyscript56.3 KB
engine/result_handler.pyscript5.6 KB
engine/scanner.pyscript52.0 KB
engine/smart_recovery.pyscript3.6 KB
engine/submitter.pyscript26.2 KB
engine/v1_monitor.pyscript2.7 KB
engine/vasp_submission.pyscript29.0 KB
graph_converter.pyscript6.5 KB
mcp_tools.pyscript7.5 KB
provenance.pyscript5.8 KB
reference.pyscript1.1 KB
service.pyscript4.0 KB
state_map.pyscript1.5 KB
states.pyscript7.5 KB
task_decorator.pyscript2.5 KB
task_ids.pyscript986 B
v1_compat.pyscript2.8 KB
workflow.pyscript12.0 KB

engine/scanner.py

View on GitHub
"""WorkflowEngine — stateless periodic scanner.

Each scan_cycle() reads DB, advances task states, and returns.
No in-memory state between cycles. Crash and restart safely.
"""

from __future__ import annotations
import asyncio
import json
import logging
from datetime import datetime, timezone, timedelta
from typing import Any

import catgo.workflow.builtins  # noqa: F401 — register built-in task types

from catgo.workflow.db import WorkflowDB
from catgo.workflow.states import TaskState, WorkflowState
from catgo.workflow.engine.advancer import advance_waiting_tasks
from catgo.workflow.engine.broadcast import broadcast as _broadcast
from catgo.workflow.engine.control_flow import handle_while_task, handle_zone_task
from catgo.workflow.engine.error_handler import handle_errors
from catgo.workflow.provenance import record_provenance as _record_provenance

logger = logging.getLogger(__name__)

# V2 (TaskState enum) → V1 step status string used by the frontend's
# StepInfo / workflow_steps table. Used by _update_workflow_status to
# reflect engine truth into the legacy table the UI polls.
_V2_TO_V1_STATUS: dict[str, str] = {
    "WAITING": "pending",
    "READY": "pending",
    "PENDING_REVIEW": "pending",
    "GENERATING": "submitting",
    "UPLOADING": "submitting",
    "SUBMITTED": "queued",
    "QUEUED": "queued",
    "RUNNING": "running",
    "COMPLETED_REMOTE": "completed",
    "COMPLETED": "completed",
    "SKIPPED": "completed",
    "REMOTE_ERROR": "failed",  # surfaced as failed while retries happen
    "FAILED": "failed",
    "PAUSED": "paused",
}


def _poscar_to_json(poscar_str: str) -> str:
    """Convert POSCAR/CONTCAR string to pymatgen JSON dict string.

    If the input is already valid JSON, returns it unchanged.
    This ensures downstream nodes always get JSON-format structures.
    """
    import json as _json
    # Already JSON?
    s = poscar_str.strip()
    if s.startswith("{"):
        return poscar_str
    # Parse POSCAR via pymatgen
    try:
        from pymatgen.io.vasp import Poscar
        structure = Poscar.from_str(poscar_str).structure
        return _json.dumps(structure.as_dict())
    except Exception as e:
        logger.warning("_poscar_to_json: failed to parse POSCAR (%s), storing raw", e)
        return poscar_str


class WorkflowEngine:
    """Stateless workflow engine. Call scan_cycle() periodically.

    Tracks in-flight async tasks per workflow so they can be cancelled on reset.
    """

    def __init__(self, db: WorkflowDB, config: dict[str, Any] | None = None):
        self.db = db
        self.config = config or {}
        self.poll_interval = self.config.get("engine", {}).get("poll_interval", 30)
        # Track in-flight async tasks per workflow for cancellation on reset
        self._inflight: dict[str, list[asyncio.Task]] = {}
        # Set by lifecycle.start_engine once the dedicated engine loop is up.
        # Used to schedule task cancellation across threads.
        self._loop: asyncio.AbstractEventLoop | None = None

    def cancel_inflight(self, workflow_id: str) -> int:
        """Cancel all in-flight async tasks for a workflow. Called on reset.

        Safe to call from any thread: tasks belong to the engine's dedicated
        loop, so cancellation is scheduled via call_soon_threadsafe when
        invoked from a different thread (e.g. a FastAPI request).

        Returns the number of cancelled tasks.
        """
        tasks = self._inflight.pop(workflow_id, [])
        cancelled = 0
        loop = self._loop
        for t in tasks:
            if t.done():
                continue
            if loop is not None and loop.is_running():
                loop.call_soon_threadsafe(t.cancel)
            else:
                t.cancel()
            cancelled += 1
        if cancelled:
            logger.info("Cancelled %d in-flight tasks for workflow %s", cancelled, workflow_id)
        return cancelled

    def _track_task(self, workflow_id: str, task: asyncio.Task) -> None:
        """Track an async task for later cancellation on reset."""
        if workflow_id not in self._inflight:
            self._inflight[workflow_id] = []
        # Clean up completed tasks
        self._inflight[workflow_id] = [t for t in self._inflight[workflow_id] if not t.done()]
        self._inflight[workflow_id].append(task)

    async def scan_cycle(self) -> None:
        """One pass of the state machine. Reads DB, advances states."""
        await self._ensure_hpc_connections()

        workflows = self.db.list_workflows()
        active = [w for w in workflows if w["status"] == "running"]

        # Safety net: also process FAILED workflows that still have
        # recoverable tasks (REMOTE_ERROR with transient errors).
        # This handles edge cases where the workflow was marked FAILED
        # before the from_task_states() fix, or due to a race condition.
        failed = [w for w in workflows if w["status"] == "failed"]
        for fw in failed:
            tasks = self.db.get_all_tasks(fw["id"])
            if any(
                t.get("status") == TaskState.REMOTE_ERROR.value
                and t.get("error_type") == "transient"
                for t in tasks
            ):
                active.append(fw)
                logger.info(
                    "Workflow %s: FAILED but has recoverable transient-error tasks, processing",
                    fw["id"],
                )

        # Safety net: recover COMPLETED workflows that still have non-terminal
        # tasks.  This catches cases where the frontend incorrectly pushed
        # "completed" while tasks were still WAITING/RUNNING/REMOTE_ERROR.
        completed = [w for w in workflows if w["status"] == "completed"]
        for cw in completed:
            tasks = self.db.get_all_tasks(cw["id"])
            states = {t.get("status") for t in tasks}
            non_terminal = states - {
                TaskState.COMPLETED.value, TaskState.FAILED.value,
                TaskState.CANCELLED.value, TaskState.SKIPPED.value,
                TaskState.MAPPED.value,
            }
            if non_terminal:
                self.db.update_workflow(cw["id"], status=WorkflowState.RUNNING.value)
                active.append(cw)
                logger.warning(
                    "Workflow %s: COMPLETED but has non-terminal tasks %s, reverting to RUNNING",
                    cw["id"], non_terminal,
                )

        for wf in active:
            wf_id = wf["id"]
            try:
                await self._process_workflow(wf_id)
            except Exception as e:
                logger.error("Error processing workflow %s: %s", wf_id, e, exc_info=True)

    async def _ensure_hpc_connections(self) -> None:
        """Restore HPC connections from DB if pool is empty or unhealthy.

        Only auto-reconnects sessions that don't require user input
        (ssh_config, key with key_file). Tracks consecutive failures
        per session and removes stale entries after 3 failures.

        Also triggers reconnection when existing connections appear alive
        but tasks are stuck in REMOTE_ERROR with transient errors — this
        catches half-open sockets that passed is_alive but fail on actual use.
        """
        from catgo.utils.hpc_client import pool, LOCAL_SESSION_ID

        # Check if we already have any live remote connections
        has_remote = any(
            sid != LOCAL_SESSION_ID and conn.is_alive
            for sid, conn in pool.connections.items()
        )
        if has_remote:
            # Even with live connections, check if dead sessions need reconnecting.
            # A connection may appear alive (is_alive=True) while the TCP socket
            # is actually half-open. If there are dead sessions in the pool that
            # haven't been reconnected, try now.
            has_dead = bool(pool._dead_connections)
            if not has_dead:
                return  # All good, nothing to do

        # Try to restore from DB
        sessions = self.db.get_hpc_sessions()
        if not sessions:
            return

        # Track reconnect failures: maps session_id -> first failure timestamp
        if not hasattr(self, "_reconnect_first_failure"):
            self._reconnect_first_failure: dict[str, datetime] = {}

        # Collect all non-terminal tasks to check if sessions are still needed
        all_tasks = []
        for wf in self.db.list_workflows():
            all_tasks.extend(self.db.get_all_tasks(wf["id"]))

        from catgo.workflow.states import _TERMINAL_STATES

        # Engine-thread reconnects must create the asyncssh connection on
        # the FastAPI loop, because that is where the connection's loop-
        # bound Futures will be awaited by FastAPI handlers. Without this
        # hop, the new connection would be born on the engine loop and
        # only engine code could use it — breaking UI file browser,
        # terminal, etc.
        from catgo.utils.hpc_client import get_fastapi_loop
        fastapi_loop = get_fastapi_loop()
        current_loop = asyncio.get_running_loop()
        needs_hop = (
            fastapi_loop is not None
            and fastapi_loop is not current_loop
            and not fastapi_loop.is_closed()
        )

        async def _dispatch(factory):
            if needs_hop:
                fut = asyncio.run_coroutine_threadsafe(factory(), fastapi_loop)
                return await asyncio.wrap_future(fut)
            return await factory()

        for s in sessions:
            sid = s["session_id"]
            auth = s["auth_method"]

            # Only auto-reconnect methods that don't need interactive input
            if auth == "ssh_config":
                try:
                    from catgo.models.hpc import HPCConnectionConfig, AuthMethod, SchedulerType
                    config = HPCConnectionConfig(
                        host=s["host"],
                        username=s["username"],
                        port=s.get("port", 22),
                        auth_method=AuthMethod.SSH_CONFIG,
                        ssh_alias=s.get("ssh_alias"),
                        scheduler=SchedulerType(s.get("scheduler", "slurm")),
                    )
                    await _dispatch(lambda cfg=config, sid=sid: pool.connect_ssh_config(cfg, sid))
                    self._reconnect_first_failure.pop(sid, None)
                    logger.info(
                        "Auto-reconnected HPC session %s (%s@%s) via ssh_config",
                        sid[:8], s["username"], s["host"],
                    )
                    return  # One connection is enough
                except Exception as e:
                    self._handle_reconnect_failure(sid, e, all_tasks)

            elif auth == "key" and s.get("key_file"):
                try:
                    from catgo.models.hpc import HPCConnectionConfig, AuthMethod, SchedulerType
                    config = HPCConnectionConfig(
                        host=s["host"],
                        username=s["username"],
                        port=s.get("port", 22),
                        auth_method=AuthMethod.KEY,
                        key_file=s["key_file"],
                        scheduler=SchedulerType(s.get("scheduler", "slurm")),
                    )
                    await _dispatch(lambda cfg=config, sid=sid: pool.connect(cfg, sid))
                    self._reconnect_first_failure.pop(sid, None)
                    logger.info(
                        "Auto-reconnected HPC session %s (%s@%s) via key",
                        sid[:8], s["username"], s["host"],
                    )
                    return  # One connection is enough
                except Exception as e:
                    self._handle_reconnect_failure(sid, e, all_tasks)

    _SESSION_STALE_HOURS = 24

    def _handle_reconnect_failure(
        self, session_id: str, error: Exception, all_tasks: list[dict],
    ) -> None:
        """Track reconnect failures with time-based retention.

        Sessions are only removed after 24 hours of continuous failure AND
        when no non-terminal tasks reference the session.
        """
        from catgo.workflow.states import _TERMINAL_STATES

        now = datetime.now(timezone.utc)
        if session_id not in self._reconnect_first_failure:
            self._reconnect_first_failure[session_id] = now
            logger.debug(
                "Auto-reconnect failed for %s (first failure): %s",
                session_id[:8], error,
            )
            return

        first_failure = self._reconnect_first_failure[session_id]
        elapsed = now - first_failure

        # Never remove a session while non-terminal tasks still reference it
        has_active_tasks = any(
            t.get("hpc_session_id") == session_id
            and TaskState(t["status"]) not in _TERMINAL_STATES
            for t in all_tasks
        )
        if has_active_tasks:
            logger.debug(
                "Auto-reconnect failed for %s (%.0fh elapsed, tasks still active): %s",
                session_id[:8], elapsed.total_seconds() / 3600, error,
            )
            return

        if elapsed > timedelta(hours=self._SESSION_STALE_HOURS):
            logger.warning(
                "Removing stale HPC session %s after %.0fh of failures: %s",
                session_id[:8], elapsed.total_seconds() / 3600, error,
            )
            self.db.delete_hpc_session(session_id)
            self._reconnect_first_failure.pop(session_id, None)
        else:
            logger.debug(
                "Auto-reconnect failed for %s (%.0fh / %dh until removal): %s",
                session_id[:8], elapsed.total_seconds() / 3600,
                self._SESSION_STALE_HOURS, error,
            )

    def _merged_config(self, workflow_id: str) -> dict[str, Any]:
        """Merge workflow-level config_json (from run panel) into global config.

        Workflow config provides: calc_templates, cluster_configs, job params,
        account, partition, etc. that the user set in RunConfigDialog.
        """
        import copy
        merged = copy.deepcopy(self.config)
        try:
            wf = self.db.get_workflow(workflow_id)
            # Read from run_config_json which stores the WorkflowRunConfig from the Run Configuration dialog
            raw = wf.get("run_config_json") or wf.get("config_json", "{}")
            wf_config = json.loads(raw) if isinstance(raw, str) else (raw or {})
            if not wf_config:
                return merged

            if "hpc" not in merged:
                merged["hpc"] = {}
            hpc = merged["hpc"]

            # Top-level job_script_template → hpc.job_script_template
            # Also check nested hpc section (V2 engine config format)
            jst = wf_config.get("job_script_template") or wf_config.get("hpc", {}).get("job_script_template")
            if jst:
                hpc["job_script_template"] = jst

            # default_job_params → hpc.job_defaults
            # Check top-level (V1 WorkflowRunConfig format) first, then nested
            # hpc.job_defaults (V2 engine config from _run_config_to_engine_config)
            djp = wf_config.get("default_job_params", {})
            if not djp:
                djp = wf_config.get("hpc", {}).get("job_defaults", {})
            logger.info(f"[_merged_config] Read from config_json: default_job_params={djp}")
            if djp:
                if "job_defaults" not in hpc:
                    hpc["job_defaults"] = {}
                hpc["job_defaults"].update(djp)
                logger.info(f"[_merged_config] Applied to hpc.job_defaults: {hpc['job_defaults']}")

            # Resolve cluster-specific config from session
            session_id = wf_config.get("default_session_id", "")
            cluster_cfgs = wf_config.get("cluster_configs", {})
            cc = cluster_cfgs.get(session_id, {})
            if cc:
                # Cluster template overrides global if set
                if cc.get("default_template"):
                    hpc["job_script_template"] = cc["default_template"]
                # Cluster-specific fields. cp2k_data_dir / cp2k_command land
                # in hpc.job_defaults so _gen_cp2k can pull them into the task
                # params and _generate_cp2k_input_content can emit absolute
                # BASIS / POTENTIAL paths; job_script.py reads cp2k_command
                # as a run_command override.
                for key in ("orca_dir", "potcar_root", "potcar_functional",
                            "vasp_command", "module_loads", "account",
                            "cp2k_data_dir", "cp2k_command"):
                    if cc.get(key):
                        hpc.setdefault("job_defaults", {})[key] = cc[key]
                # POTCAR root/functional are read at hpc ROOT level by the VASP
                # submitter (submitter._submit_one / batch_submitter), not from
                # job_defaults. Without this, a configured POTCAR dir lands only
                # in job_defaults, the submitter reads "" from root, and POTCAR
                # generation is silently skipped.
                for key in ("potcar_root", "potcar_functional"):
                    if cc.get(key):
                        hpc[key] = cc[key]

            # Direct hpc-level keys (backward compat for legacy configs)
            for key in ("run_commands",):
                if key in wf_config:
                    hpc[key] = wf_config[key]

            # base_work_dir: check nested hpc section (from RunConfigDialog via _run_config_to_engine_config)
            wf_hpc_config = wf_config.get("hpc", {})
            if wf_hpc_config.get("base_work_dir"):
                hpc["base_work_dir"] = wf_hpc_config["base_work_dir"]
            # Fallback: check top-level for backward compat with legacy configs
            elif wf_config.get("base_work_dir"):
                hpc["base_work_dir"] = wf_config["base_work_dir"]

            # Preserve explicit False. V1 stores these at the top level; the V2
            # adapter stores use_custodian under hpc. Losing either shape made
            # the UI default (True) silently execute as False.
            for key in ("use_custodian", "custodian_max_errors"):
                if key in wf_config:
                    hpc[key] = wf_config[key]
                elif key in wf_hpc_config:
                    hpc[key] = wf_hpc_config[key]

            # Top-level keys
            for key in ("calc_templates", "cluster_configs",
                        "default_session_id", "orca_binary"):
                if key in wf_config:
                    merged[key] = wf_config[key]

            # Propagate default_session_id into hpc section so
            # get_hpc_connection (which reads config["hpc"]["default_session_id"])
            # can find it.  Also check the nested V2 engine config format.
            sid = wf_config.get("default_session_id") or wf_config.get("hpc", {}).get("default_session_id")
            if sid:
                hpc["default_session_id"] = sid

            # Merge defaults
            if "defaults" in wf_config:
                if "defaults" not in merged:
                    merged["defaults"] = {}
                for sw, defs in wf_config["defaults"].items():
                    if sw not in merged["defaults"]:
                        merged["defaults"][sw] = {}
                    merged["defaults"][sw].update(defs)

        except Exception as e:
            logger.warning("Failed to merge workflow config for %s: %s", workflow_id, e)
        return merged

    async def _process_workflow(self, workflow_id: str) -> None:
        """Process one workflow: advance states, submit, poll, collect, handle errors."""
        # Merge workflow-level config (from run panel) with global config
        config = self._merged_config(workflow_id)

        # 1. WAITING -> READY
        advance_waiting_tasks(self.db, workflow_id)

        # 2. Execute READY local tasks immediately
        self._execute_ready_local_tasks(workflow_id)

        # 2b. Recover transient-error tasks (SSH back?)
        await self._recover_transient_errors(workflow_id)

        # 3. Submit READY HPC tasks
        from catgo.workflow.engine.submitter import submit_ready_tasks
        await submit_ready_tasks(self.db, workflow_id, config)

        # 4. Poll SUBMITTED/QUEUED/RUNNING tasks
        from catgo.workflow.engine.poller import poll_active_tasks
        await poll_active_tasks(self.db, workflow_id, config)

        # 5. Collect results from COMPLETED_REMOTE tasks
        from catgo.workflow.engine.collector import collect_completed_tasks
        await collect_completed_tasks(self.db, workflow_id, config)

        # 6. Handle REMOTE_ERROR -> retry or FAILED (skips transient errors)
        handle_errors(self.db, workflow_id, config)

        # 7. Update workflow-level status
        self._update_workflow_status(workflow_id)

    async def _recover_transient_errors(self, workflow_id: str) -> None:
        """Try to recover REMOTE_ERROR tasks whose error_type is 'transient'.

        When SSH reconnects, these tasks should resume rather than stay stuck:
        - If the task has an hpc_job_id, probe the job status first:
          * Job completed on HPC -> COMPLETED_REMOTE (results will be collected)
          * Job still running/queued -> restore to SUBMITTED/RUNNING
          * Job failed -> change error_type to "compute" so error_handler retries it
        - If no hpc_job_id (submit never finished), reset to READY for re-submission.
        """
        from catgo.workflow.engine.hpc_utils import get_hpc_connection
        from catgo.workflow.engine.poller import _check_job

        error_tasks = self.db.get_tasks_by_status(
            workflow_id, TaskState.REMOTE_ERROR.value,
        )
        # Also recover FAILED tasks that were caused by transient errors
        # (e.g., "No HPC connection" that exhausted retries before the fix)
        failed_tasks = self.db.get_tasks_by_status(
            workflow_id, TaskState.FAILED.value,
        )
        _TRANSIENT_PATTERNS = ["No HPC connection", "connection lost", "SSH", "connection unavailable"]
        for t in failed_tasks:
            msg = t.get("error_message", "") or ""
            if t.get("error_type") == "transient" or any(p.lower() in msg.lower() for p in _TRANSIENT_PATTERNS):
                error_tasks.append(t)

        transient = [t for t in error_tasks if
            t.get("error_type") == "transient" or
            t.get("status") == TaskState.FAILED.value  # FAILED from transient patterns
        ]
        if not transient:
            return

        for task in transient:
            task_id = task["id"]
            hpc = await get_hpc_connection(task, self.config)
            if not hpc:
                # Still disconnected — leave in REMOTE_ERROR, try next cycle
                continue

            job_id = task.get("hpc_job_id")
            if not job_id:
                # Submit never completed — go back to READY for re-submission
                self.db.update_task(task_id,
                    status=TaskState.READY.value,
                    error_message=None,
                    error_type=None,
                )
                logger.info(
                    "Task %s: transient recovery -> READY (no job_id, will re-submit)",
                    task_id,
                )
                continue

            # Task was submitted before disconnect — check what happened to the job
            try:
                job_status = await _check_job(hpc, job_id)
            except Exception as e:
                logger.warning("Task %s: transient recovery probe failed: %s", task_id, e)
                continue

            if job_status == "COMPLETED_REMOTE":
                self.db.update_task(task_id,
                    status=TaskState.COMPLETED_REMOTE.value,
                    error_message=None,
                    error_type=None,
                )
                logger.info(
                    "Task %s: transient recovery -> COMPLETED_REMOTE (job %s finished while disconnected)",
                    task_id, job_id,
                )
            elif job_status == "FAILED":
                # The HPC job itself failed — reclassify as compute error
                # so the error_handler can apply smart recovery / retries.
                self.db.update_task(task_id,
                    error_type="compute",
                    error_message="HPC job failed (detected after reconnect)",
                )
                logger.info(
                    "Task %s: transient -> compute error (job %s failed on HPC)",
                    task_id, job_id,
                )
            elif job_status in ("RUNNING", "QUEUED"):
                self.db.update_task(task_id,
                    status=TaskState.SUBMITTED.value if job_status == "QUEUED" else TaskState.RUNNING.value,
                    error_message=None,
                    error_type=None,
                )
                logger.info(
                    "Task %s: transient recovery -> %s (job %s still active)",
                    task_id, job_status, job_id,
                )
            else:
                # UNKNOWN — leave in REMOTE_ERROR, try again next cycle
                logger.debug(
                    "Task %s: transient recovery got UNKNOWN status for job %s, will retry",
                    task_id, job_id,
                )

    def _get_workflow_execution_mode(self, workflow_id: str) -> str:
        """Read execution_mode from per-workflow config_json."""
        try:
            wf = self.db.get_workflow(workflow_id)
            import json as _json
            cfg = _json.loads(wf.get("config_json", "{}") or "{}")
            return cfg.get("execution_mode", "hpc")
        except Exception:
            return "hpc"

    def _execute_ready_local_tasks(self, workflow_id: str) -> None:
        """Execute local tasks — the core of local workflow execution.

        LOCAL EXECUTION ARCHITECTURE
        ============================

        When execution_mode is 'local', calculations run on the user's machine
        (e.g., MACE subprocess) instead of being submitted to an HPC cluster.

        Task routing priority (checked in order):
          1. MLP nodes + local mode → _execute_mlp_local_task (MACE/CHGNet subprocess)
          2. Analysis nodes (always local) → _execute_analysis_local_task
          3. @task(local=True) builtins → defn.func() (structure_input, slab_gen, etc.)
          4. LOCAL_NODES/BUILD_NODES without @task → _execute_v1_local_task
          5. Everything else → skipped (submitter handles HPC submission)

        V1→V2 RESULT BRIDGE
        ====================

        The V1 executors (mlp.py, analysis.py, local.py) store results in an
        in-memory step_results dict and the V1 workflow_steps table. The V2
        engine stores results in the task_results table. Each bridge method:
          1. Builds a V1-style step_results dict from V2 task_results (_build_step_results)
          2. Calls the V1 executor
          3. Extracts results and stores them in V2 task_results (structure_json, energy, etc.)
          4. Stores contcar/stdout/work_dir in outputs_json for frontend display
          5. Persists work_dir on the V2 tasks table for file listing

        FAN-OUT (BATCH EXECUTION)
        =========================

        When a parent node (e.g., batch_slab_gen) produces multiple structures
        (_fan_out=True, structures=[list]), the MLP bridge detects this and
        dispatches to execute_mlp_local_batch() which runs all structures in
        parallel. Results are stored as a batch array in outputs_json.
        """
        from catgo.workflow.task_decorator import get_task_definition
        from catgo.workflow.engine.resolver import resolve_task_inputs
        from catgo.workflow.engine.resolver import _KEY_TO_COLUMN
        from workflow.node_sets import MLP_NODES, ANALYSIS_NODES, LOCAL_NODES, BUILD_NODES, _resolve_software, UNIFIED_CALC_NODES
        import json

        execution_mode = self._get_workflow_execution_mode(workflow_id)

        ready = self.db.get_tasks_by_status(workflow_id, TaskState.READY.value)
        for task in ready:
            task_id = task["id"]

            # --- Control-flow tasks: delegate to control_flow module ---
            if task["task_type"] == "__while__":
                handle_while_task(self.db, task, workflow_id)
                continue
            if task["task_type"] == "__zone__":
                handle_zone_task(self.db, task, workflow_id)
                continue

            task_type = task["task_type"]
            params = json.loads(task.get("params_json", "{}") or "{}")

            # Resolve unified calc types (geo_opt+mlp → mlp_relax) and the
            # generic "analysis" node (carries its concrete kind in params.type
            # → e.g. surface_energy/elastic_analysis). Without including
            # "analysis" here the resolver never ran and the node fell through
            # to the HPC submitter ("No HPC connection available").
            resolved_type = task_type
            if task_type in UNIFIED_CALC_NODES or task_type == "analysis":
                resolved_type, _ = _resolve_software(task_type, params)

            # --- MLP local execution (when execution_mode == "local") ---
            if execution_mode == "local" and resolved_type in MLP_NODES:
                self._execute_mlp_local_task(workflow_id, task, resolved_type, params)
                continue

            # --- Analysis local execution (always local, no HPC needed) ---
            if resolved_type in ANALYSIS_NODES:
                self._execute_analysis_local_task(workflow_id, task, resolved_type, params)
                continue

            # --- @task(local=True) builtins run via their registered function ---
            defn = get_task_definition(task_type)
            if defn and defn.local:
                # Fall through to the existing defn.func() executor below
                pass
            elif resolved_type in LOCAL_NODES or resolved_type in BUILD_NODES:
                # V1 local nodes without @task registration (batch_slab_gen, etc.)
                self._execute_v1_local_task(workflow_id, task, resolved_type, params)
                continue
            else:
                continue  # Not a local task — submitter handles it

            self.db.update_task(task_id, status=TaskState.RUNNING.value)
            asyncio.get_event_loop().create_task(
                _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "RUNNING"})
            )

            try:
                inputs = resolve_task_inputs(self.db, task_id)
                params = json.loads(task.get("params_json", "{}") or "{}")
                all_inputs = {**inputs, **params}

                if defn.func:
                    result = defn.func(**all_inputs)
                else:
                    result = {}

                if isinstance(result, dict):
                    # Map result keys to DB column names, extras go to outputs_json
                    _VALID_COLUMNS = {
                        "energy", "structure_json", "real_freqs_json",
                        "imag_freqs_json", "positions_json", "masses_json",
                        "gibbs", "zpe", "ts_correction", "outputs_json",
                    }
                    db_result = {}
                    extras = {}
                    for k, v in result.items():
                        col = _KEY_TO_COLUMN.get(k, k)
                        if col in _VALID_COLUMNS:
                            db_result[col] = v
                        else:
                            extras[k] = v
                    if extras:
                        db_result["outputs_json"] = json.dumps(extras)
                    self.db.store_result(task_id, workflow_id, **db_result)
                    try:
                        _record_provenance(self.db, workflow_id, task_id, db_result, task)
                    except Exception as prov_err:
                        logger.warning("Provenance recording failed for task %s: %s", task_id, prov_err)

                self.db.update_task(task_id, status=TaskState.COMPLETED.value)
                asyncio.get_event_loop().create_task(
                    _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "COMPLETED"})
                )
                logger.info("Task %s (%s): local execution completed", task_id, task["task_type"])

            except Exception as e:
                self.db.update_task(task_id,
                    status=TaskState.FAILED.value,
                    error_message=str(e),
                )
                asyncio.get_event_loop().create_task(
                    _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "FAILED"})
                )
                logger.error("Task %s (%s): local execution failed: %s",
                             task_id, task["task_type"], e)

    def _build_step_results(self, task_id: str) -> dict[str, dict]:
        """Build V1-style step_results dict from parent task DB results."""
        import json as _json
        step_results: dict[str, dict] = {}
        parent_links = self.db.get_task_parents(task_id)
        for link in parent_links:
            pid = link["source_task_id"]
            result_row = self.db.get_result(pid)
            if not result_row:
                continue
            # Merge result columns + outputs_json into a flat dict
            r: dict = {}
            for k, v in result_row.items():
                if k in ("task_id", "workflow_id") or v is None:
                    continue
                if k == "outputs_json" and isinstance(v, str):
                    try:
                        r.update(_json.loads(v))
                    except Exception:
                        pass
                else:
                    r[k] = v
            # Also pull node_type from the parent task
            parent_task = self.db.get_task(pid)
            if parent_task:
                r["node_type"] = parent_task.get("task_type", "")
            step_results[pid] = r
        return step_results

    def _get_parent_ids_for_task(self, step_id: str, _edges: list) -> list[str]:
        """Get parent task IDs from the DB (ignores edges arg for V2 compat)."""
        links = self.db.get_task_parents(step_id)
        return [link["source_task_id"] for link in links]

    def _execute_mlp_local_task(
        self, workflow_id: str, task: dict, resolved_type: str, params: dict,
    ) -> None:
        """Launch MLP local execution as an async task (MACE/CHGNet/M3GNet)."""
        task_id = task["id"]
        self.db.update_task(task_id, status=TaskState.RUNNING.value)
        asyncio.get_event_loop().create_task(
            _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "RUNNING"})
        )

        async def _run() -> None:
            try:
                step_results = self._build_step_results(task_id)

                async def _bcast(wf_id: str, msg: dict) -> None:
                    await _broadcast(wf_id, msg)

                # Detect fan-out: parent has multiple structures (e.g. from batch_slab_gen)
                fan_out_structures = None
                fan_out_labels = None
                parent_links = self.db.get_task_parents(task_id)
                for link in parent_links:
                    pid = link["source_task_id"]
                    parent_data = step_results.get(pid, {})
                    structs = parent_data.get("structures")
                    if parent_data.get("_fan_out") and structs and len(structs) > 1:
                        fan_out_structures = structs
                        fan_out_labels = parent_data.get("labels")
                        break

                if fan_out_structures:
                    # Use parent labels if available, otherwise generate generic ones
                    labels = fan_out_labels or [f"slab_{i}" for i in range(len(fan_out_structures))]
                    from workflow.engines.mlp import execute_mlp_local_batch
                    await execute_mlp_local_batch(
                        workflow_id, task_id, resolved_type, params,
                        fan_out_structures,
                        labels,
                        step_results,
                        type("Config", (), {"local_work_dir": ""})(),
                        _bcast,
                    )
                else:
                    from workflow.engines.mlp import execute_mlp_local
                    await execute_mlp_local(
                        workflow_id, task_id, resolved_type, params,
                        [], step_results,
                        type("Config", (), {"local_work_dir": ""})(),
                        _bcast, self._get_parent_ids_for_task,
                    )

                # Bridge MLP results into V2 task_results table so
                # downstream tasks can find them via resolve_task_inputs().
                mlp_result = step_results.get(task_id, {})
                logger.info(
                    "Task %s: bridging MLP result to V2 DB (keys=%s)",
                    task_id, list(mlp_result.keys()) if mlp_result else "EMPTY",
                )
                import json as _json_bridge

                db_fields: dict[str, Any] = {}

                if mlp_result.get("_fan_out") and mlp_result.get("results"):
                    # --- Batch result: multiple structures processed ---
                    batch_results = mlp_result["results"]
                    # Re-expose the optimized structures as a flat `structures`
                    # list (with parallel `labels`) so a DOWNSTREAM fan-out
                    # consumer (e.g. geo_opt → single_point/freq chain) detects
                    # the fan-out again and re-fans. Without this the batch result
                    # only carries `results`, which the fan-out detector ignores,
                    # so the chain silently collapses to the first structure.
                    fanned_structs: list = []
                    fanned_labels: list = []
                    for entry in batch_results:
                        if entry.get("status") != "completed":
                            continue
                        r = entry.get("result", {})
                        contcar = r.get("contcar")
                        if contcar:
                            fanned_structs.append(_poscar_to_json(contcar))
                            fanned_labels.append(entry.get("label", f"structure_{entry.get('index', len(fanned_structs))}"))
                    if len(fanned_structs) > 1:
                        mlp_result = {
                            **mlp_result,
                            "structures": fanned_structs,
                            "labels": fanned_labels,
                        }
                    # Store ALL results in outputs_json for frontend display
                    db_fields["outputs_json"] = _json_bridge.dumps(mlp_result, default=str)
                    # Use first completed structure as primary structure_json for downstream
                    for entry in batch_results:
                        if entry.get("status") == "completed":
                            r = entry.get("result", {})
                            if r.get("contcar"):
                                db_fields["structure_json"] = _poscar_to_json(r["contcar"])
                            if r.get("energy") is not None:
                                db_fields["energy"] = r["energy"]
                            break
                else:
                    # --- Single result ---
                    contcar = mlp_result.get("contcar")
                    if contcar:
                        db_fields["structure_json"] = _poscar_to_json(contcar)
                    if mlp_result.get("energy") is not None:
                        db_fields["energy"] = mlp_result["energy"]
                    freqs = mlp_result.get("frequencies")
                    if freqs is not None:
                        db_fields["real_freqs_json"] = _json_bridge.dumps(freqs) if not isinstance(freqs, str) else freqs
                    if mlp_result.get("zpe") is not None:
                        db_fields["zpe"] = mlp_result["zpe"]
                    # Everything else (contcar, stdout, work_dir) goes into outputs_json
                    _skip = {"energy", "frequencies", "zpe"}
                    extras = {k: v for k, v in mlp_result.items() if k not in _skip and v is not None}
                    if "stdout" in extras and isinstance(extras["stdout"], str):
                        extras["stdout"] = extras["stdout"][-5000:]
                    if extras:
                        db_fields["outputs_json"] = _json_bridge.dumps(extras, default=str)

                if db_fields:
                    self.db.store_result(task_id, workflow_id, **db_fields)
                # Persist work_dir on V2 tasks table for file listing endpoint
                work_dir = mlp_result.get("work_dir")
                if not work_dir:
                    # Batch runs store work_dir via V1 update_step_work_dir
                    task_row = self.db.get_task(task_id)
                    work_dir = task_row.get("work_dir") if task_row else None
                self.db.update_task(task_id,
                    status=TaskState.COMPLETED.value,
                    **({"work_dir": work_dir} if work_dir else {}))
                logger.info("Task %s (%s): MLP local execution completed", task_id, resolved_type)
            except Exception as e:
                self.db.update_task(task_id,
                    status=TaskState.FAILED.value,
                    error_message=str(e)[:500],
                )
                asyncio.get_event_loop().create_task(
                    _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "FAILED"})
                )
                logger.error("Task %s (%s): MLP local failed: %s", task_id, resolved_type, e)

        self._track_task(workflow_id, asyncio.get_event_loop().create_task(_run()))

    def _execute_analysis_local_task(
        self, workflow_id: str, task: dict, resolved_type: str, params: dict,
    ) -> None:
        """Execute analysis nodes locally (surface_energy, adsorption_energy, etc.)."""
        task_id = task["id"]
        self.db.update_task(task_id, status=TaskState.RUNNING.value)
        asyncio.get_event_loop().create_task(
            _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "RUNNING"})
        )

        async def _run() -> None:
            try:
                step_results = self._build_step_results(task_id)

                async def _bcast(wf_id: str, msg: dict) -> None:
                    await _broadcast(wf_id, msg)

                from workflow.engines.analysis import execute_analysis_node
                await execute_analysis_node(
                    workflow_id, task_id, resolved_type, params,
                    [], step_results,
                    type("Config", (), {"execution_mode": "local"})(),
                    _bcast, self._get_parent_ids_for_task,
                )

                # Bridge analysis results into V2 task_results table
                ana_result = step_results.get(task_id, {})
                db_fields: dict[str, Any] = {}
                for struct_key in ("contcar", "structure_json", "structure"):
                    val = ana_result.get(struct_key)
                    if val:
                        db_fields["structure_json"] = _poscar_to_json(val)
                        break
                if ana_result.get("energy") is not None:
                    db_fields["energy"] = ana_result["energy"]
                # Keep all other fields (including work_dir, stdout) for frontend
                _skip = {"energy"}
                extras = {k: v for k, v in ana_result.items() if k not in _skip and v is not None}
                if extras:
                    import json as _json_ana
                    db_fields["outputs_json"] = _json_ana.dumps(extras, default=str)
                if db_fields:
                    self.db.store_result(task_id, workflow_id, **db_fields)

                # Persist work_dir if analysis produced one
                ana_work_dir = ana_result.get("work_dir")
                self.db.update_task(task_id,
                    status=TaskState.COMPLETED.value,
                    **({"work_dir": ana_work_dir} if ana_work_dir else {}))
                asyncio.get_event_loop().create_task(
                    _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "COMPLETED"})
                )
                logger.info("Task %s (%s): analysis execution completed", task_id, resolved_type)
            except Exception as e:
                self.db.update_task(task_id,
                    status=TaskState.FAILED.value,
                    error_message=str(e)[:500],
                )
                asyncio.get_event_loop().create_task(
                    _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "FAILED"})
                )
                logger.error("Task %s (%s): analysis failed: %s", task_id, resolved_type, e)

        self._track_task(workflow_id, asyncio.get_event_loop().create_task(_run()))

    def _execute_v1_local_task(
        self, workflow_id: str, task: dict, resolved_type: str, params: dict,
    ) -> None:
        """Execute V1 local/build nodes via the V1 local executor engine."""
        task_id = task["id"]
        self.db.update_task(task_id, status=TaskState.RUNNING.value)
        asyncio.get_event_loop().create_task(
            _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "RUNNING"})
        )

        async def _run() -> None:
            try:
                step_results = self._build_step_results(task_id)
                import json as _json

                # Read edges from DB for the V1 executor
                edges = []
                parent_links = self.db.get_task_parents(task_id)
                for link in parent_links:
                    edges.append({
                        "source": link["source_task_id"],
                        "target": task_id,
                        "sourceKey": link.get("source_key", "structure"),
                        "targetKey": link.get("target_key", "structure"),
                    })

                async def _bcast(wf_id: str, msg: dict) -> None:
                    await _broadcast(wf_id, msg)

                from workflow.engines.local import execute_local_node
                await execute_local_node(
                    workflow_id, task_id, resolved_type, params,
                    edges, step_results,
                    type("Config", (), {"local_work_dir": ""})(),
                    _bcast, self._get_parent_ids_for_task,
                )

                # Bridge results into V2 task_results table
                v1_result = step_results.get(task_id, {})
                db_fields: dict[str, Any] = {}
                for struct_key in ("contcar", "structure_json", "structure"):
                    val = v1_result.get(struct_key)
                    if val:
                        db_fields["structure_json"] = _poscar_to_json(val)
                        break
                if v1_result.get("energy") is not None:
                    db_fields["energy"] = v1_result["energy"]
                # Keep all fields (structures, _fan_out, work_dir, etc.) for frontend + downstream
                _skip = {"energy"}
                extras = {k: v for k, v in v1_result.items() if k not in _skip and v is not None}
                if extras:
                    db_fields["outputs_json"] = _json.dumps(extras, default=str)
                if db_fields:
                    self.db.store_result(task_id, workflow_id, **db_fields)

                v1_work_dir = v1_result.get("work_dir")
                self.db.update_task(task_id,
                    status=TaskState.COMPLETED.value,
                    **({"work_dir": v1_work_dir} if v1_work_dir else {}))
                asyncio.get_event_loop().create_task(
                    _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "COMPLETED"})
                )
                logger.info("Task %s (%s): V1 local execution completed", task_id, resolved_type)
            except Exception as e:
                self.db.update_task(task_id,
                    status=TaskState.FAILED.value,
                    error_message=str(e)[:500],
                )
                asyncio.get_event_loop().create_task(
                    _broadcast(workflow_id, {"type": "task_status", "task_id": task_id, "status": "FAILED"})
                )
                logger.error("Task %s (%s): V1 local failed: %s", task_id, resolved_type, e)

        self._track_task(workflow_id, asyncio.get_event_loop().create_task(_run()))

    def _update_workflow_status(self, workflow_id: str) -> None:
        """Derive workflow status from task states.

        Also reflects per-task state into the V1 `workflow_steps` table so
        the frontend (which still reads V1) shows the live status — without
        this, V1 steps stay frozen at whatever they were when the workflow
        was first created, and the user sees stale "pending" / "completed"
        cells even though the V2 engine has moved on.
        """
        tasks = self.db.get_all_tasks(workflow_id)
        if not tasks:
            return

        current = self.db.get_workflow(workflow_id)

        # Sync V2 task statuses → V1 workflow_steps every cycle. Cheap (one
        # UPDATE per task, indexed by id) and idempotent. Done BEFORE the
        # workflow-level draft early-return so it runs even for draft
        # workflows that have completed tasks. Wrapped per-task so one
        # failure (e.g. step never registered in V1) doesn't skip the rest.
        try:
            from catgo.utils.workflow_db import update_step as update_v1_step
            for t in tasks:
                v1_status = _V2_TO_V1_STATUS.get(t["status"], "pending")
                try:
                    update_v1_step(workflow_id, t.get("node_id") or t["id"], {
                        "status": v1_status,
                        "hpc_job_id": t.get("hpc_job_id"),
                        "hpc_session_id": t.get("hpc_session_id"),
                        "work_dir": t.get("work_dir"),
                        "error_message": t.get("error_message"),
                        "started_at": t.get("started_at"),
                        "completed_at": t.get("completed_at"),
                    })
                except Exception:
                    continue
        except Exception:
            pass

        # Never override an explicitly-set "draft" status — only the user
        # (via submit_workflow) should transition out of draft.
        if current["status"] == WorkflowState.DRAFT.value:
            return

        states = [TaskState(t["status"]) for t in tasks]
        new_status = WorkflowState.from_task_states(states)

        status_changed = current["status"] != new_status.value
        if status_changed:
            self.db.update_workflow(workflow_id, status=new_status.value)
            try:
                loop = asyncio.get_running_loop()
            except RuntimeError:
                loop = None
            if loop is not None:
                loop.create_task(
                    _broadcast(
                        workflow_id,
                        {"type": "workflow_status", "status": new_status.value},
                    )
                )
            logger.info("Workflow %s: status → %s", workflow_id, new_status.value)

        # Always reconcile V1 workflow status from V2 — even when V2 hasn't
        # transitioned this cycle. Without this, a stale V1 row (e.g. left
        # over as "completed" after the engine restarted mid-run) sticks
        # forever because the change-detector above never fires. Idempotent
        # UPDATE keyed on workflow_id is cheap. Errors are still swallowed
        # since V1 is the legacy DB and not critical to engine correctness.
        try:
            from catgo.utils.workflow_db import update_workflow as update_v1_workflow
            update_v1_workflow(workflow_id, {"status": new_status.value})
        except Exception:
            pass

    async def run_forever(self) -> None:
        """Run the scanner in a loop."""
        logger.info("WorkflowEngine started (poll_interval=%ds)", self.poll_interval)
        while True:
            try:
                await self.scan_cycle()
            except Exception as e:
                logger.error("Scan cycle failed: %s", e, exc_info=True)
            await asyncio.sleep(self.poll_interval)
planning-and-task-breakdownaddyosmani/agent-skillsBreaks work into ordered tasks. Use when you have a spec or clear requirements and need to break work into implementable tasks. Use when a task feels too large to start, when you need to estimate scope, or when parallel work is possible.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
screenpipethedotmack/claude-memSearch screen and audio history recorded by Screenpipe and optionally save selected context into claude-mem. Use when the user asks to recall desktop activity, a meeting, or something seen in another app through Screenpipe.Stars: 118kUpdated Claude CodeCodexVerifiedInstall
contact-widgetnexu-io/open-designSelf-contained floating chat widget with welcome screen, social links, meeting button, and message input. Single HTML file, zero dependencies.Stars: 108kUpdated Claude CodeCodexVerifiedInstall
meeting-notesnexu-io/open-designMeeting notes page — title bar with attendees, agenda checklist, decisions block, action items table with owners + dates, and a "next meeting" footer. Use when the brief mentions "meeting notes", "minutes", "1:1 notes", "all-hands recap", or "会议纪要".Stars: 108kUpdated Claude CodeCodexVerifiedInstall