Skip to content

Hello-QM/catgo-LRG / server/catgo/workflow

catgo-workflow

Verified

Create and manage computational chemistry workflows with CatGo. Supports VASP, CP2K, ORCA, MLP, LAMMPS. Build OER/HER/CO2RR workflows, geometry optimization, frequency analysis, Gibbs energy calculations.

FileKindSize
SKILL.mdskill md3.8 KB
__init__.pyscript1.1 KB
builtins.pyscript4.7 KB
builtins_impl.pyscript21.2 KB
config.pyscript6.5 KB
db.pyscript25.3 KB
engine/__init__.pyscript615 B
engine/advancer.pyscript9.3 KB
engine/ai_diagnosis.pyscript3.8 KB
engine/batch_submitter.pyscript9.7 KB
engine/broadcast.pyscript1.8 KB
engine/collector.pyscript8.3 KB
engine/control_flow.pyscript5.7 KB
engine/engine_builtins.pyscript19.6 KB
engine/engine_registry.pyscript1.5 KB
engine/error_handler.pyscript3.6 KB
engine/hpc_utils.pyscript4.0 KB
engine/job_script.pyscript11.6 KB
engine/lifecycle.pyscript10.8 KB
engine/orca_progress.pyscript3.4 KB
engine/poller.pyscript11.7 KB
engine/resolver.pyscript2.7 KB
engine/result_collector.pyscript56.3 KB
engine/result_handler.pyscript5.6 KB
engine/scanner.pyscript52.0 KB
engine/smart_recovery.pyscript3.6 KB
engine/submitter.pyscript26.2 KB
engine/v1_monitor.pyscript2.7 KB
engine/vasp_submission.pyscript29.0 KB
graph_converter.pyscript6.5 KB
mcp_tools.pyscript7.5 KB
provenance.pyscript5.8 KB
reference.pyscript1.1 KB
service.pyscript4.0 KB
state_map.pyscript1.5 KB
states.pyscript7.5 KB
task_decorator.pyscript2.5 KB
task_ids.pyscript986 B
v1_compat.pyscript2.8 KB
workflow.pyscript12.0 KB

engine/batch_submitter.py

View on GitHub
"""Batch submission: submit multiple tasks as a single SLURM array job.

When the submitter detects a fan-out (multiple READY tasks of the same type),
it auto-promotes them to a single ``sbatch --array=0-N%max_concurrent`` call
instead of N individual sbatch submissions. This matches the V1 engine's
batch_execute.py pattern but integrates with the V2 scanner/submitter.

Reuses existing engine_registry generators for input file creation and
job_script.py for SLURM script generation.
"""

from __future__ import annotations

import json
import logging
from typing import Any

from catgo.workflow.db import WorkflowDB
from catgo.workflow.states import TaskState
from catgo.workflow.engine.hpc_utils import (
    get_hpc_connection,
    resolve_work_dir,
    map_task_type_to_engine,
)
from catgo.workflow.engine.resolver import resolve_task_inputs, primary_structure_input
from catgo.workflow.engine.job_script import generate_job_script
from catgo.workflow.engine.engine_registry import get_engine_generator
from catgo.workflow.engine.vasp_submission import (
    resolve_vasp_command,
    resolve_vasp_input_policy,
    resolve_vasp_use_custodian,
    write_vasp_input_manifest,
)

logger = logging.getLogger(__name__)

# Minimum number of same-type tasks before promoting to array job
ARRAY_JOB_THRESHOLD = 3


async def submit_batch_tasks(
    db: WorkflowDB,
    task_ids: list[str],
    workflow_id: str,
    config: dict[str, Any],
) -> str | None:
    """Submit multiple tasks as a single SLURM array job.

    All tasks must be the same task_type. Generates inputs into numbered
    subdirectories (000000/, 000001/, ...) under a shared batch directory,
    then submits one ``sbatch --array`` call.

    Returns the SLURM job_id, or None on failure.
    """
    n = len(task_ids)
    if n == 0:
        return None

    tasks = [db.get_task(tid) for tid in task_ids]
    first_task = tasks[0]
    task_params_list = [
        json.loads(task.get("params_json", "{}") or "{}")
        for task in tasks
    ]
    params = task_params_list[0]
    resolved_type, engine_key = map_task_type_to_engine(first_task["task_type"], params)

    # One array script executes one command in every directory. Mixed VASP
    # command/binary/custodian fingerprints would make manifests lie.
    vasp_resolutions = []
    vasp_input_policies = []
    if engine_key == "vasp":
        vasp_resolutions = [
            resolve_vasp_command(task_params, config)
            for task_params in task_params_list
        ]
        vasp_input_policies = [
            resolve_vasp_input_policy(task_params, config)
            for task_params in task_params_list
        ]
        fingerprints = {
            (
                resolution.command,
                resolution.binary_token,
                resolve_vasp_use_custodian(task_params, config),
            )
            for resolution, task_params in zip(
                vasp_resolutions, task_params_list,
            )
        }
        if len(fingerprints) != 1:
            raise ValueError(
                "VASP array submission requires identical resolved run commands, "
                "binary tokens, and custodian modes for every task"
            )

    # Get HPC connection from the first task
    hpc = await get_hpc_connection(first_task, config)
    if not hpc:
        logger.error("No HPC connection for batch submission")
        return None

    # Batch directory: base_dir/workflow_id/batch_<task_type>.
    # Resolve base_dir the same way resolve_work_dir does — the run config sets
    # base_work_dir (promoted to hpc.base_work_dir by the scanner), NOT
    # paths.base_dir. Reading only paths.base_dir left base_dir empty, so the
    # batch dir became "/<workflow_id>/..." and mkdir failed with permission
    # denied at the filesystem root.
    base_dir = (
        config.get("paths", {}).get("base_dir")
        or config.get("hpc", {}).get("base_work_dir")
        or config.get("base_work_dir")
        or "~/calculations"
    )
    batch_dir = f"{base_dir}/{workflow_id}/batch_{first_task['task_type']}"

    # ── Phase 1: Generate inputs into numbered subdirectories ──
    generator = get_engine_generator(engine_key)
    if not generator:
        logger.error("No engine registered for '%s'", engine_key)
        return None

    await hpc.run_on_owner(lambda: hpc.conn.run(f"mkdir -p {batch_dir}", check=True))

    for i, (task, task_params) in enumerate(zip(tasks, task_params_list)):
        inputs = resolve_task_inputs(db, task["id"])
        structure_str = primary_structure_input(inputs.get("structure")) or ""
        if engine_key == "lammps":
            task_params = {**task_params, "_resolved_workflow_inputs": inputs}

        work_dir = f"{batch_dir}/{i:06d}"
        db.update_task(task["id"], status=TaskState.GENERATING.value, work_dir=work_dir)
        await hpc.run_on_owner(lambda wd=work_dir: hpc.conn.run(f"mkdir -p {wd}", check=True))
        await hpc.run_on_owner(lambda wd=work_dir, tp=task_params, ss=structure_str, t=task:
            generator(hpc, wd, resolved_type, tp, ss, config, t)
        )

        # Upload custodian script if needed (same as _submit_one in submitter.py)
        if engine_key == "vasp":
            from catgo.workflow.engine.job_script import generate_custodian_script
            vasp_resolution = vasp_resolutions[i]
            custodian_py = generate_custodian_script(
                vasp_resolution.command, task_params, config,
            )
            if custodian_py:
                await hpc.run_on_owner(lambda wd=work_dir, py=custodian_py: hpc.conn.run(
                    f"cat > {wd}/run_custodian.py << 'CATGO_EOF'\n{py}\nCATGO_EOF",
                    check=True,
                ))
            # Generate POTCAR
            from catgo.workflow.engine.submitter import (
                _generate_potcar,
                _resolve_potcar_settings,
            )
            potcar_root, potcar_func = _resolve_potcar_settings(config)
            if potcar_root:
                await _generate_potcar(hpc, work_dir, potcar_root, potcar_func)
            await write_vasp_input_manifest(
                hpc, work_dir, vasp_resolution,
                use_custodian=resolve_vasp_use_custodian(task_params, config),
                input_policy=vasp_input_policies[i],
            )

        db.update_task(task["id"], status=TaskState.UPLOADING.value)

    # ── Phase 2: Build array job script ──
    max_concurrent = config.get("hpc", {}).get("job_defaults", {}).get("max_concurrent", 50)

    # Generate a base job script from the first task
    job_script = generate_job_script(engine_key, "$TASK_DIR", first_task, params, config)

    # Inject --array directive and array-aware cd
    lines = job_script.split("\n")
    array_line = f"#SBATCH --array=0-{n - 1}%{max_concurrent}"
    cd_line = f'cd {batch_dir}/$(printf "%06d" $SLURM_ARRAY_TASK_ID)'

    # Find the last #SBATCH line to insert array directive after it
    last_sbatch = -1
    for i_line, line in enumerate(lines):
        if line.strip().startswith("#SBATCH"):
            last_sbatch = i_line

    if last_sbatch >= 0:
        lines.insert(last_sbatch + 1, array_line)
        # Fix output/error paths for array jobs: write logs into each subdirectory
        for i_line, line in enumerate(lines):
            if "#SBATCH --output=" in line:
                lines[i_line] = f'#SBATCH --output=%A_%a.out'
            elif "#SBATCH --error=" in line:
                lines[i_line] = f'#SBATCH --error=%A_%a.err'
        # Replace the existing "cd ..." line with the array-aware version
        replaced_cd = False
        for i_line, line in enumerate(lines):
            if line.strip().startswith("cd ") and "$TASK_DIR" in line:
                lines[i_line] = cd_line
                replaced_cd = True
                break
        if not replaced_cd:
            # Insert cd after the last SBATCH + array line
            insert_at = last_sbatch + 2
            while insert_at < len(lines) and (
                not lines[insert_at].strip() or lines[insert_at].strip().startswith("#")
            ):
                insert_at += 1
            lines.insert(insert_at, cd_line)
    else:
        lines = [f"#!/bin/bash", array_line, "", cd_line] + lines

    # Replace any remaining $TASK_DIR references
    array_script = "\n".join(lines).replace(
        "$TASK_DIR", f'{batch_dir}/$(printf "%06d" $SLURM_ARRAY_TASK_ID)'
    )

    # ── Phase 3: Write script and submit ──
    script_path = f"{batch_dir}/submit_array.sh"
    await hpc.run_on_owner(lambda: hpc.conn.run(
        f"cat > {script_path} << 'CATGO_EOF'\n{array_script}\nCATGO_EOF",
        check=True,
    ))
    await hpc.run_on_owner(lambda: hpc.conn.run(f"chmod +x {script_path}", check=True))

    result = await hpc.run_on_owner(
        lambda: hpc.conn.run(f"cd {batch_dir} && sbatch submit_array.sh", check=False)
    )
    stdout = result.stdout.strip() if hasattr(result, "stdout") else str(result)

    # Parse job ID from "Submitted batch job 12345"
    job_id = ""
    for word in stdout.split():
        if word.isdigit():
            job_id = word
            break

    if not job_id:
        logger.error("Batch submission failed: %s", stdout)
        for task in tasks:
            db.update_task(
                task["id"],
                status=TaskState.REMOTE_ERROR.value,
                error_message=f"Batch sbatch failed: {stdout}",
                error_type="transient",
            )
        return None

    # ── Phase 4: Update all tasks with array job IDs ──
    for i, task in enumerate(tasks):
        db.update_task(
            task["id"],
            status=TaskState.SUBMITTED.value,
            hpc_job_id=f"{job_id}_{i}",  # SLURM array task ID format
        )

    logger.info("Batch submitted %d tasks as array job %s", n, job_id)
    return job_id
planning-and-task-breakdownaddyosmani/agent-skillsBreaks work into ordered tasks. Use when you have a spec or clear requirements and need to break work into implementable tasks. Use when a task feels too large to start, when you need to estimate scope, or when parallel work is possible.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
screenpipethedotmack/claude-memSearch screen and audio history recorded by Screenpipe and optionally save selected context into claude-mem. Use when the user asks to recall desktop activity, a meeting, or something seen in another app through Screenpipe.Stars: 118kUpdated Claude CodeCodexVerifiedInstall
contact-widgetnexu-io/open-designSelf-contained floating chat widget with welcome screen, social links, meeting button, and message input. Single HTML file, zero dependencies.Stars: 108kUpdated Claude CodeCodexVerifiedInstall
meeting-notesnexu-io/open-designMeeting notes page — title bar with attendees, agenda checklist, decisions block, action items table with owners + dates, and a "next meeting" footer. Use when the brief mentions "meeting notes", "minutes", "1:1 notes", "all-hands recap", or "会议纪要".Stars: 108kUpdated Claude CodeCodexVerifiedInstall