me-model sub-workflow (#6)

doc/source/registry.rst

@@ -4,6 +4,12 @@
 Registry
 ========
 
+memodel|neurons_memodel|v3
+**************************
+
+.. image:: generated/memodel__neurons_memodel__v3.svg
+
+
 connectome_filtering|synapses|v2
 ********************************
 

src/blue_cwl/core/exceptions.py

@@ -13,3 +13,7 @@ class ReferenceResolutionError(CWLError):
 
 class CWLValidationError(CWLError):
  """CWL Validation error."""
+
+
+class InputConcretizationError(CWLError):
+ """Input concretization error."""

src/blue_cwl/core/process.py

@@ -8,7 +8,7 @@
 import blue_cwl.core.cwl
 from blue_cwl.core.common import CustomBaseModel
 from blue_cwl.core.cwl_types import Directory, File, NexusResource
-from blue_cwl.core.exceptions import CWLError
+from blue_cwl.core.exceptions import CWLError, InputConcretizationError, ReferenceResolutionError
 from blue_cwl.core.executor import Executor, LocalExecutor
 from blue_cwl.core.resolve import resolve_parameter_references
 from blue_cwl.core.types import EnvVarDict, InputValue, InputValueObject, OutputValueObject
@@ -113,7 +113,12 @@ def build_command_line_tool_process(
  """
  L.debug("CommandLineTool input values: %s", input_values)
 
- concretized_inputs: dict[str, InputValueObject] = _concretize_inputs(tool.inputs, input_values)
+ try:
+ concretized_inputs: dict[str, InputValueObject] = _concretize_inputs(
+ tool.inputs, input_values
+ )
+ except Exception as e:
+ raise CWLError(f"CommandLineTool'{tool.id}' inputs concretization failed.\n") from e
  L.debug("Concretized CommandLineTool inputs: %s", concretized_inputs)
 
  concretized_outputs: dict[str, OutputValueObject] = _concretize_tool_outputs(
@@ -153,7 +158,7 @@ def _concretize_inputs(
  concretized_inputs[name] = _input_value_to_object(inp.type, value)
 
  if not set(inputs).issubset(set(concretized_inputs)):
- raise CWLError(
+ raise InputConcretizationError(
  "Concretized input values are not consistent with the input template.\n"
  f"Expected: {sorted(inputs.keys())}.\n"
  f"Got : {sorted(concretized_inputs.keys())}"
@@ -321,14 +326,32 @@ def _get_source_value(source, input_objects, sources):
  # The valueFrom fields are evaluated after the the source fields.
  for name, inp in step.inputs.items():
  if inp.valueFrom:
- step_input_values[name] = resolve_parameter_references(
- expression=inp.valueFrom,
- inputs=step_input_values,
- context=step_sources[name],
- runtime={},
- )
-
- step_process = build_command_line_tool_process(step.run, step_input_values)
+ try:
+ step_input_values[name] = resolve_parameter_references(
+ expression=inp.valueFrom,
+ inputs=step_input_values,
+ context=step_sources[name],
+ runtime={},
+ )
+ except ReferenceResolutionError as e:
+ raise CWLError(
+ f"Workflow step '{step_name}' parameter reference resolution failed.\n"
+ f"Input : {name}\n"
+ f"valueFrom: {inp.valueFrom}\n"
+ f"self type: {type(step_sources[name])}"
+ ) from e
+
+ try:
+ step_process = build_command_line_tool_process(step.run, step_input_values)
+ except InputConcretizationError as e:
+ raise CWLError(
+ f"Workflow step '{step_name}' process build failed.\n"
+ f"Workflow step input values : {sorted(step_input_values.keys())}\n"
+ f"CommandLineTool expected inputs: {sorted(step.run.inputs.keys())}"
+ ) from e
+ except Exception as e:
+ raise CWLError(f"Workflow step '{step_name}' process build failed.\n") from e
+
  return step_process
 
 

src/blue_cwl/generators/memodel/neurons_memodel/v3/definition.cwl

@@ -0,0 +1,134 @@
+cwlVersion: v1.2
+class: Workflow
+
+id: workflow_me_model
+label: workflow-me-model
+
+inputs:
+
+ - id: configuration_id
+ type: NexusType
+
+ - id: circuit_id
+ type: NexusType
+
+ - id: output_dir
+ type: Directory
+
+outputs:
+
+ - id: circuit
+ type: NexusType
+ outputSource: register/circuit
+
+steps:
+
+ - id: setup
+ in:
+ output_dir: output_dir
+ out:
+ - stage_dir
+ - build_dir
+ run: ./setup.cwl
+
+ - id: stage
+ in:
+ stage_dir: setup/stage_dir
+ circuit_id: circuit_id
+ configuration_id: configuration_id
+ out:
+ - config_file
+ - circuit_file
+ - nodes_file
+ - morphologies_dir
+ run: ./stage.cwl
+
+ - id: recipe
+ run: ./recipe.cwl
+ in:
+ config_file: stage/config_file
+ output_file:
+ source: setup/build_dir
+ valueFrom: $(self.path)/recipe.json
+ out:
+ - recipe_file
+
+ - id: emodel_prepare
+ run: ./emodel_prepare.cwl
+ in:
+ config_file: recipe/recipe_file
+ work_dir:
+ source: setup/build_dir
+ valueFrom: $(self.path)/configs
+ out_mechanisms_dir:
+ source: setup/build_dir
+ valueFrom: $(self.path)/mechanisms
+ out:
+ - work_dir
+ - mechanisms_dir
+
+ - id: emodel_assign
+ run: ./emodel_assign.cwl
+ in:
+ nodes_file: stage/nodes_file
+ recipe_file: recipe/recipe_file
+ out_nodes_file:
+ source: setup/build_dir
+ valueFrom: $(self.path)/assign_nodes.h5
+ out_configs_dir:
+ source: setup/build_dir
+ valueFrom: $(self.path)/configs
+ out:
+ - nodes_file
+ - configs_dir
+
+ - id: emodel_adapt
+ run: ./emodel_adapt.cwl
+ in:
+ nodes_file: emodel_assign/nodes_file
+ recipe_file: recipe/recipe_file
+ configs_dir: emodel_assign/configs_dir
+ mechanisms_dir: emodel_prepare/mechanisms_dir
+ morphologies_dir: stage/morphologies_dir
+ work_dir:
+ source: setup/build_dir
+ valueFrom: $(self.path)/adapt_workdir
+ out_nodes_file:
+ source: setup/build_dir
+ valueFrom: $(self.path)/adapt_nodes.h5
+ out_hoc_dir:
+ source: setup/build_dir
+ valueFrom: $(self.path)/hoc
+ parallel_lib:
+ valueFrom: multiprocessing
+ out:
+ - hoc_dir
+ - nodes_file
+
+ - id: emodel_currents
+ run: ./emodel_currents.cwl
+ in:
+ hoc_dir: emodel_adapt/hoc_dir
+ recipe_file: recipe/recipe_file
+ circuit_file: stage/circuit_file
+ nodes_file: emodel_adapt/nodes_file
+ morphologies_dir: stage/morphologies_dir
+ mechanisms_dir: emodel_prepare/mechanisms_dir
+ out_nodes_file:
+ source: setup/build_dir
+ valueFrom: $(self.path)/nodes.h5
+ parallel_lib:
+ valueFrom: multiprocessing
+ out:
+ - nodes_file
+
+ - id: register
+ run: ./register.cwl
+ in:
+ circuit_id: circuit_id
+ circuit_file: stage/circuit_file
+ hoc_dir: emodel_adapt/hoc_dir
+ nodes_file: emodel_currents/nodes_file
+ output_dir: output_dir
+ out:
+ - circuit

src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_adapt.cwl

@@ -0,0 +1,87 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+id: emodel_adapt
+label: emodel-adapt
+
+environment:
+ env_type: MODULE
+ modules:
+ - unstable
+ - py-emodel-generalisation
+
+executor:
+ type: slurm
+ slurm_config:
+ partition: prod
+ account: proj134
+ exclusive: true
+ time: '11:00:00'
+ constraint: nvme
+ nodes: 1
+ mem: 0
+ remote_config:
+ host: bbpv1.epfl.ch
+ env_vars:
+ NEURON_MODULE_OPTIONS: --nogui
+
+baseCommand: ["emodel-generalisation", "-v", "--no-progress", "adapt", "--no-reuse"]
+
+inputs:
+
+ - id: nodes_file
+ type: File
+ inputBinding:
+ prefix: --input-node-path
+
+ - id: out_nodes_file
+ type: string
+ inputBinding:
+ prefix: --output-node-path
+
+ - id: morphologies_dir
+ type: Directory
+ inputBinding:
+ prefix: --morphology-path
+
+ - id: mechanisms_dir
+ type: Directory
+ inputBinding:
+ prefix: --mech-path
+
+ - id: recipe_file
+ type: File
+ inputBinding:
+ prefix: --config-path
+
+ - id: out_hoc_dir
+ type: string
+ inputBinding:
+ prefix: --output-hoc-path
+
+ - id: configs_dir
+ type: Directory
+ inputBinding:
+ prefix: --local-config-path
+
+ - id: work_dir
+ type: string
+ inputBinding:
+ prefix: --local-dir
+
+ - id: parallel_lib
+ type: string
+ inputBinding:
+ prefix: --parallel-lib
+
+outputs:
+
+ - id: nodes_file
+ type: File
+ outputBinding:
+ glob: $(inputs.out_nodes_file)
+
+ - id: hoc_dir
+ type: Directory
+ outputBinding:
+ glob: $(inputs.out_hoc_dir)

src/blue_cwl/generators/memodel/neurons_memodel/v3/emodel_assign.cwl

@@ -0,0 +1,61 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+id: emodel_assign
+label: emodel-assign
+
+environment:
+ env_type: MODULE
+ modules:
+ - unstable
+ - py-emodel-generalisation
+
+executor:
+ type: slurm
+ slurm_config:
+ partition: prod
+ account: proj134
+ exclusive: true
+ time: '01:00:00'
+ nodes: 1
+ mem: 0
+ remote_config:
+ host: bbpv1.epfl.ch
+ env_vars:
+ NEURON_MODULE_OPTIONS: --nogui
+
+baseCommand: ["emodel-generalisation", "-v", "--no-progress", "assign"]
+
+inputs:
+
+ - id: nodes_file
+ type: File
+ inputBinding:
+ prefix: --input-node-path
+
+ - id: recipe_file
+ type: File
+ inputBinding:
+ prefix: --config-path
+
+ - id: out_nodes_file
+ type: string
+ inputBinding:
+ prefix: --output-node-path
+
+ - id: out_configs_dir
+ type: string
+ inputBinding:
+ prefix: --local-config-path
+
+outputs:
+
+ - id: nodes_file
+ type: File
+ outputBinding:
+ glob: $(inputs.out_nodes_file)
+
+ - id: configs_dir
+ type: Directory
+ outputBinding:
+ glob: $(inputs.out_configs_dir)