Add mount options from the CLI (#279)

New CLI argument `--mount` that is a shortcut to override mount options for `input` parameters. Works with docker and singularity runners. This is useful to ensure (in a convenient way) that input directories and files are not accidently modified by MLCube tasks.

---------

Co-authored-by: Sergey Serebryakov <[email protected]>

mlcube/mlcube/__main__.py

@@ -16,6 +16,7 @@
     UsageExamples,
     parse_cli_args,
 )
+from mlcube.config import MountType
 from mlcube.errors import ExecutionError, IllegalParameterValueError, MLCubeError
 from mlcube.parser import CliParser
 from mlcube.shell import Shell
@@ -123,6 +124,14 @@ def parser_process(value: str, state: click.parser.ParsingState):
     default=None,
     help="CPU options defined during MLCube container execution.",
 )
+mount_option = click.option(
+    "--mount",
+    required=False,
+    type=click.Choice([MountType.RW, MountType.RO]),
+    default=None,
+    help="Mount options for all input parameters. These mount options override any other mount options defined for "
+    "each input parameters. A typical use case is to ensure that inputs are mounted in read-only (ro) mode.",
+)
 
 
 @click.group(name="mlcube", add_help_option=False)
@@ -279,6 +288,7 @@ def configure(mlcube: t.Optional[str], platform: str, p: t.Tuple[str]) -> None:
 @gpus_option
 @memory_option
 @cpu_option
+@mount_option
 @Options.help
 @click.pass_context
 def run(
@@ -292,6 +302,7 @@ def run(
     gpus: str,
     memory: str,
     cpu: str,
+    mount: str,
 ) -> None:
     """Run MLCube task(s).
 
@@ -307,9 +318,12 @@ def run(
         gpus: GPU usage options defined during MLCube container execution.
         memory: Memory RAM options defined during MLCube container execution.
         cpu: CPU options defined during MLCube container execution.
+        mount: Mount (global) options defined for all input parameters in all tasks to be executed. They override any
+            mount options defined for individual parameters.
     """
     logger.info(
-        "run input_arg mlcube=%s, platform=%s, task=%s, workspace=%s, network=%s, security=%s, gpus=%s, memory=%s, "
+        "run input_arg mlcube=%s, platform=%s, task=%s, workspace=%s, network=%s, security=%s, gpus=%s, "
+        "memory=%s, mount=%s"
         "cpu=%s",
         mlcube,
         platform,
@@ -320,6 +334,7 @@ def run(
         gpus,
         memory,
         cpu,
+        mount,
     )
     runner_cls, mlcube_config = parse_cli_args(
         unparsed_args=ctx.args,
@@ -332,6 +347,7 @@ def run(
             "gpus": gpus,
             "memory": memory,
             "cpu": cpu,
+            "mount": mount,
         },
         resolve=True,
     )

mlcube/mlcube/config.py

@@ -261,7 +261,7 @@ def check_parameters(parameters: DictConfig, task_cli_args: t.Dict) -> None:
             )
         #
         for name in parameters.keys():
-            # The `_param_name` is anyway there, so check it's not None.
+            # The `name` is anyway there, so check it's not None.
             [param_def] = MLCubeConfig.ensure_values_exist(parameters, name, dict)
             # Deal with the case when value is a string (default value).
             if isinstance(param_def, str):

mlcube/mlcube/parser.py

@@ -78,7 +78,7 @@ def parse_list_arg(
 
     @staticmethod
     def parse_extra_arg(
-            unparsed_args: t.List[str], parsed_args: t.Dict[str, t.Optional[str]]
+        unparsed_args: t.List[str], parsed_args: t.Dict[str, t.Optional[str]]
     ) -> t.Tuple[DictConfig, t.Dict]:
         """Parse extra arguments on a command line.
 
@@ -111,31 +111,79 @@ def parse_extra_arg(
             [arg[2:] for arg in unparsed_args if arg.startswith("-P")]
         )
 
-        task_args = [arg.split("=") for arg in unparsed_args if not arg.startswith("-P")]
+        task_args = [
+            arg.split("=") for arg in unparsed_args if not arg.startswith("-P")
+        ]
         task_args = {arg[0]: arg[1] for arg in task_args}
 
         # Parse unparsed arguments
-        platform: t.Optional[str] = parsed_args.get('platform', None)
-        if platform in {'docker', 'singularity'}:
+        platform: t.Optional[str] = parsed_args.get("platform", None)
+        if platform in {"docker", "singularity"}:
             runner_run_args = {}
-            if parsed_args.get('network', None):
-                runner_run_args["--network"] = parsed_args['network']
-            if parsed_args.get('security', None):
+            if parsed_args.get("network", None):
+                runner_run_args["--network"] = parsed_args["network"]
+            if parsed_args.get("security", None):
                 key = "--security-opt" if platform == "docker" else "--security"
-                runner_run_args[key] = parsed_args['security']
-            if parsed_args.get('gpus', None):
+                runner_run_args[key] = parsed_args["security"]
+            if parsed_args.get("gpus", None):
                 if platform == "docker":
-                    runner_run_args["--gpus"] = parsed_args['gpus']
+                    runner_run_args["--gpus"] = parsed_args["gpus"]
                 else:
                     runner_run_args["--nv"] = ""
-                    os.environ['SINGULARITYENV_CUDA_VISIBLE_DEVICES'] = parsed_args['gpus']
-            if parsed_args.get('memory', None):
+                    os.environ["SINGULARITYENV_CUDA_VISIBLE_DEVICES"] = parsed_args[
+                        "gpus"
+                    ]
+            if parsed_args.get("memory", None):
                 key = "--memory" if platform == "docker" else "--vm-ram"
-                runner_run_args[key] = parsed_args['memory']
-            if parsed_args.get('cpu', None):
+                runner_run_args[key] = parsed_args["memory"]
+            if parsed_args.get("cpu", None):
                 key = "--cpuset-cpus" if platform == "docker" else "--vm-cpu"
-                runner_run_args[key] = parsed_args['cpu']
+                runner_run_args[key] = parsed_args["cpu"]
+            runner_run_args["--mount_opts"] = parsed_args["mount"]
 
             mlcube_args.merge_with({platform: runner_run_args})
 
         return mlcube_args, task_args
+
+    @staticmethod
+    def parse_optional_arg(
+        platform: t.Optional[str],
+        network_option: t.Optional[str],
+        security_option: t.Optional[str],
+        gpus_option: t.Optional[str],
+        memory_option: t.Optional[str],
+        cpu_option: t.Optional[str],
+        mount_option: t.Optional[str],
+    ) -> t.Tuple[DictConfig, t.Dict]:
+        """platform: Platform to use to run this MLCube (docker, singularity, gcp, k8s etc).
+        network_option: Networking options defined during MLCube container execution.
+        security_option: Security options defined during MLCube container execution.
+        gpus_option: GPU usage options defined during MLCube container execution.
+        memory_option: Memory RAM options defined during MLCube container execution.
+        cpu_option: CPU options defined during MLCube container execution.
+        mount_option: Mount options for paths.
+        """
+        mlcube_args, opts = {}, {}
+
+        opts["--mount_opts"] = mount_option
+        if network_option is not None:
+            opts["--network"] = network_option
+
+        if security_option is not None:
+            key = "--security-opt" if platform == "docker" else "--security"
+            opts[key] = security_option
+
+        if gpus_option is not None:
+            key = "--gpus" if platform == "docker" else "--nv"
+            opts[key] = gpus_option
+
+        if memory_option is not None:
+            key = "--memory" if platform == "docker" else "--vm-ram"
+            opts[key] = memory_option
+
+        if cpu_option is not None:
+            key = "--cpu-shares" if platform == "docker" else "--vm-cpu"
+            opts[key] = cpu_option
+
+        mlcube_args[platform] = opts
+        return mlcube_args, {}