feat: add support for excluding otherwise-selected recipes (#962)

The nightly rebuild job fails on bioconda-repodata-patches -- which is
built by the same workflow immediately before the rebuild job. This
allows us to exclude individual recipes from the CLI.

---------

Co-authored-by: Martin Grigorov <[email protected]>
Co-authored-by: aliciaaevans <[email protected]>

bioconda_utils/build.py

@@ -300,7 +300,9 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str],
                   mulled_conda_image: str = pkg_test.MULLED_CONDA_IMAGE,
                   record_build_failures: bool = False,
                   skiplist_leafs: bool = False,
-                  live_logs: bool = True):
+                  live_logs: bool = True,
+                  exclude: List[str] = None,
+                  ):
     """
     Build one or many bioconda packages.
 
@@ -329,6 +331,8 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str],
       keep_old_work: Do not remove anything from environment, even after successful build and test.
       skiplist_leafs: If True, blacklist leaf packages that fail to build
       live_logs: If True, enable live logging during the build process
+      exclude: list of recipes to exclude. Typically used for
+        temporary exclusion; otherwise consider adding recipe to skiplist.
     """
     if not recipes:
         logger.info("Nothing to be done.")
@@ -358,6 +362,10 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str],
     failed = []
 
     dag, name2recipes = graph.build(recipes, config=config_path, blacklist=blacklist)
+    if exclude:
+        for name in exclude:
+            dag.remove_node(name)
+
     if not dag:
         logger.info("Nothing to be done.")
         return True

bioconda_utils/cli.py

@@ -435,6 +435,7 @@ def do_lint(recipe_folder, config, packages="*", cache=None, list_checks=False,
 @arg("--record-build-failures", action="store_true", help="Record build failures in build_failure.yaml next to the recipe.")
 @arg("--skiplist-leafs", action="store_true", help="Skiplist leaf recipes (i.e. ones that are not depended on by any other recipes) that fail to build.")
 @arg('--disable-live-logs', action='store_true', help="Disable live logging during the build process")
+@arg('--exclude', nargs='+', help='Packages to exclude during this run')
 @enable_logging()
 def build(recipe_folder, config, packages="*", git_range=None, testonly=False,
           force=False, docker=None, mulled_test=False, build_script_template=None,
@@ -445,7 +446,8 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False,
           docker_base_image=None,
           record_build_failures=False,
           skiplist_leafs=False,
-          disable_live_logs=False):
+          disable_live_logs=False,
+          exclude=None):
     cfg = utils.load_config(config)
     setup = cfg.get('setup', None)
     if setup:
@@ -506,7 +508,9 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False,
                             mulled_conda_image=mulled_conda_image,
                             record_build_failures=record_build_failures,
                             skiplist_leafs=skiplist_leafs,
-                            live_logs=(not disable_live_logs))
+                            live_logs=(not disable_live_logs),
+                            exclude=exclude,
+                            )
     exit(0 if success else 1)
 
 

test/test_utils.py

@@ -156,6 +156,37 @@ def multi_build(request, recipes_fixture, config_fixture):
             ensure_missing(pkg)
 
 
+@pytest.fixture(scope='module', params=PARAMS, ids=IDS)
+def multi_build_exclude(request, recipes_fixture, config_fixture):
+    """
+    Builds the "one" and "two" recipes; provides (but then excludes) the
+    "three" recipe.
+    """
+    if request.param:
+        docker_builder = docker_utils.RecipeBuilder(
+            use_host_conda_bld=True,
+            docker_base_image=DOCKER_BASE_IMAGE)
+        mulled_test = True
+    else:
+        docker_builder = None
+        mulled_test = False
+    logger.error("Fixture: Building one/two (and not three) %s",
+                 "within docker" if docker_builder else "locally")
+    build.build_recipes(recipes_fixture.basedir, config_fixture,
+                        recipes_fixture.recipe_dirnames,
+                        docker_builder=docker_builder,
+                        mulled_test=mulled_test,
+                        exclude=['three'],
+                        )
+    logger.error("Fixture: Building one/two (and not three) %s -- DONE",
+                 "within docker" if docker_builder else "locally")
+    built_packages = recipes_fixture.pkgs
+    yield built_packages
+    for pkgs in built_packages.values():
+        for pkg in pkgs:
+            ensure_missing(pkg)
+
+
 @pytest.fixture(scope='module')
 def single_upload():
     """
@@ -216,6 +247,7 @@ def test_upload(single_upload):
 def test_single_build_only(single_build):
     for pkg in single_build:
         assert os.path.exists(pkg)
+        ensure_missing(pkg)
 
 
 @pytest.mark.skipif(SKIP_DOCKER_TESTS, reason='skipping on osx')
@@ -229,6 +261,18 @@ def test_multi_build(multi_build):
     for v in multi_build.values():
         for pkg in v:
             assert os.path.exists(pkg)
+            ensure_missing(pkg)
+
+
+@pytest.mark.long_running_1
+def test_multi_build_exclude(multi_build_exclude):
+    for (k, v) in multi_build_exclude.items():
+        for pkg in v:
+            if k == 'three':
+                assert not os.path.exists(pkg)
+            else:
+                assert os.path.exists(pkg)
+                ensure_missing(pkg)
 
 
 @pytest.mark.skipif(SKIP_DOCKER_TESTS, reason='skipping on osx')
@@ -268,6 +312,7 @@ def test_docker_builder_build(recipes_fixture):
                                 build_args='', env={})
     for pkg in pkgs:
         assert os.path.exists(pkg)
+        ensure_missing(pkg)
 
 
 @pytest.mark.skipif(SKIP_DOCKER_TESTS, reason='skipping on osx')