diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index 1253123805..748c6770fe 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -34,6 +34,7 @@ def build( force=False, channels=None, docker_builder=None, + mulled_upload_target='biocontainers', ): """ Build a single recipe for a single env @@ -66,6 +67,10 @@ def build( docker_builder : docker_utils.RecipeBuilder object Use this docker builder to build the recipe, copying over the built recipe to the host's conda-bld directory. + + mulled_upload_target: biocontainers + Namespace for docker container + """ # Clean provided env and exisiting os.environ to only allow whitelisted env @@ -161,7 +166,10 @@ def build( base_image = 'bioconda/extended-base-image' if use_base_image else None try: - res = pkg_test.test_package(pkg_path, base_image=base_image) + if channels: + res = pkg_test.test_package(pkg_path, base_image=base_image, mulled_upload_target=mulled_upload_target, channels=channels) + else: + res = pkg_test.test_package(pkg_path, base_image=base_image, mulled_upload_target=mulled_upload_target) logger.info("TEST SUCCESS %s, %s", recipe, utils.envstr(_env)) mulled_image = pkg_test.get_image_name(pkg_path) @@ -181,7 +189,7 @@ def build_recipes( docker_builder=None, label=None, anaconda_upload=False, - mulled_upload_target=None, + mulled_upload_target='biocontainers', check_channels=None, ): """ @@ -222,7 +230,7 @@ def build_recipes( anaconda_upload : bool If True, upload the package to anaconda.org. - mulled_upload_target : None + mulled_upload_target : biocontainers If not None, upload the mulled docker image to the given target on quay.io. check_channels : list @@ -350,6 +358,7 @@ def build_recipes( force=force, channels=config['channels'], docker_builder=docker_builder, + mulled_upload_target=mulled_upload_target, ) all_success &= res.success diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index 3b9921b3b2..eafaf41882 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -346,7 +346,7 @@ def build( build_script_template=None, pkg_dir=None, anaconda_upload=False, - mulled_upload_target=None, + mulled_upload_target='biocontainers', keep_image=False, ): setup_logger(loglevel) diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py index e3e2c23db3..871614bde6 100644 --- a/bioconda_utils/docker_utils.py +++ b/bioconda_utils/docker_utils.py @@ -88,7 +88,9 @@ # exists before adding the channel. mkdir -p {self.container_staging}/linux-64 mkdir -p {self.container_staging}/noarch +mkdir -p {self.container_package_cache} conda config --add channels file://{self.container_staging} > /dev/null 2>&1 +export CONDA_PKGS_DIRS={self.container_package_cache} # The actual building... # we explicitly point to the meta.yaml, in order to keep @@ -192,6 +194,7 @@ def __init__( tag='tmp-bioconda-builder', container_recipe='/opt/recipe', container_staging="/opt/host-conda-bld", + container_package_cache="/opt/anaconda-pkg-cache", requirements=None, build_script_template=BUILD_SCRIPT_TEMPLATE, dockerfile_template=DOCKERFILE_TEMPLATE, @@ -219,6 +222,9 @@ def __init__( Upon successful building container-built packages will be copied over. Mounted as read-write. + container_package_cache: str + If you are using package caches put them here + requirements : None or str Path to a "requirements.txt" file which will be installed with conda in a newly-created container. If None, then use the default @@ -322,6 +328,8 @@ def __init__( self.container_recipe = container_recipe self.container_staging = container_staging + ##TODO Clean this up - should get it from the environmental variable + self.container_package_cache = container_package_cache self.host_conda_bld = get_host_conda_bld() if use_host_conda_bld: @@ -450,14 +458,20 @@ def build_recipe(self, recipe_dir, build_args, env, pkg, noarch=False): env_list.append('-e') env_list.append('{0}={1}'.format(k, v)) + volume_mounts = [ + '-v', '{0}:/opt/build_script.bash'.format(build_script), + '-v', '{0}:{1}'.format(self.pkg_dir, self.container_staging), + '-v', '{0}:{1}'.format(recipe_dir, self.container_recipe), + ] + + if os.environ.get('CONDA_PKGS_DIRS'): + volume_mounts = volume_mounts + ['-v', '{0}:{1}'.format(os.environ.get('CONDA_PKGS_DIRS'), self.container_package_cache)] + cmd = [ 'docker', 'run', '--net', 'host', '--rm', - '-v', '{0}:/opt/build_script.bash'.format(build_script), - '-v', '{0}:{1}'.format(self.pkg_dir, self.container_staging), - '-v', '{0}:{1}'.format(recipe_dir, self.container_recipe), - ] + env_list + [ + ] + volume_mounts + env_list + [ self.tag, '/bin/bash', '/opt/build_script.bash', ] diff --git a/bioconda_utils/pkg_test.py b/bioconda_utils/pkg_test.py index 18af573b33..fdf947f538 100644 --- a/bioconda_utils/pkg_test.py +++ b/bioconda_utils/pkg_test.py @@ -79,7 +79,8 @@ def test_package( name_override=None, channels=["conda-forge", "defaults"], mulled_args="", - base_image=None + base_image=None, + mulled_upload_target='biocontainers' ): """ Tests a built package in a minimal docker container. @@ -104,6 +105,9 @@ def test_package( base_image : None | str Specify custom base image. Busybox is used in the default case. + mulled_upload_target : biocontainers | str + Namespace for docker container + """ assert path.endswith('.tar.bz2'), "Unrecognized path {0}".format(path) @@ -130,9 +134,10 @@ def test_package( 'mulled-build', 'build-and-test', spec, - '-n', 'biocontainers', + '-n', mulled_upload_target, '--test', tests ] + logger.debug('Cmd to run: %s' % cmd) if name_override: cmd += ['--name-override', name_override] cmd += channel_args