Accumulated tweaks from Marc M.

Be less specific about gettext with ~libxml2

can use libc+iconv or gettext with or without libxml2

initial round of ups compatability from prev release

readonly bootstrap (cvmfs) workaround

package updates from fnal-develop

added ups compat files

more missing files

moderncmakedomain package

per_os_scope

Redo of FNALssi/spack #94

redo of FNALssi #162

Revert "redo of FNALssi #162"

This reverts commit f92ed6e.

Someone fixed the geant4 recipe with a for loop in the recipe instead,
so not doing this after all.

redo of FNALssi #161

more cleanups from fnal-develop

More lib64 fun

speling of startswith

version

lib/spack/llnl/util/filesystem.py

@@ -2351,7 +2351,7 @@ def find_libraries(libraries, root, shared=True, recursive=False, runtime=True):
     # perform first non-recursive search in root/lib then in root/lib64 and
     # finally search all of root recursively. The search stops when the first
     # match is found.
-    common_lib_dirs = ["lib", "lib64"]
+    common_lib_dirs = ["lib64", "lib"]
     if sys.platform == "win32":
         common_lib_dirs.extend(["bin", "Lib"])
 

lib/spack/spack/build_systems/cmake.py

@@ -16,8 +16,10 @@
 import spack.build_environment
 import spack.builder
 import spack.package_base
+import spack.util.path
 from spack.directives import build_system, conflicts, depends_on, variant
 from spack.multimethod import when
+import archspec.cpu
 
 from ._checks import BaseBuilder, execute_build_time_tests
 
@@ -274,8 +276,13 @@ def std_args(pkg, generator=None):
             generator,
             define("CMAKE_INSTALL_PREFIX", pathlib.Path(pkg.prefix).as_posix()),
             define("CMAKE_BUILD_TYPE", build_type),
+            define("BUILD_TESTING", pkg.run_tests),
         ]
 
+        # if we're building for a 64 bit system, prefer lib64 paths
+        if str(archspec.cpu.host().generic).find("_64_") > 0:
+            args.append(define('FIND_LIBRARY_USE_LIB64_PATHS', True))
+
         # CMAKE_INTERPROCEDURAL_OPTIMIZATION only exists for CMake >= 3.9
         if pkg.spec.satisfies("^[email protected]:"):
             args.append(define("CMAKE_INTERPROCEDURAL_OPTIMIZATION", ipo))
@@ -450,6 +457,7 @@ def cmake_args(self):
 
             * CMAKE_INSTALL_PREFIX
             * CMAKE_BUILD_TYPE
+            * BUILD_TESTING
 
         which will be set automatically.
         """

lib/spack/spack/config.py

@@ -154,7 +154,10 @@ def _write_section(self, section):
             with open(filename, "w") as f:
                 syaml.dump_config(data, stream=f, default_flow_style=False)
         except (syaml.SpackYAMLError, IOError) as e:
-            raise ConfigFileError(f"cannot write to '{filename}'") from e
+            if hasattr(e, 'errno') and e.errno in [13,30]:
+                tty.warn("Ignoring write error on readonly %s" % filename)
+            else:
+                raise ConfigFileError(f"cannot write to '{filename}'") from e
 
     def clear(self):
         """Empty cached config information."""
@@ -753,9 +756,9 @@ def _add_platform_scope(cfg, scope_type, name, path):
 def _add_os_scope(cfg, scope_type, name, path):
     """Add an os-specific subdirectory for the current platform."""
     host_platform = spack.platforms.host()
-    oss = host_platform.operating_system("frontend")
-    os_name = "%s/%s" % (name, oss)
-    os_path = "%s/%s" % (path, oss)
+    oss = str(host_platform.operating_system("frontend"))
+    os_name = os.path.join(name, oss)
+    os_path = os.path.join(path, oss)
     cfg.push_scope(scope_type(os_name, os_path))
 
 

lib/spack/spack/main.py

@@ -820,7 +820,12 @@ def shell_set(var, value):
     shell_set("_sp_sys_type", str(spack.spec.ArchSpec.default_arch()))
     shell_set("_sp_compatible_sys_types", ":".join(_compatible_sys_types()))
     # print roots for all module systems
-    module_to_roots = {"tcl": list(), "lmod": list(), "ups_table": list(), "ups_version": list()}
+    module_to_roots = {
+      "tcl": list(),
+      "lmod": list(),
+      "ups_table": list(),
+      "ups_version": list(),
+    }
     for name in module_to_roots.keys():
         path = spack.modules.common.root_path(name, "default")
         module_to_roots[name].append(path)

lib/spack/spack/modules/ups_table.py

@@ -37,7 +37,9 @@ def make_configuration(spec, module_set_name, explicit):
     try:
         return configuration_registry[key]
     except KeyError:
-        return configuration_registry.setdefault(key, UpsTableConfiguration(spec, module_set_name))
+        return configuration_registry.setdefault(
+            key, UpsTableConfiguration(spec, module_set_name, explicit)
+        )
 
 
 def make_layout(spec, module_set_name, explicit):
@@ -96,7 +98,9 @@ def conflicts(self):
         for item in self.conf.conflicts:
             naming_scheme = self.conf["naming_scheme"]
             if len([x for x in f.parse(item)]) > 1:
-                for naming_dir, conflict_dir in zip(naming_scheme.split("/"), item.split("/")):
+                for naming_dir, conflict_dir in zip(
+                    naming_scheme.split("/"), item.split("/")
+                ):
                     if naming_dir != conflict_dir:
                         message = "conflict scheme does not match naming "
                         message += "scheme [{spec}]\n\n"
@@ -105,7 +109,9 @@ def conflicts(self):
                         message += "** You may want to check your "
                         message += "`modules.yaml` configuration file **\n"
                         tty.error(
-                            message.format(spec=self.spec, nformat=naming_scheme, cformat=item)
+                            message.format(
+                                spec=self.spec, nformat=naming_scheme, cformat=item
+                            )
                         )
                         raise SystemExit("Module generation aborted.")
                 item = self.spec.format(item)

lib/spack/spack/modules/ups_version.py

@@ -38,7 +38,7 @@ def make_configuration(spec, module_set_name, explicit):
         return configuration_registry[key]
     except KeyError:
         return configuration_registry.setdefault(
-            key, UpsVersionConfiguration(spec, module_set_name)
+            key, UpsVersionConfiguration(spec, module_set_name, explicit)
         )
 
 
@@ -96,7 +96,9 @@ def conflicts(self):
         for item in self.conf.conflicts:
             naming_scheme = self.conf["naming_scheme"]
             if len([x for x in f.parse(item)]) > 1:
-                for naming_dir, conflict_dir in zip(naming_scheme.split("/"), item.split("/")):
+                for naming_dir, conflict_dir in zip(
+                    naming_scheme.split("/"), item.split("/")
+                ):
                     if naming_dir != conflict_dir:
                         message = "conflict scheme does not match naming "
                         message += "scheme [{spec}]\n\n"
@@ -105,7 +107,9 @@ def conflicts(self):
                         message += "** You may want to check your "
                         message += "`modules.yaml` configuration file **\n"
                         tty.error(
-                            message.format(spec=self.spec, nformat=naming_scheme, cformat=item)
+                            message.format(
+                                spec=self.spec, nformat=naming_scheme, cformat=item
+                            )
                         )
                         raise SystemExit("Module generation aborted.")
                 item = self.spec.format(item)

lib/spack/spack/package_base.py

@@ -1236,10 +1236,10 @@ def extendee_spec(self):
                 deps.append(dep)
 
         if deps:
-            # assert len(deps) == 1
+            # we may not still need this, but just in case... mengel
             if len(deps) == 2:
                 assert repr(deps[0]) == repr(deps[1])
-                tty.debug("duplicated extendee deps?! {0}".format(repr(deps[0])))
+                tty.debug("Duplicated extendee deps?! {0}".format(repr(deps[0])))
             else:
                 assert len(deps) == 1
             return deps[0]

lib/spack/spack/relocate.py

@@ -496,7 +496,7 @@ def new_relocate_elf_binaries(binaries, prefix_to_prefix):
 
     # Transform to binary string
     prefix_to_prefix = OrderedDict(
-        (k.encode("utf-8"), v.encode("utf-8")) for (k, v) in prefix_to_prefix.items()
+        (k.encode("utf-8"), v.encode("utf-8")) for (k, v) in prefix_to_prefix.items() if k and v
     )
 
     for path in binaries:

lib/spack/spack/relocate_text.py

@@ -20,7 +20,9 @@ def encode_path(p: Prefix) -> bytes:
 
 
 def _prefix_to_prefix_as_bytes(prefix_to_prefix) -> Dict[bytes, bytes]:
-    return OrderedDict((encode_path(k), encode_path(v)) for (k, v) in prefix_to_prefix.items())
+    return OrderedDict(
+        (encode_path(k), encode_path(v)) for (k, v) in prefix_to_prefix.items() if k and v
+    )
 
 
 def utf8_path_to_binary_regex(prefix: str):

lib/spack/spack/spec.py

@@ -2848,6 +2848,7 @@ def inject_patches_variant(root):
                 # don't blow up if there is a package in the buildcache we do not have
                 # a recipe for -- mengel
                 continue
+
             if dspec.spec.name not in pkg_deps:
                 continue
 

lib/spack/spack/test/modules/ups_table.py

@@ -67,7 +67,9 @@ def test_autoload_all(self, modulefile_content, module_configuration):
         assert len([x for x in content if "FILE = Table" in x]) == 1
         assert len([x for x in content if "setupRequired" in x]) == 3
 
-    def test_prerequisites_direct(self, modulefile_content, module_configuration):
+    def test_prerequisites_direct(
+            self, modulefile_content, module_configuration
+    ):
         """Tests asking direct dependencies as prerequisites."""
 
         module_configuration("prerequisites_direct")
@@ -89,15 +91,25 @@ def test_alter_environment(self, modulefile_content, module_configuration):
         module_configuration("alter_environment")
         content = modulefile_content("mpileaks platform=test target=x86_64")
 
-        assert len([x for x in content if x.startswith("PathPrepend(CMAKE_PREFIX_PATH")]) == 0
+        assert len([x for x in content
+                    if x.startswith("PathPrepend(CMAKE_PREFIX_PATH")
+                    ]) == 0
         assert len([x for x in content if 'EnvSet(FOO, "foo"' in x]) == 1
-        assert len([x for x in content if 'EnvSet(OMPI_MCA_mpi_leave_pinned, "1"' in x]) == 1
-        assert len([x for x in content if 'EnvSet(OMPI_MCA_MPI_LEAVE_PINNED, "1"' in x]) == 0
+        assert len([
+            x for x in content if 'EnvSet(OMPI_MCA_mpi_leave_pinned, "1"' in x
+        ]) == 1
+        assert len([
+            x for x in content if 'EnvSet(OMPI_MCA_MPI_LEAVE_PINNED, "1"' in x
+        ]) == 0
         assert len([x for x in content if "EnvSet(MPILEAKS_ROOT" in x]) == 1
 
-        content = modulefile_content("libdwarf %clang platform=test target=x86_32")
+        content = modulefile_content(
+            "libdwarf %clang platform=test target=x86_32"
+        )
 
-        assert len([x for x in content if x.startswith("PathPrepend(CMAKE_PREFIX_PATH")]) == 0
+        assert len([x for x in content
+                    if x.startswith("PathPrepend(CMAKE_PREFIX_PATH")
+                    ]) == 0
         assert len([x for x in content if 'EnvSet(FOO, "foo"' in x]) == 0
         assert len([x for x in content if "unEnvSet(BAR" in x]) == 0
         assert len([x for x in content if "FILE = Table" in x]) == 1
@@ -152,7 +164,8 @@ def test_invalid_token_in_env_name(self, factory, module_configuration):
         with pytest.raises(RuntimeError):
             writer.write()
 
-    def test_module_index(self, module_configuration, factory, tmpdir_factory):
+    def test_module_index(
+            self, module_configuration, factory, tmpdir_factory):
         module_configuration("suffix")
 
         w1, s1 = factory("mpileaks")
@@ -191,17 +204,23 @@ def test_setup_environment(self, modulefile_content, module_configuration):
         content = modulefile_content(str(spec["callpath"]))
 
         assert len([x for x in content if "EnvSet(FOOBAR" in x]) == 1
-        assert len([x for x in content if 'EnvSet(FOOBAR, "callpath"' in x]) == 1
+        assert len(
+            [x for x in content if 'EnvSet(FOOBAR, "callpath"' in x]
+        ) == 1
 
-    def test_override_template_in_package(self, modulefile_content, module_configuration):
+    def test_override_template_in_package(
+            self, modulefile_content, module_configuration
+    ):
         """Tests overriding a template from and attribute in the package."""
 
         module_configuration("autoload_direct")
         content = modulefile_content("override-module-templates")
 
         assert "PRODUCT = override-module-templates" in content
 
-    def test_override_template_in_modules_yaml(self, modulefile_content, module_configuration):
+    def test_override_template_in_modules_yaml(
+            self, modulefile_content, module_configuration
+    ):
         """Tests overriding a template from `modules.yaml`"""
         module_configuration("override_template")
 
@@ -213,7 +232,9 @@ def test_override_template_in_modules_yaml(self, modulefile_content, module_conf
 
     @pytest.mark.regression("4400")
     @pytest.mark.db
-    def test_blacklist_implicits(self, modulefile_content, module_configuration, database):
+    def test_blacklist_implicits(
+            self, modulefile_content, module_configuration, database
+    ):
         module_configuration("blacklist_implicits")
 
         # mpileaks has been installed explicitly when setting up
@@ -232,7 +253,9 @@ def test_blacklist_implicits(self, modulefile_content, module_configuration, dat
 
     @pytest.mark.regression("9624")
     @pytest.mark.db
-    def test_autoload_with_constraints(self, modulefile_content, module_configuration, database):
+    def test_autoload_with_constraints(
+            self, modulefile_content, module_configuration, database
+    ):
         """Tests the automatic loading of direct dependencies."""
 
         module_configuration("autoload_with_constraints")

lib/spack/spack/test/modules/ups_version.py

@@ -65,7 +65,9 @@ def test_autoload_all(self, modulefile_content, module_configuration):
 
         assert len([x for x in content if "FILE = version" in x]) == 1
 
-    def test_prerequisites_direct(self, modulefile_content, module_configuration):
+    def test_prerequisites_direct(
+            self, modulefile_content, module_configuration
+    ):
         """Tests asking direct dependencies as prerequisites."""
 
         module_configuration("prerequisites_direct")
@@ -88,7 +90,9 @@ def test_alter_environment(self, modulefile_content, module_configuration):
         content = modulefile_content("mpileaks platform=test target=x86_64")
         assert len([x for x in content if "FILE = version" in x]) == 1
 
-        content = modulefile_content("libdwarf %clang platform=test target=x86_32")
+        content = modulefile_content(
+            "libdwarf %clang platform=test target=x86_32"
+        )
         assert len([x for x in content if "FILE = version" in x]) == 1
 
     def test_blacklist(self, modulefile_content, module_configuration):
@@ -140,7 +144,8 @@ def test_invalid_token_in_env_name(self, factory, module_configuration):
         with pytest.raises(RuntimeError):
             writer.write()
 
-    def test_module_index(self, module_configuration, factory, tmpdir_factory):
+    def test_module_index(
+            self, module_configuration, factory, tmpdir_factory):
         module_configuration("suffix")
 
         w1, s1 = factory("mpileaks")
@@ -172,7 +177,9 @@ def test_setup_environment(self, modulefile_content, module_configuration):
         spec = spack.spec.Spec("mpileaks")
         spec.concretize()
 
-    def test_override_template_in_modules_yaml(self, modulefile_content, module_configuration):
+    def test_override_template_in_modules_yaml(
+            self, modulefile_content, module_configuration
+    ):
         """Tests overriding a template from `modules.yaml`"""
         module_configuration("override_template")
 
@@ -182,7 +189,9 @@ def test_override_template_in_modules_yaml(self, modulefile_content, module_conf
         content = modulefile_content("mpileaks arch=x86-linux")
         assert "Override even better!" in content
 
-    def test_extend_context(self, modulefile_content, module_configuration):
+    def test_extend_context(
+            self, modulefile_content, module_configuration
+    ):
         """Tests using a package defined context"""
         module_configuration("autoload_direct")
         content = modulefile_content("override-context-templates")
@@ -191,7 +200,9 @@ def test_extend_context(self, modulefile_content, module_configuration):
 
     @pytest.mark.regression("9624")
     @pytest.mark.db
-    def test_autoload_with_constraints(self, modulefile_content, module_configuration, database):
+    def test_autoload_with_constraints(
+            self, modulefile_content, module_configuration, database
+    ):
         """Tests the automatic loading of direct dependencies."""
 
         module_configuration("autoload_with_constraints")

lib/spack/spack/util/environment.py

@@ -678,7 +678,7 @@ def deprioritize_system_paths(
             separator: separator for the paths (default: os.pathsep)
         """
         item = DeprioritizeSystemPaths(
-            name, separator=separator, trace=self._trace(), target=target
+            name, separator=separator, trace=self._trace(), target=target,
         )
         self.env_modifications.append(item)
 

share/spack/templates/modules/modulefile.ups_table

@@ -25,7 +25,7 @@ QUALIFIERS=""
 
 
 {%set spack_root = os.environ['SPACK_ROOT'] %}
-{%set install_root =  spack_root[0:spack_root.find('/spack/')] %}
+{%set install_root = spack_root[0:spack_root.find('/spack/')] %}
 {%if spec.prefix.startswith(install_root) %}
 
 Action = setup

var/spack/repos/builtin/packages/libgd/package.py

@@ -47,3 +47,8 @@ def patch(self):
             "configure",
             string=True,
         )
+        filter_file(
+            "#include *<string.h>",
+            "#include <string.h>\n#include <limits.h>",
+            "src/gd_gd2.c"
+        )