Skip to content

Commit

Permalink
Merge pull request #472 from FNALssi/maintenance/merge-from-upstream-…
Browse files Browse the repository at this point in the history
…2024-12-04

Merge from upstream/develop 2024-12-04
  • Loading branch information
marcmengel authored Dec 4, 2024
2 parents 4797901 + 14512a7 commit e4effb5
Show file tree
Hide file tree
Showing 108 changed files with 1,440 additions and 552 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build-containers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ jobs:
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Build & Deploy ${{ matrix.dockerfile[0] }}
uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75
uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355
with:
context: dockerfiles/${{ matrix.dockerfile[0] }}
platforms: ${{ matrix.dockerfile[1] }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/valid-style.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ jobs:
. .github/workflows/bin/setup_git.sh
- name: Run style tests
run: |
share/spack/qa/run-style-tests -s import
share/spack/qa/run-style-tests
audit:
uses: ./.github/workflows/audit.yaml
secrets: inherit
Expand Down
2 changes: 2 additions & 0 deletions etc/spack/defaults/packages.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1271,6 +1271,8 @@ packages:
buildable: false
cray-mvapich2:
buildable: false
egl:
buildable: false
fujitsu-mpi:
buildable: false
hpcx-mpi:
Expand Down
8 changes: 8 additions & 0 deletions lib/spack/docs/getting_started.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1326,6 +1326,7 @@ Required:
* Microsoft Visual Studio
* Python
* Git
* 7z

Optional:
* Intel Fortran (needed for some packages)
Expand Down Expand Up @@ -1391,6 +1392,13 @@ as the project providing Git support on Windows. This is additionally the recomm
for installing Git on Windows, a link to which can be found above. Spack requires the
utilities vendored by this project.

"""
7zip
"""

A tool for extracting ``.xz`` files is required for extracting source tarballs. The latest 7zip
can be located at https://sourceforge.net/projects/sevenzip/.

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Step 2: Install and setup Spack
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Expand Down
2 changes: 1 addition & 1 deletion lib/spack/docs/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ python-levenshtein==0.26.1
docutils==0.21.2
pygments==2.18.0
urllib3==2.2.3
pytest==8.3.3
pytest==8.3.4
isort==5.13.2
black==24.10.0
flake8==7.1.1
Expand Down
36 changes: 20 additions & 16 deletions lib/spack/llnl/util/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
Callable,
Deque,
Dict,
Generator,
Iterable,
List,
Match,
Expand Down Expand Up @@ -2772,22 +2773,6 @@ def prefixes(path):
return paths


@system_path_filter
def md5sum(file):
"""Compute the MD5 sum of a file.
Args:
file (str): file to be checksummed
Returns:
MD5 sum of the file's content
"""
md5 = hashlib.md5()
with open(file, "rb") as f:
md5.update(f.read())
return md5.digest()


@system_path_filter
def remove_directory_contents(dir):
"""Remove all contents of a directory."""
Expand Down Expand Up @@ -2838,6 +2823,25 @@ def temporary_dir(
remove_directory_contents(tmp_dir)


@contextmanager
def edit_in_place_through_temporary_file(file_path: str) -> Generator[str, None, None]:
"""Context manager for modifying ``file_path`` in place, preserving its inode and hardlinks,
for functions or external tools that do not support in-place editing. Notice that this function
is unsafe in that it works with paths instead of a file descriptors, but this is by design,
since we assume the call site will create a new inode at the same path."""
tmp_fd, tmp_path = tempfile.mkstemp(
dir=os.path.dirname(file_path), prefix=f"{os.path.basename(file_path)}."
)
# windows cannot replace a file with open fds, so close since the call site needs to replace.
os.close(tmp_fd)
try:
shutil.copyfile(file_path, tmp_path, follow_symlinks=True)
yield tmp_path
shutil.copyfile(tmp_path, file_path, follow_symlinks=True)
finally:
os.unlink(tmp_path)


def filesummary(path, print_bytes=16) -> Tuple[int, bytes]:
"""Create a small summary of the given file. Does not error
when file does not exist.
Expand Down
4 changes: 3 additions & 1 deletion lib/spack/spack/binary_distribution.py
Original file line number Diff line number Diff line change
Expand Up @@ -2334,7 +2334,9 @@ def is_backup_file(file):
if not codesign:
return
for binary in changed_files:
codesign("-fs-", binary)
# preserve the original inode by running codesign on a copy
with fsys.edit_in_place_through_temporary_file(binary) as tmp_binary:
codesign("-fs-", tmp_binary)

# If we are installing back to the same location
# relocate the sbang location if the spack directory changed
Expand Down
22 changes: 13 additions & 9 deletions lib/spack/spack/build_systems/cmake.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import re
import sys
from itertools import chain
from typing import Any, List, Optional, Set, Tuple
from typing import Any, List, Optional, Tuple

import archspec.cpu

Expand All @@ -23,6 +23,7 @@
import spack.phase_callbacks
import spack.spec
import spack.util.prefix
from spack import traverse
from spack.directives import build_system, conflicts, depends_on, variant
from spack.multimethod import when
from spack.util.environment import filter_system_paths
Expand Down Expand Up @@ -168,15 +169,18 @@ def _values(x):
def get_cmake_prefix_path(pkg: spack.package_base.PackageBase) -> List[str]:
"""Obtain the CMAKE_PREFIX_PATH entries for a package, based on the cmake_prefix_path package
attribute of direct build/test and transitive link dependencies."""
# Add direct build/test deps
selected: Set[str] = {s.dag_hash() for s in pkg.spec.dependencies(deptype=dt.BUILD | dt.TEST)}
# Add transitive link deps
selected.update(s.dag_hash() for s in pkg.spec.traverse(root=False, deptype=dt.LINK))
# Separate out externals so they do not shadow Spack prefixes
externals, spack_built = stable_partition(
(s for s in pkg.spec.traverse(root=False, order="topo") if s.dag_hash() in selected),
lambda x: x.external,
edges = traverse.traverse_topo_edges_generator(
traverse.with_artificial_edges([pkg.spec]),
visitor=traverse.MixedDepthVisitor(
direct=dt.BUILD | dt.TEST, transitive=dt.LINK, key=traverse.by_dag_hash
),
key=traverse.by_dag_hash,
root=False,
all_edges=False, # cover all nodes, not all edges
)
ordered_specs = [edge.spec for edge in edges]
# Separate out externals so they do not shadow Spack prefixes
externals, spack_built = stable_partition((s for s in ordered_specs), lambda x: x.external)

return filter_system_paths(
path for spec in chain(spack_built, externals) for path in spec.package.cmake_prefix_paths
Expand Down
2 changes: 0 additions & 2 deletions lib/spack/spack/cmd/style.py
Original file line number Diff line number Diff line change
Expand Up @@ -323,8 +323,6 @@ def process_files(file_list, is_args):
rewrite_and_print_output(output, args, pat, replacement)

packages_isort_args = (
"--rm",
"spack",
"--rm",
"spack.pkgkit",
"--rm",
Expand Down
4 changes: 2 additions & 2 deletions lib/spack/spack/compilers/intel.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ def setup_custom_environment(self, pkg, env):
# Edge cases for Intel's oneAPI compilers when using the legacy classic compilers:
# Always pass flags to disable deprecation warnings, since these warnings can
# confuse tools that parse the output of compiler commands (e.g. version checks).
if self.real_version >= Version("2021") and self.real_version <= Version("2023"):
if self.real_version >= Version("2021") and self.real_version < Version("2024"):
env.append_flags("SPACK_ALWAYS_CFLAGS", "-diag-disable=10441")
env.append_flags("SPACK_ALWAYS_CXXFLAGS", "-diag-disable=10441")
if self.real_version >= Version("2021") and self.real_version <= Version("2024"):
if self.real_version >= Version("2021") and self.real_version < Version("2025"):
env.append_flags("SPACK_ALWAYS_FFLAGS", "-diag-disable=10448")
4 changes: 2 additions & 2 deletions lib/spack/spack/compilers/oneapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,10 +155,10 @@ def setup_custom_environment(self, pkg, env):
# icx+icpx+ifx or icx+icpx+ifort. But to be on the safe side (some users may
# want to try to swap icpx against icpc, for example), and since the Intel LLVM
# compilers accept these diag-disable flags, we apply them for all compilers.
if self.real_version >= Version("2021") and self.real_version <= Version("2023"):
if self.real_version >= Version("2021") and self.real_version < Version("2024"):
env.append_flags("SPACK_ALWAYS_CFLAGS", "-diag-disable=10441")
env.append_flags("SPACK_ALWAYS_CXXFLAGS", "-diag-disable=10441")
if self.real_version >= Version("2021") and self.real_version <= Version("2024"):
if self.real_version >= Version("2021") and self.real_version < Version("2025"):
env.append_flags("SPACK_ALWAYS_FFLAGS", "-diag-disable=10448")

# 2024 release bumped the libsycl version because of an ABI
Expand Down
11 changes: 6 additions & 5 deletions lib/spack/spack/relocate.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import macholib.mach_o
import macholib.MachO

import llnl.util.filesystem as fs
import llnl.util.lang
import llnl.util.tty as tty
from llnl.util.lang import memoized
Expand Down Expand Up @@ -275,10 +276,10 @@ def modify_macho_object(cur_path, rpaths, deps, idpath, paths_to_paths):

# Deduplicate and flatten
args = list(itertools.chain.from_iterable(llnl.util.lang.dedupe(args)))
install_name_tool = executable.Executable("install_name_tool")
if args:
args.append(str(cur_path))
install_name_tool = executable.Executable("install_name_tool")
install_name_tool(*args)
with fs.edit_in_place_through_temporary_file(cur_path) as temp_path:
install_name_tool(*args, temp_path)


def macholib_get_paths(cur_path):
Expand Down Expand Up @@ -717,8 +718,8 @@ def fixup_macos_rpath(root, filename):
# No fixes needed
return False

args.append(abspath)
executable.Executable("install_name_tool")(*args)
with fs.edit_in_place_through_temporary_file(abspath) as temp_path:
executable.Executable("install_name_tool")(*args, temp_path)
return True


Expand Down
4 changes: 2 additions & 2 deletions lib/spack/spack/schema/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,8 @@
{
"names": ["install_missing_compilers"],
"message": "The config:install_missing_compilers option has been deprecated in "
"Spack v0.23, and is currently ignored. It will be removed from config in "
"Spack v0.25.",
"Spack v0.23, and is currently ignored. It will be removed from config after "
"Spack v1.0.",
"error": False,
},
],
Expand Down
2 changes: 1 addition & 1 deletion lib/spack/spack/stage.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@ def _generate_fetchers(self, mirror_only=False) -> Generator["fs.FetchStrategy",
# Insert fetchers in the order that the URLs are provided.
fetchers[:0] = (
fs.from_url_scheme(
url_util.join(mirror.fetch_url, self.mirror_layout.path),
url_util.join(mirror.fetch_url, *self.mirror_layout.path.split(os.sep)),
checksum=digest,
expand=expand,
extension=extension,
Expand Down
11 changes: 11 additions & 0 deletions lib/spack/spack/test/llnl/util/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -1249,3 +1249,14 @@ def test_find_input_types(tmp_path: pathlib.Path):

with pytest.raises(TypeError):
fs.find(1, "file.txt") # type: ignore


def test_edit_in_place_through_temporary_file(tmp_path):
(tmp_path / "example.txt").write_text("Hello")
current_ino = os.stat(tmp_path / "example.txt").st_ino
with fs.edit_in_place_through_temporary_file(tmp_path / "example.txt") as temporary:
os.unlink(temporary)
with open(temporary, "w") as f:
f.write("World")
assert (tmp_path / "example.txt").read_text() == "World"
assert os.stat(tmp_path / "example.txt").st_ino == current_ino
63 changes: 60 additions & 3 deletions lib/spack/spack/test/traverse.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,8 @@ def create_dag(nodes, edges):
"""
specs = {name: Spec(name) for name in nodes}
for parent, child, deptypes in edges:
specs[parent].add_dependency_edge(
specs[child], depflag=dt.canonicalize(deptypes), virtuals=()
)
depflag = deptypes if isinstance(deptypes, dt.DepFlag) else dt.canonicalize(deptypes)
specs[parent].add_dependency_edge(specs[child], depflag=depflag, virtuals=())
return specs


Expand Down Expand Up @@ -454,3 +453,61 @@ def test_topo_is_bfs_for_trees(cover):
assert list(traverse.traverse_nodes([binary_tree["A"]], order="topo", cover=cover)) == list(
traverse.traverse_nodes([binary_tree["A"]], order="breadth", cover=cover)
)


@pytest.mark.parametrize("roots", [["A"], ["A", "B"], ["B", "A"], ["A", "B", "A"]])
@pytest.mark.parametrize("order", ["breadth", "post", "pre"])
@pytest.mark.parametrize("include_root", [True, False])
def test_mixed_depth_visitor(roots, order, include_root):
"""Test that the MixedDepthVisitor lists unique edges that are reachable either directly from
roots through build type edges, or transitively through link type edges. The tests ensures that
unique edges are listed exactly once."""
my_graph = create_dag(
nodes=["A", "B", "C", "D", "E", "F", "G", "H", "I"],
edges=(
("A", "B", dt.LINK | dt.RUN),
("A", "C", dt.BUILD),
("A", "D", dt.BUILD | dt.RUN),
("A", "H", dt.LINK),
("A", "I", dt.RUN),
("B", "D", dt.BUILD | dt.LINK),
("C", "E", dt.BUILD | dt.LINK | dt.RUN),
("D", "F", dt.LINK),
("D", "G", dt.BUILD | dt.RUN),
("H", "B", dt.LINK),
),
)
starting_points = traverse.with_artificial_edges([my_graph[root] for root in roots])
visitor = traverse.MixedDepthVisitor(direct=dt.BUILD, transitive=dt.LINK)

if order == "pre":
edges = traverse.traverse_depth_first_edges_generator(
starting_points, visitor, post_order=False, root=include_root
)
elif order == "post":
edges = traverse.traverse_depth_first_edges_generator(
starting_points, visitor, post_order=True, root=include_root
)
elif order == "breadth":
edges = traverse.traverse_breadth_first_edges_generator(
starting_points, visitor, root=include_root
)

artificial_edges = [(None, root) for root in roots] if include_root else []
simple_edges = [
(None if edge.parent is None else edge.parent.name, edge.spec.name) for edge in edges
]

# make sure that every edge is listed exactly once and that the right edges are listed
assert len(simple_edges) == len(set(simple_edges))
assert set(simple_edges) == {
# the roots
*artificial_edges,
("A", "B"),
("A", "C"),
("A", "D"),
("A", "H"),
("B", "D"),
("D", "F"),
("H", "B"),
}
3 changes: 3 additions & 0 deletions lib/spack/spack/test/versions.py
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,9 @@ def test_stringify_version(version_str):
v.string = None
assert str(v) == version_str

v.string = None
assert v.string == version_str


def test_len():
a = Version("1.2.3.4")
Expand Down
Loading

0 comments on commit e4effb5

Please sign in to comment.