Feature/corbett/dense

CMakeLists.txt

@@ -35,6 +35,7 @@ if( NOT is_submodule )
 
  option( ENABLE_ADDR2LINE "Enable addr2line usage in stacktraces" ON )
  option( ENABLE_CUDA "Build with CUDA" OFF )
+ option( ENABLE_HIP "Build with HIP" OFF )
  option( ENABLE_UMPIRE "Build with UMPIRE" OFF )
  option( ENABLE_CHAI "Build with CHAI" OFF )
  option( ENABLE_CALIPER "Build with Caliper" OFF )
@@ -80,6 +81,8 @@ blt_list_append( TO lvarray_dependencies ELEMENTS chai IF ENABLE_CHAI )
 
 blt_list_append( TO lvarray_dependencies ELEMENTS cuda IF ENABLE_CUDA )
 
+blt_list_append( TO lvarray_dependencies ELEMENTS blt::hip IF ENABLE_HIP )
+
 blt_list_append( TO lvarray_dependencies ELEMENTS caliper IF ENABLE_CALIPER )
 
 

Notes.txt

@@ -0,0 +1 @@
+./scripts/uberenv/uberenv.py --prefix=../uberenv-libs/ --spack-config-dir=./scripts/uberenv/spack_configs/toss_3_x86_64_ib_python/ --spec="%[email protected] +umpire +chai +caliper +pylvarray +benchmarks +examples ^caliper ~adiak ~mpi ~libunwind ~libdw ~papi"

cmake/CMakeBasics.cmake

@@ -12,9 +12,9 @@ option( ENABLE_TOTALVIEW_OUTPUT "" OFF )
 set( LVARRAY_BUILD_OBJ_LIBS OFF CACHE BOOL "" )
 
 
-if( NOT BLT_CXX_STD STREQUAL c++14 )
- MESSAGE(FATAL_ERROR "c++14 is NOT enabled. LvArray requires c++14")
-endif()
+# if( NOT BLT_CXX_STD STREQUAL c++14 )
+#  MESSAGE(FATAL_ERROR "c++14 is NOT enabled. LvArray requires c++14")
+# endif()
 
 
 blt_append_custom_compiler_flag( FLAGS_VAR CMAKE_CXX_FLAGS DEFAULT "${OpenMP_CXX_FLAGS}")

cmake/Config.cmake

@@ -2,8 +2,10 @@
 set( PREPROCESSOR_DEFINES UMPIRE
  CHAI
  CUDA
+ HIP
  TOTALVIEW_OUTPUT
- CALIPER )
+ CALIPER
+ MAGMA )
 
 set( USE_CONFIGFILE ON CACHE BOOL "" )
 foreach( DEP in ${PREPROCESSOR_DEFINES})

cmake/SetupTPL.cmake

@@ -1,19 +1,79 @@
+macro(find_and_register)
+ set(singleValueArgs NAME HEADER)
+ set(multiValueArgs INCLUDE_DIRECTORIES
+ LIBRARY_DIRECTORIES
+ LIBRARIES
+ EXTRA_LIBRARIES
+ DEPENDS )
+
+ ## parse the arguments
+ cmake_parse_arguments(arg
+ "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN})
+
+ if(NOT DEFINED arg_NAME)
+ message(FATAL_ERROR "The find_and_register required parameter NAME specifies the name of the library to register.")
+ endif()
+
+ if(NOT DEFINED arg_INCLUDE_DIRECTORIES)
+ message(FATAL_ERROR "The find_and_register required parameter INCLUDE_DIRECTORIES specifies the directories to search for the given header.")
+ endif()
+
+ if(NOT DEFINED arg_LIBRARY_DIRECTORIES)
+ message(FATAL_ERROR "The find_and_register required parameter LIBRARY_DIRECTORIES specifies the directories to search for the given libraries.")
+ endif()
+
+ if(NOT DEFINED arg_HEADER)
+ message(FATAL_ERROR "The find_and_register required parameter HEADER specifies the header to search for.")
+ endif()
+
+ if(NOT DEFINED arg_LIBRARIES)
+ message(FATAL_ERROR "The find_and_register required parameter LIBRARIES specifies the libraries to search for.")
+ endif()
+
+ find_path(${arg_NAME}_INCLUDE_DIR ${arg_HEADER}
+ PATHS ${arg_INCLUDE_DIRECTORIES}
+ NO_DEFAULT_PATH
+ NO_CMAKE_ENVIRONMENT_PATH
+ NO_CMAKE_PATH
+ NO_SYSTEM_ENVIRONMENT_PATH
+ NO_CMAKE_SYSTEM_PATH)
+
+ if(${arg_NAME}_INCLUDE_DIR STREQUAL ${arg_NAME}_INCLUDE_DIR-NOTFOUND)
+ message(FATAL_ERROR "Could not find '${arg_HEADER}' in '${arg_INCLUDE_DIRECTORIES}'")
+ endif()
+
+ blt_find_libraries(FOUND_LIBS ${arg_NAME}_LIBRARIES
+ NAMES ${arg_LIBRARIES}
+ PATHS ${arg_LIBRARY_DIRECTORIES}
+ REQUIRED ON)
+
+ blt_import_library(NAME ${arg_NAME}
+ INCLUDES ${${arg_NAME}_INCLUDE_DIR}
+ LIBRARIES ${${arg_NAME}_LIBRARIES} ${arg_EXTRA_LIBRARIES}
+ TREAT_INCLUDES_AS_SYSTEM ON
+ DEPENDS_ON ${arg_DEPENDS})
+
+endmacro(find_and_register)
+
 set(thirdPartyLibs "")
 
-################################
+###############################
 # CAMP
-################################
-if(NOT EXISTS ${CAMP_DIR})
- message(FATAL_ERROR "CAMP_DIR must be defined and point to a valid directory when using CAMP.")
-endif()
+###############################
+if(CAMP_DIR STREQUAL RAJA_DIR)
+ message(STATUS "LvArray using CAMP from RAJA.")
+else()
+ if(NOT EXISTS ${CAMP_DIR})
+ message(FATAL_ERROR "CAMP_DIR must be defined and point to a valid directory when using CAMP.")
+ endif()
 
-message(STATUS "Using CAMP from ${CAMP_DIR}")
+ message(STATUS "LvArray using CAMP from ${CAMP_DIR}")
 
-find_package(camp REQUIRED PATHS ${CAMP_DIR})
+ find_package(camp REQUIRED PATHS ${CAMP_DIR})
 
-set(ENABLE_CAMP ON CACHE BOOL "")
+ set(thirdPartyLibs ${thirdPartyLibs} camp)
+endif()
 
-set(thirdPartyLibs ${thirdPartyLibs} camp)
 
 ################################
 # RAJA
@@ -22,7 +82,7 @@ if(NOT EXISTS ${RAJA_DIR})
  message(FATAL_ERROR "RAJA_DIR must be defined and point to a valid directory when using RAJA.")
 endif()
 
-message(STATUS "Using RAJA from ${RAJA_DIR}")
+message(STATUS "LvArray using RAJA from ${RAJA_DIR}")
 
 find_package(RAJA REQUIRED PATHS ${RAJA_DIR})
 
@@ -39,20 +99,26 @@ if(ENABLE_UMPIRE)
  message(FATAL_ERROR "UMPIRE_DIR must be defined and point to a valid directory when using Umpire.")
  endif()
 
- message(STATUS "Using Umpire from ${UMPIRE_DIR}")
+ message(STATUS "LvArray using Umpire from ${UMPIRE_DIR}")
 
  find_package(umpire REQUIRED
  PATHS ${UMPIRE_DIR})
 
  set(thirdPartyLibs ${thirdPartyLibs} umpire)
 else()
- message(STATUS "Not using Umpire.")
+ message(STATUS "LvArray not using Umpire.")
 endif()
 
 ################################
 # CHAI
 ################################
 if(ENABLE_CHAI)
+ if(NOT EXISTS ${CHAI_DIR})
+ message(FATAL_ERROR "CHAI_DIR must be defined and point to a valid directory when using CHAI.")
+ endif()
+
+ message(STATUS "Using CHAI from ${CHAI_DIR}")
+
  if(NOT ENABLE_UMPIRE)
  message(FATAL_ERROR "Umpire must be enabled to use CHAI.")
  endif()
@@ -65,32 +131,32 @@ if(ENABLE_CHAI)
  message(FATAL_ERROR "CHAI_DIR must be defined and point to a valid directory when using CHAI.")
  endif()
 
- message(STATUS "Using CHAI from ${CHAI_DIR}")
+ message(STATUS "LvArray using CHAI from ${CHAI_DIR}")
 
  find_package(chai REQUIRED
  PATHS ${CHAI_DIR})
-
- # If this isn't done chai will add -lRAJA to the link line, but we don't link to RAJA like that.
- get_target_property(CHAI_LINK_LIBRARIES chai INTERFACE_LINK_LIBRARIES)
- list(REMOVE_ITEM CHAI_LINK_LIBRARIES RAJA)
- set_target_properties(chai
- PROPERTIES INTERFACE_LINK_LIBRARIES "${CHAI_LINK_LIBRARIES}")
+ 
+ # # If this isn't done chai will add -lRAJA to the link line, but we don't link to RAJA like that.
+ # get_target_property(CHAI_LINK_LIBRARIES chai INTERFACE_LINK_LIBRARIES)
+ # list(REMOVE_ITEM CHAI_LINK_LIBRARIES RAJA)
+ # set_target_properties(chai
+ #  PROPERTIES INTERFACE_LINK_LIBRARIES "${CHAI_LINK_LIBRARIES}")
 
  set(thirdPartyLibs ${thirdPartyLibs} chai)
 else()
- message(STATUS "Not using CHAI.")
+ message(STATUS "LvArray not using CHAI.")
 endif()
 
 
-################################
+###############################
 # CALIPER
-################################
+###############################
 if(ENABLE_CALIPER)
  if(NOT EXISTS ${CALIPER_DIR})
  message(FATAL_ERROR "CALIPER_DIR must be defined and point to a valid directory when using caliper.")
  endif()
 
- message(STATUS "Using caliper from ${CALIPER_DIR}")
+ message(STATUS "LvArray using caliper from ${CALIPER_DIR}")
 
  find_package(caliper REQUIRED
  PATHS ${CALIPER_DIR})
@@ -102,22 +168,66 @@ if(ENABLE_CALIPER)
 
  set(thirdPartyLibs ${thirdPartyLibs} caliper)
 else()
- message(STATUS "Not using caliper.")
+ message(STATUS "LvArray not using caliper.")
 endif()
 
 ################################
 # Python
 ################################
-if ( ENABLE_PYLVARRAY )
- message( STATUS "Python3_EXECUTABLE=${Python3_EXECUTABLE}" )
- find_package( Python3 REQUIRED
-  COMPONENTS Development NumPy )
+if(ENABLE_PYLVARRAY)
+ message(STATUS "Python3_EXECUTABLE=${Python3_EXECUTABLE}")
+ find_package(Python3 REQUIRED
+ COMPONENTS Development NumPy)
 
- message( STATUS "Python3_INCLUDE_DIRS = ${Python3_INCLUDE_DIRS}" )
- message( STATUS "Python3_LIBRARY_DIRS = ${Python3_LIBRARY_DIRS}" )
- message( STATUS "Python3_NumPy_INCLUDE_DIRS = ${Python3_NumPy_INCLUDE_DIRS}" )
+ message(STATUS "Python3_INCLUDE_DIRS = ${Python3_INCLUDE_DIRS}")
+ message(STATUS "Python3_LIBRARY_DIRS = ${Python3_LIBRARY_DIRS}")
+ message(STATUS "Python3_NumPy_INCLUDE_DIRS = ${Python3_NumPy_INCLUDE_DIRS}")
 
- set( thirdPartyLibs ${thirdPartyLibs} Python3::Python Python3::NumPy )
+ set(thirdPartyLibs ${thirdPartyLibs} Python3::Python Python3::NumPy)
+else()
+ message(STATUS "Not building pylvarray")
+endif()
+
+################################
+# LAPACK/BLAS
+################################
+if(ENABLE_LAPACK)
+ message(STATUS "BLAS_LIBRARIES = ${BLAS_LIBRARIES}")
+ message(STATUS "LAPACK_LIBRARIES = ${LAPACK_LIBRARIES}")
+
+ blt_import_library(NAME blas
+ TREAT_INCLUDES_AS_SYSTEM ON
+ LIBRARIES ${BLAS_LIBRARIES})
+
+ blt_import_library(NAME lapack
+ DEPENDS_ON blas
+ TREAT_INCLUDES_AS_SYSTEM ON
+ LIBRARIES ${LAPACK_LIBRARIES})
+
+ set(thirdPartyLibs ${thirdPartyLibs} blas lapack)
+else()
+ message(STATUS "Not using LAPACK or BLAS.")
+endif()
+
+################################
+# MAGMA
+################################
+if(ENABLE_MAGMA)
+ message(STATUS "Using MAGMA from ${MAGMA_DIR}")
+
+ if(NOT ENABLE_LAPACK)
+ message(FATAL_ERROR "LAPACK must be enabled to use MAGMA.")
+ endif()
+
+ find_and_register(NAME magma
+ INCLUDE_DIRECTORIES ${MAGMA_DIR}/include
+ LIBRARY_DIRECTORIES ${MAGMA_DIR}/lib
+ HEADER magma.h
+ LIBRARIES magma)
+
+ set(thirdPartyLibs ${thirdPartyLibs} magma)
+else()
+ message(STATUS "Not using MAGMA.")
 endif()
 
 set( thirdPartyLibs ${thirdPartyLibs} CACHE STRING "" )

host-configs/LLNL/lassen-base.cmake

@@ -21,14 +21,14 @@ set(ENABLE_CUDA ON CACHE BOOL "")
 set(CUDA_TOOLKIT_ROOT_DIR /usr/tce/packages/cuda/cuda-10.1.243 CACHE STRING "")
 set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "")
 set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
-set(CUDA_ARCH sm_70 CACHE STRING "")
+set(CUDA_ARCHITECTURES sm_70 CACHE STRING "")
 set(CMAKE_CUDA_STANDARD 14 CACHE STRING "")
-set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "")
+set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCHITECTURES} --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le" CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")
 
-set(CHAI_CUDA_FLAGS "-arch ${CUDA_ARCH}" CACHE STRING "" FORCE)
+set(CHAI_CUDA_FLAGS "-arch ${CUDA_ARCHITECTURES}" CACHE STRING "" FORCE)
 
 # Uncomment this line to make nvcc output register usage for each kernel.
 # set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --resource-usage" CACHE STRING "" FORCE)

host-configs/ORNL/crusher-base.cmake

@@ -0,0 +1,25 @@
+
+set(CMAKE_CXX_STANDARD 14 CACHE STRING "")
+
+set( ENABLE_MPI ON CACHE BOOL "" FORCE )
+set( ENABLE_FIND_MPI ON CACHE BOOL "" FORCE )
+
+# HIP Options
+set( ENABLE_HIP ON CACHE BOOL "" FORCE )
+
+# suppress -Werror for now
+set( ENABLE_WARNINGS_AS_ERRORS FALSE CACHE BOOL "" FORCE )
+
+# GTEST
+set(ENABLE_GTEST_DEATH_TESTS OFF CACHE BOOL "")
+set(gtest_disable_pthreads ON CACHE BOOL "")
+
+# disable most binaries and doc generation
+set(ENABLE_TESTS OFF CACHE BOOL "" FORCE)
+set(DISABLE_UNIT_TESTS ON CACHE BOOL "" FORCE)
+set(ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE)
+set(ENABLE_BENCHMARKS OFF CACHE BOOL "" FORCE)
+set(ENABLE_DOCS OFF CACHE BOOL "" FORCE)
+
+# BLT trying to find MPI fails on cray with cce
+set(ENABLE_FIND_MPI FALSE CACHE BOOL "")

host-configs/ORNL/crusher-cce@13.0.1.cmake

@@ -0,0 +1,39 @@
+
+set(CONFIG_NAME "[email protected]" CACHE PATH "")
+include( ${CMAKE_CURRENT_LIST_DIR}/crusher-base.cmake )
+
+# Set up the tpls
+set(GEOSX_TPL_ROOT_DIR "/gpfs/alpine/geo127/world-shared/cray-sles15-zen2/cce-13.0.1" CACHE PATH "")
+set(GEOSX_TPL_DIR ${GEOSX_TPL_ROOT_DIR} CACHE PATH "")
+set(GEOSX_TPL_DIR2 "/gpfs/alpine/geo127/world-shared/cray-sles15-zen3/cce-13.0.1" CACHE PATH "")
+
+set(CAMP_DIR "${GEOSX_TPL_DIR2}/camp-0.2.2-oogry5gz2fts7jufeykxzmowajtmgzi3" CACHE PATH "" )
+
+set(RAJA_DIR "${GEOSX_TPL_DIR2}/raja-2022.03.0-ex5v5y6jtotfxxvwcs7bblwvy4ktjykq" CACHE PATH "" )
+
+set(ENABLE_UMPIRE TRUE CACHE BOOL "" )
+set(UMPIRE_DIR "${GEOSX_TPL_DIR2}/umpire-develop-jqqth57w2ets75sljw7lc5uxoi5wwi3c" CACHE PATH "" )
+
+set(ENABLE_CHAI TRUE CACHE BOOL "" )
+set(CHAI_DIR "${GEOSX_TPL_DIR2}/chai-2022.03.0-w7lka3bkp36mbk5kzucgtp3eowomllgl" CACHE PATH "" )
+
+set(METIS_DIR "${GEOSX_TPL_DIR}/metis-5.1.0-zcfkawg5ifqpzcihrc3i6cdrrijusc2p/" CACHE PATH "" )
+set(PARMETIS_DIR "${GEOSX_TPL_DIR}/parmetis-4.0.3-t2amifl5hh7yewre24gn2x3mlrz7qkl5/" CACHE PATH "" )
+
+# C++ options
+set(CMAKE_C_COMPILER "/opt/cray/pe/craype/2.7.13/bin/cc" CACHE PATH "")
+set(CMAKE_CXX_COMPILER "/opt/cray/pe/craype/2.7.13/bin/CC" CACHE PATH "")
+set(CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/2.7.13/bin/ftn" CACHE PATH "")
+
+
+if( ENABLE_HIP )
+ set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE ) # don't invoke hipcc, rely on cce link-time compilation
+
+ set( HIP_VERSION_STRING "4.5.2" CACHE STRING "" )
+ set( HIP_ROOT "/opt/rocm-4.5.2" CACHE PATH "" )
+
+ set( CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "" FORCE )
+ set( AMDGPU_TARGETS "${CMAKE_HIP_ARCHITECTURES}" CACHE STRING "" FORCE )
+ set( CMAKE_CXX_FLAGS "-fgpu-rdc" CACHE STRING "" FORCE )
+ set( CMAKE_CXX_LINK_FLAGS "-fgpu-rdc --hip-link" CACHE STRING "" FORCE )
+endif()