Errors related to LvArray when trying to build GEOSX with GPU support

[jspk19140]

Hello,

I get a lot of errors related to LvArray when I try to build GEOSX with GPU support. There is no problem when I build without GPU support. I use Rockylinux 8 x86_64 with GCC 11.2.0, OpenMPI 4.1.1, CUDA 11.6.0, Python 3.9.6.

Below the 2 first errors and my config files.

Have you ever seen this errors ?

[ 19%] Building CUDA object coreComponents/LvArray/benchmarks/CMakeFiles/benchmarkOuterProduct.dir/benchmarkOuterProduct.cpp.o
cd /geosx/codes/GEOSX/build-user-foss-2021b-cuda-sm60-el8-x86_64-release/coreComponents/LvArray/benchmarks && /easybuild/software/CUDA/11.6.0/bin/nvcc -forward-unknown-to-host-compiler -ccbin=/easybuild/software/GCCcore/11.2.0/bin/g++ -DBENCHMARK_STATIC_DEFINE -DCAMP_HAVE_CUDA -DGTEST_HAS_DEATH_TEST=1 -DbenchmarkOuterProduct_EXPORTS --options-file CMakeFiles/benchmarkOuterProduct.dir/includes_CUDA.rsp -restrict -arch sm_60 --expt-relaxed-constexpr --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations -Xcompiler -std=c++14 -O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -std=c++14 "--generate-code=arch=compute_60,code=[compute_60,sm_60]" -Xcompiler=-fPIE -Xcompiler=-fopenmp -MD -MT coreComponents/LvArray/benchmarks/CMakeFiles/benchmarkOuterProduct.dir/benchmarkOuterProduct.cpp.o -MF CMakeFiles/benchmarkOuterProduct.dir/benchmarkOuterProduct.cpp.o.d -x cu -c /geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/benchmarkOuterProduct.cpp -o CMakeFiles/benchmarkOuterProduct.dir/benchmarkOuterProduct.cpp.o
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/benchmarkReduceKernels.hpp(86): error: template instantiation resulted in unexpected function type of "RAJA::internal::ViewBase<LvArray::benchmarking::VALUE_TYPE, LvArray::benchmarking::VALUE_TYPE *, RAJA::detail::LayoutBase_impl<camp::int_seq<camp::idx_t, 0L>, LvArray::benchmarking::INDEX_TYPE, 0L>> (const LvArray::Array<LvArray::benchmarking::VALUE_TYPE, 1, RAJA::PERM_I, LvArray::benchmarking::INDEX_TYPE, LvArray::testing::DEFAULT_BUFFER> &)" (the meaning of a name may have changed since the template declaration -- the type of the template is "LvArray::benchmarking::RajaView<T, PERMUTATION> (const LvArray::benchmarking::ArrayT<T, PERMUTATION> &)")
          detected during instantiation of "LvArray::benchmarking::makeRajaView" based on template arguments <LvArray::benchmarking::VALUE_TYPE, RAJA::PERM_I>
(86): here

(...)

[ 19%] Linking CXX executable ../../../tests/exampleArrayOfArrays
cd /geosx/codes/GEOSX/build-user-foss-2021b-cuda-sm60-el8-x86_64-release/coreComponents/LvArray/examples && /geosx/cmake-3.27.1-linux-x86_64/bin/cmake -E cmake_link_script CMakeFiles/exampleArrayOfArrays.dir/link.txt --verbose=1
/easybuild/software/GCCcore/11.2.0/bin/g++  -Wall -Wextra      -fopenmp  -Wall -Wextra -Wpedantic -pedantic-errors -Wshadow -Wfloat-equal -Wcast-align -Wcast-qual  -fopenmp  -Wall -Wextra -Wpedantic -pedantic-errors -Wshadow -Wfloat-equal -Wcast-align -Wcast-qual  -O3 -DNDEBUG -Wl,--export-dynamic -rdynamic -fopenmp -L/easybuild/software/CUDA/11.6.0/lib64 CMakeFiles/exampleArrayOfArrays.dir/exampleArrayOfArrays.cpp.o -o ../../../tests/exampleArrayOfArrays   -L/easybuild/software/CUDA/11.6.0/targets/x86_64-linux/lib/stubs  -Wl,-rpath,::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ../../../lib/libgtest_main.a ../../../lib/libgtest.a -lpthread ../../../lib/liblvarray.a -ldl /geosx/codes/thirdPartyLibs/install-user-foss-2021b-cuda-sm60-el8-x86_64-release/raja/lib/libcamp.a /geosx/codes/thirdPartyLibs/install-user-foss-2021b-cuda-sm60-el8-x86_64-release/raja/lib/libRAJA.a /geosx/codes/thirdPartyLibs/install-user-foss-2021b-cuda-sm60-el8-x86_64-release/chai/lib/libumpire.a /geosx/codes/thirdPartyLibs/install-user-foss-2021b-cuda-sm60-el8-x86_64-release/chai/lib/libchai.a /geosx/codes/thirdPartyLibs/install-user-foss-2021b-cuda-sm60-el8-x86_64-release/raja/lib/libRAJA.a /geosx/codes/thirdPartyLibs/install-user-foss-2021b-cuda-sm60-el8-x86_64-release/chai/lib/libumpire.a /geosx/codes/thirdPartyLibs/install-user-foss-2021b-cuda-sm60-el8-x86_64-release/raja/lib/libcamp.a -lpthread /easybuild/software/CUDA/11.6.0/lib64/libcudart_static.a -ldl /usr/lib64/librt.so -lcudadevrt -lcudart_static -lrt -lpthread -ldl
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/benchmarkReduceKernels.hpp(162): error: more than one conversion function from "LvArray::Array<LvArray::benchmarking::VALUE_TYPE, 1, RAJA::PERM_I, LvArray::benchmarking::INDEX_TYPE, LvArray::testing::DEFAULT_BUFFER>" to "<error-type>" applies:
            function "LvArray::Array<T, NDIM, PERMUTATION, INDEX_TYPE, BUFFER_TYPE>::operator LvArray::ArrayView<const T, NDIM, LvArray::ArrayView<T, NDIM, <expression>, INDEX_TYPE, BUFFER_TYPE>::USD, INDEX_TYPE, BUFFER_TYPE>() const & [with T=LvArray::benchmarking::VALUE_TYPE, NDIM=1, PERMUTATION=RAJA::PERM_I, INDEX_TYPE=LvArray::benchmarking::INDEX_TYPE, BUFFER_TYPE=LvArray::testing::DEFAULT_BUFFER]"
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/../src/Array.hpp(290): here
            function "LvArray::ArrayView<T, NDIM_TPARAM, USD_TPARAM, INDEX_TYPE, BUFFER_TYPE>::operator LvArray::ArraySlice<T, LvArray::ArrayView<T, NDIM_TPARAM, USD_TPARAM, INDEX_TYPE, BUFFER_TYPE>::NDIM, LvArray::ArrayView<T, NDIM_TPARAM, USD_TPARAM, INDEX_TYPE, BUFFER_TYPE>::USD, INDEX_TYPE>() const & [with T=LvArray::benchmarking::VALUE_TYPE, NDIM_TPARAM=1, USD_TPARAM=0, INDEX_TYPE=LvArray::benchmarking::INDEX_TYPE, BUFFER_TYPE=LvArray::testing::DEFAULT_BUFFER]"
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/../src/ArrayView.hpp(355): here
          detected during:
            instantiation of "void LvArray::benchmarking::ReduceRAJA<POLICY>::RAJAView() [with POLICY=LvArray::testing::serialPolicy]"
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/benchmarkReduce.cpp(109): here
            instantiation of "void LvArray::benchmarking::rajaViewRAJA<POLICY>(benchmark::State &) [with POLICY=LvArray::testing::serialPolicy]"
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/benchmarkReduce.cpp(148): here

/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/benchmarkReduceKernels.hpp(162): error: more than one conversion function from "LvArray::Array<LvArray::benchmarking::VALUE_TYPE, 1, RAJA::PERM_I, LvArray::benchmarking::INDEX_TYPE, LvArray::testing::DEFAULT_BUFFER>" to "<error-type>" applies:
            function "LvArray::Array<T, NDIM, PERMUTATION, INDEX_TYPE, BUFFER_TYPE>::operator LvArray::ArrayView<const T, NDIM, LvArray::ArrayView<T, NDIM, <expression>, INDEX_TYPE, BUFFER_TYPE>::USD, INDEX_TYPE, BUFFER_TYPE>() const & [with T=LvArray::benchmarking::VALUE_TYPE, NDIM=1, PERMUTATION=RAJA::PERM_I, INDEX_TYPE=LvArray::benchmarking::INDEX_TYPE, BUFFER_TYPE=LvArray::testing::DEFAULT_BUFFER]"
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/../src/Array.hpp(290): here
            function "LvArray::ArrayView<T, NDIM_TPARAM, USD_TPARAM, INDEX_TYPE, BUFFER_TYPE>::operator LvArray::ArraySlice<T, LvArray::ArrayView<T, NDIM_TPARAM, USD_TPARAM, INDEX_TYPE, BUFFER_TYPE>::NDIM, LvArray::ArrayView<T, NDIM_TPARAM, USD_TPARAM, INDEX_TYPE, BUFFER_TYPE>::USD, INDEX_TYPE>() const & [with T=LvArray::benchmarking::VALUE_TYPE, NDIM_TPARAM=1, USD_TPARAM=0, INDEX_TYPE=LvArray::benchmarking::INDEX_TYPE, BUFFER_TYPE=LvArray::testing::DEFAULT_BUFFER]"
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/../src/ArrayView.hpp(355): here
          detected during:
            instantiation of "void LvArray::benchmarking::ReduceRAJA<POLICY>::RAJAView() [with POLICY=LvArray::testing::parallelHostPolicy]"
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/benchmarkReduce.cpp(109): here
            instantiation of "void LvArray::benchmarking::rajaViewRAJA<POLICY>(benchmark::State &) [with POLICY=LvArray::testing::parallelHostPolicy]"
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/benchmarkReduce.cpp(148): here

/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/benchmarkReduceKernels.hpp(162): error: more than one conversion function from "LvArray::Array<LvArray::benchmarking::VALUE_TYPE, 1, RAJA::PERM_I, LvArray::benchmarking::INDEX_TYPE, LvArray::testing::DEFAULT_BUFFER>" to "<error-type>" applies:
            function "LvArray::Array<T, NDIM, PERMUTATION, INDEX_TYPE, BUFFER_TYPE>::operator LvArray::ArrayView<const T, NDIM, LvArray::ArrayView<T, NDIM, <expression>, INDEX_TYPE, BUFFER_TYPE>::USD, INDEX_TYPE, BUFFER_TYPE>() const & [with T=LvArray::benchmarking::VALUE_TYPE, NDIM=1, PERMUTATION=RAJA::PERM_I, INDEX_TYPE=LvArray::benchmarking::INDEX_TYPE, BUFFER_TYPE=LvArray::testing::DEFAULT_BUFFER]"
/geosx/codes/GEOSX/src/coreComponents/LvArray/benchmarks/../src/Array.hpp(290): here
            function "LvArray::ArrayView<T, NDIM_TPARAM, USD_TPARAM, INDEX_TYPE, BUFFER_TYPE>::operator LvArray::ArraySlice<T, LvArray::ArrayView<T, NDIM_TPARAM, USD_TPARAM, INDEX_TYPE, BUFFER_TYPE>::NDIM, LvArray::ArrayView<T, NDIM_TPARAM, USD_TPARAM, INDEX_TYPE, BUFFER_TYPE>::USD, INDEX_TYPE>() const & [with T=LvArray::benchmarking::VALUE_TYPE, NDIM_TPARAM=1, USD_TPARAM=0, INDEX_TYPE=LvArray::benchmarking::INDEX_TYPE, BUFFER_TYPE=LvArray::testing::DEFAULT_BUFFER]"

(...)

# file : user-foss-2021b-cuda-sm60-el8-x86_64.cmake

# detect host and name the configuration file
site_name(HOST_NAME)
set(CONFIG_NAME "${HOST_NAME}-user-foss-2021b-cuda-sm60-el8-x86_64" CACHE PATH "")
message("CONFIG_NAME = ${CONFIG_NAME}")

# Now let's add what's dedicated to GPU.
set(ENABLE_CUDA ON CACHE PATH "" FORCE)
set(CUDA_TOOLKIT_ROOT_DIR "$ENV{CUDA_HOME}" CACHE PATH "")
set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "")
set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")

set(CMAKE_CUDA_ARCHITECTURES "60" CACHE STRING "")
set(CUDA_ARCH sm_60 CACHE STRING "")
set(CMAKE_CUDA_STANDARD 14 CACHE STRING "")
### The inclusion of -std=c++14 is a workaround for a cuda10/gcc8 bug ###
set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-relaxed-constexpr --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations -Xcompiler -std=c++14" CACHE STRING "")
set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "")
set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")

# Current version of hypre does not build with GPU support.
# Most recent version does build. Let's wait for an upgrade on our side.
set(ENABLE_HYPRE_DEVICE "CPU" CACHE STRING "" FORCE)

# define the path to your compiled installation directory
set(GEOSX_TPL_DIR "${CMAKE_SOURCE_DIR}/../../thirdPartyLibs/install-user-foss-2021b-cuda-sm60-el8-x86_64-release" CACHE PATH "")

include("${CMAKE_CURRENT_LIST_DIR}/user-foss-2021b-base-x86_64.cmake")

# file : user-foss-2021b-base-x86_64.cmake

# set paths to C, C++, and Fortran compilers. Note that while GEOSX does not contain any Fortran code,
# some of the third-party libraries do contain Fortran code. Thus a Fortran compiler must be specified.
set(CMAKE_C_COMPILER "$ENV{EBROOTGCCCORE}/bin/gcc" CACHE PATH "")
set(CMAKE_CXX_COMPILER "$ENV{EBROOTGCCCORE}/bin/g++" CACHE PATH "")
set(CMAKE_Fortran_COMPILER "$ENV{EBROOTGCCCORE}/bin/gfortran" CACHE PATH "")
set(ENABLE_FORTRAN OFF CACHE BOOL "" FORCE)

# enable MPI and set paths to compilers and executable.
# Note that the MPI compilers are wrappers around standard serial compilers.
# Therefore, the MPI compilers must wrap the appropriate serial compilers specified
# in CMAKE_C_COMPILER, CMAKE_CXX_COMPILER, and CMAKE_Fortran_COMPILER.
set(ENABLE_MPI ON CACHE BOOL "")
set(MPI_C_COMPILER "$ENV{EBROOTOPENMPI}/bin/mpicc" CACHE PATH "")
set(MPI_CXX_COMPILER "$ENV{EBROOTOPENMPI}/bin/mpicxx" CACHE PATH "")
set(MPI_Fortran_COMPILER "$ENV{EBROOTOPENMPI}/bin/mpif90" CACHE PATH "")
set(MPIEXEC "$ENV{EBROOTOPENMPI}/bin/mpirun" CACHE PATH "")

# enable OpenMP
set(ENABLE_OPENMP ON CACHE BOOL "" FORCE)

# enable PAMELA and PVTPackage
set(ENABLE_PAMELA ON CACHE BOOL "" FORCE)
set(ENABLE_PVTPackage ON CACHE BOOL "" FORCE)

# enable tests
set(ENABLE_GTEST_DEATH_TESTS ON CACHE BOOL "" FORCE)

# do not treat warnings as errors to fix :
# src/coreComponents/mesh/mpiCommunications/NeighborCommunicator.cpp:509:5: error: ‘location’ may be used uninitialized [-Werror=maybe-uninitialized]
set(ENABLE_WARNINGS_AS_ERRORS OFF CACHE BOOL "" FORCE)

set(BLT_MPI_LINK_FLAGS "-Wl,-rpath,${EBROOTHWLOC}/lib -Wl,-rpath,${EBROOTLIBEVENT}/lib64 -Wl,-rpath,${EBROOTOPENMPI}/lib -Wl,--enable-new-dtags" CACHE PATH "" FORCE)

# enable/disable solvers
set(ENABLE_PETSC OFF CACHE BOOL "" FORCE)
set(ENABLE_TRILINOS OFF CACHE BOOL "" FORCE)
set(ENABLE_HYPRE ON CACHE BOOL "" FORCE)

# let GEOSX define some third party libraries information for you
include(${CMAKE_CURRENT_LIST_DIR}/tpls.cmake)

Thanks for your help !

Best regards !

[wrtobin]

GEOS-DEV/LvArray#294

[paveltomin]

@wrtobin any update?