diff --git a/CMakeLists.txt b/CMakeLists.txt index f0b6a6830..10cb4ad5a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -350,7 +350,7 @@ TARGET_LINK_LIBRARIES(${TARGETLIB} PUBLIC ${DFTFE_MPI_LIBRARIES}) #Dependencies # -FIND_PACKAGE(deal.II 9.1.0 REQUIRED +FIND_PACKAGE(deal.II 9.5.1 REQUIRED HINTS ${deal.II_DIR} ${DEAL_II_DIR} ../ ../../ $ENV{DEAL_II_DIR} ) IF(NOT ${deal.II_FOUND}) @@ -388,11 +388,11 @@ ENDIF() #DEAL_II_INITIALIZE_CACHED_VARIABLES() -TARGET_INCLUDE_DIRECTORIES(${TARGETLIB} PUBLIC ${DEAL_II_INCLUDE_DIRS}) +TARGET_INCLUDE_DIRECTORIES(${TARGETLIB} PUBLIC ${DEAL_II_INCLUDE_RELDIR}) IF("${CMAKE_BUILD_TYPE}" STREQUAL "Release") - TARGET_LINK_LIBRARIES (${TARGETLIB} PUBLIC ${DEAL_II_LIBRARIES_RELEASE}) + TARGET_LINK_LIBRARIES (${TARGETLIB} PUBLIC dealii::dealii_release) ELSE() - TARGET_LINK_LIBRARIES (${TARGETLIB} PUBLIC ${DEAL_II_LIBRARIES_DEBUG}) + TARGET_LINK_LIBRARIES (${TARGETLIB} PUBLIC dealii::dealii_debug) ENDIF() IF (WITH_CUSTOMIZED_DEALII) @@ -619,7 +619,7 @@ IF (WITH_TESTING) ENABLE_TESTING() INCLUDE(CTest) # default 600sec is not enough for some tests, increase: - SET (TEST_TIME_LIMIT 3600) + SET (TEST_TIME_LIMIT 5000) # IF (WITH_COMPLEX) ADD_SUBDIRECTORY(tests/dft/pseudopotential/complex) diff --git a/README.md b/README.md index 540472809..b13c99bfe 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ DFT-FE code builds on top of the deal.II library for everything that has to do w The steps to install the necessary dependencies and DFT-FE itself are described in the *Installation* section of the DFT-FE manual (compile doc/manual/manual.tex or download the development version manual [here](https://github.com/dftfeDevelopers/dftfe/blob/manual/manual-develop.pdf)). We have created several shell based installation scripts for the development version of DFT-FE (`publicGithubDevelop` branch) on various machines: - - [OLCF Frontier](https://github.com/dftfeDevelopers/install_DFTFE/tree/frontierScriptROCM6) - - [NERSC Perlmutter](https://github.com/dftfeDevelopers/install_DFTFE/tree/perlmutterScript) + - [OLCF Frontier](https://github.com/dftfeDevelopers/install_DFTFE/tree/frontierScriptROCM6Dealii9.5.2) + - [NERSC Perlmutter](https://github.com/dftfeDevelopers/install_DFTFE/tree/perlmutterScriptDealii9.5.2) - [ALCF Polaris](https://github.com/dftfeDevelopers/install_DFTFE/tree/polarisScript) - [UMICH Greatlakes](https://github.com/dftfeDevelopers/install_DFTFE/tree/greatlakesScript) diff --git a/doc/manual/installation.tex b/doc/manual/installation.tex index 6154f69e1..5b53c9544 100644 --- a/doc/manual/installation.tex +++ b/doc/manual/installation.tex @@ -18,9 +18,9 @@ \subsection{Compiling and installing external libraries} \dftfe{} is primarily based on the open source finite element library \href{http://www.dealii.org/}{deal.II}, through which external dependencies -on \href{http://p4est.org/}{p4est} and BLAS-LAPACK are set. The other required external libraries, which are +on \href{http://p4est.org/}{p4est}, \href{https://github.com/kokkos/kokkos}{kokkos} and BLAS-LAPACK are set. The other required external libraries, which are not interfaced via deal.II are \href{http://www.alglib.net/}{ALGLIB}, \href{http://www.tddft.org/programs/libxc/}{Libxc}, \href{https://atztogo.github.io/spglib/}{spglib}, \href{http://www.xmlsoft.org/}{Libxml2} and \href{https://elpa.mpcdf.mpg.de/}{ELPA} (ELPA requires \href{http://www.netlib.org/scalapack/}{ScaLAPACK}). DFT-FE also optionally links to \href{https://www.mcs.anl.gov/petsc/}{PETSc} (via deali.II), \href{http://slepc.upv.es/}{SLEPc} (via deali.II) and to \href{https://developer.nvidia.com/nccl}{nccl} (for GPU compilation). The optional dependencies of PETSc and SLEPc are only required for all-electron calculations using DFT-FE, which uses the more stable Gram-Schmidt orthogonalization routine instead of the default Cholesky-Gram-Schmidt orthognalization. For pseudopotential calculations, PETSc and SLEPc dependencies are not required as the default Cholesky-Gram-Schmidt orthogonalization is very robust. \dftfe{} can also be linked to \href{https://github.com/awvwgk/simple-dftd3}{simple-dftd3} and \href{https://github.com/dftd4/dftd4}{dftd4} in order to provide support for the dftd family of dispersion corrections by Grimme et. al.. Below, we give brief installation instructions for each of the above libraries. -\subsubsection{Instructions for dependencies: ALGLIB, Libxc, spglib, Libxml2, ScaLAPACK, ELPA, p4est and nccl (nccl is optional)} +\subsubsection{Instructions for dependencies: ALGLIB, Libxc, spglib, Libxml2, ScaLAPACK, ELPA, p4est, kokkos, and nccl (nccl is optional)} \begin{enumerate} \item {\bf ALGLIB}: Used by \dftfe{} for spline fitting for various radial data. Download the current release of the Alglib free C++ edition from \url{http://www.alglib.net/download.php}. After downloading and unpacking, go to \verb|cpp/src|, and create a shared library using a C++ compiler. For example, using GCC compiler do \begin{verbatim} @@ -135,7 +135,7 @@ \subsubsection{Instructions for dependencies: ALGLIB, Libxc, spglib, Libxml2, Sc $LD_LIBRARY_PATH:$ROCM_PATH/lib:$ROCM_PATH/hip/lib:$ROCM_PATH/lib:$ROCM_PATH/rocblas/lib \end{verbatim} -\item {\bf p4est}: This library is used by deal.II to create and distribute finite-element meshes across multiple processors. Download the v2.2 tarball of p4est from \url{https://github.com/p4est/p4est.github.io/raw/master/release/p4est-2.2.tar.gz}. Next download the \verb|p4est-setup.sh| script from \url{https://raw.githubusercontent.com/dftfeDevelopers/dftfe/manual/p4est-setup.sh}. Use the script to automatically compile and install a debug and optimized version of p4est by doing +\item {\bf p4est}: This library is used by deal.II to create and distribute finite-element meshes across multiple processors. Download the latest release tarball of p4est from \url{https://www.p4est.org}. Next download the \verb|p4est-setup.sh| script from \url{https://raw.githubusercontent.com/dftfeDevelopers/dftfe/manual/p4est-setup.sh}. Use the script to automatically compile and install a debug and optimized version of p4est by doing \begin{verbatim} $ chmod u+x p4est-setup.sh $ ./p4est-setup.sh p4est-x-y-z.tar.gz p4est_install_dir_path @@ -143,6 +143,20 @@ \subsubsection{Instructions for dependencies: ALGLIB, Libxc, spglib, Libxml2, Sc %Also download the script from \url{https://github.com/dftfeDevelopers/dftfe/raw/manual/p4est-setup.sh} if using Intel compilers, or from \url{https://dealii.org/developer/external-libs/p4est.html} if using GCC compilers. Use the script to automatically compile and install a debug and optimized version of p4est by doing +\item {\bf kokkos}: This library is used by deal.II to performance portability across architectures. Download the latest release tarball of p4est from \url{https://github.com/kokkos/kokkos}, and use cmake based compilation. For example, +\begin{verbatim} +$ cd kokkos-4.3.00 +$ mkdir build && cd build +$ cmake -DCMAKE_C_COMPILER=cc -DCMAKE_C_FLAGS="-O2 -fPIC" + -DCMAKE_CXX_COMPILER=CC + -DCMAKE_CXX_FLAGS="-O2 -fPIC" + -DCMAKE_INSTALL_PREFIX=kokkos_install_dir_path + .. +$ make -j 4 +$ make install +\end{verbatim} + + \item {\bf nccl/rccl (optional)}: nccl (CUDA) and rccl (ROCm) libraries are optional dependency for DFT-FE for optimal GPU Direct MPI collective communication calls. This library is recommended for running very large system sizes (greater than 20,000 electrons) on GPUs using DFT-FE. Caution: nccl/rccl library requires appropriate hardware support for GPU Direct MPI communication and GPU Aware MPI library. To install nccl/rccl, use the appropriate release version (corresponding to the CUDA/ROCm compiler version) from \url{https://github.com/NVIDIA/nccl} (CUDA) \url{https://github.com/ROCmSoftwarePlatform/rccl} (ROCm) and follow installation instructions therein. \end{enumerate} @@ -153,12 +167,9 @@ \subsubsection{Instructions for deal.II} \begin{enumerate} -\item Obtain the customized version of deal.II library via -\begin{verbatim} -$ git clone -b dealiiCustomizedCUDARelease https://github.com/dftfeDevelopers/dealii.git -\end{verbatim} +\item Obtain the latest release version (9.5.2) of deal.II -\item In addition to requiring C, C++ and Fortran compilers, MPI library, and CMake, deal.II additionally requires BOOST library. If not found externally, cmake will resort to the bundled BOOST that comes along with deal.II. Based on our experience, we recommend to use the deal.II's bundled boost (enforced by unsetting/unloading external BOOST library environment paths) to avoid compilation issues. Please do not install deal.II with GPU support as that is not needed by DFT-FE. Further, when installing deal.II for (DFT-FE with GPU support), one must use \verb|-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF| to avoid compilation failure of DFT-FE. +\item In addition to requiring C, C++ and Fortran compilers, MPI library, CMake, and Kokkos. deal.II additionally requires BOOST library. If not found externally, deal.II will resort to the bundled BOOST. Based on our experience, we recommend to use the deal.II's bundled boost (enforced by unsetting/unloading external BOOST library environment paths). Please do not install deal.II with GPU support as that is not needed by DFT-FE. Further, when installing deal.II for (DFT-FE with GPU support), one must use \verb|-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF| to avoid compilation failure of DFT-FE. \item \begin{verbatim} @@ -182,6 +193,7 @@ \subsubsection{Instructions for deal.II} -DCMAKE_C_FLAGS=c_flags -DDEAL_II_WITH_MPI=ON -DDEAL_II_WITH_64BIT_INDICES=ON -DDEAL_II_WITH_P4EST=ON -DP4EST_DIR=p4est_install_dir_path +-DKOKKOS_DIR=kokkos_install_dir_path -DDEAL_II_WITH_LAPACK=ON -DLAPACK_DIR=lapack_dir_paths (both BLAS and LAPACK directory paths) -DLAPACK_FOUND=true @@ -190,6 +202,7 @@ \subsubsection{Instructions for deal.II} -DDEAL_II_WITH_TASKFLOW=OFF -DDEAL_II_COMPONENT_EXAMPLES=OFF -DDEAL_II_FORCE_BUNDLED_BOOST=ON +-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF \end{verbatim} @@ -216,6 +229,7 @@ \subsubsection{Instructions for deal.II} -DDEAL_II_WITH_TBB=OFF -DDEAL_II_WITH_TASKFLOW=OFF -DDEAL_II_FORCE_BUNDLED_BOOST=ON +-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF -DDEAL_II_WITH_P4EST=ON -DP4EST_DIR=p4est_install_path -DDEAL_II_WITH_LAPACK=ON -DLAPACK_DIR="${MKLROOT}/lib/intel64" diff --git a/helpers/Crusher/setupUserCrusher.sh b/helpers/Crusher/setupUserCrusher.sh deleted file mode 100755 index 27f1af010..000000000 --- a/helpers/Crusher/setupUserCrusher.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/bin/bash -# script to setup and build DFT-FE. - -set -e -set -o pipefail - -if [ -s CMakeLists.txt ]; then - echo "This script must be run from the build directory!" - exit 1 -fi - -# Path to project source -SRC=`dirname $0` # location of source directory - -######################################################################## -#Provide paths below for external libraries, compiler options and flags, -# and optimization flag - -#Paths for required external libraries -dealiiDir="/ccs/proj/mat187/dsambit/softwareDFTFEGccMpichLatest/dealii/installopenblas" -alglibDir="/ccs/proj/mat187/dsambit/softwareDFTFEGcc/alglib/cpp/src" -libxcDir="/ccs/proj/mat187/dsambit/softwareDFTFEGcc/libxc/install" -spglibDir="/ccs/proj/mat187/dsambit/softwareDFTFEGcc/spglib/install" -xmlIncludeDir="/usr/include/libxml2" -xmlLibDir="/usr/lib64" -ELPA_PATH="/ccs/proj/mat187/dsambit/softwareDFTFEGccMpichLatest/elpa/installgpu" - -#Paths for optional external libraries -# path for NCCL/RCCL libraries -DCCL_PATH="/ccs/proj/mat187/dsambit/softwareDFTFEGccMpichLatest/rccl/build" -mdiPath="" - -#Toggle GPU compilation -withGPU=ON -gpuLang="hip" # Use "cuda"/"hip" -gpuVendor="amd" # Use "nvidia/amd" -withGPUAwareMPI=ON #Please use this option with care - #Only use if the machine supports - #device aware MPI and is profiled - #to be fast - -#Option to link to DCCL library (Only for GPU compilation) -withDCCL=ON -withMDI=OFF - -#Compiler options and flags -cxx_compiler=CC #sets DCMAKE_CXX_COMPILER -cxx_flags="-march=znver3 -fPIC -I${MPICH_DIR}/include -I${ROCM_PATH}/include" #sets DCMAKE_CXX_FLAGS -cxx_flagsRelease="-O2" #sets DCMAKE_CXX_FLAGS_RELEASE -device_flags="-march=znver3 -O2 -munsafe-fp-atomics -I${MPICH_DIR}/include -I${ROCM_PATH}/include" - #setDCMAKE_CXX_CUDA/HIP_FLAGS - #(only applicable for withGPU=ON) -device_architectures="gfx90a" # set DCMAKE_CXX_CUDA/HIP_ARCHITECTURES - #(only applicable for withGPU=ON) - - -#Option to compile with default or higher order quadrature for storing pseudopotential data -#ON is recommended for MD simulations with hard pseudopotentials -withHigherQuadPSP=OFF - -# build type: "Release" or "Debug" -build_type=Release - -testing=OFF -minimal_compile=ON -########################################################################### -#Usually, no changes are needed below this line -# - -#if [[ x"$build_type" == x"Release" ]]; then -# c_flags="$c_flagsRelease" -# cxx_flags="$c_flagsRelease" -#else -#fi -out=`echo "$build_type" | tr '[:upper:]' '[:lower:]'` - -function cmake_real() { - mkdir -p real && cd real - if [ "$gpuLang" = "cuda" ]; then - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=OFF -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_CUDA_FLAGS="$device_flags" -DCMAKE_CUDA_ARCHITECTURES="$device_architectures"\ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - elif [ "$gpuLang" = "hip" ]; then - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=OFF -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_HIP_FLAGS="$device_flags" -DCMAKE_HIP_ARCHITECTURES="$device_architectures"\ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DCMAKE_SHARED_LINKER_FLAGS="-L${ROCM_PATH}/lib -lamdhip64 -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa"\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - else - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=OFF\ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - fi -} - -function cmake_cplx() { - mkdir -p complex && cd complex - if [ "$gpuLang" = "cuda" ]; then - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=ON -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_CUDA_FLAGS="$device_flags" -DCMAKE_CUDA_ARCHITECTURES="$device_architectures"\ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - elif [ "$gpuLang" = "hip" ]; then - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=ON -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_HIP_FLAGS="$device_flags" -DCMAKE_HIP_ARCHITECTURES="$device_architectures"\ - -DCMAKE_SHARED_LINKER_FLAGS="-L${ROCM_PATH}/lib -lamdhip64 -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa"\ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - else - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=ON \ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - fi - -} - -RCol='\e[0m' -Blu='\e[0;34m'; -if [ -d "$out" ]; then # build directory exists - echo -e "${Blu}$out directory already present${RCol}" -else - rm -rf "$out" - echo -e "${Blu}Creating $out ${RCol}" - mkdir -p "$out" -fi - -cd $out - -echo -e "${Blu}Building Real executable in $build_type mode...${RCol}" -cmake_real "$SRC" && make -j8 -cd .. - -echo -e "${Blu}Building Complex executable in $build_type mode...${RCol}" -cmake_cplx "$SRC" && make -j8 -cd .. - -echo -e "${Blu}Build complete.${RCol}" diff --git a/helpers/NERSCCoriKNL/READMENerscCori b/helpers/NERSCCoriKNL/READMENerscCori deleted file mode 100644 index 3ec2b97a6..000000000 --- a/helpers/NERSCCoriKNL/READMENerscCori +++ /dev/null @@ -1,10 +0,0 @@ -NERSC Cori is a Cray sytem. -Before compiling dftfe or the required libraries do: - -export XTPE_LINK_TYPE=dynamic -export CRAYPE_LINK_TYPE=dynamic -module swap craype-haswell craype-mic-knl -module load craype-hugepages2M -module load intel/18.0.1.163 - -Recommeneded to put the above in your bash script diff --git a/helpers/NERSCCoriKNL/p4est-setup.sh b/helpers/NERSCCoriKNL/p4est-setup.sh deleted file mode 100644 index 3ccc108a3..000000000 --- a/helpers/NERSCCoriKNL/p4est-setup.sh +++ /dev/null @@ -1,147 +0,0 @@ -#! /bin/bash -# -# This file is part of p4est [1]. -# p4est is a C library to manage a collection (a forest) of multiple -# connected adaptive quadtrees or octrees in parallel. -# -# Copyright (C) 2010 The University of Texas System -# Written by Carsten Burstedde, Lucas C. Wilcox, and Tobin Isaac -# Modified 2010 by Wolfgang Bangerth -# Modified 2010 by Timo Heister -# Modified 2013 by Matthias Maier -# Modified 2018 by Sambit Das (to fix OpenMP linking with intel compilers) -# -# p4est is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# [1] http://www.p4est.org -# - -# This program comes with ABSOLUTELY NO WARRANTY. - -# error message when zlib is not found -MISSING_ZLIB_MESSAGE="deal.II requires that p4est be built with zlib support. Please \ -either ensure that zlib is installed in a standard location or add appropriate \ -flags to LDFLAGS and CPPFLAGS to both calls to configure to describe where zlib's \ -shared object files and headers are (e.g., LDFLAGS=\"-L/path/to/shared-objects/\" \ -and CPPFLAGS=\"-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL -I/path/to/headers/\")." - -# unpack under current directory -UNPACK=`pwd` - -# choose names for fast and debug compilation directories -BUILD_DIR="$UNPACK/p4est-build" -BUILD_FAST="$BUILD_DIR/FAST" -BUILD_DEBUG="$BUILD_DIR/DEBUG" - -function busage() { - echo "Usage: `basename $0` []" - echo " or: `basename $0` /path/to/p4est-src/ []" -} -function bdie () { - echo "Error: $@" - exit 1 -} - -if test -z "$CFLAGS" -a -z "$P4EST_CFLAGS_FAST" ; then - export CFLAGS_FAST="-O2 -qopenmp" -else - export CFLAGS_FAST="$CFLAGS $P4EST_CFLAGS_FAST" -fi -echo "CFLAGS_FAST: $CFLAGS_FAST" -if test -z "$CFLAGS" -a -z "$P4EST_CFLAGS_DEBUG" ; then - export CFLAGS_DEBUG="-O0 -g -qopenmp" -else - export CFLAGS_DEBUG="$CFLAGS $P4EST_CFLAGS_DEBUG" -fi -echo "CFLAGS_DEBUG: $CFLAGS_DEBUG" - -TGZ="$1"; shift -if test -d "$TGZ" ; then - SRCDIR="$TGZ" - echo "using existing source dir '$SRCDIR'" -else - if test ! -f "$TGZ" ; then - busage - bdie "File not found" - fi - if ! (echo "$TGZ" | grep -q 'p4est.*.tar.gz') ; then - busage - bdie "File name mismatch" - fi -fi - -# choose names for fast and debug installation directories -INSTALL_DIR="$1"; shift -if test -z "$INSTALL_DIR" ; then - INSTALL_DIR="$UNPACK/p4est-install" -fi -INSTALL_FAST="$INSTALL_DIR/FAST" -INSTALL_DEBUG="$INSTALL_DIR/DEBUG" - -echo -echo "This script tries to unpack, configure and build the p4est library." -echo "Build FAST: $BUILD_FAST" -echo "Build DEBUG: $BUILD_DEBUG" -echo "Install FAST: $INSTALL_FAST" -echo "Install DEBUG: $INSTALL_DEBUG" -echo "Checking environment: CFLAGS P4EST_CFLAGS_FAST P4EST_CFLAGS_DEBUG" - -# remove old versions -if test -d "$BUILD_DIR" ; then - rm -rf "$BUILD_DIR" -fi - -if test -f "$TGZ" ; then - DIR=`echo "$TGZ" | sed 's/\(p4est.*\).tar.gz/\1/'` - DIR=`basename $DIR` - echo "Unpack directory: $UNPACK/$DIR" - if test -d "$UNPACK/$DIR" ; then - echo "Source directory found (remove it to unpack anew)" - else - echo -n "Unpacking... " - tar -xvz -f "$TGZ" -C "$UNPACK" >/dev/null - echo "done" - fi - SRCDIR=$UNPACK/$DIR -fi -test -f "$SRCDIR/src/p4est.h" || bdie "Main header file missing" -test -f "$SRCDIR/configure" || bdie "Configure script missing" - -echo "See output in files .../config.output and .../make.output" -echo -echo "Build FAST version in $BUILD_FAST" -mkdir -p "$BUILD_FAST" -cd "$BUILD_FAST" -"$SRCDIR/configure" CC=cc CXX=CC FC=ftn F77=ftn --enable-mpi --enable-shared\ - --disable-vtk-binary --without-blas --enable-openmp=-qopenmp \ - --prefix="$INSTALL_FAST" CFLAGS="$CFLAGS_FAST" \ - CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \ - "$@" > config.output || bdie "Error in configure" -make -C sc -j 8 > make.output || bdie "Error in make sc" -make -j 8 >> make.output || bdie "Error in make p4est" -# ensure that we built p4est with zlib -grep -q 'P4EST_HAVE_ZLIB *1' "$BUILD_FAST/src/p4est_config.h" \ - || bdie "$MISSING_ZLIB_MESSAGE" -make install >> make.output || bdie "Error in make install" -echo "FAST version installed in $INSTALL_FAST" - -echo -echo "Build DEBUG version in $BUILD_DEBUG" -mkdir -p "$BUILD_DEBUG" -cd "$BUILD_DEBUG" -"$SRCDIR/configure" CC=cc CXX=CC FC=ftn F77=ftn --enable-debug --enable-mpi --enable-shared\ - --disable-vtk-binary --without-blas --enable-openmp=-qopenmp \ - --prefix="$INSTALL_DEBUG" CFLAGS="$CFLAGS_DEBUG" \ - CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \ - "$@" > config.output || bdie "Error in configure" -make -C sc -j 8 > make.output || bdie "Error in make sc" -make -j 8 >> make.output || bdie "Error in make p4est" -# ensure that we built p4est with zlib -grep -q 'P4EST_HAVE_ZLIB *1' "$BUILD_DEBUG/src/p4est_config.h" \ - || bdie "$MISSING_ZLIB_MESSAGE" -make install >> make.output || bdie "Error in make install" -echo "DEBUG version installed in $INSTALL_DEBUG" -echo diff --git a/helpers/NERSCCoriKNL/sampleJobScript.slurm b/helpers/NERSCCoriKNL/sampleJobScript.slurm deleted file mode 100644 index 7c11a2d56..000000000 --- a/helpers/NERSCCoriKNL/sampleJobScript.slurm +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -l - -#SBATCH -q debug -#SBATCH -N 512 -#SBATCH -C knl -#SBATCH -S 4 -#SBATCH --switches=2@20:00:00 -#SBATCH -t 00:25:00 -#SBATCH -A m2360 -#SBATCH --mail-user=user@umich.edu -#SBATCH --mail-type=begin -#SBATCH --mail-type=end -#SBATCH -J 6x6x6 -#SBATCH -o 6x6x6.o%j - -# setup and run -echo "Job $SLURM_JOB_NAME, ID $SLURM_JOB_ID" -cd $SLURM_SUBMIT_DIR - -module load craype-hugepages2M - -export OMP_NUM_THREADS=2 -export OMP_PLACES=threads -export OMP_PROC_BIND=spread -export PMI_MMAP_SYNC_WAIT_TIME=600 - -sbcast --compress=lz4 /global/homes/d/user/dftfeProduction/build/release/real/main /tmp/main - -srun -n 16384 -c 8 --cpu_bind=cores /tmp/main parameterFile.prm > output diff --git a/helpers/NERSCCoriKNL/setupNerscCoriKnl.sh b/helpers/NERSCCoriKNL/setupNerscCoriKnl.sh deleted file mode 100755 index fa0e663ad..000000000 --- a/helpers/NERSCCoriKNL/setupNerscCoriKnl.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail -#script to setup and build DFT-FE - -######################################################################## -#Provide paths below for external libraries, compiler options and flags, -# and optimization flag - -#Paths for external libraries -dealiiPetscRealDir="/project/projectdirs/m2360/softwaresDFTFE/dealiiKNL/intel18_petscReal_64Bit_scalapackMpichHugepages" -dealiiPetscComplexDir="/project/projectdirs/m2360/softwaresDFTFE/dealiiKNL/intel18_petscComplex_64Bit_scalapack" -alglibDir="/global/project/projectdirs/m2360/softwaresDFTFE/alglib/cpp/src" -libxcDir="/project/projectdirs/m2360/softwaresDFTFE/libxc/install_libxc2.2.0_intel18" -spglibDir="/project/projectdirs/m2360/softwaresDFTFE/spglib" -xmlIncludeDir="/usr/include/libxml2/libxml" -xmlLibDir="/usr/lib64" -elpaIncludeDir="/global/project/projectdirs/m2360/softwaresDFTFE/elpa/install/include/elpa_openmp-2018.05.001" -elpaLibDir="/global/project/projectdirs/m2360/softwaresDFTFE/elpa/install/lib" - -#If you have installed dealii by linking with intel mkl library set underlying flag to "ON", -#otherwise set it to "OFF" -withIntelMkl=ON - -#Compiler options and flags -c_compiler=cc -cxx_compiler=CC -c_flagsRelease=-O2 -cxx_flagsRelease=-O2 - -#Option to link to ELPA -withELPA=ON - -#Optmization flag: 1 for optimized mode and 0 for debug mode compilation -optimizedMode=1 - -########################################################################### -#Usually, no changes are needed below this line -# -RCol='\e[0m' -Blu='\e[0;34m'; -if [ $optimizedMode == 1 ]; then - if [ -d "build/release" ]; then - echo -e "${Blu}build/release directory already present${RCol}" - # Control will enter here if build directory exists. - cd build - cd release - echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}" - mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd .. - echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}" - mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. - else - rm -rf build/release - echo -e "${Blu}Creating build directory...${RCol}" - mkdir -p build && cd build - mkdir -p release && cd release - echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}" - mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd .. - echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}" - mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. - fi -else - if [ -d "build/debug" ]; then - echo -e "${Blu}build/debug directory already present${RCol}" - # Control will enter here if build directory exists. - cd build - cd debug - echo -e "${Blu}Building Real executable in Debug mode...${RCol}" - mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd .. - echo -e "${Blu}Building Complex executable in Debug mode...${RCol}" - mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. - else - rm -rf build/debug - echo -e "${Blu}Creating build directory...${RCol}" - mkdir -p build && cd build - mkdir -p debug && cd debug - echo -e "${Blu}Building Real executable in Debug mode...${RCol}" - mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd .. - echo -e "${Blu}Building Complex executable in Debug mode...${RCol}" - mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. - fi -fi -echo -e "${Blu}Build complete.${RCol}" diff --git a/helpers/Summit/setupPSP.sh b/helpers/Summit/setupPSP.sh deleted file mode 100755 index 2e46163d7..000000000 --- a/helpers/Summit/setupPSP.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash -# script to setup and build DFT-FE. - -set -e -set -o pipefail - -if [ -s CMakeLists.txt ]; then - echo "This script must be run from the build directory!" - exit 1 -fi - -# Path to project source -SRC=`dirname $0` # location of source directory - -######################################################################## -#Provide paths below for external libraries, compiler options and flags, -# and optimization flag - -#Paths for required external libraries -dealiiDir="/ccs/proj/mat239/software/dealiiDevCustomized/installDealiiGcc9.1.0NOCUDA" -alglibDir="/ccs/proj/mat239/software/alglib/cpp/src" -libxcDir="/ccs/proj/mat239/software/libxc/installGcc9.1.0" -spglibDir="/ccs/proj/mat239/software/spglib/installGcc9.1.0" -xmlIncludeDir="/usr/include/libxml2" -xmlLibDir="/usr/lib64" -ELPA_PATH="/ccs/proj/mat239/software/elpa/install202211" - -#Paths for optional external libraries -# path for NCCL/RCCL libraries -DCCL_PATH="/ccs/proj/mat239/software/ncclnew/build" -mdiPath="" - -#Toggle GPU compilation -withGPU=ON -gpuLang="cuda" # Use "cuda"/"hip" -gpuVendor="nvidia" # Use "nvidia/amd" -withGPUAwareMPI=OFF #Please use this option with care - #Only use if the machine supports - #device aware MPI and is profiled - #to be fast - -#Option to link to DCCL library (Only for GPU compilation) -withDCCL=ON -withMDI=OFF - -#Compiler options and flags -cxx_compiler=mpic++ #sets DCMAKE_CXX_COMPILER -cxx_flags="-fPIC" #sets DCMAKE_CXX_FLAGS -cxx_flagsRelease="-O2" #sets DCMAKE_CXX_FLAGS_RELEASE -device_flags="-arch=sm_70" # set DCMAKE_CXX_CUDA/HIP_FLAGS - #(only applicable for withGPU=ON) -device_architectures="70" # set DCMAKE_CXX_CUDA/HIP_ARCHITECTURES - #(only applicable for withGPU=ON) - - -#Option to compile with default or higher order quadrature for storing pseudopotential data -#ON is recommended for MD simulations with hard pseudopotentials -withHigherQuadPSP=OFF - -# build type: "Release" or "Debug" -build_type=Release - -testing=OFF -minimal_compile=ON -########################################################################### -#Usually, no changes are needed below this line -# - -#if [[ x"$build_type" == x"Release" ]]; then -# c_flags="$c_flagsRelease" -# cxx_flags="$c_flagsRelease" -#else -#fi -out=`echo "$build_type" | tr '[:upper:]' '[:lower:]'` - -function cmake_real() { - mkdir -p real && cd real - if [ "$gpuLang" = "cuda" ]; then - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=OFF -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_CUDA_FLAGS="$device_flags" -DCMAKE_CUDA_ARCHITECTURES="$device_architectures"\ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - elif [ "$gpuLang" = "hip" ]; then - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=OFF -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_HIP_FLAGS="$device_flags" -DCMAKE_HIP_ARCHITECTURES="$device_architectures"\ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - else - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=OFF\ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - fi -} - -function cmake_cplx() { - mkdir -p complex && cd complex - if [ "$gpuLang" = "cuda" ]; then - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=ON -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_CUDA_FLAGS="$device_flags" -DCMAKE_CUDA_ARCHITECTURES="$device_architectures"\ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - elif [ "$gpuLang" = "hip" ]; then - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=ON -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_HIP_FLAGS="$device_flags" -DCMAKE_HIP_ARCHITECTURES="$device_architectures"\ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - else - cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\ - -DCMAKE_CXX_FLAGS="$cxx_flags"\ - -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \ - -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \ - -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \ - -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \ - -DXML_INCLUDE_DIR=$xmlIncludeDir\ - -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \ - -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\ - -DWITH_COMPLEX=ON \ - -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\ - -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 - fi -} - -RCol='\e[0m' -Blu='\e[0;34m'; -if [ -d "$out" ]; then # build directory exists - echo -e "${Blu}$out directory already present${RCol}" -else - rm -rf "$out" - echo -e "${Blu}Creating $out ${RCol}" - mkdir -p "$out" -fi - -cd $out - -echo -e "${Blu}Building Real executable in $build_type mode...${RCol}" -cmake_real "$SRC" && make -j8 -cd .. - -echo -e "${Blu}Building Complex executable in $build_type mode...${RCol}" -cmake_cplx "$SRC" && make -j8 -cd .. - -echo -e "${Blu}Build complete.${RCol}" diff --git a/helpers/XSEDEStampede2KNL/p4est-setup.sh b/helpers/XSEDEStampede2KNL/p4est-setup.sh deleted file mode 100644 index 2c23c569e..000000000 --- a/helpers/XSEDEStampede2KNL/p4est-setup.sh +++ /dev/null @@ -1,134 +0,0 @@ -#! /bin/bash -# -# This file is part of p4est [1]. -# p4est is a C library to manage a collection (a forest) of multiple -# connected adaptive quadtrees or octrees in parallel. -# -# Copyright (C) 2010 The University of Texas System -# Written by Carsten Burstedde, Lucas C. Wilcox, and Tobin Isaac -# Modified 2010 by Wolfgang Bangerth -# Modified 2010 by Timo Heister -# Modified 2013 by Matthias Maier -# Modified 2018 by Sambit Das (to fix OpenMP linking with intel compilers) -# -# p4est is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# [1] http://www.p4est.org -# - -# This program comes with ABSOLUTELY NO WARRANTY. - -# unpack under current directory -UNPACK=`pwd` - -# choose names for fast and debug compilation directories -BUILD_DIR="$UNPACK/p4est-build" -BUILD_FAST="$BUILD_DIR/FAST" -BUILD_DEBUG="$BUILD_DIR/DEBUG" - -function busage() { - echo "Usage: `basename $0` []" - echo " or: `basename $0` /path/to/p4est-src/ []" -} -function bdie () { - echo "Error: $@" - exit 1 -} - -if test -z "$CFLAGS" -a -z "$P4EST_CFLAGS_FAST" ; then - export CFLAGS_FAST="-O2 -qopenmp" -else - export CFLAGS_FAST="$CFLAGS $P4EST_CFLAGS_FAST" -fi -echo "CFLAGS_FAST: $CFLAGS_FAST" -if test -z "$CFLAGS" -a -z "$P4EST_CFLAGS_DEBUG" ; then - export CFLAGS_DEBUG="-O0 -g -qopenmp" -else - export CFLAGS_DEBUG="$CFLAGS $P4EST_CFLAGS_DEBUG" -fi -echo "CFLAGS_DEBUG: $CFLAGS_DEBUG" - -TGZ="$1"; shift -if test -d "$TGZ" ; then - SRCDIR="$TGZ" - echo "using existing source dir '$SRCDIR'" -else - if test ! -f "$TGZ" ; then - busage - bdie "File not found" - fi - if ! (echo "$TGZ" | grep -q 'p4est.*.tar.gz') ; then - busage - bdie "File name mismatch" - fi -fi - -# choose names for fast and debug installation directories -INSTALL_DIR="$1"; shift -if test -z "$INSTALL_DIR" ; then - INSTALL_DIR="$UNPACK/p4est-install" -fi -INSTALL_FAST="$INSTALL_DIR/FAST" -INSTALL_DEBUG="$INSTALL_DIR/DEBUG" - -echo -echo "This script tries to unpack, configure and build the p4est library." -echo "Build FAST: $BUILD_FAST" -echo "Build DEBUG: $BUILD_DEBUG" -echo "Install FAST: $INSTALL_FAST" -echo "Install DEBUG: $INSTALL_DEBUG" -echo "Checking environment: CFLAGS P4EST_CFLAGS_FAST P4EST_CFLAGS_DEBUG" - -# remove old versions -if test -d "$BUILD_DIR" ; then - rm -rf "$BUILD_DIR" -fi - -if test -f "$TGZ" ; then - DIR=`echo "$TGZ" | sed 's/\(p4est.*\).tar.gz/\1/'` - DIR=`basename $DIR` - echo "Unpack directory: $UNPACK/$DIR" - if test -d "$UNPACK/$DIR" ; then - echo "Source directory found (remove it to unpack anew)" - else - echo -n "Unpacking... " - tar -xvz -f "$TGZ" -C "$UNPACK" >/dev/null - echo "done" - fi - SRCDIR=$UNPACK/$DIR -fi -test -f "$SRCDIR/src/p4est.h" || bdie "Main header file missing" -test -f "$SRCDIR/configure" || bdie "Configure script missing" - -echo "See output in files .../config.output and .../make.output" -echo -echo "Build FAST version in $BUILD_FAST" -mkdir -p "$BUILD_FAST" -cd "$BUILD_FAST" -"$SRCDIR/configure" --enable-mpi --enable-shared \ - --disable-vtk-binary --without-blas --enable-openmp=-qopenmp\ - --prefix="$INSTALL_FAST" CFLAGS="$CFLAGS_FAST" \ - CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \ - "$@" > config.output || bdie "Error in configure" -make -C sc -j 8 > make.output || bdie "Error in make sc" -make -j 8 >> make.output || bdie "Error in make p4est" -make install >> make.output || bdie "Error in make install" -echo "FAST version installed in $INSTALL_FAST" - -echo -echo "Build DEBUG version in $BUILD_DEBUG" -mkdir -p "$BUILD_DEBUG" -cd "$BUILD_DEBUG" -"$SRCDIR/configure" --enable-debug --enable-mpi --enable-shared \ - --disable-vtk-binary --without-blas --enable-openmp=-qopenmp\ - --prefix="$INSTALL_DEBUG" CFLAGS="$CFLAGS_DEBUG" \ - CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \ - "$@" > config.output || bdie "Error in configure" -make -C sc -j 8 > make.output || bdie "Error in make sc" -make -j 8 >> make.output || bdie "Error in make p4est" -make install >> make.output || bdie "Error in make install" -echo "DEBUG version installed in $INSTALL_DEBUG" -echo diff --git a/helpers/XSEDEStampede2KNL/setupStampede2.sh b/helpers/XSEDEStampede2KNL/setupStampede2.sh deleted file mode 100755 index 7bb5c6e7a..000000000 --- a/helpers/XSEDEStampede2KNL/setupStampede2.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail -#script to setup and build DFT-FE - -######################################################################## -#Provide paths below for external libraries, compiler options and flags, -# and optimization flag - -#Paths for external libraries -dealiiPetscRealDir="/work/05316/dsambit/publicSharedSoftware/dealiiDev/intel_18.0.0_petscReal_scalapack_64Bit" -dealiiPetscComplexDir="/work/05316/dsambit/publicSharedSoftware/dealiiDev/intel_18.0.0_petscComplex_scalapack_64Bit" -alglibDir="/work/05316/dsambit/publicSharedSoftware/alglib/cppIntel18.0.0/src" -libxcDir="/work/05316/dsambit/publicSharedSoftware/libxc/install_intel18.0.0" -spglibDir="/work/05316/dsambit/publicSharedSoftware/spglib/install_intel18.0.0" -xmlIncludeDir="/usr/include/libxml2" -xmlLibDir="/usr/lib64" - -#If you have installed dealii by linking with intel mkl library set underlying flag to "ON", -#otherwise set it to "OFF" -withIntelMkl=ON - -#Compiler options and flags -c_compiler=mpicc -cxx_compiler=mpicxx -c_flagsRelease=-O2 -cxx_flagsRelease=-O2 - -#Optmization flag: 1 for optimized mode and 0 for debug mode compilation -optimizedMode=1 - -########################################################################### -#Usually, no changes are needed below this line -# -RCol='\e[0m' -Blu='\e[0;34m'; -if [ $optimizedMode == 1 ]; then - if [ -d "build/release" ]; then - echo -e "${Blu}build/release directory already present${RCol}" - # Control will enter here if build directory exists. - cd build - cd release - echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}" - mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd .. - echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}" - mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. - else - rm -rf build/release - echo -e "${Blu}Creating build directory...${RCol}" - mkdir -p build && cd build - mkdir -p release && cd release - echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}" - mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd .. - echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}" - mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. - fi -else - if [ -d "build/debug" ]; then - echo -e "${Blu}build/debug directory already present${RCol}" - # Control will enter here if build directory exists. - cd build - cd debug - echo -e "${Blu}Building Real executable in Debug mode...${RCol}" - mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd .. - echo -e "${Blu}Building Complex executable in Debug mode...${RCol}" - mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. - else - rm -rf build/debug - echo -e "${Blu}Creating build directory...${RCol}" - mkdir -p build && cd build - mkdir -p debug && cd debug - echo -e "${Blu}Building Real executable in Debug mode...${RCol}" - mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd .. - echo -e "${Blu}Building Complex executable in Debug mode...${RCol}" - mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. - fi -fi -echo -e "${Blu}Build complete.${RCol}" diff --git a/helpers/p4est-setup.sh b/helpers/p4est-setup.sh index 9b126f531..bc6ccd1f0 100755 --- a/helpers/p4est-setup.sh +++ b/helpers/p4est-setup.sh @@ -15,7 +15,7 @@ # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # -# [1] http://www.p4est.org +# [1] https://www.p4est.org # # This program comes with ABSOLUTELY NO WARRANTY. @@ -114,7 +114,8 @@ echo echo "Build FAST version in $BUILD_FAST" mkdir -p "$BUILD_FAST" cd "$BUILD_FAST" -"$SRCDIR/configure" -host=x86_64 CC=mpicc CXX=mpicxx FC=mpif90 F77=mpif90 --enable-mpi --enable-shared \ +"$SRCDIR/configure" -host=x86_64 CC=mpicc CXX=mpicxx FC=mpif90 F77=mpif90 \ + --enable-mpi --enable-shared \ --disable-vtk-binary --without-blas \ --prefix="$INSTALL_FAST" CFLAGS="$CFLAGS_FAST" \ CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \ @@ -132,7 +133,8 @@ echo echo "Build DEBUG version in $BUILD_DEBUG" mkdir -p "$BUILD_DEBUG" cd "$BUILD_DEBUG" -"$SRCDIR/configure" -host=x86_64 CC=mpicc CXX=mpicxx FC=mpif90 F77=mpif90 --enable-debug --enable-mpi --enable-shared \ +"$SRCDIR/configure" -host=x86_64 CC=mpicc CXX=mpicxx FC=mpif90 F77=mpif90 \ + --enable-debug --enable-mpi --enable-shared \ --disable-vtk-binary --without-blas \ --prefix="$INSTALL_DEBUG" CFLAGS="$CFLAGS_DEBUG" \ CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \ diff --git a/setupDevelopPetsc.sh b/setupDevelopPetsc.sh index 1da8bf1b5..8db5bf55a 100755 --- a/setupDevelopPetsc.sh +++ b/setupDevelopPetsc.sh @@ -17,8 +17,8 @@ SRC=`dirname $0` # location of source directory # and optimization flag #Paths for required external libraries -dealiiPetscRealDir="/home/vikramg/DFT-softwares-gcc/dealii/install_real_cpu" -dealiiPetscComplexDir="/home/vikramg/DFT-softwares-gcc/dealii/install_complex_cpu" +dealiiPetscRealDir="/home/vikramg/DFT-softwares-gcc/dealii9.5.2/install_real_cpu" +dealiiPetscComplexDir="/home/vikramg/DFT-softwares-gcc/dealii9.5.2/install_complex_cpu" alglibDir="/home/vikramg/DFT-softwares-gcc/alglib/alglib-cpp/src" libxcDir="/home/vikramg/DFT-softwares-gcc/libxc/libxc-5.2.3/install_libxc5.2.3" spglibDir="/home/vikramg/DFT-softwares-gcc/spglib/install" @@ -46,7 +46,7 @@ withGPUAwareMPI=OFF #Please use this option with care withDCCL=OFF withMDI=OFF withTorch=OFF -withCustomizedDealii=ON +withCustomizedDealii=OFF #Compiler options and flags cxx_compiler=/sw/pkgs/arc/stacks/gcc/10.3.0/openmpi/4.1.6/bin/mpicxx #sets DCMAKE_CXX_COMPILER diff --git a/setupUser.sh b/setupUser.sh index b5528278c..7f5079022 100755 --- a/setupUser.sh +++ b/setupUser.sh @@ -47,7 +47,7 @@ withGPUAwareMPI=OFF #Please use this option with care withDCCL=ON withMDI=OFF withTorch=OFF -withCustomizedDealii=ON +withCustomizedDealii=OFF #Compiler options and flags cxx_compiler=mpic++ #sets DCMAKE_CXX_COMPILER diff --git a/setupUserPetsc.sh b/setupUserPetsc.sh index 9dbc9166d..f95d6e3f5 100755 --- a/setupUserPetsc.sh +++ b/setupUserPetsc.sh @@ -46,7 +46,7 @@ withGPUAwareMPI=OFF #Please use this option with care withDCCL=OFF withMDI=OFF withTorch=OFF -withCustomizedDealii=ON +withCustomizedDealii=OFF #Compiler options and flags cxx_compiler=mpic++ #sets DCMAKE_CXX_COMPILER diff --git a/src/dft/initBoundaryConditions.cc b/src/dft/initBoundaryConditions.cc index 0513b6ca4..461d36744 100644 --- a/src/dft/initBoundaryConditions.cc +++ b/src/dft/initBoundaryConditions.cc @@ -182,6 +182,7 @@ namespace dftfe // push back into Constraint Matrices d_constraintsVector.push_back(&constraintsNone); +#ifdef DFTFE_WITH_CUSTOMIZED_DEALII if (d_dftParamsPtr->constraintsParallelCheck) { dealii::IndexSet locally_active_dofs_debug; @@ -199,7 +200,7 @@ namespace dftfe dealii::ExcMessage( "DFT-FE Error: Constraints are not consistent in parallel.")); } - +#endif // // create matrix free structure // diff --git a/src/dft/pRefinedDoFHandler.cc b/src/dft/pRefinedDoFHandler.cc index df67491cf..78ac7f984 100644 --- a/src/dft/pRefinedDoFHandler.cc +++ b/src/dft/pRefinedDoFHandler.cc @@ -336,7 +336,7 @@ namespace dftfe d_constraintsVectorElectro.push_back(&d_constraintsForPhiPrimeElectro); d_phiPrimeDofHandlerIndexElectro = d_constraintsVectorElectro.size() - 1; - +#ifdef DFTFE_WITH_CUSTOMIZED_DEALII if (d_dftParamsPtr->constraintsParallelCheck) { dealii::IndexSet locally_active_dofs_debug; @@ -355,6 +355,7 @@ namespace dftfe dealii::ExcMessage( "DFT-FE Error: Constraints are not consistent in parallel.")); } +#endif // Fill dofHandler vector std::vector *> matrixFreeDofHandlerVectorInput; diff --git a/src/dftfeWrapper.cc b/src/dftfeWrapper.cc index 74c6fcaab..8fdee2f6a 100644 --- a/src/dftfeWrapper.cc +++ b/src/dftfeWrapper.cc @@ -21,6 +21,7 @@ // deal.II header // #include +#include #include #ifdef USE_PETSC diff --git a/src/poisson/poissonSolverProblem.cc b/src/poisson/poissonSolverProblem.cc index 1bde4c148..16fed24ef 100644 --- a/src/poisson/poissonSolverProblem.cc +++ b/src/poisson/poissonSolverProblem.cc @@ -333,7 +333,16 @@ namespace dftfe fe_eval_sc.distribute_local_to_global(rhs); if (d_isStoreSmearedChargeRhs) - fe_eval_sc.distribute_local_to_global(d_rhsSmearedCharge); + { + fe_eval_sc.reinit(macrocell); + for (unsigned int q = 0; q < fe_eval_sc.n_q_points; ++q) + { + fe_eval_sc.submit_value(smearedbQuads[q], q); + } + fe_eval_sc.integrate(true, false); + + fe_eval_sc.distribute_local_to_global(d_rhsSmearedCharge); + } } } } diff --git a/src/poisson/poissonSolverProblemDevice.cc b/src/poisson/poissonSolverProblemDevice.cc index 11ce651e6..bbcc11a7c 100644 --- a/src/poisson/poissonSolverProblemDevice.cc +++ b/src/poisson/poissonSolverProblemDevice.cc @@ -372,7 +372,16 @@ namespace dftfe fe_eval_sc.distribute_local_to_global(rhs); if (d_isStoreSmearedChargeRhs) - fe_eval_sc.distribute_local_to_global(d_rhsSmearedCharge); + { + fe_eval_sc.reinit(macrocell); + for (unsigned int q = 0; q < fe_eval_sc.n_q_points; ++q) + { + fe_eval_sc.submit_value(smearedbQuads[q], q); + } + fe_eval_sc.integrate(true, false); + + fe_eval_sc.distribute_local_to_global(d_rhsSmearedCharge); + } } } } diff --git a/src/triangulation/triangulationManager/generateMesh.cc b/src/triangulation/triangulationManager/generateMesh.cc index 628157786..2023b6f59 100644 --- a/src/triangulation/triangulationManager/generateMesh.cc +++ b/src/triangulation/triangulationManager/generateMesh.cc @@ -1400,11 +1400,12 @@ namespace dftfe pcout << "Hanging node and periodic constraints parallel consistency not achieved." << std::endl; - +#ifdef DFTFE_WITH_CUSTOMIZED_DEALII AssertThrow( d_dftParams.createConstraintsFromSerialDofhandler, dealii::ExcMessage( "DFT-FE error: this is due to a known issue related to hanging node constraints in dealii. Please set CONSTRAINTS FROM SERIAL DOFHANDLER = true under the Boundary conditions subsection in the input parameters file to circumvent this issue.")); +#endif } } diff --git a/testsGPU/pseudopotential/complex/jobscripts/perlmutter.slurm b/testsGPU/pseudopotential/complex/jobscripts/perlmutter.slurm index 537065377..b3351ccf4 100644 --- a/testsGPU/pseudopotential/complex/jobscripts/perlmutter.slurm +++ b/testsGPU/pseudopotential/complex/jobscripts/perlmutter.slurm @@ -16,7 +16,7 @@ OMP_NUM_THREADS=1 MPICH_GPU_SUPPORT_ENABLED=1 LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib64 -module load nccl/2.15.5-ofi +module load nccl srun ./dftfe parameterFileMg2x_1.prm > outputMg2x_1 @@ -35,5 +35,4 @@ srun ./dftfe parameterFileMg2x_7.prm > outputMg2x_7 srun ./dftfe parameterFileMg2x_8.prm > outputMg2x_8 -srun -n 6 -c 1 ./dftfe parameterFileBe.prm > outputBe - +srun ./dftfe parameterFileBe.prm > outputBe diff --git a/testsGPU/pseudopotential/real/jobscripts/perlmutter18Tasks18GPUs.slurm b/testsGPU/pseudopotential/real/jobscripts/perlmutter18Tasks18GPUs.slurm index 5c14c45ad..7e5cf206c 100644 --- a/testsGPU/pseudopotential/real/jobscripts/perlmutter18Tasks18GPUs.slurm +++ b/testsGPU/pseudopotential/real/jobscripts/perlmutter18Tasks18GPUs.slurm @@ -15,7 +15,7 @@ OMP_NUM_THREADS=1 MPICH_GPU_SUPPORT_ENABLED=1 LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib64 -module load nccl/2.15.5-ofi +module load nccl srun ./dftfe Input_MD_0.prm > output_MD_0 srun ./dftfe Input_MD_1.prm > output_MD_1 diff --git a/testsGPU/pseudopotential/real/jobscripts/perlmutter6Tasks6GPUs.slurm b/testsGPU/pseudopotential/real/jobscripts/perlmutter6Tasks6GPUs.slurm index fc7b39268..d2b72d064 100644 --- a/testsGPU/pseudopotential/real/jobscripts/perlmutter6Tasks6GPUs.slurm +++ b/testsGPU/pseudopotential/real/jobscripts/perlmutter6Tasks6GPUs.slurm @@ -15,7 +15,7 @@ OMP_NUM_THREADS=1 MPICH_GPU_SUPPORT_ENABLED=1 LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib64 -module load nccl/2.15.5-ofi +module load nccl srun ./dftfe parameterFileN2_1.prm > outputN2_1 srun ./dftfe parameterFileN2_2.prm > outputN2_2 diff --git a/utils/dftParameters.cc b/utils/dftParameters.cc index a6fa9d58b..5b012fd32 100644 --- a/utils/dftParameters.cc +++ b/utils/dftParameters.cc @@ -1066,7 +1066,7 @@ namespace dftfe "[Advanced] Toggle GPU MODE in Poisson solve."); prm.declare_entry("VSELF GPU MODE", - "false", + "true", dealii::Patterns::Bool(), "[Advanced] Toggle GPU MODE in vself Poisson solve."); } @@ -1177,6 +1177,7 @@ namespace dftfe numSCFIterations = 1; maxLinearSolverIterations = 1; poissonGPU = true; + vselfGPU = true; mixingHistory = 1; npool = 1; maxLinearSolverIterationsHelmholtz = 1;