From 2d8e20e9743bc1b8cc222d5755d555e093c4fbdf Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Mon, 22 Apr 2024 10:07:28 -0400
Subject: [PATCH 01/14] compilation with dealii9.5.2 successful

---
 CMakeLists.txt              | 6 +++---
 doc/manual/installation.tex | 5 +----
 setupDevelopPetsc.sh        | 6 +++---
 setupUser.sh                | 2 +-
 src/dft/dft.cc              | 3 ++-
 src/dftfeWrapper.cc         | 1 +
 6 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 36a99ae47..269d4e536 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -385,11 +385,11 @@ ENDIF()
 #DEAL_II_INITIALIZE_CACHED_VARIABLES()
 
 
-TARGET_INCLUDE_DIRECTORIES(${TARGETLIB} PUBLIC ${DEAL_II_INCLUDE_DIRS})
+TARGET_INCLUDE_DIRECTORIES(${TARGETLIB} PUBLIC ${DEAL_II_INCLUDE_RELDIR})
 IF("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
-    TARGET_LINK_LIBRARIES (${TARGETLIB} PUBLIC ${DEAL_II_LIBRARIES_RELEASE})
+    TARGET_LINK_LIBRARIES (${TARGETLIB} PUBLIC dealii::dealii_release)
 ELSE()
-   TARGET_LINK_LIBRARIES (${TARGETLIB} PUBLIC ${DEAL_II_LIBRARIES_DEBUG})
+   TARGET_LINK_LIBRARIES (${TARGETLIB} PUBLIC dealii::dealii_debug)
 ENDIF()
 
 IF (WITH_CUSTOMIZED_DEALII)
diff --git a/doc/manual/installation.tex b/doc/manual/installation.tex
index 4c1fc0cbc..0323dda15 100644
--- a/doc/manual/installation.tex
+++ b/doc/manual/installation.tex
@@ -153,10 +153,7 @@ \subsubsection{Instructions for deal.II}
 
 \begin{enumerate}
 
-\item Obtain the customized version of deal.II library via
-\begin{verbatim}
-$ git clone -b dealiiCustomizedCUDARelease https://github.com/dftfeDevelopers/dealii.git
-\end{verbatim}
+\item Obtain the latest release version (9.5.2) of deal.II
 
 \item In addition to requiring C, C++ and Fortran compilers, MPI library, and CMake, deal.II additionally requires BOOST library. If not found externally, cmake will resort to the bundled BOOST that comes along with deal.II. Based on our experience, we recommend to use the deal.II's bundled boost (enforced by unsetting/unloading external BOOST library environment paths) to avoid compilation issues. Please do not install deal.II with GPU support as that is not needed by DFT-FE. Further, when installing deal.II for (DFT-FE with GPU support), one must use \verb|-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF| to avoid compilation failure of DFT-FE.
 
diff --git a/setupDevelopPetsc.sh b/setupDevelopPetsc.sh
index 1da8bf1b5..8db5bf55a 100755
--- a/setupDevelopPetsc.sh
+++ b/setupDevelopPetsc.sh
@@ -17,8 +17,8 @@ SRC=`dirname $0` # location of source directory
 # and optimization flag
 
 #Paths for required external libraries
-dealiiPetscRealDir="/home/vikramg/DFT-softwares-gcc/dealii/install_real_cpu"
-dealiiPetscComplexDir="/home/vikramg/DFT-softwares-gcc/dealii/install_complex_cpu"
+dealiiPetscRealDir="/home/vikramg/DFT-softwares-gcc/dealii9.5.2/install_real_cpu"
+dealiiPetscComplexDir="/home/vikramg/DFT-softwares-gcc/dealii9.5.2/install_complex_cpu"
 alglibDir="/home/vikramg/DFT-softwares-gcc/alglib/alglib-cpp/src"
 libxcDir="/home/vikramg/DFT-softwares-gcc/libxc/libxc-5.2.3/install_libxc5.2.3"
 spglibDir="/home/vikramg/DFT-softwares-gcc/spglib/install"
@@ -46,7 +46,7 @@ withGPUAwareMPI=OFF #Please use this option with care
 withDCCL=OFF
 withMDI=OFF
 withTorch=OFF
-withCustomizedDealii=ON
+withCustomizedDealii=OFF
 
 #Compiler options and flags
 cxx_compiler=/sw/pkgs/arc/stacks/gcc/10.3.0/openmpi/4.1.6/bin/mpicxx  #sets DCMAKE_CXX_COMPILER
diff --git a/setupUser.sh b/setupUser.sh
index b5528278c..7f5079022 100755
--- a/setupUser.sh
+++ b/setupUser.sh
@@ -47,7 +47,7 @@ withGPUAwareMPI=OFF #Please use this option with care
 withDCCL=ON
 withMDI=OFF
 withTorch=OFF
-withCustomizedDealii=ON
+withCustomizedDealii=OFF
 
 #Compiler options and flags
 cxx_compiler=mpic++  #sets DCMAKE_CXX_COMPILER
diff --git a/src/dft/dft.cc b/src/dft/dft.cc
index 2804b55b6..f3a39643e 100644
--- a/src/dft/dft.cc
+++ b/src/dft/dft.cc
@@ -4429,7 +4429,8 @@ namespace dftfe
             unsigned int              FEOrderElectro,
             dftfe::utils::MemorySpace memorySpace>
   const std::vector<std::vector<double>> &
-  dftClass<FEOrder, FEOrderElectro, memorySpace>::getImageAtomLocationsCart() const
+  dftClass<FEOrder, FEOrderElectro, memorySpace>::getImageAtomLocationsCart()
+    const
   {
     return d_imagePositionsTrunc;
   }
diff --git a/src/dftfeWrapper.cc b/src/dftfeWrapper.cc
index d75d028b8..7a2e2bcd0 100644
--- a/src/dftfeWrapper.cc
+++ b/src/dftfeWrapper.cc
@@ -21,6 +21,7 @@
 // deal.II header
 //
 #include <deal.II/base/data_out_base.h>
+#include <deal.II/base/multithread_info.h>
 #include <p4est_bits.h>
 
 #ifdef USE_PETSC

From 9edcae13a481e0a59ff0aba13f0195b0c7bbb438 Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Mon, 22 Apr 2024 20:14:42 -0400
Subject: [PATCH 02/14] cleanups

---
 CMakeLists.txt                              |   2 +-
 doc/manual/installation.tex                 |   2 +-
 helpers/Crusher/setupUserCrusher.sh         | 188 --------------------
 helpers/NERSCCoriKNL/READMENerscCori        |  10 --
 helpers/NERSCCoriKNL/p4est-setup.sh         | 147 ---------------
 helpers/NERSCCoriKNL/sampleJobScript.slurm  |  29 ---
 helpers/NERSCCoriKNL/setupNerscCoriKnl.sh   |  83 ---------
 helpers/Summit/setupPSP.sh                  | 184 -------------------
 helpers/XSEDEStampede2KNL/p4est-setup.sh    | 134 --------------
 helpers/XSEDEStampede2KNL/setupStampede2.sh |  78 --------
 helpers/p4est-setup.sh                      |   8 +-
 11 files changed, 7 insertions(+), 858 deletions(-)
 delete mode 100755 helpers/Crusher/setupUserCrusher.sh
 delete mode 100644 helpers/NERSCCoriKNL/READMENerscCori
 delete mode 100644 helpers/NERSCCoriKNL/p4est-setup.sh
 delete mode 100644 helpers/NERSCCoriKNL/sampleJobScript.slurm
 delete mode 100755 helpers/NERSCCoriKNL/setupNerscCoriKnl.sh
 delete mode 100755 helpers/Summit/setupPSP.sh
 delete mode 100644 helpers/XSEDEStampede2KNL/p4est-setup.sh
 delete mode 100755 helpers/XSEDEStampede2KNL/setupStampede2.sh

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 269d4e536..e9de62391 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -347,7 +347,7 @@ TARGET_LINK_LIBRARIES(${TARGETLIB} PUBLIC ${DFTFE_MPI_LIBRARIES})
 #Dependencies
 #
 
-FIND_PACKAGE(deal.II 9.1.0 REQUIRED
+FIND_PACKAGE(deal.II 9.5.2 REQUIRED
   HINTS ${deal.II_DIR} ${DEAL_II_DIR} ../ ../../ $ENV{DEAL_II_DIR}
   )
 IF(NOT ${deal.II_FOUND})
diff --git a/doc/manual/installation.tex b/doc/manual/installation.tex
index 0323dda15..341ee4358 100644
--- a/doc/manual/installation.tex
+++ b/doc/manual/installation.tex
@@ -135,7 +135,7 @@ \subsubsection{Instructions for dependencies: ALGLIB, Libxc, spglib, Libxml2, Sc
 $LD_LIBRARY_PATH:$ROCM_PATH/lib:$ROCM_PATH/hip/lib:$ROCM_PATH/lib:$ROCM_PATH/rocblas/lib
 \end{verbatim}
 
-\item   {\bf p4est}: This library is used by deal.II to create and distribute finite-element meshes across multiple processors. Download the v2.2 tarball of p4est from \url{https://github.com/p4est/p4est.github.io/raw/master/release/p4est-2.2.tar.gz}. Next download the \verb|p4est-setup.sh| script from \url{https://raw.githubusercontent.com/dftfeDevelopers/dftfe/manual/p4est-setup.sh}. Use the script to automatically compile and install a debug and optimized version of p4est by doing
+\item   {\bf p4est}: This library is used by deal.II to create and distribute finite-element meshes across multiple processors. Download the latest release tarball of p4est from \url{https://www.p4est.org}. Next download the \verb|p4est-setup.sh| script from \url{https://raw.githubusercontent.com/dftfeDevelopers/dftfe/manual/p4est-setup.sh}. Use the script to automatically compile and install a debug and optimized version of p4est by doing
 \begin{verbatim}
 $ chmod u+x p4est-setup.sh
 $ ./p4est-setup.sh p4est-x-y-z.tar.gz p4est_install_dir_path
diff --git a/helpers/Crusher/setupUserCrusher.sh b/helpers/Crusher/setupUserCrusher.sh
deleted file mode 100755
index 27f1af010..000000000
--- a/helpers/Crusher/setupUserCrusher.sh
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/bin/bash
-# script to setup and build DFT-FE.
-
-set -e
-set -o pipefail
-
-if [ -s CMakeLists.txt ]; then
-    echo "This script must be run from the build directory!"
-    exit 1
-fi
-
-# Path to project source
-SRC=`dirname $0` # location of source directory
-
-########################################################################
-#Provide paths below for external libraries, compiler options and flags,
-# and optimization flag
-
-#Paths for required external libraries
-dealiiDir="/ccs/proj/mat187/dsambit/softwareDFTFEGccMpichLatest/dealii/installopenblas"
-alglibDir="/ccs/proj/mat187/dsambit/softwareDFTFEGcc/alglib/cpp/src"
-libxcDir="/ccs/proj/mat187/dsambit/softwareDFTFEGcc/libxc/install"
-spglibDir="/ccs/proj/mat187/dsambit/softwareDFTFEGcc/spglib/install"
-xmlIncludeDir="/usr/include/libxml2"
-xmlLibDir="/usr/lib64"
-ELPA_PATH="/ccs/proj/mat187/dsambit/softwareDFTFEGccMpichLatest/elpa/installgpu"
-
-#Paths for optional external libraries
-# path for NCCL/RCCL libraries
-DCCL_PATH="/ccs/proj/mat187/dsambit/softwareDFTFEGccMpichLatest/rccl/build" 
-mdiPath=""
-
-#Toggle GPU compilation
-withGPU=ON
-gpuLang="hip"     # Use "cuda"/"hip"
-gpuVendor="amd" # Use "nvidia/amd"
-withGPUAwareMPI=ON #Please use this option with care
-                   #Only use if the machine supports 
-                   #device aware MPI and is profiled
-                   #to be fast
-
-#Option to link to DCCL library (Only for GPU compilation)
-withDCCL=ON
-withMDI=OFF
-
-#Compiler options and flags
-cxx_compiler=CC  #sets DCMAKE_CXX_COMPILER
-cxx_flags="-march=znver3 -fPIC -I${MPICH_DIR}/include -I${ROCM_PATH}/include" #sets DCMAKE_CXX_FLAGS
-cxx_flagsRelease="-O2" #sets DCMAKE_CXX_FLAGS_RELEASE
-device_flags="-march=znver3 -O2 -munsafe-fp-atomics -I${MPICH_DIR}/include -I${ROCM_PATH}/include"
-                           #setDCMAKE_CXX_CUDA/HIP_FLAGS 
-                           #(only applicable for withGPU=ON)
-device_architectures="gfx90a" # set DCMAKE_CXX_CUDA/HIP_ARCHITECTURES 
-                           #(only applicable for withGPU=ON)
-
-
-#Option to compile with default or higher order quadrature for storing pseudopotential data
-#ON is recommended for MD simulations with hard pseudopotentials
-withHigherQuadPSP=OFF
-
-# build type: "Release" or "Debug"
-build_type=Release
-
-testing=OFF
-minimal_compile=ON
-###########################################################################
-#Usually, no changes are needed below this line
-#
-
-#if [[ x"$build_type" == x"Release" ]]; then
-#  c_flags="$c_flagsRelease"
-#  cxx_flags="$c_flagsRelease"
-#else
-#fi
-out=`echo "$build_type" | tr '[:upper:]' '[:lower:]'`
-
-function cmake_real() {
-  mkdir -p real && cd real
-  if [ "$gpuLang" = "cuda" ]; then
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=OFF -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_CUDA_FLAGS="$device_flags" -DCMAKE_CUDA_ARCHITECTURES="$device_architectures"\
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1
-  elif [ "$gpuLang" = "hip" ]; then
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=OFF -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_HIP_FLAGS="$device_flags" -DCMAKE_HIP_ARCHITECTURES="$device_architectures"\
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DCMAKE_SHARED_LINKER_FLAGS="-L${ROCM_PATH}/lib -lamdhip64 -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa"\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1  
-  else
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=OFF\
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1    
-  fi  
-}
-
-function cmake_cplx() {
-  mkdir -p complex && cd complex
-  if [ "$gpuLang" = "cuda" ]; then
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=ON -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_CUDA_FLAGS="$device_flags" -DCMAKE_CUDA_ARCHITECTURES="$device_architectures"\
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1
-  elif [ "$gpuLang" = "hip" ]; then
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=ON -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_HIP_FLAGS="$device_flags" -DCMAKE_HIP_ARCHITECTURES="$device_architectures"\
-    -DCMAKE_SHARED_LINKER_FLAGS="-L${ROCM_PATH}/lib -lamdhip64 -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa"\
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1
-  else
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=ON \
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1    
-  fi
-
-}
-
-RCol='\e[0m'
-Blu='\e[0;34m';
-if [ -d "$out" ]; then # build directory exists
-    echo -e "${Blu}$out directory already present${RCol}"
-else
-    rm -rf "$out"
-    echo -e "${Blu}Creating $out ${RCol}"
-    mkdir -p "$out"
-fi
-
-cd $out
-
-echo -e "${Blu}Building Real executable in $build_type mode...${RCol}"
-cmake_real "$SRC" && make -j8
-cd ..
-
-echo -e "${Blu}Building Complex executable in $build_type mode...${RCol}"
-cmake_cplx "$SRC" && make -j8
-cd ..
-
-echo -e "${Blu}Build complete.${RCol}"
diff --git a/helpers/NERSCCoriKNL/READMENerscCori b/helpers/NERSCCoriKNL/READMENerscCori
deleted file mode 100644
index 3ec2b97a6..000000000
--- a/helpers/NERSCCoriKNL/READMENerscCori
+++ /dev/null
@@ -1,10 +0,0 @@
-NERSC Cori is a Cray sytem.
-Before compiling dftfe or the required libraries do:
-
-export XTPE_LINK_TYPE=dynamic
-export CRAYPE_LINK_TYPE=dynamic
-module swap craype-haswell craype-mic-knl
-module load craype-hugepages2M
-module load intel/18.0.1.163
-
-Recommeneded to put the above in your bash script
diff --git a/helpers/NERSCCoriKNL/p4est-setup.sh b/helpers/NERSCCoriKNL/p4est-setup.sh
deleted file mode 100644
index 3ccc108a3..000000000
--- a/helpers/NERSCCoriKNL/p4est-setup.sh
+++ /dev/null
@@ -1,147 +0,0 @@
-#! /bin/bash
-#
-# This file is part of p4est [1].
-# p4est is a C library to manage a collection (a forest) of multiple
-# connected adaptive quadtrees or octrees in parallel.
-#
-# Copyright (C) 2010 The University of Texas System
-# Written by Carsten Burstedde, Lucas C. Wilcox, and Tobin Isaac
-# Modified 2010 by Wolfgang Bangerth
-# Modified 2010 by Timo Heister
-# Modified 2013 by Matthias Maier
-# Modified 2018 by Sambit Das (to fix OpenMP linking with intel compilers)
-#
-# p4est is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# [1] http://www.p4est.org
-#
-
-# This program comes with ABSOLUTELY NO WARRANTY.
-
-# error message when zlib is not found
-MISSING_ZLIB_MESSAGE="deal.II requires that p4est be built with zlib support. Please \
-either ensure that zlib is installed in a standard location or add appropriate \
-flags to LDFLAGS and CPPFLAGS to both calls to configure to describe where zlib's \
-shared object files and headers are (e.g., LDFLAGS=\"-L/path/to/shared-objects/\" \
-and CPPFLAGS=\"-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL -I/path/to/headers/\")."
-
-# unpack under current directory
-UNPACK=`pwd`
-
-# choose names for fast and debug compilation directories
-BUILD_DIR="$UNPACK/p4est-build"
-BUILD_FAST="$BUILD_DIR/FAST"
-BUILD_DEBUG="$BUILD_DIR/DEBUG"
-
-function busage() {
-        echo "Usage: `basename $0` <p4est_tar.gz_file> [<install location>]"
-        echo "   or: `basename $0` /path/to/p4est-src/ [<install location>]"
-}
-function bdie () {
-        echo "Error: $@"
-        exit 1
-}
-
-if test -z "$CFLAGS" -a -z "$P4EST_CFLAGS_FAST" ; then
-        export CFLAGS_FAST="-O2 -qopenmp"
-else
-        export CFLAGS_FAST="$CFLAGS $P4EST_CFLAGS_FAST"
-fi
-echo "CFLAGS_FAST: $CFLAGS_FAST"
-if test -z "$CFLAGS" -a -z "$P4EST_CFLAGS_DEBUG" ; then
-        export CFLAGS_DEBUG="-O0 -g -qopenmp"
-else
-        export CFLAGS_DEBUG="$CFLAGS $P4EST_CFLAGS_DEBUG"
-fi
-echo "CFLAGS_DEBUG: $CFLAGS_DEBUG"
-
-TGZ="$1"; shift
-if test -d "$TGZ" ; then
-  SRCDIR="$TGZ"  
-  echo "using existing source dir '$SRCDIR'"
-else
-    if test ! -f "$TGZ" ; then
-        busage
-        bdie "File not found"
-    fi
-    if ! (echo "$TGZ" | grep -q 'p4est.*.tar.gz') ; then
-        busage
-        bdie "File name mismatch"
-    fi
-fi
-
-# choose names for fast and debug installation directories
-INSTALL_DIR="$1"; shift
-if test -z "$INSTALL_DIR" ; then
-        INSTALL_DIR="$UNPACK/p4est-install"
-fi
-INSTALL_FAST="$INSTALL_DIR/FAST"
-INSTALL_DEBUG="$INSTALL_DIR/DEBUG"
-
-echo
-echo "This script tries to unpack, configure and build the p4est library."
-echo "Build FAST: $BUILD_FAST"
-echo "Build DEBUG: $BUILD_DEBUG"
-echo "Install FAST: $INSTALL_FAST"
-echo "Install DEBUG: $INSTALL_DEBUG"
-echo "Checking environment: CFLAGS P4EST_CFLAGS_FAST P4EST_CFLAGS_DEBUG"
-
-# remove old versions
-if test -d "$BUILD_DIR" ; then
-        rm -rf "$BUILD_DIR"
-fi
-
-if test -f "$TGZ" ; then
-    DIR=`echo "$TGZ" | sed 's/\(p4est.*\).tar.gz/\1/'`
-    DIR=`basename $DIR`
-    echo "Unpack directory: $UNPACK/$DIR"
-    if test -d "$UNPACK/$DIR" ; then
-        echo "Source directory found (remove it to unpack anew)"
-    else
-        echo -n "Unpacking... "
-        tar -xvz -f "$TGZ" -C "$UNPACK" >/dev/null
-        echo "done"
-    fi
-    SRCDIR=$UNPACK/$DIR
-fi
-test -f "$SRCDIR/src/p4est.h" || bdie "Main header file missing"
-test -f "$SRCDIR/configure" || bdie "Configure script missing"
-
-echo "See output in files .../config.output and .../make.output"
-echo
-echo "Build FAST version in $BUILD_FAST"
-mkdir -p "$BUILD_FAST"
-cd "$BUILD_FAST"
-"$SRCDIR/configure" CC=cc CXX=CC FC=ftn F77=ftn --enable-mpi --enable-shared\
-        --disable-vtk-binary --without-blas --enable-openmp=-qopenmp \
-        --prefix="$INSTALL_FAST" CFLAGS="$CFLAGS_FAST" \
-        CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \
-        "$@" > config.output || bdie "Error in configure"
-make -C sc -j 8 > make.output || bdie "Error in make sc"
-make -j 8 >> make.output || bdie "Error in make p4est"
-# ensure that we built p4est with zlib
-grep -q 'P4EST_HAVE_ZLIB *1' "$BUILD_FAST/src/p4est_config.h" \
-    || bdie "$MISSING_ZLIB_MESSAGE"
-make install >> make.output || bdie "Error in make install"
-echo "FAST version installed in $INSTALL_FAST"
-
-echo
-echo "Build DEBUG version in $BUILD_DEBUG"
-mkdir -p "$BUILD_DEBUG"
-cd "$BUILD_DEBUG"
-"$SRCDIR/configure" CC=cc CXX=CC FC=ftn F77=ftn --enable-debug --enable-mpi --enable-shared\
-        --disable-vtk-binary --without-blas --enable-openmp=-qopenmp \
-        --prefix="$INSTALL_DEBUG" CFLAGS="$CFLAGS_DEBUG" \
-        CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \
-        "$@" > config.output || bdie "Error in configure"
-make -C sc -j 8 > make.output || bdie "Error in make sc"
-make -j 8 >> make.output || bdie "Error in make p4est"
-# ensure that we built p4est with zlib
-grep -q 'P4EST_HAVE_ZLIB *1' "$BUILD_DEBUG/src/p4est_config.h" \
-    || bdie "$MISSING_ZLIB_MESSAGE"
-make install >> make.output || bdie "Error in make install"
-echo "DEBUG version installed in $INSTALL_DEBUG"
-echo
diff --git a/helpers/NERSCCoriKNL/sampleJobScript.slurm b/helpers/NERSCCoriKNL/sampleJobScript.slurm
deleted file mode 100644
index 7c11a2d56..000000000
--- a/helpers/NERSCCoriKNL/sampleJobScript.slurm
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash -l
-
-#SBATCH -q debug
-#SBATCH -N 512
-#SBATCH -C knl
-#SBATCH -S 4
-#SBATCH --switches=2@20:00:00
-#SBATCH -t 00:25:00
-#SBATCH -A m2360
-#SBATCH --mail-user=user@umich.edu
-#SBATCH --mail-type=begin
-#SBATCH --mail-type=end
-#SBATCH -J 6x6x6
-#SBATCH -o 6x6x6.o%j
-
-# setup and run
-echo "Job $SLURM_JOB_NAME, ID $SLURM_JOB_ID"
-cd $SLURM_SUBMIT_DIR
-
-module load craype-hugepages2M
-
-export OMP_NUM_THREADS=2
-export OMP_PLACES=threads
-export OMP_PROC_BIND=spread
-export PMI_MMAP_SYNC_WAIT_TIME=600
-
-sbcast --compress=lz4 /global/homes/d/user/dftfeProduction/build/release/real/main /tmp/main
-
-srun -n 16384 -c 8 --cpu_bind=cores /tmp/main parameterFile.prm > output
diff --git a/helpers/NERSCCoriKNL/setupNerscCoriKnl.sh b/helpers/NERSCCoriKNL/setupNerscCoriKnl.sh
deleted file mode 100755
index fa0e663ad..000000000
--- a/helpers/NERSCCoriKNL/setupNerscCoriKnl.sh
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/bin/bash
-set -e
-set -o pipefail
-#script to setup and build DFT-FE
-
-########################################################################
-#Provide paths below for external libraries, compiler options and flags,
-# and optimization flag
-
-#Paths for external libraries
-dealiiPetscRealDir="/project/projectdirs/m2360/softwaresDFTFE/dealiiKNL/intel18_petscReal_64Bit_scalapackMpichHugepages"
-dealiiPetscComplexDir="/project/projectdirs/m2360/softwaresDFTFE/dealiiKNL/intel18_petscComplex_64Bit_scalapack"
-alglibDir="/global/project/projectdirs/m2360/softwaresDFTFE/alglib/cpp/src"
-libxcDir="/project/projectdirs/m2360/softwaresDFTFE/libxc/install_libxc2.2.0_intel18"
-spglibDir="/project/projectdirs/m2360/softwaresDFTFE/spglib"
-xmlIncludeDir="/usr/include/libxml2/libxml"
-xmlLibDir="/usr/lib64"
-elpaIncludeDir="/global/project/projectdirs/m2360/softwaresDFTFE/elpa/install/include/elpa_openmp-2018.05.001"
-elpaLibDir="/global/project/projectdirs/m2360/softwaresDFTFE/elpa/install/lib"
-
-#If you have installed dealii by linking with intel mkl library set underlying flag to "ON",
-#otherwise set it to "OFF"
-withIntelMkl=ON
-
-#Compiler options and flags
-c_compiler=cc
-cxx_compiler=CC
-c_flagsRelease=-O2
-cxx_flagsRelease=-O2
-
-#Option to link to ELPA
-withELPA=ON
-
-#Optmization flag: 1 for optimized mode and 0 for debug mode compilation
-optimizedMode=1
-
-###########################################################################
-#Usually, no changes are needed below this line
-#
-RCol='\e[0m'
-Blu='\e[0;34m';
-if [ $optimizedMode == 1 ]; then
-  if [ -d "build/release" ]; then
-    echo -e "${Blu}build/release directory already present${RCol}"
-    # Control will enter here if build directory exists.
-    cd build
-    cd release
-    echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}"
-    mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
-    echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}"
-    mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
-  else
-    rm -rf build/release
-    echo -e "${Blu}Creating build directory...${RCol}"
-    mkdir -p build && cd build
-    mkdir -p release && cd release
-    echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}"
-    mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
-    echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}"
-    mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl  ../../../. && make -j 4 && cd ../..
-  fi
-else
-  if [ -d "build/debug" ]; then
-    echo -e "${Blu}build/debug directory already present${RCol}"
-    # Control will enter here if build directory exists.
-    cd build
-    cd debug
-    echo -e "${Blu}Building Real executable in Debug mode...${RCol}"
-    mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
-    echo -e "${Blu}Building Complex executable in Debug mode...${RCol}"
-    mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
-  else
-    rm -rf build/debug
-    echo -e "${Blu}Creating build directory...${RCol}"
-    mkdir -p build && cd build
-    mkdir -p debug && cd debug
-    echo -e "${Blu}Building Real executable in Debug mode...${RCol}"
-    mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
-    echo -e "${Blu}Building Complex executable in Debug mode...${RCol}"
-    mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
-  fi
-fi
-echo -e "${Blu}Build complete.${RCol}"
diff --git a/helpers/Summit/setupPSP.sh b/helpers/Summit/setupPSP.sh
deleted file mode 100755
index 2e46163d7..000000000
--- a/helpers/Summit/setupPSP.sh
+++ /dev/null
@@ -1,184 +0,0 @@
-#!/bin/bash
-# script to setup and build DFT-FE.
-
-set -e
-set -o pipefail
-
-if [ -s CMakeLists.txt ]; then
-    echo "This script must be run from the build directory!"
-    exit 1
-fi
-
-# Path to project source
-SRC=`dirname $0` # location of source directory
-
-########################################################################
-#Provide paths below for external libraries, compiler options and flags,
-# and optimization flag
-
-#Paths for required external libraries
-dealiiDir="/ccs/proj/mat239/software/dealiiDevCustomized/installDealiiGcc9.1.0NOCUDA"
-alglibDir="/ccs/proj/mat239/software/alglib/cpp/src"
-libxcDir="/ccs/proj/mat239/software/libxc/installGcc9.1.0"
-spglibDir="/ccs/proj/mat239/software/spglib/installGcc9.1.0"
-xmlIncludeDir="/usr/include/libxml2"
-xmlLibDir="/usr/lib64"
-ELPA_PATH="/ccs/proj/mat239/software/elpa/install202211"
-
-#Paths for optional external libraries
-# path for NCCL/RCCL libraries
-DCCL_PATH="/ccs/proj/mat239/software/ncclnew/build"
-mdiPath=""
-
-#Toggle GPU compilation
-withGPU=ON
-gpuLang="cuda"     # Use "cuda"/"hip"
-gpuVendor="nvidia" # Use "nvidia/amd"
-withGPUAwareMPI=OFF #Please use this option with care
-                   #Only use if the machine supports 
-                   #device aware MPI and is profiled
-                   #to be fast
-
-#Option to link to DCCL library (Only for GPU compilation)
-withDCCL=ON
-withMDI=OFF
-
-#Compiler options and flags
-cxx_compiler=mpic++  #sets DCMAKE_CXX_COMPILER
-cxx_flags="-fPIC" #sets DCMAKE_CXX_FLAGS
-cxx_flagsRelease="-O2" #sets DCMAKE_CXX_FLAGS_RELEASE
-device_flags="-arch=sm_70" # set DCMAKE_CXX_CUDA/HIP_FLAGS 
-                           #(only applicable for withGPU=ON)
-device_architectures="70" # set DCMAKE_CXX_CUDA/HIP_ARCHITECTURES 
-                           #(only applicable for withGPU=ON)
-
-
-#Option to compile with default or higher order quadrature for storing pseudopotential data
-#ON is recommended for MD simulations with hard pseudopotentials
-withHigherQuadPSP=OFF
-
-# build type: "Release" or "Debug"
-build_type=Release
-
-testing=OFF
-minimal_compile=ON
-###########################################################################
-#Usually, no changes are needed below this line
-#
-
-#if [[ x"$build_type" == x"Release" ]]; then
-#  c_flags="$c_flagsRelease"
-#  cxx_flags="$c_flagsRelease"
-#else
-#fi
-out=`echo "$build_type" | tr '[:upper:]' '[:lower:]'`
-
-function cmake_real() {
-  mkdir -p real && cd real
-  if [ "$gpuLang" = "cuda" ]; then
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=OFF -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_CUDA_FLAGS="$device_flags" -DCMAKE_CUDA_ARCHITECTURES="$device_architectures"\
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1
-  elif [ "$gpuLang" = "hip" ]; then
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=OFF -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_HIP_FLAGS="$device_flags" -DCMAKE_HIP_ARCHITECTURES="$device_architectures"\
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1  
-  else
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=OFF\
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1    
-  fi  
-}
-
-function cmake_cplx() {
-  mkdir -p complex && cd complex
-  if [ "$gpuLang" = "cuda" ]; then
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=ON -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_CUDA_FLAGS="$device_flags" -DCMAKE_CUDA_ARCHITECTURES="$device_architectures"\
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1
-  elif [ "$gpuLang" = "hip" ]; then
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=ON -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_HIP_FLAGS="$device_flags" -DCMAKE_HIP_ARCHITECTURES="$device_architectures"\
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1
-  else
-    cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
-    -DCMAKE_CXX_FLAGS="$cxx_flags"\
-    -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
-    -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
-    -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
-    -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
-    -DXML_INCLUDE_DIR=$xmlIncludeDir\
-    -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath \
-    -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH"\
-    -DWITH_COMPLEX=ON \
-    -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
-    -DHIGHERQUAD_PSP=$withHigherQuadPSP $1    
-  fi
-}
-
-RCol='\e[0m'
-Blu='\e[0;34m';
-if [ -d "$out" ]; then # build directory exists
-    echo -e "${Blu}$out directory already present${RCol}"
-else
-    rm -rf "$out"
-    echo -e "${Blu}Creating $out ${RCol}"
-    mkdir -p "$out"
-fi
-
-cd $out
-
-echo -e "${Blu}Building Real executable in $build_type mode...${RCol}"
-cmake_real "$SRC" && make -j8
-cd ..
-
-echo -e "${Blu}Building Complex executable in $build_type mode...${RCol}"
-cmake_cplx "$SRC" && make -j8
-cd ..
-
-echo -e "${Blu}Build complete.${RCol}"
diff --git a/helpers/XSEDEStampede2KNL/p4est-setup.sh b/helpers/XSEDEStampede2KNL/p4est-setup.sh
deleted file mode 100644
index 2c23c569e..000000000
--- a/helpers/XSEDEStampede2KNL/p4est-setup.sh
+++ /dev/null
@@ -1,134 +0,0 @@
-#! /bin/bash
-#
-# This file is part of p4est [1].
-# p4est is a C library to manage a collection (a forest) of multiple
-# connected adaptive quadtrees or octrees in parallel.
-#
-# Copyright (C) 2010 The University of Texas System
-# Written by Carsten Burstedde, Lucas C. Wilcox, and Tobin Isaac
-# Modified 2010 by Wolfgang Bangerth
-# Modified 2010 by Timo Heister
-# Modified 2013 by Matthias Maier
-# Modified 2018 by Sambit Das (to fix OpenMP linking with intel compilers)
-#
-# p4est is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# [1] http://www.p4est.org
-#
-
-# This program comes with ABSOLUTELY NO WARRANTY.
-
-# unpack under current directory
-UNPACK=`pwd`
-
-# choose names for fast and debug compilation directories
-BUILD_DIR="$UNPACK/p4est-build"
-BUILD_FAST="$BUILD_DIR/FAST"
-BUILD_DEBUG="$BUILD_DIR/DEBUG"
-
-function busage() {
-        echo "Usage: `basename $0` <p4est_tar.gz_file> [<install location>]"
-        echo "   or: `basename $0` /path/to/p4est-src/ [<install location>]"
-}
-function bdie () {
-        echo "Error: $@"
-        exit 1
-}
-
-if test -z "$CFLAGS" -a -z "$P4EST_CFLAGS_FAST" ; then
-        export CFLAGS_FAST="-O2 -qopenmp"
-else
-        export CFLAGS_FAST="$CFLAGS $P4EST_CFLAGS_FAST"
-fi
-echo "CFLAGS_FAST: $CFLAGS_FAST"
-if test -z "$CFLAGS" -a -z "$P4EST_CFLAGS_DEBUG" ; then
-        export CFLAGS_DEBUG="-O0 -g -qopenmp"
-else
-        export CFLAGS_DEBUG="$CFLAGS $P4EST_CFLAGS_DEBUG"
-fi
-echo "CFLAGS_DEBUG: $CFLAGS_DEBUG"
-
-TGZ="$1"; shift
-if test -d "$TGZ" ; then
-  SRCDIR="$TGZ"  
-  echo "using existing source dir '$SRCDIR'"
-else
-    if test ! -f "$TGZ" ; then
-        busage
-        bdie "File not found"
-    fi
-    if ! (echo "$TGZ" | grep -q 'p4est.*.tar.gz') ; then
-        busage
-        bdie "File name mismatch"
-    fi
-fi
-
-# choose names for fast and debug installation directories
-INSTALL_DIR="$1"; shift
-if test -z "$INSTALL_DIR" ; then
-        INSTALL_DIR="$UNPACK/p4est-install"
-fi
-INSTALL_FAST="$INSTALL_DIR/FAST"
-INSTALL_DEBUG="$INSTALL_DIR/DEBUG"
-
-echo
-echo "This script tries to unpack, configure and build the p4est library."
-echo "Build FAST: $BUILD_FAST"
-echo "Build DEBUG: $BUILD_DEBUG"
-echo "Install FAST: $INSTALL_FAST"
-echo "Install DEBUG: $INSTALL_DEBUG"
-echo "Checking environment: CFLAGS P4EST_CFLAGS_FAST P4EST_CFLAGS_DEBUG"
-
-# remove old versions
-if test -d "$BUILD_DIR" ; then
-        rm -rf "$BUILD_DIR"
-fi
-
-if test -f "$TGZ" ; then
-    DIR=`echo "$TGZ" | sed 's/\(p4est.*\).tar.gz/\1/'`
-    DIR=`basename $DIR`
-    echo "Unpack directory: $UNPACK/$DIR"
-    if test -d "$UNPACK/$DIR" ; then
-        echo "Source directory found (remove it to unpack anew)"
-    else
-        echo -n "Unpacking... "
-        tar -xvz -f "$TGZ" -C "$UNPACK" >/dev/null
-        echo "done"
-    fi
-    SRCDIR=$UNPACK/$DIR
-fi
-test -f "$SRCDIR/src/p4est.h" || bdie "Main header file missing"
-test -f "$SRCDIR/configure" || bdie "Configure script missing"
-
-echo "See output in files .../config.output and .../make.output"
-echo
-echo "Build FAST version in $BUILD_FAST"
-mkdir -p "$BUILD_FAST"
-cd "$BUILD_FAST"
-"$SRCDIR/configure" --enable-mpi --enable-shared \
-        --disable-vtk-binary --without-blas --enable-openmp=-qopenmp\
-        --prefix="$INSTALL_FAST" CFLAGS="$CFLAGS_FAST" \
-        CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \
-        "$@" > config.output || bdie "Error in configure"
-make -C sc -j 8 > make.output || bdie "Error in make sc"
-make -j 8 >> make.output || bdie "Error in make p4est"
-make install >> make.output || bdie "Error in make install"
-echo "FAST version installed in $INSTALL_FAST"
-
-echo
-echo "Build DEBUG version in $BUILD_DEBUG"
-mkdir -p "$BUILD_DEBUG"
-cd "$BUILD_DEBUG"
-"$SRCDIR/configure" --enable-debug --enable-mpi --enable-shared \
-        --disable-vtk-binary --without-blas --enable-openmp=-qopenmp\
-        --prefix="$INSTALL_DEBUG" CFLAGS="$CFLAGS_DEBUG" \
-        CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \
-        "$@" > config.output || bdie "Error in configure"
-make -C sc -j 8 > make.output || bdie "Error in make sc"
-make -j 8 >> make.output || bdie "Error in make p4est"
-make install >> make.output || bdie "Error in make install"
-echo "DEBUG version installed in $INSTALL_DEBUG"
-echo
diff --git a/helpers/XSEDEStampede2KNL/setupStampede2.sh b/helpers/XSEDEStampede2KNL/setupStampede2.sh
deleted file mode 100755
index 7bb5c6e7a..000000000
--- a/helpers/XSEDEStampede2KNL/setupStampede2.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/bin/bash
-set -e
-set -o pipefail
-#script to setup and build DFT-FE
-
-########################################################################
-#Provide paths below for external libraries, compiler options and flags,
-# and optimization flag
-
-#Paths for external libraries
-dealiiPetscRealDir="/work/05316/dsambit/publicSharedSoftware/dealiiDev/intel_18.0.0_petscReal_scalapack_64Bit"
-dealiiPetscComplexDir="/work/05316/dsambit/publicSharedSoftware/dealiiDev/intel_18.0.0_petscComplex_scalapack_64Bit"
-alglibDir="/work/05316/dsambit/publicSharedSoftware/alglib/cppIntel18.0.0/src"
-libxcDir="/work/05316/dsambit/publicSharedSoftware/libxc/install_intel18.0.0"
-spglibDir="/work/05316/dsambit/publicSharedSoftware/spglib/install_intel18.0.0"
-xmlIncludeDir="/usr/include/libxml2"
-xmlLibDir="/usr/lib64"
-
-#If you have installed dealii by linking with intel mkl library set underlying flag to "ON",
-#otherwise set it to "OFF"
-withIntelMkl=ON
-
-#Compiler options and flags
-c_compiler=mpicc
-cxx_compiler=mpicxx
-c_flagsRelease=-O2
-cxx_flagsRelease=-O2
-
-#Optmization flag: 1 for optimized mode and 0 for debug mode compilation
-optimizedMode=1
-
-###########################################################################
-#Usually, no changes are needed below this line
-#
-RCol='\e[0m'
-Blu='\e[0;34m';
-if [ $optimizedMode == 1 ]; then
-  if [ -d "build/release" ]; then
-    echo -e "${Blu}build/release directory already present${RCol}"
-    # Control will enter here if build directory exists.
-    cd build
-    cd release
-    echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}"
-    mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ..
-    echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}"
-    mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
-  else
-    rm -rf build/release
-    echo -e "${Blu}Creating build directory...${RCol}"
-    mkdir -p build && cd build
-    mkdir -p release && cd release
-    echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}"
-    mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ..
-    echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}"
-    mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
-  fi
-else
-  if [ -d "build/debug" ]; then
-    echo -e "${Blu}build/debug directory already present${RCol}"
-    # Control will enter here if build directory exists.
-    cd build
-    cd debug
-    echo -e "${Blu}Building Real executable in Debug mode...${RCol}"
-    mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ..
-    echo -e "${Blu}Building Complex executable in Debug mode...${RCol}"
-    mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
-  else
-    rm -rf build/debug
-    echo -e "${Blu}Creating build directory...${RCol}"
-    mkdir -p build && cd build
-    mkdir -p debug && cd debug
-    echo -e "${Blu}Building Real executable in Debug mode...${RCol}"
-    mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ..
-    echo -e "${Blu}Building Complex executable in Debug mode...${RCol}"
-    mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
-  fi
-fi
-echo -e "${Blu}Build complete.${RCol}"
diff --git a/helpers/p4est-setup.sh b/helpers/p4est-setup.sh
index 9b126f531..bc6ccd1f0 100755
--- a/helpers/p4est-setup.sh
+++ b/helpers/p4est-setup.sh
@@ -15,7 +15,7 @@
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 #
-# [1] http://www.p4est.org
+# [1] https://www.p4est.org
 #
 
 # This program comes with ABSOLUTELY NO WARRANTY.
@@ -114,7 +114,8 @@ echo
 echo "Build FAST version in $BUILD_FAST"
 mkdir -p "$BUILD_FAST"
 cd "$BUILD_FAST"
-"$SRCDIR/configure" -host=x86_64 CC=mpicc CXX=mpicxx FC=mpif90 F77=mpif90 --enable-mpi --enable-shared \
+"$SRCDIR/configure" -host=x86_64 CC=mpicc CXX=mpicxx FC=mpif90 F77=mpif90 \
+        --enable-mpi --enable-shared \
         --disable-vtk-binary --without-blas \
         --prefix="$INSTALL_FAST" CFLAGS="$CFLAGS_FAST" \
         CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \
@@ -132,7 +133,8 @@ echo
 echo "Build DEBUG version in $BUILD_DEBUG"
 mkdir -p "$BUILD_DEBUG"
 cd "$BUILD_DEBUG"
-"$SRCDIR/configure" -host=x86_64 CC=mpicc CXX=mpicxx FC=mpif90 F77=mpif90 --enable-debug --enable-mpi --enable-shared \
+"$SRCDIR/configure" -host=x86_64 CC=mpicc CXX=mpicxx FC=mpif90 F77=mpif90 \
+                    --enable-debug --enable-mpi --enable-shared \
         --disable-vtk-binary --without-blas \
         --prefix="$INSTALL_DEBUG" CFLAGS="$CFLAGS_DEBUG" \
         CPPFLAGS="-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL" \

From 9f57544b379de50c489242d971d84db3302e454d Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Wed, 24 Apr 2024 10:08:14 -0400
Subject: [PATCH 03/14] update docs

---
 doc/manual/installation.tex | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/doc/manual/installation.tex b/doc/manual/installation.tex
index 341ee4358..df0d7235b 100644
--- a/doc/manual/installation.tex
+++ b/doc/manual/installation.tex
@@ -155,7 +155,7 @@ \subsubsection{Instructions for deal.II}
 
 \item Obtain the latest release version (9.5.2) of deal.II
 
-\item In addition to requiring C, C++ and Fortran compilers, MPI library, and CMake, deal.II additionally requires BOOST library. If not found externally, cmake will resort to the bundled BOOST that comes along with deal.II. Based on our experience, we recommend to use the deal.II's bundled boost (enforced by unsetting/unloading external BOOST library environment paths) to avoid compilation issues. Please do not install deal.II with GPU support as that is not needed by DFT-FE. Further, when installing deal.II for (DFT-FE with GPU support), one must use \verb|-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF| to avoid compilation failure of DFT-FE.
+\item In addition to requiring C, C++ and Fortran compilers, MPI library, and CMake, deal.II additionally requires BOOST and Kokkos libraries. If not found externally, deal.II will resort to the bundled BOOST and Kokkos libraries that comes along with deal.II. Based on our experience, we recommend to use the deal.II's bundled boost (enforced by unsetting/unloading external BOOST library environment paths) and bundled Kokkos to avoid compilation issues. Please do not install deal.II with GPU support as that is not needed by DFT-FE. Further, when installing deal.II for (DFT-FE with GPU support), one must use \verb|-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF| to avoid compilation failure of DFT-FE.
 
 \item
 \begin{verbatim}
@@ -165,7 +165,10 @@ \subsubsection{Instructions for deal.II}
         otherCmakeOptions ../deal.II
 $ make -j 8        
 $ make install
+$ cp -r ../bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics dealii_install_dir_path
+/include/deal.II/bundled/desul/
 \end{verbatim}
+The last step of copying the atomics include files is required to avoid compilation issues when linking deal.II to DFT-FE.
 {\bf ``otherCmakeOptions'' include} the following options for CPU installation:
 \begin{verbatim}
 -DCMAKE_CXX_STANDARD=17
@@ -187,6 +190,7 @@ \subsubsection{Instructions for deal.II}
 -DDEAL_II_WITH_TASKFLOW=OFF
 -DDEAL_II_COMPONENT_EXAMPLES=OFF
 -DDEAL_II_FORCE_BUNDLED_BOOST=ON
+-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF
 \end{verbatim}
 
 
@@ -213,6 +217,7 @@ \subsubsection{Instructions for deal.II}
 -DDEAL_II_WITH_TBB=OFF
 -DDEAL_II_WITH_TASKFLOW=OFF 
 -DDEAL_II_FORCE_BUNDLED_BOOST=ON
+-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF
 -DDEAL_II_WITH_P4EST=ON 
 -DP4EST_DIR=p4est_install_path 
 -DDEAL_II_WITH_LAPACK=ON -DLAPACK_DIR="${MKLROOT}/lib/intel64"

From 86a6f023390ce7599fb076583b337698501ecf92 Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Wed, 24 Apr 2024 10:21:04 -0400
Subject: [PATCH 04/14] set VSELF GPU=true by default

---
 src/dft/dos.cc                | 8 ++++----
 src/dft/kohnShamEigenSolve.cc | 8 +++++---
 utils/dftParameters.cc        | 3 ++-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/dft/dos.cc b/src/dft/dos.cc
index e2d006159..53989e5c6 100644
--- a/src/dft/dos.cc
+++ b/src/dft/dos.cc
@@ -179,8 +179,8 @@ namespace dftfe
     const double totalEigenValues  = eigenValuesAllkPoints.size();
     const double intervalSize      = 0.001;
     const double sigma             = C_kb * d_dftParamsPtr->TVal;
-    double lowerBoundEpsilon = 1.5 * eigenValuesAllkPoints[0];
-    double upperBoundEpsilon =
+    double       lowerBoundEpsilon = 1.5 * eigenValuesAllkPoints[0];
+    double       upperBoundEpsilon =
       eigenValuesAllkPoints[totalEigenValues - 1] * 1.5;
 
     MPI_Allreduce(MPI_IN_PLACE,
@@ -415,7 +415,7 @@ namespace dftfe
                   dataTypes::mpi_type_id(&upperBoundEpsilon),
                   MPI_MAX,
                   interpoolcomm);
-    
+
     unsigned int numberIntervals =
       std::ceil((upperBoundEpsilon - lowerBoundEpsilon) / intervalSize);
     unsigned int numberGlobalAtoms = atomLocations.size();
@@ -1009,7 +1009,7 @@ namespace dftfe
                   1,
                   dataTypes::mpi_type_id(&upperBoundEpsilon),
                   MPI_MAX,
-                  interpoolcomm);    
+                  interpoolcomm);
 
     unsigned int numberIntervals =
       std::ceil((upperBoundEpsilon - lowerBoundEpsilon) / intervalSize);
diff --git a/src/dft/kohnShamEigenSolve.cc b/src/dft/kohnShamEigenSolve.cc
index 1fa463d4e..a32df3718 100644
--- a/src/dft/kohnShamEigenSolve.cc
+++ b/src/dft/kohnShamEigenSolve.cc
@@ -1022,13 +1022,15 @@ namespace dftfe
                 double functionValue;
                 if (factor <= 0.0)
                   {
-                    double temp2  = 1.0 / (1.0 + exp(factor));
-                    functionValue = (2.0 - d_dftParamsPtr->spinPolarized)*temp2;
+                    double temp2 = 1.0 / (1.0 + exp(factor));
+                    functionValue =
+                      (2.0 - d_dftParamsPtr->spinPolarized) * temp2;
                   }
                 else
                   {
                     double temp2  = 1.0 / (1.0 + exp(-factor));
-                    functionValue = (2.0 - d_dftParamsPtr->spinPolarized) * exp(-factor) * temp2;
+                    functionValue = (2.0 - d_dftParamsPtr->spinPolarized) *
+                                    exp(-factor) * temp2;
                   }
                 if (functionValue > 1e-3)
                   highestOccupiedState = i;
diff --git a/utils/dftParameters.cc b/utils/dftParameters.cc
index db7db967d..348e0d401 100644
--- a/utils/dftParameters.cc
+++ b/utils/dftParameters.cc
@@ -1055,7 +1055,7 @@ namespace dftfe
                           "[Advanced] Toggle GPU MODE in Poisson solve.");
 
         prm.declare_entry("VSELF GPU MODE",
-                          "false",
+                          "true",
                           dealii::Patterns::Bool(),
                           "[Advanced] Toggle GPU MODE in vself Poisson solve.");
       }
@@ -1166,6 +1166,7 @@ namespace dftfe
     numSCFIterations                           = 1;
     maxLinearSolverIterations                  = 1;
     poissonGPU                                 = true;
+    vselfGPU                                   = true;
     mixingHistory                              = 1;
     npool                                      = 1;
     maxLinearSolverIterationsHelmholtz         = 1;

From 9ec7ceaac928ee10583c6a3c52b2da68982269db Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Wed, 24 Apr 2024 11:01:42 -0700
Subject: [PATCH 05/14] cleanups

---
 testsGPU/pseudopotential/complex/jobscripts/perlmutter.slurm  | 4 ++--
 .../real/jobscripts/perlmutter18Tasks18GPUs.slurm             | 2 +-
 .../real/jobscripts/perlmutter6Tasks6GPUs.slurm               | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/testsGPU/pseudopotential/complex/jobscripts/perlmutter.slurm b/testsGPU/pseudopotential/complex/jobscripts/perlmutter.slurm
index bfff8ad16..0434ad540 100644
--- a/testsGPU/pseudopotential/complex/jobscripts/perlmutter.slurm
+++ b/testsGPU/pseudopotential/complex/jobscripts/perlmutter.slurm
@@ -16,7 +16,7 @@ OMP_NUM_THREADS=1
 MPICH_GPU_SUPPORT_ENABLED=1
 LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib
 LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib64
-module load nccl/2.15.5-ofi
+module load nccl
 
 
 srun ./dftfe parameterFileMg2x_1.prm > outputMg2x_1
@@ -31,5 +31,5 @@ srun ./dftfe parameterFileMg2x_5.prm > outputMg2x_5
 
 srun ./dftfe parameterFileMg2x_6.prm > outputMg2x_6
 
-srun -n 6 -c 1 ./dftfe parameterFileBe.prm > outputBe
+srun ./dftfe parameterFileBe.prm > outputBe
 
diff --git a/testsGPU/pseudopotential/real/jobscripts/perlmutter18Tasks18GPUs.slurm b/testsGPU/pseudopotential/real/jobscripts/perlmutter18Tasks18GPUs.slurm
index 5c14c45ad..7e5cf206c 100644
--- a/testsGPU/pseudopotential/real/jobscripts/perlmutter18Tasks18GPUs.slurm
+++ b/testsGPU/pseudopotential/real/jobscripts/perlmutter18Tasks18GPUs.slurm
@@ -15,7 +15,7 @@ OMP_NUM_THREADS=1
 MPICH_GPU_SUPPORT_ENABLED=1
 LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib
 LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib64
-module load nccl/2.15.5-ofi
+module load nccl
 
 srun  ./dftfe Input_MD_0.prm > output_MD_0
 srun  ./dftfe Input_MD_1.prm > output_MD_1
diff --git a/testsGPU/pseudopotential/real/jobscripts/perlmutter6Tasks6GPUs.slurm b/testsGPU/pseudopotential/real/jobscripts/perlmutter6Tasks6GPUs.slurm
index fc7b39268..d2b72d064 100644
--- a/testsGPU/pseudopotential/real/jobscripts/perlmutter6Tasks6GPUs.slurm
+++ b/testsGPU/pseudopotential/real/jobscripts/perlmutter6Tasks6GPUs.slurm
@@ -15,7 +15,7 @@ OMP_NUM_THREADS=1
 MPICH_GPU_SUPPORT_ENABLED=1
 LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib
 LD_LIBRARY_PATH = $LD_LIBRARY_PATH:$WD/env2/lib64
-module load nccl/2.15.5-ofi
+module load nccl
 
 srun ./dftfe parameterFileN2_1.prm > outputN2_1
 srun ./dftfe parameterFileN2_2.prm > outputN2_2

From 16e38cd2927ae00cd3d80dd1e284d2f0efd8a617 Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Thu, 25 Apr 2024 14:38:20 -0400
Subject: [PATCH 06/14] wip on manual

---
 doc/manual/installation.tex | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/doc/manual/installation.tex b/doc/manual/installation.tex
index df0d7235b..74f0544b5 100644
--- a/doc/manual/installation.tex
+++ b/doc/manual/installation.tex
@@ -155,7 +155,7 @@ \subsubsection{Instructions for deal.II}
 
 \item Obtain the latest release version (9.5.2) of deal.II
 
-\item In addition to requiring C, C++ and Fortran compilers, MPI library, and CMake, deal.II additionally requires BOOST and Kokkos libraries. If not found externally, deal.II will resort to the bundled BOOST and Kokkos libraries that comes along with deal.II. Based on our experience, we recommend to use the deal.II's bundled boost (enforced by unsetting/unloading external BOOST library environment paths) and bundled Kokkos to avoid compilation issues. Please do not install deal.II with GPU support as that is not needed by DFT-FE. Further, when installing deal.II for (DFT-FE with GPU support), one must use \verb|-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF| to avoid compilation failure of DFT-FE.
+\item In addition to requiring C, C++ and Fortran compilers, MPI library, CMake, and Kokkos. deal.II additionally requires BOOST library. If not found externally, deal.II will resort to the bundled BOOST. Based on our experience, we recommend to use the deal.II's bundled boost (enforced by unsetting/unloading external BOOST library environment paths). Please do not install deal.II with GPU support as that is not needed by DFT-FE. Further, when installing deal.II for (DFT-FE with GPU support), one must use \verb|-DDEAL_II_ALLOW_PLATFORM_INTROSPECTION=OFF| to avoid compilation failure of DFT-FE.
 
 \item
 \begin{verbatim}
@@ -165,10 +165,7 @@ \subsubsection{Instructions for deal.II}
         otherCmakeOptions ../deal.II
 $ make -j 8        
 $ make install
-$ cp -r ../bundled/kokkos-3.7.00/tpls/desul/include/desul/atomics dealii_install_dir_path
-/include/deal.II/bundled/desul/
 \end{verbatim}
-The last step of copying the atomics include files is required to avoid compilation issues when linking deal.II to DFT-FE.
 {\bf ``otherCmakeOptions'' include} the following options for CPU installation:
 \begin{verbatim}
 -DCMAKE_CXX_STANDARD=17

From 5d7c8423c354194f6a39e7b795e3c57765d7d423 Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Tue, 18 Jun 2024 11:52:41 -0400
Subject: [PATCH 07/14] do not check constraints consistency in parallel when
 using uncustomized dealii

---
 src/dft/initBoundaryConditions.cc | 3 ++-
 src/dft/pRefinedDoFHandler.cc     | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/dft/initBoundaryConditions.cc b/src/dft/initBoundaryConditions.cc
index 0513b6ca4..461d36744 100644
--- a/src/dft/initBoundaryConditions.cc
+++ b/src/dft/initBoundaryConditions.cc
@@ -182,6 +182,7 @@ namespace dftfe
     // push back into Constraint Matrices
     d_constraintsVector.push_back(&constraintsNone);
 
+#ifdef DFTFE_WITH_CUSTOMIZED_DEALII
     if (d_dftParamsPtr->constraintsParallelCheck)
       {
         dealii::IndexSet locally_active_dofs_debug;
@@ -199,7 +200,7 @@ namespace dftfe
           dealii::ExcMessage(
             "DFT-FE Error: Constraints are not consistent in parallel."));
       }
-
+#endif
     //
     // create matrix free structure
     //
diff --git a/src/dft/pRefinedDoFHandler.cc b/src/dft/pRefinedDoFHandler.cc
index df67491cf..78ac7f984 100644
--- a/src/dft/pRefinedDoFHandler.cc
+++ b/src/dft/pRefinedDoFHandler.cc
@@ -336,7 +336,7 @@ namespace dftfe
     d_constraintsVectorElectro.push_back(&d_constraintsForPhiPrimeElectro);
     d_phiPrimeDofHandlerIndexElectro = d_constraintsVectorElectro.size() - 1;
 
-
+#ifdef DFTFE_WITH_CUSTOMIZED_DEALII
     if (d_dftParamsPtr->constraintsParallelCheck)
       {
         dealii::IndexSet locally_active_dofs_debug;
@@ -355,6 +355,7 @@ namespace dftfe
           dealii::ExcMessage(
             "DFT-FE Error: Constraints are not consistent in parallel."));
       }
+#endif
 
     // Fill dofHandler vector
     std::vector<const dealii::DoFHandler<3> *> matrixFreeDofHandlerVectorInput;

From 52d6b5b72bc9cbb36dd5d6b7ee12fc65a9d21332 Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Tue, 18 Jun 2024 13:51:47 -0400
Subject: [PATCH 08/14] update manual

---
 doc/manual/installation.tex | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/doc/manual/installation.tex b/doc/manual/installation.tex
index b0b708d1b..5b53c9544 100644
--- a/doc/manual/installation.tex
+++ b/doc/manual/installation.tex
@@ -18,9 +18,9 @@
 
 \subsection{Compiling and installing external libraries}
 \dftfe{} is primarily based on the open source finite element library \href{http://www.dealii.org/}{deal.II}, through which external dependencies
-on \href{http://p4est.org/}{p4est} and BLAS-LAPACK are set. The other required external libraries, which are
+on \href{http://p4est.org/}{p4est}, \href{https://github.com/kokkos/kokkos}{kokkos} and BLAS-LAPACK are set. The other required external libraries, which are
 not interfaced via deal.II are \href{http://www.alglib.net/}{ALGLIB}, \href{http://www.tddft.org/programs/libxc/}{Libxc}, \href{https://atztogo.github.io/spglib/}{spglib}, \href{http://www.xmlsoft.org/}{Libxml2} and \href{https://elpa.mpcdf.mpg.de/}{ELPA} (ELPA requires \href{http://www.netlib.org/scalapack/}{ScaLAPACK}). DFT-FE also optionally links to \href{https://www.mcs.anl.gov/petsc/}{PETSc} (via deali.II), \href{http://slepc.upv.es/}{SLEPc} (via deali.II) and to \href{https://developer.nvidia.com/nccl}{nccl} (for GPU compilation). The optional dependencies of PETSc and SLEPc are only required for all-electron calculations using DFT-FE, which uses the more stable Gram-Schmidt orthogonalization routine instead of the default Cholesky-Gram-Schmidt orthognalization. For pseudopotential calculations, PETSc and SLEPc dependencies are not required as the default Cholesky-Gram-Schmidt orthogonalization is very robust. \dftfe{} can also be linked to \href{https://github.com/awvwgk/simple-dftd3}{simple-dftd3} and \href{https://github.com/dftd4/dftd4}{dftd4} in order to provide support for the dftd family of dispersion corrections by Grimme et. al.. Below, we give brief installation instructions for each of the above libraries.
-\subsubsection{Instructions for dependencies: ALGLIB, Libxc, spglib, Libxml2, ScaLAPACK, ELPA, p4est and nccl (nccl is optional)}
+\subsubsection{Instructions for dependencies: ALGLIB, Libxc, spglib, Libxml2, ScaLAPACK, ELPA, p4est, kokkos, and nccl (nccl is optional)}
 \begin{enumerate}
 	\item   {\bf ALGLIB}: Used by \dftfe{} for spline fitting for various radial data. Download the current release of the Alglib free C++ edition from \url{http://www.alglib.net/download.php}. After downloading and unpacking, go to \verb|cpp/src|, and create a shared library using a C++ compiler. For example, using GCC compiler do
 \begin{verbatim}
@@ -143,6 +143,20 @@ \subsubsection{Instructions for dependencies: ALGLIB, Libxc, spglib, Libxml2, Sc
 
 %Also download the script from \url{https://github.com/dftfeDevelopers/dftfe/raw/manual/p4est-setup.sh} if using Intel compilers, or from \url{https://dealii.org/developer/external-libs/p4est.html} if using GCC compilers. Use the script to automatically compile and install a debug and optimized version of p4est by doing
 
+\item   {\bf kokkos}: This library is used by deal.II to performance portability across architectures. Download the latest release tarball of p4est from \url{https://github.com/kokkos/kokkos}, and use cmake based compilation. For example,
+\begin{verbatim}
+$ cd kokkos-4.3.00
+$ mkdir build && cd build
+$ cmake -DCMAKE_C_COMPILER=cc -DCMAKE_C_FLAGS="-O2 -fPIC"
+        -DCMAKE_CXX_COMPILER=CC 
+        -DCMAKE_CXX_FLAGS="-O2 -fPIC" 
+        -DCMAKE_INSTALL_PREFIX=kokkos_install_dir_path
+        ..
+$ make -j 4
+$ make install
+\end{verbatim}
+
+
 \item {\bf nccl/rccl (optional)}: nccl (CUDA) and rccl (ROCm) libraries are optional dependency for DFT-FE for optimal GPU Direct MPI collective communication calls. This library is recommended for running very large system sizes (greater than 20,000 electrons) on GPUs using DFT-FE. Caution: nccl/rccl library requires appropriate hardware support for GPU Direct MPI communication and GPU Aware MPI library. To install nccl/rccl, use the appropriate release version (corresponding to the CUDA/ROCm compiler version) from \url{https://github.com/NVIDIA/nccl} (CUDA) \url{https://github.com/ROCmSoftwarePlatform/rccl} (ROCm) and follow installation instructions therein.
 
 \end{enumerate}
@@ -179,6 +193,7 @@ \subsubsection{Instructions for deal.II}
 -DCMAKE_C_FLAGS=c_flags
 -DDEAL_II_WITH_MPI=ON -DDEAL_II_WITH_64BIT_INDICES=ON
 -DDEAL_II_WITH_P4EST=ON -DP4EST_DIR=p4est_install_dir_path
+-DKOKKOS_DIR=kokkos_install_dir_path
 -DDEAL_II_WITH_LAPACK=ON
 -DLAPACK_DIR=lapack_dir_paths (both BLAS and LAPACK directory paths)
 -DLAPACK_FOUND=true

From 339b51809d2e3a51b0090b71ad4ebc76b8490316 Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Tue, 18 Jun 2024 21:18:26 -0400
Subject: [PATCH 09/14] cleanups

---
 CMakeLists.txt                                         | 2 +-
 src/triangulation/triangulationManager/generateMesh.cc | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4e29c3446..e540e07d4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -619,7 +619,7 @@ IF (WITH_TESTING)
   ENABLE_TESTING()
   INCLUDE(CTest)
   # default 600sec is not enough for some tests, increase:
-  SET (TEST_TIME_LIMIT 3600)
+  SET (TEST_TIME_LIMIT 5000)
   #
   IF (WITH_COMPLEX)
      ADD_SUBDIRECTORY(tests/dft/pseudopotential/complex)
diff --git a/src/triangulation/triangulationManager/generateMesh.cc b/src/triangulation/triangulationManager/generateMesh.cc
index 628157786..2023b6f59 100644
--- a/src/triangulation/triangulationManager/generateMesh.cc
+++ b/src/triangulation/triangulationManager/generateMesh.cc
@@ -1400,11 +1400,12 @@ namespace dftfe
               pcout
                 << "Hanging node and periodic constraints parallel consistency not achieved."
                 << std::endl;
-
+#ifdef DFTFE_WITH_CUSTOMIZED_DEALII
             AssertThrow(
               d_dftParams.createConstraintsFromSerialDofhandler,
               dealii::ExcMessage(
                 "DFT-FE error: this is due to a known issue related to hanging node constraints in dealii. Please set CONSTRAINTS FROM SERIAL DOFHANDLER = true under the Boundary conditions subsection in the input parameters file to circumvent this issue."));
+#endif
           }
       }
 

From d87a7996d57e393dd717835965ae7fdd5028a0ad Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Tue, 25 Jun 2024 20:58:25 -0400
Subject: [PATCH 10/14] bug fix related to inconsistency with dealii9.5.2

---
 src/poisson/poissonSolverProblem.cc       | 9 +++++++++
 src/poisson/poissonSolverProblemDevice.cc | 9 +++++++++
 2 files changed, 18 insertions(+)

diff --git a/src/poisson/poissonSolverProblem.cc b/src/poisson/poissonSolverProblem.cc
index 1bde4c148..19b1d38f0 100644
--- a/src/poisson/poissonSolverProblem.cc
+++ b/src/poisson/poissonSolverProblem.cc
@@ -333,7 +333,16 @@ namespace dftfe
                 fe_eval_sc.distribute_local_to_global(rhs);
 
                 if (d_isStoreSmearedChargeRhs)
+                {
+                  fe_eval_sc.reinit(macrocell);
+                  for (unsigned int q = 0; q < fe_eval_sc.n_q_points; ++q)
+                  {
+                    fe_eval_sc.submit_value(smearedbQuads[q], q);
+                  }
+                  fe_eval_sc.integrate(true, false);
+     
                   fe_eval_sc.distribute_local_to_global(d_rhsSmearedCharge);
+                }
               }
           }
       }
diff --git a/src/poisson/poissonSolverProblemDevice.cc b/src/poisson/poissonSolverProblemDevice.cc
index 11ce651e6..ce2bc104e 100644
--- a/src/poisson/poissonSolverProblemDevice.cc
+++ b/src/poisson/poissonSolverProblemDevice.cc
@@ -372,7 +372,16 @@ namespace dftfe
                 fe_eval_sc.distribute_local_to_global(rhs);
 
                 if (d_isStoreSmearedChargeRhs)
+                {
+                  fe_eval_sc.reinit(macrocell);
+                  for (unsigned int q = 0; q < fe_eval_sc.n_q_points; ++q)
+                  {
+                    fe_eval_sc.submit_value(smearedbQuads[q], q);
+                  }
+                  fe_eval_sc.integrate(true, false);
+
                   fe_eval_sc.distribute_local_to_global(d_rhsSmearedCharge);
+                }
               }
           }
       }

From 8f6031ee5b776bad80f1f5b9a19cd2a4ed069ed2 Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Tue, 25 Jun 2024 21:00:01 -0400
Subject: [PATCH 11/14] apply indentation standard

---
 src/poisson/poissonSolverProblem.cc       | 16 ++++++++--------
 src/poisson/poissonSolverProblemDevice.cc | 16 ++++++++--------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/poisson/poissonSolverProblem.cc b/src/poisson/poissonSolverProblem.cc
index 19b1d38f0..16fed24ef 100644
--- a/src/poisson/poissonSolverProblem.cc
+++ b/src/poisson/poissonSolverProblem.cc
@@ -333,16 +333,16 @@ namespace dftfe
                 fe_eval_sc.distribute_local_to_global(rhs);
 
                 if (d_isStoreSmearedChargeRhs)
-                {
-                  fe_eval_sc.reinit(macrocell);
-                  for (unsigned int q = 0; q < fe_eval_sc.n_q_points; ++q)
                   {
-                    fe_eval_sc.submit_value(smearedbQuads[q], q);
+                    fe_eval_sc.reinit(macrocell);
+                    for (unsigned int q = 0; q < fe_eval_sc.n_q_points; ++q)
+                      {
+                        fe_eval_sc.submit_value(smearedbQuads[q], q);
+                      }
+                    fe_eval_sc.integrate(true, false);
+
+                    fe_eval_sc.distribute_local_to_global(d_rhsSmearedCharge);
                   }
-                  fe_eval_sc.integrate(true, false);
-     
-                  fe_eval_sc.distribute_local_to_global(d_rhsSmearedCharge);
-                }
               }
           }
       }
diff --git a/src/poisson/poissonSolverProblemDevice.cc b/src/poisson/poissonSolverProblemDevice.cc
index ce2bc104e..bbcc11a7c 100644
--- a/src/poisson/poissonSolverProblemDevice.cc
+++ b/src/poisson/poissonSolverProblemDevice.cc
@@ -372,16 +372,16 @@ namespace dftfe
                 fe_eval_sc.distribute_local_to_global(rhs);
 
                 if (d_isStoreSmearedChargeRhs)
-                {
-                  fe_eval_sc.reinit(macrocell);
-                  for (unsigned int q = 0; q < fe_eval_sc.n_q_points; ++q)
                   {
-                    fe_eval_sc.submit_value(smearedbQuads[q], q);
+                    fe_eval_sc.reinit(macrocell);
+                    for (unsigned int q = 0; q < fe_eval_sc.n_q_points; ++q)
+                      {
+                        fe_eval_sc.submit_value(smearedbQuads[q], q);
+                      }
+                    fe_eval_sc.integrate(true, false);
+
+                    fe_eval_sc.distribute_local_to_global(d_rhsSmearedCharge);
                   }
-                  fe_eval_sc.integrate(true, false);
-
-                  fe_eval_sc.distribute_local_to_global(d_rhsSmearedCharge);
-                }
               }
           }
       }

From 0e078ed3a1d6998dfa0ace88539b0aba45a192f1 Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Wed, 26 Jun 2024 12:03:41 -0400
Subject: [PATCH 12/14] update README

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 540472809..b13c99bfe 100644
--- a/README.md
+++ b/README.md
@@ -15,8 +15,8 @@ DFT-FE code builds on top of the deal.II library for everything that has to do w
 The steps to install the necessary dependencies and DFT-FE itself are described in the *Installation* section of the DFT-FE manual (compile doc/manual/manual.tex or download the development version manual [here](https://github.com/dftfeDevelopers/dftfe/blob/manual/manual-develop.pdf)). 
 
 We have created several shell based installation scripts for the development version of DFT-FE (`publicGithubDevelop` branch) on various machines:
-  - [OLCF Frontier](https://github.com/dftfeDevelopers/install_DFTFE/tree/frontierScriptROCM6)
-  - [NERSC Perlmutter](https://github.com/dftfeDevelopers/install_DFTFE/tree/perlmutterScript)
+  - [OLCF Frontier](https://github.com/dftfeDevelopers/install_DFTFE/tree/frontierScriptROCM6Dealii9.5.2)
+  - [NERSC Perlmutter](https://github.com/dftfeDevelopers/install_DFTFE/tree/perlmutterScriptDealii9.5.2)
   - [ALCF Polaris](https://github.com/dftfeDevelopers/install_DFTFE/tree/polarisScript)
   - [UMICH Greatlakes](https://github.com/dftfeDevelopers/install_DFTFE/tree/greatlakesScript)
     

From d681254247407a924b447e0f1b502bc680a02d2f Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Wed, 26 Jun 2024 12:21:09 -0400
Subject: [PATCH 13/14] update setup

---
 setupUserPetsc.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setupUserPetsc.sh b/setupUserPetsc.sh
index 9dbc9166d..f95d6e3f5 100755
--- a/setupUserPetsc.sh
+++ b/setupUserPetsc.sh
@@ -46,7 +46,7 @@ withGPUAwareMPI=OFF #Please use this option with care
 withDCCL=OFF
 withMDI=OFF
 withTorch=OFF
-withCustomizedDealii=ON
+withCustomizedDealii=OFF
 
 #Compiler options and flags
 cxx_compiler=mpic++  #sets DCMAKE_CXX_COMPILER

From 60441fb049b73ccf527ae072d85d292ae4265851 Mon Sep 17 00:00:00 2001
From: Sambit Das <dsambit@umich.edu>
Date: Wed, 26 Jun 2024 14:07:58 -0400
Subject: [PATCH 14/14] allow linking to dealii9.5.1 as that is the lastest
 version supported by spack

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e540e07d4..10cb4ad5a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -350,7 +350,7 @@ TARGET_LINK_LIBRARIES(${TARGETLIB} PUBLIC ${DFTFE_MPI_LIBRARIES})
 #Dependencies
 #
 
-FIND_PACKAGE(deal.II 9.5.2 REQUIRED
+FIND_PACKAGE(deal.II 9.5.1 REQUIRED
   HINTS ${deal.II_DIR} ${DEAL_II_DIR} ../ ../../ $ENV{DEAL_II_DIR}
   )
 IF(NOT ${deal.II_FOUND})