Merged in nscfStrategy (pull request #515)

nscf implementation WIP

Approved-by: Sambit Das
Approved-by: Kartick Ramakrishnan

CMakeLists.txt

@@ -130,11 +130,11 @@ SET(TARGET_SRC
  ./src/dft/moveAtoms.cc
  ./src/dft/moveMeshToAtoms.cc
  ./src/dft/nodalDensityMixingSchemes.cc
- ./src/dft/nscf.cc
  ./src/dft/pRefinedDoFHandler.cc
  ./src/dft/psiInitialGuess.cc
  ./src/dft/publicMethods.cc
  ./src/dft/restart.cc
+ ./src/dft/solveNSCF.cc
  ./src/dft/lowrankApproxScfDielectricMatrixInv.cc
  ./src/dft/lowrankApproxScfDielectricMatrixInvSpinPolarized.cc
  ./src/dft/computeOutputDensityDirectionalDerivative.cc

helpers/MATRIXcluster/setupDevelopPetsc_MATRIX.sh

@@ -0,0 +1,152 @@
+#!/bin/bash
+# script to setup and build DFT-FE.
+
+set -e
+set -o pipefail
+
+if [ -s CMakeLists.txt ]; then
+ echo "This script must be run from the build directory!"
+ exit 1
+fi
+
+# Path to project source
+SRC=`dirname $0` # location of source directory
+
+########################################################################
+#Provide paths below for external libraries, compiler options and flags,
+# and optimization flag
+
+#Paths for required external libraries
+dealiiPetscRealDir="/shared/dftfesoftwares2021/dealii/installgcc9.2cudaawarempiWithPetscSlpec"
+dealiiPetscComplexDir="/shared/dftfesoftwares2021/dealii/installgcc9.2cudaawarempiWithPetscSlpecComplex"
+alglibDir="/shared/dftfesoftwares2021/alglib/cpp/src"
+libxcDir="/shared/dftfesoftwares2021/libxc/gcc9.2_libxc_5.1.5"
+spglibDir="/shared/dftfesoftwares2021/spglib/gcc9.2_spglib"
+xmlIncludeDir="/usr/include/libxml2"
+xmlLibDir="/usr/lib64"
+ELPA_PATH="/shared/dftfesoftwares2021/elpa2022/installgcc9.2elpa2022withcudaawarempi"
+numdiffdir="/shared/dftfesoftwares2021/numdiff/build"
+
+#Paths for optional external libraries
+# path for NCCL/RCCL libraries
+DCCL_PATH=""
+mdiPath=""
+
+#Toggle GPU compilation
+withGPU=ON
+gpuLang="cuda" # Use "cuda"/"hip"
+gpuVendor="nvidia" # Use "nvidia/amd"
+withGPUAwareMPI=OFF #Please use this option with care
+ #Only use if the machine supports 
+ #device aware MPI and is profiled
+ #to be fast
+
+#Option to link to DCCL library (Only for GPU compilation)
+withDCCL=OFF
+withMDI=OFF
+withTorch=OFF
+withCustomizedDealii=ON
+
+#Compiler options and flags
+cxx_compiler=mpic++ #sets DCMAKE_CXX_COMPILER
+cxx_flags="-march=native -fPIC" #sets DCMAKE_CXX_FLAGS
+cxx_flagsRelease="-O2" #sets DCMAKE_CXX_FLAGS_RELEASE
+device_flags="-arch=sm_70" # set DCMAKE_CXX_CUDA_FLAGS 
+ #(only applicable for withGPU=ON)
+device_architectures="70" # set DCMAKE_CXX_CUDA_ARCHITECTURES 
+ #(only applicable for withGPU=ON)
+
+
+#Option to compile with default or higher order quadrature for storing pseudopotential data
+#ON is recommended for MD simulations with hard pseudopotentials
+withHigherQuadPSP=OFF
+
+# build type: "Release" or "Debug"
+build_type=Release
+
+testing=ON
+minimal_compile=ON
+###########################################################################
+#Usually, no changes are needed below this line
+#
+
+#if [[ x"$build_type" == x"Release" ]]; then
+# c_flags="$c_flagsRelease"
+# cxx_flags="$c_flagsRelease"
+#else
+#fi
+out=`echo "$build_type" | tr '[:upper:]' '[:lower:]'`
+
+function cmake_configure() {
+ if [ "$gpuLang" = "cuda" ]; then
+ cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
+ -DCMAKE_CXX_FLAGS="$cxx_flags"\
+ -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
+ -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
+ -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
+ -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
+ -DXML_INCLUDE_DIR=$xmlIncludeDir\
+ -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath -DWITH_TORCH=$withTorch \
+ -DWITH_CUSTOMIZED_DEALII=$withCustomizedDealii\
+ -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH;$numdiffdir"\
+ -DWITH_COMPLEX=$withComplex -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_CUDA_FLAGS="$device_flags" -DCMAKE_CUDA_ARCHITECTURES="$device_architectures"\
+ -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
+ -DHIGHERQUAD_PSP=$withHigherQuadPSP $1
+ elif [ "$gpuLang" = "hip" ]; then
+ cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
+ -DCMAKE_CXX_FLAGS="$cxx_flags"\
+ -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
+ -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
+ -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
+ -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
+ -DXML_INCLUDE_DIR=$xmlIncludeDir\
+ -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath -DWITH_TORCH=$withTorch \
+ -DWITH_CUSTOMIZED_DEALII=$withCustomizedDealii\
+ -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH;$numdiffdir"\
+ -DWITH_COMPLEX=$withComplex -DWITH_GPU=$withGPU -DGPU_LANG=$gpuLang -DGPU_VENDOR=$gpuVendor -DWITH_GPU_AWARE_MPI=$withGPUAwareMPI -DCMAKE_HIP_FLAGS="$device_flags" -DCMAKE_HIP_ARCHITECTURES="$device_architectures"\
+ -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
+ -DHIGHERQUAD_PSP=$withHigherQuadPSP $1
+ else
+ cmake -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_COMPILER=$cxx_compiler\
+ -DCMAKE_CXX_FLAGS="$cxx_flags"\
+ -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" \
+ -DCMAKE_BUILD_TYPE=$build_type -DDEAL_II_DIR=$dealiiDir \
+ -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir \
+ -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir \
+ -DXML_INCLUDE_DIR=$xmlIncludeDir\
+ -DWITH_MDI=$withMDI -DMDI_PATH=$mdiPath -DWITH_TORCH=$withTorch \
+ -DWITH_CUSTOMIZED_DEALII=$withCustomizedDealii\
+ -DWITH_DCCL=$withDCCL -DCMAKE_PREFIX_PATH="$ELPA_PATH;$DCCL_PATH;$numdiffdir"\
+ -DWITH_COMPLEX=$withComplex \
+ -DWITH_TESTING=$testing -DMINIMAL_COMPILE=$minimal_compile\
+ -DHIGHERQUAD_PSP=$withHigherQuadPSP $1 
+ fi
+}
+
+RCol='\e[0m'
+Blu='\e[0;34m';
+if [ -d "$out" ]; then # build directory exists
+ echo -e "${Blu}$out directory already present${RCol}"
+else
+ rm -rf "$out"
+ echo -e "${Blu}Creating $out ${RCol}"
+ mkdir -p "$out"
+fi
+
+cd $out
+
+withComplex=OFF
+dealiiDir=$dealiiPetscRealDir
+echo -e "${Blu}Building Real executable in $build_type mode...${RCol}"
+mkdir -p real && cd real
+cmake_configure "$SRC" && make -j8
+cd ..
+
+withComplex=ON
+dealiiDir=$dealiiPetscComplexDir
+echo -e "${Blu}Building Complex executable in $build_type mode...${RCol}"
+mkdir -p complex && cd complex
+cmake_configure "$SRC" && make -j8
+cd ..
+
+echo -e "${Blu}Build complete.${RCol}"

include/dft.h

@@ -1737,6 +1737,17 @@ namespace dftfe
  const double _fermiEnergy);
 
 
+ /**
+ * @brief compute the maximum of the residual norm of the highest state of interest among all k points
+ */
+ double
+ computeMaximumHighestOccupiedStateResidualNorm(
+ const std::vector<std::vector<double>>
+ &residualNormWaveFunctionsAllkPoints,
+ const std::vector<std::vector<double>> &eigenValuesAllkPoints,
+ const unsigned int highestState);
+
+
  void
  kohnShamEigenSpaceCompute(
  const unsigned int s,

include/dftParameters.h

@@ -52,7 +52,8 @@ namespace dftfe
 
  bool isPseudopotential, periodicX, periodicY, periodicZ, useSymm,
  timeReversal, pseudoTestsFlag, constraintMagnetization, writeDosFile,
- writeLdosFile, writeLocalizationLengths, pinnedNodeForPBC, writePdosFile;
+ writeLdosFile, writeBandsFile, writeLocalizationLengths, pinnedNodeForPBC,
+ writePdosFile;
 
 
  /** parameters for LRD preconditioner **/
@@ -162,6 +163,7 @@ namespace dftfe
  double pspCutoffImageCharges;
  bool reuseLanczosUpperBoundFromFirstCall;
  bool allowMultipleFilteringPassesAfterFirstScf;
+ unsigned int highestStateOfInterestForChebFiltering;
  bool useELPADeviceKernel;
  bool deviceMemOptMode;
 

include/xmlTodftfeParser.h

@@ -61,9 +61,9 @@ namespace dftfe
  std::vector<std::string> coreDensity;
  std::vector<std::string> mesh;
  std::vector<std::tuple<size_t, size_t, std::vector<std::string>>>
-  projectors;
+ projectors;
  std::vector<std::tuple<size_t, std::string, std::vector<std::string>>>
-  PSwfc;
+ PSwfc;
 
  std::vector<std::tuple<size_t, size_t, size_t, double>> d_ij;
 

pseudoConverters/upfToxml.cc

@@ -316,7 +316,7 @@ namespace dftfe
 
  // std::cerr << " upf_nproj = " << upf_nproj << std::endl;
 
- //std::vector<int> upf_l(upf_nwfc);
+ // std::vector<int> upf_l(upf_nwfc);
 
  // read mesh
  find_start_element("PP_MESH", upfFile);
@@ -436,12 +436,12 @@ namespace dftfe
  find_start_element("PP_PSWFC", upfFile);
  std::vector<std::vector<double>> upf_pswfcdata;
  upf_pswfcdata.resize(upf_nwfc);
- std::vector<int> upf_wfc_l(upf_nwfc);
+ std::vector<int>  upf_wfc_l(upf_nwfc);
  std::vector<std::string> upf_wfc_orbital(upf_nwfc);
- int upf_lwfcmax=0;
+ int  upf_lwfcmax = 0;
  for (int j = 0; j < upf_nwfc; j++)
  {
- int index, angular_momentum;
+ int  index, angular_momentum;
  std::string orbital;
  os.str("");
  os << j + 1;
@@ -465,7 +465,7 @@ namespace dftfe
  is.clear();
  is.str(buf);
  is >> orbital;
- upf_lwfcmax = std::max(upf_lwfcmax,angular_momentum);
+ upf_lwfcmax  = std::max(upf_lwfcmax, angular_momentum);
  upf_wfc_l[index - 1] = angular_momentum;
  upf_wfc_orbital[index - 1] = orbital;
  upf_pswfcdata[j].resize(upf_mesh_size);
@@ -474,7 +474,7 @@ namespace dftfe
 
  find_end_element(element_name, upfFile);
  }
- 
+
  // compute number of projectors for each l
  // nproj_l[l] is the number of projectors having angular momentum l
  std::vector<int> npswfc_l(upf_lwfcmax + 1);
@@ -644,23 +644,23 @@ namespace dftfe
  }
 
  // PSWFC
- for(int iwave = 0; iwave < upf_nwfc; iwave++)
- {
-  xmlFile << "<PSwfc i=\"" << iwave<< "\" orbital=\"" << upf_wfc_orbital[iwave]
-  << "\" size=\"" << nplin << "\">" << std::endl;
- int count2 = 0;
-  for (int j = 0; j < nplin; j++)
-  {
-  count2 += 1;
-  xmlFile << std::setprecision(14) << upf_pswfcdata[iwave][j]
-  << " ";
- if (count2 % 4 == 0)
-  xmlFile << std::endl;
- }
-  xmlFile << "</PSwfc>" << std::endl; 
- }
-
- 
+ for (int iwave = 0; iwave < upf_nwfc; iwave++)
+  {
+ xmlFile << "<PSwfc i=\"" << iwave << "\" orbital=\""
+ << upf_wfc_orbital[iwave] << "\" size=\"" << nplin
+  << "\">" << std::endl;
+ int count2 = 0;
+ for (int j = 0; j < nplin; j++)
+ {
+ count2 += 1;
+ xmlFile << std::setprecision(14)
+  << upf_pswfcdata[iwave][j] << " ";
+ if (count2 % 4 == 0)
+ xmlFile << std::endl;
+ }
+  xmlFile << "</PSwfc>" << std::endl;
+  }
+
 
 
  // d_ij