Merged in initReinitOptimizations (pull request #546)

Initialization optimizations and OpenMP

Approved-by: Sambit Das
Approved-by: Phani Motamarri

CMakeLists.txt

@@ -116,7 +116,6 @@ SET(TARGET_SRC
  ./src/dft/charge.cc
  ./src/dft/density.cc
  ./src/dft/dos.cc
- ./src/dft/electrostaticHRefinedEnergy.cc
  ./src/dft/femUtilityFunctions.cc
  ./src/dft/fermiEnergy.cc
  ./src/dft/generateImageCharges.cc

doc/manual/parameters.tex

@@ -3,23 +3,6 @@ \subsection{Global parameters}
 
 
 \begin{itemize}
-\item {\it Parameter name:} {\tt H REFINED ELECTROSTATICS}
-\phantomsection\label{parameters:H REFINED ELECTROSTATICS}
-\label{parameters:H_20REFINED_20ELECTROSTATICS}
-
-
-\index[prmindex]{H REFINED ELECTROSTATICS}
-\index[prmindexfull]{H REFINED ELECTROSTATICS}
-{\it Value:} false
-
-
-{\it Default:} false
-
-
-{\it Description:} [Advanced] Compute electrostatic energy on a h refined mesh after each ground-state solve. Default: false.
-
-
-{\it Possible values:} A boolean value (true or false)
 \item {\it Parameter name:} {\tt KEEP SCRATCH FOLDER}
 \phantomsection\label{parameters:KEEP SCRATCH FOLDER}
 \label{parameters:KEEP_20SCRATCH_20FOLDER}

include/FEBasisOperations.h

@@ -37,7 +37,13 @@ namespace dftfe
 
  update_gradients = 0x0002,
 
- update_transpose = 0x0004
+ update_transpose = 0x0004,
+
+ update_quadpoints = 0x0008,
+
+ update_inversejacobians = 0x0010,
+
+ update_jxw = 0x0020,
  };
 
  inline UpdateFlags
@@ -100,6 +106,7 @@ namespace dftfe
  std::shared_ptr<dftfe::linearAlgebra::BLASWrapper<memorySpace>>
  BLASWrapperPtr);
 
+
  /**
  * @brief Default Destructor
  */
@@ -116,7 +123,16 @@ namespace dftfe
  void
  init(const unsigned int & dofHandlerID,
  const std::vector<unsigned int> &quadratureID,
- const UpdateFlags updateFlags = update_values);
+ const std::vector<UpdateFlags> updateFlags);
+ /**
+ * @brief fills required data structures from another FEBasisOperations object
+ * @param[in] basisOperationsSrc Source FEBasisOperations object.
+ */
+ template <dftfe::utils::MemorySpace memorySpaceSrc>
+ void
+ init(const FEBasisOperationsBase<ValueTypeBasisCoeff,
+ ValueTypeBasisData,
+ memorySpaceSrc> &basisOperationsSrc);
 
  /**
  * @brief sets internal variables and optionally resizes internal temp storage for interpolation operations
@@ -166,6 +182,14 @@ namespace dftfe
  void
  initializeConstraints();
 
+ /**
+ * @brief Reinitializes the constraintMatrixInfo object.
+ */
+ void
+ reinitializeConstraints(
+ std::vector<const dealii::AffineConstraints<ValueTypeBasisData> *>
+ &constraintsVector);
+
  /**
  * @brief Constructs the MPIPatternP2P object.
  */
@@ -255,6 +279,13 @@ namespace dftfe
  const dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace> &
  JxW() const;
 
+ /**
+ * @brief quad point coordinates for each cell.
+ */
+ const dftfe::utils::MemoryStorage<ValueTypeBasisData,
+ dftfe::utils::MemorySpace::HOST> &
+ quadPoints() const;
+
  /**
  * @brief Shape function values at quadrature points in ValueTypeBasisData.
  * @param[in] transpose if false the the data is indexed as [iQuad *
@@ -336,6 +367,13 @@ namespace dftfe
  dealii::CellId
  cellID(const unsigned int iElem) const;
 
+ /**
+ * @brief returns the cell index corresponding to given deal.ii cellID.
+ * @param[in] iElem cell Index
+ */
+ unsigned int
+ cellIndex(const dealii::CellId cellid) const;
+
  /**
  * @brief Creates a multivector.
  * @param[in] blocksize Number of vectors in the multivector.
@@ -378,48 +416,68 @@ namespace dftfe
  * @param[inout] multiVector the given multivector.
  */
  void
- distribute(
- dftfe::linearAlgebra::MultiVector<ValueTypeBasisCoeff, memorySpace>
- &multiVector) const;
+ distribute(dftfe::linearAlgebra::MultiVector<ValueTypeBasisCoeff,
+ memorySpace> &multiVector,
+ unsigned int constraintIndex =
+ std::numeric_limits<unsigned int>::max()) const;
 
 
 
- constraintInfoClass d_constraintInfo;
+ std::vector<constraintInfoClass> d_constraintInfo;
+ unsigned int d_nOMPThreads;
  std::vector<const dealii::AffineConstraints<ValueTypeBasisData> *>
  * d_constraintsVector;
  const dealii::MatrixFree<3, ValueTypeBasisData> *d_matrixFreeDataPtr;
  dftfe::utils::MemoryStorage<dftfe::global_size_type,
  dftfe::utils::MemorySpace::HOST>
  d_cellDofIndexToProcessDofIndexMap;
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisData,
+ dftfe::utils::MemorySpace::HOST>>
+ d_quadPoints;
  dftfe::utils::MemoryStorage<dftfe::global_size_type, memorySpace>
- d_flattenedCellDofIndexToProcessDofIndexMap;
- std::vector<dealii::CellId> d_cellIndexToCellIdMap;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
+ d_flattenedCellDofIndexToProcessDofIndexMap;
+ std::vector<dealii::CellId> d_cellIndexToCellIdMap;
+ std::map<dealii::CellId, unsigned int> d_cellIdToCellIndexMap;
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
  d_inverseJacobianData;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
  d_JxWData;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
  d_shapeFunctionData;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
  d_shapeFunctionGradientDataInternalLayout;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
  d_shapeFunctionGradientData;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
  d_shapeFunctionDataTranspose;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisCoeff, memorySpace>>
  d_shapeFunctionGradientDataTranspose;
 
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
  d_inverseJacobianBasisData;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
  d_JxWBasisData;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
  d_shapeFunctionBasisData;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
  d_shapeFunctionGradientBasisData;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
  d_shapeFunctionBasisDataTranspose;
- std::vector<dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
+ std::map<unsigned int,
+ dftfe::utils::MemoryStorage<ValueTypeBasisData, memorySpace>>
  d_shapeFunctionGradientBasisDataTranspose;
 
 
@@ -431,6 +489,7 @@ namespace dftfe
 
  std::vector<unsigned int> d_quadratureIDsVector;
  unsigned int d_quadratureID;
+ unsigned int d_quadratureIndex;
  std::vector<unsigned int> d_nQuadsPerCell;
  unsigned int d_dofHandlerID;
  unsigned int d_nVectors;
@@ -441,7 +500,7 @@ namespace dftfe
  unsigned int d_locallyOwnedSize;
  bool areAllCellsAffine;
  bool areAllCellsCartesian;
- UpdateFlags  d_updateFlags;
+ std::vector<UpdateFlags> d_updateFlags;
 
  std::shared_ptr<const utils::mpi::MPIPatternP2P<memorySpace>>
  mpiPatternP2P;
@@ -501,6 +560,10 @@ namespace dftfe
  ValueTypeBasisCoeff,
  ValueTypeBasisData,
  dftfe::utils::MemorySpace::HOST>::d_quadratureID;
+ using FEBasisOperationsBase<
+ ValueTypeBasisCoeff,
+ ValueTypeBasisData,
+ dftfe::utils::MemorySpace::HOST>::d_quadratureIndex;
  using FEBasisOperationsBase<
  ValueTypeBasisCoeff,
  ValueTypeBasisData,
@@ -778,6 +841,10 @@ namespace dftfe
  ValueTypeBasisCoeff,
  ValueTypeBasisData,
  dftfe::utils::MemorySpace::DEVICE>::d_quadratureID;
+ using FEBasisOperationsBase<
+ ValueTypeBasisCoeff,
+ ValueTypeBasisData,
+ dftfe::utils::MemorySpace::DEVICE>::d_quadratureIndex;
  using FEBasisOperationsBase<
  ValueTypeBasisCoeff,
  ValueTypeBasisData,

include/MemoryStorage.h

@@ -296,10 +296,10 @@ namespace dftfe
  */
  template <dftfe::utils::MemorySpace memorySpaceSrc>
  void
- copyFrom(MemoryStorage<ValueType, memorySpaceSrc> &srcMemoryStorage,
- const std::size_t N,
- const std::size_t srcOffset,
- const std::size_t dstOffset);
+ copyFrom(const MemoryStorage<ValueType, memorySpaceSrc> &srcMemoryStorage,
+ const std::size_t  N,
+ const std::size_t  srcOffset,
+ const std::size_t  dstOffset);
 
  /**
  * @brief Copies the data to a memory pointed by a raw pointer

include/dft.h

@@ -581,7 +581,8 @@ namespace dftfe
  void initUnmovedTriangulation(
  dealii::parallel::distributed::Triangulation<3> &triangulation);
  void
- initBoundaryConditions(const bool meshOnlyDeformed = false,
+ initBoundaryConditions(const bool recomputeBasisData = true,
+ const bool meshOnlyDeformed = false,
  const bool vselfPerturbationUpdateForStress = false);
  void
  initElectronicFields();
@@ -596,7 +597,8 @@ namespace dftfe
  void createpRefinedDofHandler(
  dealii::parallel::distributed::Triangulation<3> &triangulation);
  void
- initpRefinedObjects(const bool meshOnlyDeformed,
+ initpRefinedObjects(const bool recomputeBasisData,
+ const bool meshOnlyDeformed,
  const bool vselfPerturbationUpdateForStress = false);
 
  /**
@@ -975,23 +977,6 @@ namespace dftfe
  lowrankApproxScfDielectricMatrixInvSpinPolarized(
  const unsigned int scfIter);
 
- /**
- * Re solves the all electrostatics on a h refined mesh, and computes
- * the corresponding energy. This function
- * is called after reaching the ground state electron density. Currently the
- * h refinement is hardcoded to a one subdivison of carser mesh
- * FIXME: The function is not yet extened to the case when point group
- * symmetry is used. However, it works for time reversal symmetry.
- *
- */
- void
- computeElectrostaticEnergyHRefined(
-#ifdef DFTFE_WITH_DEVICE
- kohnShamDFTOperatorDeviceClass<FEOrder, FEOrderElectro>
- &kohnShamDFTEigenOperator
-#endif
- );
-
  /**
  *@brief Computes Fermi-energy obtained by imposing constraint on the number of electrons
  */
@@ -1277,6 +1262,7 @@ namespace dftfe
  unsigned int d_smearedChargeQuadratureIdElectro;
  unsigned int d_nlpspQuadratureId;
  unsigned int d_lpspQuadratureId;
+ unsigned int d_feOrderPlusOneQuadratureId;
  unsigned int d_lpspQuadratureIdElectro;
  unsigned int d_gllQuadratureId;
  unsigned int d_phiTotDofHandlerIndexElectro;
@@ -1285,18 +1271,27 @@ namespace dftfe
  unsigned int d_binsStartDofHandlerIndexElectro;
  unsigned int d_densityQuadratureId;
  unsigned int d_densityQuadratureIdElectro;
+ unsigned int d_nOMPThreads;
  dealii::MatrixFree<3, double> matrix_free_data, d_matrixFreeDataPRefined;
  std::shared_ptr<
  dftfe::basis::FEBasisOperations<dataTypes::number,
  double,
  dftfe::utils::MemorySpace::HOST>>
  basisOperationsPtrHost;
+ std::shared_ptr<
+ dftfe::basis::
+ FEBasisOperations<double, double, dftfe::utils::MemorySpace::HOST>>
+ basisOperationsPtrElectroHost;
 #if defined(DFTFE_WITH_DEVICE)
  std::shared_ptr<
  dftfe::basis::FEBasisOperations<dataTypes::number,
  double,
  dftfe::utils::MemorySpace::DEVICE>>
  basisOperationsPtrDevice;
+ std::shared_ptr<
+ dftfe::basis::
+ FEBasisOperations<double, double, dftfe::utils::MemorySpace::DEVICE>>
+ basisOperationsPtrElectroDevice;
 #endif
 
  std::shared_ptr<

include/dftParameters.h

@@ -92,7 +92,6 @@ namespace dftfe
  bool saveRhoData;
  bool loadRhoData;
  bool restartSpinFromNoSpin;
- bool electrostaticsHRefinement;
 
  bool reproducible_output;