diff --git a/CMakeLists.txt b/CMakeLists.txt index c1a853f0..f87cc11e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,7 +131,7 @@ set (MGMOL_WITH_SCALAPACK True CACHE BOOL "Build with SCALAPACK") if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) find_package(SCALAPACK) if(${SCALAPACK_FOUND}) - add_definitions(-DSCALAPACK) + add_definitions(-DMGMOL_USE_SCALAPACK) message(STATUS "SCALAPACK_INCLUDE_DIRS: ${SCALAPACK_INCLUDE_DIRS}") message(STATUS "SCALAPACK_LIBARIES: ${SCALAPACK_LIBRARIES}") else(${SCALAPACK_FOUND}) @@ -220,7 +220,8 @@ FortranCInterface_HEADER( DGETRF DGETRS DLACPY ) -FortranCInterface_HEADER( +if(${SCALAPACK_FOUND}) + FortranCInterface_HEADER( scalapack_mangle.h MACRO_NAMESPACE "FC_SCALAPACK_" SYMBOLS @@ -233,7 +234,8 @@ FortranCInterface_HEADER( pdtrtri pstrtri pdpocon pspocon pdsygst pssygst pdsyev pssyev pdelset pselset pdelget pselget pdlatra pslatra pdlaset pslaset pdgesvd psgesvd pdamax psamax -) + ) +endif(${SCALAPACK_FOUND}) FortranCInterface_HEADER( arpack_mangle.h diff --git a/src/AOMMprojector.cc b/src/AOMMprojector.cc index a35acf13..ac401490 100644 --- a/src/AOMMprojector.cc +++ b/src/AOMMprojector.cc @@ -17,11 +17,9 @@ AOMMprojector::AOMMprojector(LocGridOrbitals& phi, const std::shared_ptr& lrs) { - Control& ct = *(Control::instance()); - Mesh* mymesh = Mesh::instance(); - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); - bool with_spin = (mmpi.nspin() > 1); const short subdivx = mymesh->subdivx(); // radius of kernel functions @@ -43,10 +41,18 @@ AOMMprojector::AOMMprojector(LocGridOrbitals& phi, kernel_proj_matrices_ = new ProjectedMatricesSparse(ct.numst, ct.occ_width, lrs); else + { +#ifdef MGMOL_USE_SCALAPACK + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + bool with_spin = (mmpi.nspin() > 1); kernel_proj_matrices_ = new ProjectedMatrices>( ct.numst, with_spin, ct.occ_width); - +#else + std::cerr << "AOMMprojector requires ScaLapack" << std::endl; + abort(); +#endif + } // kernel functions use their own projected matrices and masks kernel_phi_ = new LocGridOrbitals( "AOMM", phi, kernel_proj_matrices_, kernelMasks_, nullptr); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9a1a02bd..43015b8d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,4 @@ -add_subdirectory(DistMatrix) add_subdirectory(linear_algebra) add_subdirectory(local_matrices) add_subdirectory(numerical_kernels) @@ -7,7 +6,11 @@ add_subdirectory(pb) add_subdirectory(radial) add_subdirectory(sparse_linear_algebra) add_subdirectory(tools) +if(${MGMOL_WITH_SCALAPACK}) + add_subdirectory(DistMatrix) +endif(${MGMOL_WITH_SCALAPACK}) +if(${MGMOL_WITH_SCALAPACK}) set(link_libs mgmol_distmatrix mgmol_linear_algebra mgmol_local_matrices @@ -17,6 +20,17 @@ set(link_libs mgmol_distmatrix mgmol_sparse_linear_algebra mgmol_tools ) +else(${MGMOL_WITH_SCALAPACK}) +set(link_libs + mgmol_linear_algebra + mgmol_local_matrices + mgmol_numerical_kernels + mgmol_pb + mgmol_radial + mgmol_sparse_linear_algebra + mgmol_tools + ) +endif(${MGMOL_WITH_SCALAPACK}) set(SOURCES mgmol_run.cc @@ -31,10 +45,6 @@ set(SOURCES DielectricControl.cc ReplicatedMatrix.cc ReplicatedVector.cc - SquareSubMatrix2DistMatrix.cc - LocalMatrices2DistMatrix.cc - DistMatrix2SquareLocalMatrices.cc - GrassmanCGFactory.cc DMStrategyFactory.cc manage_memory.cc SubCell.cc @@ -71,9 +81,6 @@ set(SOURCES MGmol.cc MGmol_NEB.cc ABPG.cc - GrassmanLineMinimization.cc - GrassmanCG.cc - GrassmanCGSparse.cc LBFGS.cc IonicStepper.cc Energy.cc @@ -112,8 +119,6 @@ set(SOURCES MultipoleExpansion.cc SpreadsAndCenters.cc Preconditioning.cc - OrbitalsTransform.cc - NOLMOTransform.cc LocalizationRegions.cc Hartree.cc ShiftedHartree.cc @@ -132,7 +137,6 @@ set(SOURCES lbfgsrlx.cc OrthoAndersonMix.cc AndersonMix.cc - MLWFTransform.cc Ion.cc GridMask.cc GridMaskMult.cc @@ -142,7 +146,6 @@ set(SOURCES md.cc get_vnlpsi.cc quench.cc - mlwf.cc readInput.cc Forces.cc computeHij.cc @@ -161,9 +164,26 @@ set(SOURCES magma_singleton.cc ChebyshevApproximation.cc ChebyshevApproximationInterface.cc + mlwf.cc ) -add_library(mgmol_src ${SOURCES}) +if(${MGMOL_WITH_SCALAPACK}) + set(SCALAPACK_SOURCES + SquareSubMatrix2DistMatrix.cc + LocalMatrices2DistMatrix.cc + DistMatrix2SquareLocalMatrices.cc + GrassmanCGFactory.cc + GrassmanLineMinimization.cc + GrassmanCG.cc + GrassmanCGSparse.cc + MLWFTransform.cc + NOLMOTransform.cc + OrbitalsTransform.cc + ) + add_library(mgmol_src ${SOURCES} ${SCALAPACK_SOURCES}) +else() + add_library(mgmol_src ${SOURCES}) +endif(${MGMOL_WITH_SCALAPACK}) target_include_directories(mgmol_src PRIVATE ${HDF5_INCLUDE_DIRS}) target_include_directories(mgmol_src PRIVATE ${Boost_INCLUDE_DIRS}) diff --git a/src/ChebyshevApproximation.cc b/src/ChebyshevApproximation.cc index 9ef65322..e37a4e4a 100644 --- a/src/ChebyshevApproximation.cc +++ b/src/ChebyshevApproximation.cc @@ -8,10 +8,13 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "ChebyshevApproximation.h" -#include "DistMatrix.h" #include "MPIdata.h" #include "ReplicatedMatrix.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#endif + #include template @@ -217,5 +220,7 @@ MatrixType ChebyshevApproximation::computeChebyshevApproximation( return mat; } +#ifdef MGMOL_USE_SCALAPACK template class ChebyshevApproximation>; +#endif template class ChebyshevApproximation; diff --git a/src/DFTsolver.cc b/src/DFTsolver.cc index 55e6df26..debf3cb8 100644 --- a/src/DFTsolver.cc +++ b/src/DFTsolver.cc @@ -13,13 +13,16 @@ #include "DMStrategy.h" #include "Electrostatic.h" #include "Energy.h" -#include "GrassmanCGFactory.h" #include "Ions.h" #include "MGmol.h" #include "Potentials.h" #include "ProjectedMatricesInterface.h" #include "Rho.h" +#ifdef MGMOL_USE_SCALAPACK +#include "GrassmanCGFactory.h" +#endif + template DFTsolver::DFTsolver(Hamiltonian* hamiltonian, ProjectedMatricesInterface* proj_matrices, Energy* energy, @@ -48,6 +51,7 @@ DFTsolver::DFTsolver(Hamiltonian* hamiltonian, break; } +#ifdef MGMOL_USE_SCALAPACK case OuterSolverType::NLCG: { orbitals_stepper_ = GrassmanCGFactory::create( @@ -56,6 +60,7 @@ DFTsolver::DFTsolver(Hamiltonian* hamiltonian, break; } +#endif default: std::cerr << "DFTsolver: Undefined iterative electronic structure " diff --git a/src/DMStrategyFactory.cc b/src/DMStrategyFactory.cc index 115c8cbe..8df599b3 100644 --- a/src/DMStrategyFactory.cc +++ b/src/DMStrategyFactory.cc @@ -1,6 +1,7 @@ #include "DMStrategyFactory.h" #include "ReplicatedMatrix.h" +#ifdef MGMOL_USE_SCALAPACK template <> DMStrategy>* DMStrategyFactory, @@ -35,6 +36,7 @@ DMStrategyFactory, return dm_strategy; } } +#endif template <> DMStrategy>* @@ -56,6 +58,7 @@ DMStrategyFactory, return nullptr; } +#ifdef MGMOL_USE_SCALAPACK template <> DMStrategy>* DMStrategyFactory, @@ -78,6 +81,7 @@ DMStrategyFactory, return dm_strategy; } +#endif template <> DMStrategy>* diff --git a/src/DavidsonSolver.cc b/src/DavidsonSolver.cc index 2b097be9..0ce9cf64 100644 --- a/src/DavidsonSolver.cc +++ b/src/DavidsonSolver.cc @@ -859,6 +859,8 @@ void DavidsonSolver::printTimers(std::ostream& os) target_tm_.print(os); } +#ifdef MGMOL_USE_SCALAPACK template class DavidsonSolver, dist_matrix::DistMatrix>; +#endif template class DavidsonSolver, ReplicatedMatrix>; diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index a82ec288..ef47c26b 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -9,12 +9,17 @@ #include "DensityMatrix.h" -#include "DistMatrix.h" #include "MGmol_MPI.h" #include "ReplicatedMatrix.h" #include "ReplicatedWorkSpace.h" #include "hdf_tools.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#else +typedef double DISTMATDTYPE; +#endif + #include #include #include @@ -477,5 +482,7 @@ int DensityMatrix::read(HDFrestart& h5f_file, std::string& name) return ierr; } +#ifdef MGMOL_USE_SCALAPACK template class DensityMatrix>; +#endif template class DensityMatrix; diff --git a/src/DistMatrix/BlacsContext.cc b/src/DistMatrix/BlacsContext.cc index 7610a3cb..6d056218 100644 --- a/src/DistMatrix/BlacsContext.cc +++ b/src/DistMatrix/BlacsContext.cc @@ -13,11 +13,11 @@ #include #include -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK #include "blacs.h" #endif -#ifndef SCALAPACK +#ifndef MGMOL_USE_SCALAPACK void Cblacs_pinfo(int* mypnum, int* nprocs) { diff --git a/src/DistMatrix/DistMatrix.cc b/src/DistMatrix/DistMatrix.cc index 89f19400..c35d6e74 100644 --- a/src/DistMatrix/DistMatrix.cc +++ b/src/DistMatrix/DistMatrix.cc @@ -23,14 +23,14 @@ #include -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK #include "blacs.h" #endif namespace dist_matrix { -#ifndef SCALAPACK +#ifndef MGMOL_USE_SCALAPACK int NUMROC(int* a, int* b, int* c, int* d, int* e) { return *a; } int INDXL2G(Pint indxloc, Pint nb, Pint iproc, Pint isrcproc, Pint nprocs) { @@ -135,7 +135,7 @@ void DistMatrix::resize(const int m, const int n, const int mb, const int nb) m_ = m; n_ = n; -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK mb_ = std::min(mb, m); nb_ = std::min(nb, n); #else @@ -280,7 +280,7 @@ double DistMatrix::traceProduct(const DistMatrix& x) const tsum = LinearAlgebraUtils::MPdot( size_, &val_[0], &x.val_[0]); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK MGmol_MPI& mmpi = *(MGmol_MPI::instance()); mmpi.allreduce(&tsum, &sum, 1, MPI_SUM); #else @@ -325,7 +325,7 @@ void DistMatrix::axpy(const double alpha, const DistMatrix& x) template <> void DistMatrix::identity(void) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK char uplo = 'n'; double alpha = 0.0; double beta = 1.0; @@ -342,7 +342,7 @@ void DistMatrix::identity(void) template <> void DistMatrix::identity(void) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK char uplo = 'n'; float alpha = 0.0; float beta = 1.0; @@ -371,7 +371,7 @@ DistMatrix& DistMatrix::operator=(const DistMatrix& src) exit(1); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK if (src.mb_ == mb_ && src.nb_ == nb_ && src.bc_ == bc_) #endif { @@ -390,7 +390,7 @@ DistMatrix& DistMatrix::operator=(const DistMatrix& src) memcpy(&val_[0], &src.val_[0], size_ * sizeof(double)); } } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK else { // redistribute using function pdgemr2d @@ -423,7 +423,7 @@ DistMatrix& DistMatrix::operator=(const DistMatrix& src) exit(1); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK if (src.mb_ == mb_ && src.nb_ == nb_ && src.bc_ == bc_) #endif { @@ -442,7 +442,7 @@ DistMatrix& DistMatrix::operator=(const DistMatrix& src) memcpy(&val_[0], &src.val_[0], size_ * sizeof(float)); } } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK else { // redistribute using function pdgemr2d @@ -478,7 +478,7 @@ DistMatrix& DistMatrix::assign( exit(1); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK if (src.bc_ == bc_ && src.m_ == m_ && src.n_ == n_ && src.mb_ == mb_ && src.nb_ == nb_) #endif @@ -493,7 +493,7 @@ DistMatrix& DistMatrix::assign( } } } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK else { // redistribute @@ -528,7 +528,7 @@ DistMatrix& DistMatrix::assign( exit(1); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK if (src.bc_ == bc_ && src.m_ == m_ && src.n_ == n_ && src.mb_ == mb_ && src.nb_ == nb_) #endif @@ -543,7 +543,7 @@ DistMatrix& DistMatrix::assign( } } } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK else { // redistribute @@ -589,7 +589,7 @@ void DistMatrix::gemv(const char transa, const double alpha, assert(a.n() == m_); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdgemv(&transa, &m, &n, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &ione, &beta, &val_[0], &ione, @@ -612,7 +612,7 @@ void DistMatrix::matvec( char transa = 'N'; double alpha = 1.; double beta = 0.; -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdgemv(&transa, &m_, &n_, &alpha, &val_[0], &ione, &ione, desc(), &v.val_[0], &ione, &ione, v.desc(), &ione, &beta, &y.val_[0], &ione, @@ -648,7 +648,7 @@ void DistMatrix::gemv(const char transa, const float alpha, assert(a.n() == m_); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; psgemv(&transa, &m, &n, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &ione, &beta, &val_[0], &ione, @@ -672,7 +672,7 @@ void DistMatrix::symv(const char uplo, const double alpha, { assert(a.n() == m_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdsymv(&uplo, &m_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &ione, &beta, &val_[0], &ione, @@ -696,7 +696,7 @@ void DistMatrix::symv(const char uplo, const float alpha, { assert(a.n() == m_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pssymv(&uplo, &m_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &ione, &beta, &val_[0], &ione, @@ -747,7 +747,7 @@ void DistMatrix::gemm(const char transa, const char transb, assert(k == b.n()); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdgemm(&transa, &transb, &m, &n, &k, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &beta, &val_[0], @@ -793,7 +793,7 @@ void DistMatrix::gemm(const char transa, const char transb, assert(k == b.n()); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; psgemm(&transa, &transb, &m, &n, &k, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &beta, &val_[0], @@ -834,7 +834,7 @@ void DistMatrix::symm(const char side, const char uplo, assert(a.m() == b.n()); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdsymm(&side, &uplo, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &beta, &val_[0], @@ -870,7 +870,7 @@ void DistMatrix::symm(const char side, const char uplo, assert(a.m() == b.n()); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pssymm(&side, &uplo, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &beta, &val_[0], @@ -907,7 +907,7 @@ void DistMatrix::trmm(const char side, const char uplo, { assert(a.n_ == n_); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdtrmm(&side, &uplo, &trans, &diag, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc_, &val_[0], &ione, &ione, desc_); @@ -933,7 +933,7 @@ void DistMatrix::trmm(const char side, const char uplo, const char trans, { assert(a.n_ == n_); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pstrmm(&side, &uplo, &trans, &diag, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc_, &val_[0], &ione, &ione, desc_); @@ -972,7 +972,7 @@ void DistMatrix::trsm(const char side, const char uplo, { assert(a.n_ == n_); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdtrsm(&side, &uplo, &trans, &diag, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc_, &val_[0], &ione, &ione, desc_); @@ -998,7 +998,7 @@ void DistMatrix::trsm(const char side, const char uplo, const char trans, { assert(a.n_ == n_); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pstrsm(&side, &uplo, &trans, &diag, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc_, &val_[0], &ione, &ione, desc_); @@ -1023,7 +1023,7 @@ void DistMatrix::trtrs(const char uplo, const char trans, { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdtrtrs(&uplo, &trans, &diag, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &b.val_[0], &ione, &ione, b.desc_, &info); @@ -1048,7 +1048,7 @@ void DistMatrix::trtrs(const char uplo, const char trans, { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pstrtrs(&uplo, &trans, &diag, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &b.val_[0], &ione, &ione, b.desc_, &info); @@ -1077,7 +1077,7 @@ int DistMatrix::potrf(char uplo) { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdpotrf(&uplo, &m_, val_.data(), &ione, &ione, desc_, &info); #else @@ -1105,7 +1105,7 @@ int DistMatrix::potrf(char uplo) { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pspotrf(&uplo, &m_, &val_[0], &ione, &ione, desc_, &info); #else @@ -1132,7 +1132,7 @@ void DistMatrix::getrf(std::vector& ipiv) int info; if (active_) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; ipiv.resize(mb_ + mloc_); pdgetrf(&m_, &n_, &val_[0], &ione, &ione, desc_, &ipiv[0], &info); @@ -1154,7 +1154,7 @@ void DistMatrix::getrf(std::vector& ipiv) int info; if (active_) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; ipiv.resize(mb_ + mloc_); psgetrf(&m_, &n_, &val_[0], &ione, &ione, desc_, &ipiv[0], &info); @@ -1183,7 +1183,7 @@ void DistMatrix::potrs(char uplo, DistMatrix& b) assert(m_ == n_); assert(b.m_ == m_); assert(b.n_ <= n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdpotrs(&uplo, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &b.val_[0], &ione, &ione, b.desc_, &info); @@ -1207,7 +1207,7 @@ void DistMatrix::potrs(char uplo, DistMatrix& b) assert(m_ == n_); assert(b.m_ == m_); assert(b.n_ <= n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pspotrs(&uplo, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &b.val_[0], &ione, &ione, b.desc_, &info); @@ -1235,7 +1235,7 @@ void DistMatrix::getrs( assert(m_ == n_); assert(b.m_ == m_); assert(b.n_ <= n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; assert((int)ipiv.size() == (mb_ + mloc_)); pdgetrs(&trans, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &ipiv[0], @@ -1263,7 +1263,7 @@ void DistMatrix::getrs( assert(m_ == n_); assert(b.m_ == m_); assert(b.n_ <= n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; assert((int)ipiv.size() == (mb_ + mloc_)); psgetrs(&trans, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &ipiv[0], @@ -1295,7 +1295,7 @@ int DistMatrix::potri(char uplo) { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdpotri(&uplo, &m_, &val_[0], &ione, &ione, desc_, &info); #else @@ -1322,7 +1322,7 @@ int DistMatrix::potri(char uplo) { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pspotri(&uplo, &m_, &val_[0], &ione, &ione, desc_, &info); #else @@ -1350,7 +1350,7 @@ int DistMatrix::trtri(char uplo, char diag) { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdtrtri(&uplo, &diag, &m_, &val_[0], &ione, &ione, desc_, &info); #else @@ -1383,7 +1383,7 @@ double DistMatrix::norm(char ty) double norm_val = 1.; if (active_) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; if (ty == 'I' || ty == 'i') lwork = mloc_; if (ty == '1') lwork = nloc_; @@ -1409,7 +1409,7 @@ double DistMatrix::norm(char ty) float norm_val = 1.; if (active_) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; if (ty == 'I' || ty == 'i') lwork = mloc_; if (ty == '1') lwork = nloc_; @@ -1444,7 +1444,7 @@ double DistMatrix::pocon(char uplo, double anorm) { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; int lwork = 2 * mloc_ + 3 * nloc_ + nb_ * std::min(npcol_, nprow_); lwork = std::max(lwork, 1); @@ -1490,7 +1490,7 @@ double DistMatrix::pocon(char uplo, float anorm) { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; int lwork = 2 * mloc_ + 3 * nloc_ + nb_ * std::min(npcol_, nprow_); lwork = std::max(lwork, 1); @@ -1552,7 +1552,7 @@ void DistMatrix::syrk( k = a.m(); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdsyrk(&uplo, &trans, &n, &k, &alpha, &a.val_[0], &ione, &ione, a.desc(), &beta, &val_[0], &ione, &ione, desc_); @@ -1584,7 +1584,7 @@ void DistMatrix::syrk( k = a.m(); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pssyrk(&uplo, &trans, &n, &k, &alpha, &a.val_[0], &ione, &ione, a.desc(), &beta, &val_[0], &ione, &ione, desc_); @@ -1603,7 +1603,7 @@ template <> void DistMatrix::getsub( const DistMatrix& a, int m, int n, int ia, int ja) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int iap = ia + 1; int jap = ja + 1; assert(n <= n_); @@ -1627,7 +1627,7 @@ template <> void DistMatrix::getsub( const DistMatrix& a, int m, int n, int ia, int ja) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int iap = ia + 1; int jap = ja + 1; assert(n <= n_); @@ -1651,7 +1651,7 @@ template <> void DistMatrix::getsub( const DistMatrix& a, int m, int n, int ia, int ja) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int iap = ia + 1; int jap = ja + 1; assert(n <= n_); @@ -1685,7 +1685,7 @@ void DistMatrix::transpose( assert(a.m() == n_); assert(a.n() == m_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pdtran(&m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &beta, &val_[0], &ione, &ione, desc_); @@ -1714,7 +1714,7 @@ void DistMatrix::transpose( assert(a.m() == n_); assert(a.n() == m_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; pstran(&m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &beta, &val_[0], &ione, &ione, desc_); @@ -1859,7 +1859,7 @@ double DistMatrix::sumProdElements(const DistMatrix& a) const } double sum = 0.; -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK MPI_Allreduce(&tsum, &sum, 1, MPI_DOUBLE, MPI_SUM, comm_global_); #else sum = tsum; @@ -1896,7 +1896,7 @@ DistMatrix::DistMatrix(const std::string& name, const BlacsContext& bc, template <> void DistMatrix::initFromReplicated(double* const src, const int lda) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK // all the nodes send data const int rev = 1; const int ii = 1; @@ -1921,7 +1921,7 @@ void DistMatrix::allgather(double* const a, const int lda) const assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK // all the nodes receive the data const int ii = -1; const int jj = -1; @@ -1970,7 +1970,7 @@ void DistMatrix::allgather(float* const a, const int lda) const assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK const int ii = -1; const int jj = -1; const int rev = 0; @@ -2101,7 +2101,7 @@ double DistMatrix::trace(void) const if (active_) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; trace = pdlatra(&n_, &val_[0], &ione, &ione, desc_); #else @@ -2123,7 +2123,7 @@ double DistMatrix::trace(void) const if (active_) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; trace = (double)pslatra(&n_, &val_[0], &ione, &ione, desc_); #else @@ -2154,7 +2154,7 @@ void DistMatrix::sygst( { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; double scale; pdsygst(&itype, &uplo, &m_, &val_[0], &ione, &ione, desc_, &b.val_[0], @@ -2178,7 +2178,7 @@ void DistMatrix::sygst(int itype, char uplo, const DistMatrix& b) { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ione = 1; float scale; pssygst(&itype, &uplo, &m_, &val_[0], &ione, &ione, desc_, &b.val_[0], @@ -2202,7 +2202,7 @@ void DistMatrix::syev( { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK assert(mb_ > 1); int ione = 1, izero = 0; int nn = std::max(mb_, m_); @@ -2226,7 +2226,7 @@ void DistMatrix::syev( delete[] work; } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK MPI_Bcast(&w[0], m_, MPI_DOUBLE, 0, comm_global_); #endif } @@ -2240,7 +2240,7 @@ void DistMatrix::syev( { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK assert(mb_ > 1); int ione = 1, izero = 0; int nn = std::max(mb_, m_); @@ -2264,7 +2264,7 @@ void DistMatrix::syev( delete[] work; } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK MPI_Bcast(&w[0], m_, MPI_FLOAT, 0, comm_global_); #endif } @@ -2278,7 +2278,7 @@ void DistMatrix::gesvd(char jobu, char jobvt, std::vector& s, { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK assert(mb_ > 1); int ione = 1; int lwork = -1; @@ -2309,7 +2309,7 @@ void DistMatrix::gesvd(char jobu, char jobvt, std::vector& s, delete[] work; } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int m = std::min(m_, n_); MPI_Bcast(&s[0], m, MPI_DOUBLE, 0, comm_global_); #endif @@ -2324,7 +2324,7 @@ void DistMatrix::gesvd(char jobu, char jobvt, std::vector& s, { assert(m_ == n_); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK assert(mb_ > 1); int ione = 1; int lwork = -1; @@ -2355,7 +2355,7 @@ void DistMatrix::gesvd(char jobu, char jobvt, std::vector& s, delete[] work; } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int m = std::min(m_, n_); MPI_Bcast(&s[0], m, MPI_FLOAT, 0, comm_global_); #endif @@ -2387,7 +2387,7 @@ int DistMatrix::iamax(const int j, double& val) { int indx = -1; int incx = 1; -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ix = 1; int jx = y(j) + 1; // C to fortran int proc_col = pc(j); @@ -2414,7 +2414,7 @@ int DistMatrix::iamax(const int j, float& val) { int indx = -1; int incx = 1; -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int ix = 1; int jx = y(j) + 1; // C to fortran int proc_col = pc(j); @@ -2440,7 +2440,7 @@ template <> void DistMatrix::swapColumns(const int j1, const int j2) { if (j1 == j2) return; -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int jx = j1 + 1; // C to Fortran int jy = j2 + 1; // C to Fortran int ione = 1; @@ -2458,7 +2458,7 @@ template <> void DistMatrix::swapColumns(const int j1, const int j2) { if (j1 == j2) return; -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK int jx = j1 + 1; // C to Fortran int jy = j2 + 1; // C to Fortran int ione = 1; diff --git a/src/DistMatrix/DistVector.h b/src/DistMatrix/DistVector.h index d800f89c..ba3b481a 100644 --- a/src/DistMatrix/DistVector.h +++ b/src/DistMatrix/DistVector.h @@ -96,7 +96,7 @@ class DistVector : public DistMatrix DistMatrix::val_.size(), DistMatrix::val_.data(), v.val_.data()); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK MGmol_MPI& mmpi = *(MGmol_MPI::instance()); mmpi.allreduce(&tsum, &sum, 1, MPI_SUM); #else diff --git a/src/DistMatrix/MGmol_scalapack.h b/src/DistMatrix/MGmol_scalapack.h index ae9467a4..9e685c0d 100644 --- a/src/DistMatrix/MGmol_scalapack.h +++ b/src/DistMatrix/MGmol_scalapack.h @@ -6,8 +6,8 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef MGMOL_SCALAPACK_H -#define MGMOL_SCALAPACK_H +#ifndef MGMOL_MGMOL_USE_SCALAPACK_H +#define MGMOL_MGMOL_USE_SCALAPACK_H #include "scalapack_mangle.h" @@ -18,7 +18,7 @@ typedef const float* const Pfloat; extern "C" { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK // PBLAS void pdswap( Pint, double*, Pint, Pint, Pint, Pint, double*, Pint, Pint, Pint, Pint); @@ -60,7 +60,7 @@ extern "C" void pdamax(Pint, double*, int*, double*, Pint, Pint, Pint, Pint); void psamax(Pint, float*, int*, float*, Pint, Pint, Pint, Pint); - // SCALAPACK + // MGMOL_USE_SCALAPACK void pdelset(double*, Pint, Pint, int*, Pdouble); void pselset(float*, Pint, Pint, int*, Pfloat); float pselget(Pchar, Pchar, float*, Pfloat, Pint, Pint, Pint); @@ -120,7 +120,7 @@ extern "C" int*); void psgesvd(Pchar, Pchar, int*, int*, float*, int*, int*, int*, float*, float*, int*, int*, int*, float*, int*, int*, int*, float*, int*, int*); - // SCALAPACK TOOLS + // MGMOL_USE_SCALAPACK TOOLS int NUMROC(Pint, Pint, Pint, Pint, Pint); int INDXL2G(Pint, Pint, Pint, Pint, Pint); int INDXG2L(Pint, Pint, Pint, Pint, Pint); diff --git a/src/DistMatrix/blacs.h b/src/DistMatrix/blacs.h index 6d4d2aa2..f52f9333 100644 --- a/src/DistMatrix/blacs.h +++ b/src/DistMatrix/blacs.h @@ -33,7 +33,7 @@ extern "C" int blacs_pnum(int*, int*, int*); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK extern "C" { #endif @@ -53,7 +53,7 @@ extern "C" void Cblacs_exit(int); int Cblacs_pnum(int, int, int); int Csys2blacs_handle(MPI_Comm); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK } #endif diff --git a/src/EigenDMStrategy.cc b/src/EigenDMStrategy.cc index cf803e0f..58304e9b 100644 --- a/src/EigenDMStrategy.cc +++ b/src/EigenDMStrategy.cc @@ -29,6 +29,7 @@ void EigenDMStrategy::initialize(OrbitalsType& orbitals) template int EigenDMStrategy::update(OrbitalsType& orbitals) { +#ifdef MGMOL_USE_SCALAPACK Control& ct = *(Control::instance()); dist_matrix::DistMatrix zz("Z", ct.numst, ct.numst); @@ -44,6 +45,13 @@ int EigenDMStrategy::update(OrbitalsType& orbitals) orbitals.multiply_by_matrix(zz); orbitals.setDataWithGhosts(); orbitals.trade_boundaries(); +#else + (void)orbitals; + std::cerr << "EigenDMStrategy::update(OrbitalsType& " + "orbitals) not implemented" + << std::endl; + abort(); +#endif return 0; } diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index eaf410de..b82bb31f 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -14,7 +14,9 @@ #include "DotProductManagerFactory.h" #include "GridFunc.h" #include "Laph4M.h" +#ifdef MGMOL_USE_SCALAPACK #include "LocalMatrices2DistMatrix.h" +#endif #include "LocalizationRegions.h" #include "MPIdata.h" #include "Mesh.h" @@ -407,6 +409,7 @@ void ExtendedGridOrbitals::initFourier() resetIterativeIndex(); } +#ifdef MGMOL_USE_SCALAPACK template void ExtendedGridOrbitals::multiply_by_matrix( const dist_matrix::DistMatrix& dmatrix, @@ -425,6 +428,7 @@ void ExtendedGridOrbitals::multiply_by_matrix( multiply_by_matrix(work_matrix, product, ldp); } +#endif template void ExtendedGridOrbitals::multiply_by_matrix( @@ -538,6 +542,7 @@ void ExtendedGridOrbitals::multiply_by_matrix( multiply_by_matrix(matrix, product.psi(0), product.lda_); } +#ifdef MGMOL_USE_SCALAPACK template <> template <> void ExtendedGridOrbitals::multiply_by_matrix( @@ -545,6 +550,7 @@ void ExtendedGridOrbitals::multiply_by_matrix( { multiply_by_DistMatrix(matrix); } +#endif template <> template <> @@ -554,6 +560,7 @@ void ExtendedGridOrbitals::multiply_by_matrix( multiply_by_ReplicatedMatrix(matrix); } +#ifdef MGMOL_USE_SCALAPACK template void ExtendedGridOrbitals::multiply_by_DistMatrix( const dist_matrix::DistMatrix& matrix) @@ -593,6 +600,7 @@ void ExtendedGridOrbitals::multiply_by_DistMatrix( prod_matrix_tm_.stop(); } +#endif template void ExtendedGridOrbitals::multiply_by_ReplicatedMatrix( @@ -1110,6 +1118,7 @@ void ExtendedGridOrbitals::computeDiagonalElementsDotProduct( mmpi.allreduce(&tmp[0], &ss[0], numst_, MPI_SUM); } +#ifdef MGMOL_USE_SCALAPACK template void ExtendedGridOrbitals::computeGram( dist_matrix::DistMatrix& gram_mat) @@ -1141,6 +1150,7 @@ void ExtendedGridOrbitals::computeGram( sl2dm->accumulate(ss, gram_mat); } +#endif // compute the lower-triangular part of the overlap matrix template @@ -1256,6 +1266,7 @@ void ExtendedGridOrbitals::orthonormalizeLoewdin( } if (!multbymat) { +#ifdef MGMOL_USE_SCALAPACK ProjectedMatrices>* projmatrices = dynamic_cast< ProjectedMatrices>*>( @@ -1268,6 +1279,7 @@ void ExtendedGridOrbitals::orthonormalizeLoewdin( projmatrices->setGram2Id(getIterativeIndex()); } +#endif } if (matrixTransform == nullptr) delete localP; @@ -1597,6 +1609,7 @@ void ExtendedGridOrbitals::initRand() resetIterativeIndex(); } +#ifdef MGMOL_USE_SCALAPACK template <> template <> void ExtendedGridOrbitals::addDotWithNcol2Matrix( @@ -1654,6 +1667,7 @@ void ExtendedGridOrbitals::addDotWithNcol2DistMatrix( addDot_tm_.stop(); } +#endif template <> template <> diff --git a/src/ExtendedGridOrbitals.h b/src/ExtendedGridOrbitals.h index b907995b..732c63e0 100644 --- a/src/ExtendedGridOrbitals.h +++ b/src/ExtendedGridOrbitals.h @@ -11,7 +11,9 @@ #define MGMOL_EXTENDEDGRIDORBITALS_H #include "BlockVector.h" +#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" +#endif #include "DotProductManager.h" #include "GridFunc.h" #include "HDFrestart.h" @@ -33,6 +35,9 @@ class ProjectedMatricesInterface; class LocalizationRegions; class ClusterOrbitals; +#ifndef MGMOL_USE_SCALAPACK +typedef double DISTMATDTYPE; +#endif template class ExtendedGridOrbitals : public Orbitals @@ -82,13 +87,16 @@ class ExtendedGridOrbitals : public Orbitals void projectOut(ScalarType* const, const int); void multiply_by_ReplicatedMatrix(const ReplicatedMatrix& matrix); +#ifdef MGMOL_USE_SCALAPACK void multiply_by_DistMatrix( const dist_matrix::DistMatrix& matrix); - +#endif void multiply_by_matrix( const DISTMATDTYPE* const, ScalarType*, const int) const; +#ifdef MGMOL_USE_SCALAPACK void multiply_by_matrix(const dist_matrix::DistMatrix& matrix, ScalarType* const product, const int ldp); +#endif void scal(const int i, const double alpha) { block_vector_.scal(i, alpha); } virtual void assign(const int i, const ScalarType* const v, const int n = 1) { @@ -115,8 +123,10 @@ class ExtendedGridOrbitals : public Orbitals /*! * Specialized functions */ +#ifdef MGMOL_USE_SCALAPACK void addDotWithNcol2DistMatrix( ExtendedGridOrbitals&, dist_matrix::DistMatrix&) const; +#endif void addDotWithNcol2ReplicatedMatrix( ExtendedGridOrbitals&, ReplicatedMatrix&) const; @@ -300,9 +310,11 @@ class ExtendedGridOrbitals : public Orbitals void computeGram(const int verbosity = 0); void computeGramAndInvS(const int verbosity = 0); +#ifdef MGMOL_USE_SCALAPACK void computeGram(dist_matrix::DistMatrix& gram_mat); void computeGram(const ExtendedGridOrbitals& orbitals, dist_matrix::DistMatrix& gram_mat); +#endif ScalarType maxAbsValue() const { return block_vector_.maxAbsValue(); } diff --git a/src/Forces.cc b/src/Forces.cc index ad6b3914..83df8285 100644 --- a/src/Forces.cc +++ b/src/Forces.cc @@ -422,6 +422,7 @@ SquareLocalMatrices Forces::getReplicatedDM() proj_matrices_); if (projmatrices) return projmatrices->getReplicatedDM(); } +#ifdef MGMOL_USE_SCALAPACK { ProjectedMatrices>* projmatrices = dynamic_cast< @@ -430,6 +431,7 @@ SquareLocalMatrices Forces::getReplicatedDM() assert(projmatrices); return projmatrices->getReplicatedDM(); } +#endif } // Get the nl energy as the trace of loc_kbpsi*mat_X for several loc_kbpsi diff --git a/src/GramMatrix.cc b/src/GramMatrix.cc index 1443d450..76fa1419 100644 --- a/src/GramMatrix.cc +++ b/src/GramMatrix.cc @@ -8,13 +8,18 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "GramMatrix.h" +#include "Power.h" +#include "ReplicatedMatrix.h" +#include "ReplicatedVector.h" + +#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" #include "DistMatrix2SquareLocalMatrices.h" #include "DistMatrixTools.h" #include "DistVector.h" -#include "Power.h" -#include "ReplicatedMatrix.h" -#include "ReplicatedVector.h" +#else +typedef double DISTMATDTYPE; +#endif #include #include @@ -109,6 +114,7 @@ void GramMatrix::solveLST(MatrixType& z) const ls_->trtrs('l', 't', 'n', z); } +#ifdef MGMOL_USE_SCALAPACK template <> double GramMatrix>::computeCond() { @@ -135,6 +141,7 @@ double GramMatrix>::computeCond() return cond; } +#endif template <> double GramMatrix::computeCond() @@ -325,8 +332,10 @@ void GramMatrix::applyInv(VectorType& mat) ls_->potrs('l', mat); } +#ifdef MGMOL_USE_SCALAPACK template class GramMatrix>; template void GramMatrix>::applyInv( dist_matrix::DistVector&); +#endif template class GramMatrix; template void GramMatrix::applyInv(ReplicatedVector&); diff --git a/src/GrassmanCG.h b/src/GrassmanCG.h index 01c5b394..4e7770ba 100644 --- a/src/GrassmanCG.h +++ b/src/GrassmanCG.h @@ -10,7 +10,10 @@ #ifndef MGMOL_GRASSMANCG_H #define MGMOL_GRASSMANCG_H +#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" +#endif + #include "GrassmanLineMinimization.h" #include "Hamiltonian.h" #include "Potentials.h" @@ -36,10 +39,12 @@ class GrassmanCG : public GrassmanLineMinimization void conjugate() override; double computeStepSize(T& orbitals) override; +#ifdef MGMOL_USE_SCALAPACK void computeOrbitalsProdWithH( T& orbitals1, T& orbitals2, dist_matrix::DistMatrix& mat); void computeOrbitalsProdWithH( T& orbitals, dist_matrix::DistMatrix& mat); +#endif void parallelTransportUpdate(const double lambda, T& orbitals) override; }; #endif diff --git a/src/Hamiltonian.cc b/src/Hamiltonian.cc index 28aff992..0eb2031d 100644 --- a/src/Hamiltonian.cc +++ b/src/Hamiltonian.cc @@ -171,6 +171,7 @@ void Hamiltonian::applyDeltaPot(const T& phi, T& hphi) // add to hij the elements // corresponding to the local part of the Hamiltonian +#ifdef MGMOL_USE_SCALAPACK template <> template <> void Hamiltonian>::addHlocal2matrix( @@ -206,6 +207,7 @@ void Hamiltonian>::addHlocal2matrix( // hij.print(std::cout, 0, 0, 5, 5); } +#endif template <> template <> diff --git a/src/HamiltonianMVPSolver.cc b/src/HamiltonianMVPSolver.cc index c697b5eb..f94d682e 100644 --- a/src/HamiltonianMVPSolver.cc +++ b/src/HamiltonianMVPSolver.cc @@ -350,15 +350,15 @@ void HamiltonianMVPSolver, ProjectedMatrices>, LocGridOrbitals>; - -template class HamiltonianMVPSolver, - ProjectedMatricesSparse, LocGridOrbitals>; - template class HamiltonianMVPSolver, ProjectedMatrices>, ExtendedGridOrbitals>; +#endif +template class HamiltonianMVPSolver, + ProjectedMatricesSparse, LocGridOrbitals>; template class HamiltonianMVPSolver, ExtendedGridOrbitals>; diff --git a/src/HamiltonianMVP_DMStrategy.cc b/src/HamiltonianMVP_DMStrategy.cc index 1d03a9b3..456e532f 100644 --- a/src/HamiltonianMVP_DMStrategy.cc +++ b/src/HamiltonianMVP_DMStrategy.cc @@ -9,7 +9,6 @@ #include "HamiltonianMVP_DMStrategy.h" #include "Control.h" -#include "DistMatrix.h" #include "HamiltonianMVPSolver.h" #include "Ions.h" #include "LocGridOrbitals.h" @@ -18,6 +17,10 @@ #include "ProjectedMatricesSparse.h" #include "ReplicatedMatrix.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#endif + template HamiltonianMVP_DMStrategy::HamiltonianMVP_DMStrategy(MPI_Comm comm, std::ostream& os, @@ -98,14 +101,15 @@ void HamiltonianMVP_DMStrategyreset(); } +#ifdef MGMOL_USE_SCALAPACK template class HamiltonianMVP_DMStrategy, ProjectedMatrices>, LocGridOrbitals>; -template class HamiltonianMVP_DMStrategy, - ProjectedMatricesSparse, LocGridOrbitals>; - template class HamiltonianMVP_DMStrategy, ProjectedMatrices>, ExtendedGridOrbitals>; +#endif +template class HamiltonianMVP_DMStrategy, + ProjectedMatricesSparse, LocGridOrbitals>; template class HamiltonianMVP_DMStrategy, ExtendedGridOrbitals>; diff --git a/src/KBPsiMatrixSparse.cc b/src/KBPsiMatrixSparse.cc index b12604da..d1cf791a 100644 --- a/src/KBPsiMatrixSparse.cc +++ b/src/KBPsiMatrixSparse.cc @@ -16,7 +16,10 @@ #include "Mesh.h" #include "ProjectedMatrices.h" #include "ReplicatedMatrix.h" + +#ifdef MGMOL_USE_SCALAPACK #include "SquareSubMatrix2DistMatrix.h" +#endif #include @@ -415,6 +418,7 @@ void KBPsiMatrixSparse::computeHvnlMatrix( computeHvnlMatrix(this, ions, proj_matrices); } +#ifdef MGMOL_USE_SCALAPACK template <> void KBPsiMatrixSparse::computeHvnlMatrix( const KBPsiMatrixInterface* const kbpsi2, const Ions& ions, @@ -425,6 +429,7 @@ void KBPsiMatrixSparse::computeHvnlMatrix( SquareSubMatrix2DistMatrix* ss2dm = SquareSubMatrix2DistMatrix::instance(); ss2dm->accumulate(submat, hij, 0.); } +#endif template <> void KBPsiMatrixSparse::computeHvnlMatrix( @@ -696,7 +701,9 @@ template void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, template void KBPsiMatrixSparse::computeAll( const Ions&, ExtendedGridOrbitals&); +#ifdef MGMOL_USE_SCALAPACK template double KBPsiMatrixSparse::getEvnl(const Ions& ions, ProjectedMatrices>* proj_matrices); +#endif template double KBPsiMatrixSparse::getEvnl( const Ions& ions, ProjectedMatrices* proj_matrices); diff --git a/src/LocGridOrbitals.cc b/src/LocGridOrbitals.cc index 8b22ada8..ca827ebe 100644 --- a/src/LocGridOrbitals.cc +++ b/src/LocGridOrbitals.cc @@ -13,7 +13,6 @@ #include "ColoredRegions.h" #include "Control.h" -#include "DistMatrix.h" #include "DotProductManagerFactory.h" #include "FunctionsPacking.h" #include "GridFunc.h" @@ -21,7 +20,6 @@ #include "HDFrestart.h" #include "Laph4M.h" #include "LocGridOrbitals.h" -#include "LocalMatrices2DistMatrix.h" #include "LocalizationRegions.h" #include "MPIdata.h" #include "Masks4Orbitals.h" @@ -36,6 +34,11 @@ #include "lapack_c.h" #include "memory_space.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#include "LocalMatrices2DistMatrix.h" +#endif + #include #include #include @@ -790,6 +793,7 @@ int LocGridOrbitals::packStates( return pack_->chromatic_number(); } +#ifdef MGMOL_USE_SCALAPACK template void LocGridOrbitals::multiply_by_matrix( const dist_matrix::DistMatrix& dmatrix, @@ -804,6 +808,7 @@ void LocGridOrbitals::multiply_by_matrix( multiply_by_matrix(0, chromatic_number_, work_matrix, product, ldp); } +#endif template void LocGridOrbitals::multiply_by_matrix(const int first_color, @@ -984,6 +989,7 @@ void LocGridOrbitals::multiply_by_matrix( 0, chromatic_number_, matrix, product.psi(0), product.lda_); } +#ifdef MGMOL_USE_SCALAPACK template void LocGridOrbitals::multiply_by_matrix( const dist_matrix::DistMatrix& matrix) @@ -1025,6 +1031,7 @@ void LocGridOrbitals::multiply_by_matrix( prod_matrix_tm_.stop(); } +#endif template int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) @@ -1738,6 +1745,7 @@ void LocGridOrbitals::computeDiagonalElementsDotProductLocal( } } +#ifdef MGMOL_USE_SCALAPACK template void LocGridOrbitals::computeGram( dist_matrix::DistMatrix& gram_mat) @@ -1761,6 +1769,7 @@ void LocGridOrbitals::computeGram( sl2dm->accumulate(ss, gram_mat); } +#endif // compute the lower-triangular part of the overlap matrix template @@ -1862,14 +1871,27 @@ void LocGridOrbitals::orthonormalizeLoewdin( localP = new SquareLocalMatrices( subdivx_, chromatic_number_); - ProjectedMatrices>* projmatrices - = dynamic_cast< - ProjectedMatrices>*>( - proj_matrices_); - assert(projmatrices != nullptr); - assert(localP); - projmatrices->computeLoewdinTransform( - *localP, getIterativeIndex(), update_matrices); + // try with ReplicatedMatrix first + ProjectedMatrices* projmatrices + = dynamic_cast*>(proj_matrices_); + if (projmatrices) + { + projmatrices->computeLoewdinTransform( + *localP, getIterativeIndex(), update_matrices); + } +#ifdef MGMOL_USE_SCALAPACK + else + { + ProjectedMatrices>* projmatrices + = dynamic_cast< + ProjectedMatrices>*>( + proj_matrices_); + assert(projmatrices != nullptr); + assert(localP); + projmatrices->computeLoewdinTransform( + *localP, getIterativeIndex(), update_matrices); + } +#endif multiplyByMatrix(*localP); @@ -2453,6 +2475,7 @@ void LocGridOrbitals::initRand() } // Compute nstates column of Psi^T*A*Psi starting at column 0 +#ifdef MGMOL_USE_SCALAPACK template void LocGridOrbitals::addDotWithNcol2Matrix( LocGridOrbitals& Apsi, @@ -2498,6 +2521,7 @@ void LocGridOrbitals::addDotWithNcol2Matrix( addDot_tm_.stop(); } +#endif template void LocGridOrbitals::computeGlobalIndexes( diff --git a/src/LocGridOrbitals.h b/src/LocGridOrbitals.h index ee49497c..36ad8c9b 100644 --- a/src/LocGridOrbitals.h +++ b/src/LocGridOrbitals.h @@ -37,6 +37,10 @@ class MasksSet; class Masks4Orbitals; class ReplicatedMatrix; +#ifndef MGMOL_USE_SCALAPACK +typedef double DISTMATDTYPE; +#endif + template class LocGridOrbitals : public Orbitals { @@ -103,8 +107,10 @@ class LocGridOrbitals : public Orbitals const DISTMATDTYPE* const matrix, LocGridOrbitals& product) const; void multiply_by_matrix(const int, const int, const DISTMATDTYPE* const, ScalarType*, const int) const; +#ifdef MGMOL_USE_SCALAPACK void multiply_by_matrix(const dist_matrix::DistMatrix& matrix, ScalarType* const product, const int ldp); +#endif void scal(const int i, const double alpha) { block_vector_.scal(i, alpha); } virtual void assign(const int i, const ScalarType* const v, const int n = 1) { @@ -326,9 +332,11 @@ class LocGridOrbitals : public Orbitals void computeGram(const int verbosity = 0); void computeGramAndInvS(const int verbosity = 0); +#ifdef MGMOL_USE_SCALAPACK void computeGram(dist_matrix::DistMatrix& gram_mat); void computeGram(const LocGridOrbitals& orbitals, dist_matrix::DistMatrix& gram_mat); +#endif ScalarType maxAbsValue() const { return block_vector_.maxAbsValue(); } @@ -354,8 +362,10 @@ class LocGridOrbitals : public Orbitals void getLocalOverlap(const LocGridOrbitals& orbitals, SquareLocalMatrices&); +#ifdef MGMOL_USE_SCALAPACK void addDotWithNcol2Matrix( LocGridOrbitals&, dist_matrix::DistMatrix&) const; +#endif void addDotWithNcol2Matrix(LocGridOrbitals&, ReplicatedMatrix&) const { std::cerr << "LocGridOrbitals::addDotWithNcol2Matrix not implemented " @@ -405,7 +415,9 @@ class LocGridOrbitals : public Orbitals LocGridOrbitals& product) const; void multiply_by_matrix( const DISTMATDTYPE* const matrix, LocGridOrbitals& product) const; +#ifdef MGMOL_USE_SCALAPACK void multiply_by_matrix(const dist_matrix::DistMatrix&); +#endif void multiplyByMatrix2states(const int st1, const int st2, const double* mat, LocGridOrbitals& product); diff --git a/src/LocalizationRegions.cc b/src/LocalizationRegions.cc index 47aaf209..1f9436f7 100644 --- a/src/LocalizationRegions.cc +++ b/src/LocalizationRegions.cc @@ -12,13 +12,16 @@ #include "LocGridOrbitals.h" #include "LocalizationRegions.h" #include "Mesh.h" -#include "OrbitalsTransform.h" #include "SpreadsAndCenters.h" #include "SquareLocalMatrices.h" #include "SymmetricPair.h" #include "hdf_tools.h" #include "tools.h" +#ifdef MGMOL_USE_SCALAPACK +#include "OrbitalsTransform.h" +#endif + #include #include #include @@ -389,6 +392,7 @@ float LocalizationRegions::updateRadiiConstVol(const SpreadsAndCenters& sc) return updateRadii(sc, ratio); } +#ifdef MGMOL_USE_SCALAPACK float LocalizationRegions::updateRadii( const OrbitalsTransform* ot, const float ratio) { @@ -401,6 +405,7 @@ float LocalizationRegions::updateRadii( return getMeanRadius(); } +#endif float LocalizationRegions::moveTo(const vector& target_centers) { diff --git a/src/LocalizationRegions.h b/src/LocalizationRegions.h index f835be49..702430ce 100644 --- a/src/LocalizationRegions.h +++ b/src/LocalizationRegions.h @@ -22,11 +22,14 @@ #include "mgmol_mpi_tools.h" #include "tools.h" +#ifdef MGMOL_USE_SCALAPACK +#include "OrbitalsTransform.h" +#endif + #include #include #include -class OrbitalsTransform; class SymmetricPair; typedef struct LRData @@ -398,7 +401,9 @@ class LocalizationRegions template float move(const SpreadsAndCenters& sc, const bool flag = false); +#ifdef MGMOL_USE_SCALAPACK float updateRadii(const OrbitalsTransform* ot, const float ratio); +#endif template float updateRadii(const SpreadsAndCenters& sc, const float ratio); template diff --git a/src/MGmol.cc b/src/MGmol.cc index 60771f0d..8e9df794 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -22,8 +22,6 @@ #include "DFTsolver.h" #include "DMStrategyFactory.h" #include "DavidsonSolver.h" -#include "DistMatrix.h" -#include "DistMatrix2SquareLocalMatrices.h" #include "Electrostatic.h" #include "Energy.h" #include "EnergySpreadPenalty.h" @@ -31,7 +29,6 @@ #include "FDoper.h" #include "FIRE.h" #include "Forces.h" -#include "GrassmanLineMinimization.h" #include "GridFunc.h" #include "HDFrestart.h" #include "Hamiltonian.h" @@ -45,7 +42,6 @@ #include "MGOrbitalsPreconditioning.h" #include "MGkernels.h" #include "MGmol.h" -#include "MLWFTransform.h" #include "MPIdata.h" #include "MVPSolver.h" #include "MasksSet.h" @@ -63,16 +59,24 @@ #include "ReplicatedVector.h" #include "Rho.h" #include "SP2.h" -#include "SparseDistMatrix.h" #include "SpreadPenalty.h" #include "SpreadPenaltyVolume.h" #include "SpreadsAndCenters.h" -#include "SubMatrices.h" #include "SubspaceProjector.h" #include "XCfunctionalFactory.h" #include "XConGrid.h" #include "manage_memory.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#include "DistMatrix2SquareLocalMatrices.h" +#include "DistVector.h" +#include "GrassmanLineMinimization.h" +#include "MLWFTransform.h" +#include "SparseDistMatrix.h" +#include "SubMatrices.h" +#endif + namespace mgmol { std::ostream* out = nullptr; @@ -100,8 +104,6 @@ extern Timer loopdot_tm; extern Timer loopaxpy_tm; extern Timer loopscal_tm; extern Timer loopcp_tm; -extern Timer get_NOLMO_tm; -extern Timer get_MLWF_tm; extern Timer md_iterations_tm; extern Timer md_tau_tm; extern Timer md_moveVnuc_tm; @@ -112,6 +114,10 @@ extern Timer quench_tm; extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; extern Timer updateCenters_tm; +#ifdef MGMOL_USE_SCALAPACK +extern Timer get_NOLMO_tm; +extern Timer get_MLWF_tm; +#endif #include "mgmol_Signal.h" std::set Signal::recv_; @@ -249,9 +255,15 @@ int MGmol::initial() new ProjectedMatricesMehrstellen( ct.numst, with_spin, ct.occ_width)); else + { +#ifdef MGMOL_USE_SCALAPACK proj_matrices_.reset(new ProjectedMatricesMehrstellen< dist_matrix::DistMatrix>( ct.numst, with_spin, ct.occ_width)); +#else + std::cerr << "Not implemented" << std::endl; +#endif + } } else if (ct.short_sighted) proj_matrices_.reset(new ProjectedMatricesSparse( @@ -260,9 +272,15 @@ int MGmol::initial() proj_matrices_.reset(new ProjectedMatrices( ct.numst, with_spin, ct.occ_width)); else + { +#ifdef MGMOL_USE_SCALAPACK proj_matrices_.reset( new ProjectedMatrices>( ct.numst, with_spin, ct.occ_width)); +#else + std::cerr << "Not implemented" << std::endl; +#endif + } forces_.reset(new Forces( hamiltonian_.get(), rho_.get(), proj_matrices_.get())); @@ -464,10 +482,14 @@ int MGmol::initial() hamiltonian_.get(), this, proj_matrices_.get(), current_orbitals_)); else + { +#ifdef MGMOL_USE_SCALAPACK dm_strategy_.reset(DMStrategyFactory>::create(comm_, os_, *ions_, rho_.get(), energy_.get(), electrostat_.get(), hamiltonian_.get(), this, proj_matrices_.get(), current_orbitals_)); +#endif + } // theta = invB * Hij proj_matrices_->updateThetaAndHB(); @@ -571,12 +593,17 @@ void MGmol::printMM() std::ofstream tfile("s.mm", std::ios::out); proj_matrices_->printGramMM(tfile); std::ofstream tfileh("h.mm", std::ios::out); +#ifdef MGMOL_USE_SCALAPACK ProjectedMatrices>* projmatrices = dynamic_cast< ProjectedMatrices>*>( proj_matrices_.get()); assert(projmatrices != nullptr); projmatrices->printHamiltonianMM(tfileh); +#else + std::cerr << "MGmol::printMM() not implemented" + << std::endl; +#endif } } @@ -633,6 +660,7 @@ void MGmol::write_header() << (omp_get_max_threads() > 1 ? "s " : " "); os_ << "active" << std::endl << std::endl; #endif +#ifdef MGMOL_USE_SCALAPACK if (!ct.rmatrices) { os_ << " ScaLapack block size: " @@ -645,6 +673,7 @@ void MGmol::write_header() MatricesBlacsContext& mbc(MatricesBlacsContext::instance()); mbc.print(os_); } +#endif ct.printPoissonOptions(os_); } // onpe0 @@ -745,20 +774,17 @@ void MGmol::printEigAndOcc() && ct.occupationWidthIsZero()) && onpe0) { - bool printflag = false; // try with ReplicatedMatrix first + std::shared_ptr> projmatrices + = std::dynamic_pointer_cast>( + proj_matrices_); + if (projmatrices) { - std::shared_ptr> projmatrices - = std::dynamic_pointer_cast< - ProjectedMatrices>(proj_matrices_); - if (projmatrices) - { - projmatrices->printEigenvalues(os_); - projmatrices->printOccupations(os_); - printflag = true; - } + projmatrices->printEigenvalues(os_); + projmatrices->printOccupations(os_); } - if (!printflag) +#ifdef MGMOL_USE_SCALAPACK + else { std::shared_ptr< ProjectedMatrices>> @@ -770,6 +796,7 @@ void MGmol::printEigAndOcc() projmatrices->printEigenvalues(os_); projmatrices->printOccupations(os_); } +#endif } } @@ -870,14 +897,6 @@ void MGmol::printTimers() loopscal_tm.print(os_); loopdot_tm.print(os_); - dist_matrix::SubMatrices::printTimers(os_); - - DistMatrix2SquareLocalMatrices::printTimers(os_); - - dist_matrix::SparseDistMatrix::printTimers(os_); - - dist_matrix::DistMatrix::printTimers(os_); - ReplicatedMatrix2SquareLocalMatrices::printTimers(os_); LocalMatrices2ReplicatedMatrix::printTimers(os_); @@ -894,8 +913,15 @@ void MGmol::printTimers() get_res_tm_.print(os_); comp_res_tm_.print(os_); vnlpsi_tm.print(os_); +#ifdef MGMOL_USE_SCALAPACK + dist_matrix::SubMatrices::printTimers(os_); + DistMatrix2SquareLocalMatrices::printTimers(os_); + dist_matrix::SparseDistMatrix::printTimers(os_); + dist_matrix::DistMatrix::printTimers(os_); + get_MLWF_tm.print(os_); get_NOLMO_tm.print(os_); +#endif Energy::eval_te_tm().print(os_); Electrostatic::solve_tm().print(os_); PoissonInterface::printTimers(os_); @@ -904,8 +930,10 @@ void MGmol::printTimers() ShortSightedInverse::printTimers(os_); if (std::is_same>::value) { +#ifdef MGMOL_USE_SCALAPACK MVPSolver, dist_matrix::DistMatrix>::printTimers(os_); +#endif MVPSolver, ReplicatedMatrix>::printTimers(os_); } @@ -925,8 +953,10 @@ void MGmol::printTimers() forces_->printTimers(os_); if (ct.OuterSolver() == OuterSolverType::ABPG) ABPG::printTimers(os_); +#ifdef MGMOL_USE_SCALAPACK else if (ct.OuterSolver() == OuterSolverType::NLCG) GrassmanLineMinimization::printTimers(os_); +#endif adaptLR_tm_.print(os_); updateCenters_tm.print(os_); md_iterations_tm.print(os_); @@ -950,8 +980,10 @@ void MGmol::printTimers() DavidsonSolver, ReplicatedMatrix>::printTimers(os_); ChebyshevApproximation::printTimers(os_); +#ifdef MGMOL_USE_SCALAPACK PowerGen, dist_matrix::DistVector>::printTimers(os_); +#endif BlockVector::printTimers(os_); if (ct.rmatrices) { @@ -959,6 +991,7 @@ void MGmol::printTimers() ReplicatedMatrix>::printTimers(os_); ChebyshevApproximation::printTimers(os_); } +#ifdef MGMOL_USE_SCALAPACK else { DavidsonSolver, @@ -966,6 +999,7 @@ void MGmol::printTimers() ChebyshevApproximation< dist_matrix::DistMatrix>::printTimers(os_); } +#endif MGOrbitalsPreconditioning::printTimers(os_); MGOrbitalsPreconditioning::printTimers(os_); MDfiles::printTimers(os_); @@ -1020,7 +1054,7 @@ double MGmol::get_evnl(const Ions& ions) evnl_tm_.start(); Control& ct = *(Control::instance()); - double val; + double val = std::numeric_limits::signaling_NaN(); if (ct.short_sighted) { std::shared_ptr projmatrices @@ -1041,6 +1075,7 @@ double MGmol::get_evnl(const Ions& ions) val = g_kbpsi_->getEvnl(ions, projmatrices.get()); } +#ifdef MGMOL_USE_SCALAPACK else { std::shared_ptr< @@ -1052,6 +1087,7 @@ double MGmol::get_evnl(const Ions& ions) val = g_kbpsi_->getEvnl(ions, projmatrices.get()); } +#endif } evnl_tm_.stop(); @@ -1518,6 +1554,7 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, dm_strategy->update(*dorbitals); } +#ifdef MGMOL_USE_SCALAPACK else { std::shared_ptr> dm_strategy( @@ -1528,6 +1565,7 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, dm_strategy->update(*dorbitals); } +#endif // evaluate energy and forces double ts = 0.; diff --git a/src/MGmol.h b/src/MGmol.h index 2975fe42..ff789d56 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -128,11 +128,13 @@ class MGmol : public MGmolInterface Rho& rho, const bool write_extrapolated_wf, const short count); +#ifdef MGMOL_USE_SCALAPACK void swapColumnsVect(dist_matrix::DistMatrix& evect, const dist_matrix::DistMatrix& hb2N, const std::vector& eval, dist_matrix::DistMatrix& work); void wftransform(OrbitalsType*, OrbitalsType*, Ions&); +#endif int readLRsFromInput(std::ifstream* tfile); void preWFextrapolation(); void postWFextrapolation(OrbitalsType* orbitals); @@ -233,6 +235,7 @@ class MGmol : public MGmolInterface const KBPsiMatrixSparse* const kbpsi_j, MatrixType& mat, const bool consolidate); +#ifdef MGMOL_USE_SCALAPACK void computeHij_private(OrbitalsType& orbitals_i, OrbitalsType& orbitals_j, const Ions& ions, const KBPsiMatrixSparse* const kbpsi_i, const KBPsiMatrixSparse* const kbpsi_j, @@ -245,6 +248,7 @@ class MGmol : public MGmolInterface void computeHij_private(OrbitalsType& orbitals_i, OrbitalsType& orbitals_j, const Ions& ions, const KBPsiMatrixSparse* const kbpsi_i, dist_matrix::DistMatrix& mat); +#endif void computeHij(OrbitalsType& orbitals_i, OrbitalsType& orbitals_j, const Ions& ions, const KBPsiMatrixSparse* const kbpsi, @@ -281,8 +285,12 @@ class MGmol : public MGmolInterface void addResidualSpreadPenalty(OrbitalsType& phi, OrbitalsType& res); int get_NOLMO(NOLMOTransform& noot, OrbitalsType& orbitals, OrbitalsType& work_orbitals, const double dd, const bool apply_flag); - void adaptLR(const SpreadsAndCenters* spreadf, - const OrbitalsTransform* ot); + void adaptLR(const SpreadsAndCenters* spreadf +#ifdef MGMOL_USE_SCALAPACK + , + const OrbitalsTransform* ot +#endif + ); int update_masks(); void move_orbitals(OrbitalsType** orbitals); int getMLWF2states(const int st1, const int st2, OrbitalsType& orbitals, diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 04e4954b..a26e08f1 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -10,7 +10,6 @@ #include "MVPSolver.h" #include "Control.h" -#include "DistMatrix.h" #include "Electrostatic.h" #include "Energy.h" #include "Ions.h" @@ -23,6 +22,10 @@ #include "Rho.h" #include "tools.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#endif + #include template @@ -411,10 +414,11 @@ void MVPSolver::printTimers(std::ostream& os) target_tm_.print(os); } +#ifdef MGMOL_USE_SCALAPACK template class MVPSolver, dist_matrix::DistMatrix>; -template class MVPSolver, ReplicatedMatrix>; - template class MVPSolver, dist_matrix::DistMatrix>; +#endif +template class MVPSolver, ReplicatedMatrix>; template class MVPSolver, ReplicatedMatrix>; diff --git a/src/MVP_DMStrategy.cc b/src/MVP_DMStrategy.cc index 09657755..4dcbaf78 100644 --- a/src/MVP_DMStrategy.cc +++ b/src/MVP_DMStrategy.cc @@ -72,10 +72,11 @@ void MVP_DMStrategy::dressDM() if (use_old_dm_) proj_matrices_->dressupDM(); } +#ifdef MGMOL_USE_SCALAPACK template class MVP_DMStrategy, dist_matrix::DistMatrix>; -template class MVP_DMStrategy, ReplicatedMatrix>; - template class MVP_DMStrategy, dist_matrix::DistMatrix>; +#endif +template class MVP_DMStrategy, ReplicatedMatrix>; template class MVP_DMStrategy, ReplicatedMatrix>; diff --git a/src/OrbitalsExtrapolationOrder2.cc b/src/OrbitalsExtrapolationOrder2.cc index 4fdea147..037fc84e 100644 --- a/src/OrbitalsExtrapolationOrder2.cc +++ b/src/OrbitalsExtrapolationOrder2.cc @@ -9,17 +9,21 @@ #include "OrbitalsExtrapolationOrder2.h" #include "Control.h" -#include "DistMatrixTools.h" #include "ExtendedGridOrbitals.h" #include "LocGridOrbitals.h" #include "ProjectedMatrices.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrixTools.h" +#endif + template void OrbitalsExtrapolationOrder2::extrapolate_orbitals( OrbitalsType** orbitals, OrbitalsType* new_orbitals) { Control& ct = *(Control::instance()); +#ifdef MGMOL_USE_SCALAPACK bool use_dense_proj_mat = false; if (ct.OuterSolver() != OuterSolverType::ABPG && ct.OuterSolver() != OuterSolverType::NLCG) @@ -31,6 +35,7 @@ void OrbitalsExtrapolationOrder2::extrapolate_orbitals( proj_matrices)) use_dense_proj_mat = true; } +#endif new_orbitals->assign(**orbitals); @@ -42,7 +47,8 @@ void OrbitalsExtrapolationOrder2::extrapolate_orbitals( if (ct.verbose > 1 && onpe0) (*MPIdata::sout) << "Extrapolate orbitals order 2..." << std::endl; - // align orbitals_minus1_ with new_orbitals + // align orbitals_minus1_ with new_orbitals +#ifdef MGMOL_USE_SCALAPACK if (use_dense_proj_mat) { dist_matrix::DistMatrix matQ("Q", ct.numst, ct.numst); @@ -56,9 +62,9 @@ void OrbitalsExtrapolationOrder2::extrapolate_orbitals( orbitals_minus1->axpy((ORBDTYPE)-1., *new_orbitals); orbitals_minus1->multiply_by_matrix(yyt); } - else - { // !use_dense_proj_mat - + else // !use_dense_proj_mat +#endif + { new_orbitals->scal(2.); } new_orbitals->axpy((ORBDTYPE)-1., *orbitals_minus1); diff --git a/src/OrbitalsExtrapolationOrder3.cc b/src/OrbitalsExtrapolationOrder3.cc index e0586bab..270bb5c2 100644 --- a/src/OrbitalsExtrapolationOrder3.cc +++ b/src/OrbitalsExtrapolationOrder3.cc @@ -8,11 +8,14 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "OrbitalsExtrapolationOrder3.h" -#include "DistMatrixTools.h" #include "ExtendedGridOrbitals.h" #include "LocGridOrbitals.h" #include "ProjectedMatrices.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrixTools.h" +#endif + template void OrbitalsExtrapolationOrder3::extrapolate_orbitals( OrbitalsType** orbitals, OrbitalsType* new_orbitals) @@ -21,6 +24,7 @@ void OrbitalsExtrapolationOrder3::extrapolate_orbitals( new_orbitals->assign(**orbitals); +#ifdef MGMOL_USE_SCALAPACK bool use_dense_proj_mat = false; if (ct.OuterSolver() != OuterSolverType::ABPG && ct.OuterSolver() != OuterSolverType::NLCG) @@ -32,6 +36,7 @@ void OrbitalsExtrapolationOrder3::extrapolate_orbitals( proj_matrices)) use_dense_proj_mat = true; } +#endif // do the extrapolation if previous orbitals exist (not at first step) @@ -42,7 +47,8 @@ void OrbitalsExtrapolationOrder3::extrapolate_orbitals( if (ct.verbose > 1 && onpe0) (*MPIdata::sout) << "Extrapolate orbitals using 3rd order scheme..." << std::endl; - // align orbitals_minus1 with new_orbitals + // align orbitals_minus1 with new_orbitals +#ifdef MGMOL_USE_SCALAPACK if (use_dense_proj_mat) { dist_matrix::DistMatrix matQ("Q", ct.numst, ct.numst); @@ -72,6 +78,7 @@ void OrbitalsExtrapolationOrder3::extrapolate_orbitals( } } else +#endif { tmp_orbitals_minus1.assign(*orbitals_minus1_); if (orbitals_minus2_ != nullptr) @@ -103,10 +110,6 @@ void OrbitalsExtrapolationOrder3::extrapolate_orbitals( orbitals_minus1_ = *orbitals; - if (use_dense_proj_mat) - { - } - *orbitals = new_orbitals; (*orbitals)->incrementIterativeIndex(); diff --git a/src/PolakRibiereSolver.cc b/src/PolakRibiereSolver.cc index b76d3bf0..e410930c 100644 --- a/src/PolakRibiereSolver.cc +++ b/src/PolakRibiereSolver.cc @@ -499,6 +499,7 @@ int PolakRibiereSolver::solve(OrbitalsType& orbitals, orbitals.computeGramAndInvS(); } +#ifdef MGMOL_USE_SCALAPACK // rotate pairs if smallest eigenvalue of overlap matrix below threshold if (ct.getThresholdEigenvalueGramQuench() > 0. && wolfe) { @@ -517,6 +518,7 @@ int PolakRibiereSolver::solve(OrbitalsType& orbitals, << std::endl; } } +#endif // rebuild dm with new overlap matrix dm_strategy_->dressDM(); diff --git a/src/Power.cc b/src/Power.cc index 58e268d0..b2855b29 100644 --- a/src/Power.cc +++ b/src/Power.cc @@ -15,6 +15,11 @@ #include "mputils.h" #include "random.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#include "DistVector.h" +#endif + // compute sum of squares of elements of vector y-theta*v template double diff2(VECTOR& y, VECTOR& v, const double theta, const bool verbose) @@ -97,6 +102,8 @@ void Power::computeEigenInterval(const MATRIX& A, double& emin, template class Power, SquareLocalMatrices>; +#ifdef MGMOL_USE_SCALAPACK template class Power, dist_matrix::DistMatrix>; +#endif // template class Power; diff --git a/src/Power.h b/src/Power.h index 7a929941..b296c7b5 100644 --- a/src/Power.h +++ b/src/Power.h @@ -7,7 +7,6 @@ #ifndef MGMOL_POWER_H #define MGMOL_POWER_H -#include "DistVector.h" #include "Timer.h" #include "random.h" #include diff --git a/src/PowerGen.cc b/src/PowerGen.cc index 3bd9dc74..f2eaf6c5 100644 --- a/src/PowerGen.cc +++ b/src/PowerGen.cc @@ -8,14 +8,17 @@ #include "PowerGen.h" #include "Control.h" -#include "DistMatrix.h" -#include "DistVector.h" #include "GramMatrix.h" #include "ReplicatedMatrix.h" #include "ReplicatedVector.h" #include "mputils.h" #include "random.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#include "DistVector.h" +#endif + /* Use the power method to compute the extents of the spectrum of the * generalized eigenproblem. In order to use a residual-based convergence * criterion in an efficient way, we delay normalization of the vectors to avoid @@ -185,6 +188,8 @@ void PowerGen::computeGenEigenInterval(MatrixType& mat, compute_tm_.stop(); } +#ifdef MGMOL_USE_SCALAPACK template class PowerGen, dist_matrix::DistVector>; +#endif template class PowerGen; diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index 8a3b773a..c1160ebc 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -11,10 +11,7 @@ #include "Control.h" #include "DensityMatrix.h" -#include "DistMatrix2SquareLocalMatrices.h" -#include "DistMatrixTools.h" #include "HDFrestart.h" -#include "LocalMatrices2DistMatrix.h" #include "LocalMatrices2ReplicatedMatrix.h" #include "MGmol_MPI.h" #include "Orbitals.h" @@ -25,11 +22,20 @@ #include "ReplicatedVector.h" #include "ReplicatedWorkSpace.h" #include "SP2.h" -#include "SparseDistMatrix.h" -#include "SquareSubMatrix2DistMatrix.h" #include "fermi.h" #include "hdf_tools.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix2SquareLocalMatrices.h" +#include "DistMatrixTools.h" +#include "DistVector.h" +#include "LocalMatrices2DistMatrix.h" +#include "SparseDistMatrix.h" +#include "SquareSubMatrix2DistMatrix.h" +#else +typedef double DISTMATDTYPE; +#endif + #include #include @@ -45,7 +51,9 @@ template DensityMatrix* ProjectedMatrices::dm_4dot_product_ = nullptr; +#ifdef MGMOL_USE_SCALAPACK static int sparse_distmatrix_nb_partitions = 128; +#endif template <> std::string ProjectedMatrices::getMatrixType() @@ -53,12 +61,15 @@ std::string ProjectedMatrices::getMatrixType() return "ReplicatedMatrix"; } +#ifdef MGMOL_USE_SCALAPACK template <> std::string ProjectedMatrices>::getMatrixType() { return "DistMatrix"; } +#endif +#ifdef MGMOL_USE_SCALAPACK // // conversion functions from one matrix format into another // @@ -82,6 +93,7 @@ void convert_matrix(const dist_matrix::DistMatrix& src, dst.assign(tmp); } #endif +#endif #ifndef HAVE_MAGMA void convert_matrix(const ReplicatedMatrix& src, @@ -146,6 +158,7 @@ ProjectedMatrices::~ProjectedMatrices() n_instances_--; } +#ifdef MGMOL_USE_SCALAPACK template <> void ProjectedMatrices>::convert( const SquareLocalMatrices& src, @@ -154,6 +167,7 @@ void ProjectedMatrices>::convert( LocalMatrices2DistMatrix* sl2dm = LocalMatrices2DistMatrix::instance(); sl2dm->accumulate(src, dst); } +#endif template <> void ProjectedMatrices::convert( @@ -166,6 +180,7 @@ void ProjectedMatrices::convert( sl2rm->accumulate(src, dst); } +#ifdef MGMOL_USE_SCALAPACK template <> void ProjectedMatrices>:: setupGlobalIndexes(const std::vector>& global_indexes) @@ -177,6 +192,7 @@ void ProjectedMatrices>:: comm, global_indexes, gm_->getMatrix()); LocalMatrices2DistMatrix::setup(comm, global_indexes); } +#endif template <> void ProjectedMatrices::setupGlobalIndexes( @@ -1188,6 +1204,7 @@ ProjectedMatrices::computeChemicalPotentialAndDMwithChebyshev( /* Use the power method to compute the extents of the spectrum of the * generalized eigenproblem. */ +#ifdef MGMOL_USE_SCALAPACK template <> void ProjectedMatrices>:: computeGenEigenInterval( @@ -1201,6 +1218,7 @@ void ProjectedMatrices>:: power.computeGenEigenInterval(mat, *gm_, interval, maxits, pad); } +#endif template <> void ProjectedMatrices::computeGenEigenInterval( @@ -1213,6 +1231,7 @@ void ProjectedMatrices::computeGenEigenInterval( power.computeGenEigenInterval(mat, *gm_, interval, maxits, pad); } +#ifdef MGMOL_USE_SCALAPACK template <> void ProjectedMatrices>::consolidateH() { @@ -1234,6 +1253,7 @@ void ProjectedMatrices>::consolidateH() consolidate_H_tm_.stop(); } +#endif template <> void ProjectedMatrices::consolidateH() @@ -1256,6 +1276,7 @@ void ProjectedMatrices::updateSubMatX(const MatrixType& dm) convert_matrix(dm, *localX_); } +#ifdef MGMOL_USE_SCALAPACK template <> SquareLocalMatrices ProjectedMatrices>::getReplicatedDM() @@ -1266,6 +1287,7 @@ ProjectedMatrices>::getReplicatedDM() return sldm; } +#endif template <> SquareLocalMatrices @@ -1278,5 +1300,7 @@ ProjectedMatrices::getReplicatedDM() return sldm; } +#ifdef MGMOL_USE_SCALAPACK template class ProjectedMatrices>; +#endif template class ProjectedMatrices; diff --git a/src/ProjectedMatrices2N.cc b/src/ProjectedMatrices2N.cc index d13196b2..60f2cc80 100644 --- a/src/ProjectedMatrices2N.cc +++ b/src/ProjectedMatrices2N.cc @@ -8,9 +8,12 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "ProjectedMatrices2N.h" -#include "DistMatrix.h" #include "ReplicatedMatrix.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#endif + template ProjectedMatrices2N::ProjectedMatrices2N( const int ndim, const bool with_spin, const double width) @@ -67,5 +70,7 @@ void ProjectedMatrices2N::iterativeUpdateDMwithEigenstates( ProjectedMatrices::buildDM(*work2N_); } +#ifdef MGMOL_USE_SCALAPACK template class ProjectedMatrices2N>; +#endif template class ProjectedMatrices2N; diff --git a/src/ProjectedMatricesMehrstellen.cc b/src/ProjectedMatricesMehrstellen.cc index b12aee94..2f890487 100644 --- a/src/ProjectedMatricesMehrstellen.cc +++ b/src/ProjectedMatricesMehrstellen.cc @@ -8,9 +8,12 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "ProjectedMatricesMehrstellen.h" +#include "ReplicatedMatrix.h" + +#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" #include "DistMatrixTools.h" -#include "ReplicatedMatrix.h" +#endif template ProjectedMatricesMehrstellen::ProjectedMatricesMehrstellen( @@ -97,6 +100,8 @@ void ProjectedMatricesMehrstellen::rotateAll( ProjectedMatrices::dm_->rotate(rotation_matrix, flag_eigen); } +#ifdef MGMOL_USE_SCALAPACK template class ProjectedMatricesMehrstellen< dist_matrix::DistMatrix>; +#endif template class ProjectedMatricesMehrstellen; diff --git a/src/ReplicatedMatrix.cc b/src/ReplicatedMatrix.cc index 885372d1..04c56e0d 100644 --- a/src/ReplicatedMatrix.cc +++ b/src/ReplicatedMatrix.cc @@ -600,7 +600,8 @@ void ReplicatedMatrix::syev( &lwork, &info); #endif if (info != 0) - std::cerr << "magma_dsyevd_gpu failed, info = " << info << std::endl; + std::cerr << "ReplicatedMatrix::syev() failed, info = " << info + << std::endl; // for(auto& d : evals)std::cout<::setUpperTriangularSquareMatrixToZero() } } +#ifdef MGMOL_USE_SCALAPACK template void ReplicatedWorkSpace::initSquareMatrix( const dist_matrix::DistMatrix& distmat) { distmat.allgather(square_matrix_, ndim_); } +#endif template void ReplicatedWorkSpace::initSquareMatrix( diff --git a/src/ReplicatedWorkSpace.h b/src/ReplicatedWorkSpace.h index c4d66f59..d5eaddcc 100644 --- a/src/ReplicatedWorkSpace.h +++ b/src/ReplicatedWorkSpace.h @@ -10,10 +10,13 @@ #ifndef MGMOL_REPLICATED_WORKSPACE_H #define MGMOL_REPLICATED_WORKSPACE_H -#include "DistMatrix.h" #include "ReplicatedMatrix.h" #include "Timer.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#endif + #include class ReplicatedMatrix; @@ -73,7 +76,9 @@ class ReplicatedWorkSpace void setUpperTriangularSquareMatrixToZero(); +#ifdef MGMOL_USE_SCALAPACK void initSquareMatrix(const dist_matrix::DistMatrix& tmat); +#endif void initSquareMatrix(const ReplicatedMatrix& mat); int getDim() { return ndim_; } diff --git a/src/Rho.cc b/src/Rho.cc index 7b104573..680f9e0f 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -16,12 +16,15 @@ #include "ProjectedMatrices.h" #include "ReplicatedMatrix.h" #include "SquareLocalMatrices.h" -#include "SubMatrices.h" #include "magma_singleton.h" #include "memory_space.h" #include "mputils.h" #include "numerical_kernels.h" +#ifdef MGMOL_USE_SCALAPACK +#include "SubMatrices.h" +#endif + template Timer Rho::update_tm_("Rho::update"); template @@ -576,6 +579,8 @@ template double Rho>::dotWithRho( const double* const func) const; template double Rho>::dotWithRho( const double* const func) const; + +#ifdef MGMOL_USE_SCALAPACK template void Rho>::computeRho< dist_matrix::DistMatrix>(ExtendedGridOrbitals&, ExtendedGridOrbitals&, const dist_matrix::DistMatrix&, @@ -588,6 +593,7 @@ template void Rho>::computeRho< template void Rho>::computeRho>( LocGridOrbitals&, const dist_matrix::DistMatrix&); +#endif #ifdef MGMOL_USE_MIXEDP template double Rho>::dotWithRho( const float* const func) const; diff --git a/src/SP2.cc b/src/SP2.cc index 6ff7625a..7e83eb0c 100644 --- a/src/SP2.cc +++ b/src/SP2.cc @@ -6,12 +6,15 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "SP2.h" -#include "DistMatrix.h" #include "MGmol_MPI.h" #include "MPIdata.h" #include "ReplicatedMatrix.h" #include "linear_algebra/blas3_c.h" +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" +#endif + Timer SP2::getdm_tm_("SP2::getDM"); #ifdef HAVE_BML @@ -214,6 +217,7 @@ void SP2::initializeLocalMat( reduceSumTrace(); } +#ifdef MGMOL_USE_SCALAPACK template <> void SP2::getDM(dist_matrix::DistMatrix& submatM, // output const dist_matrix::DistMatrix& invS) @@ -230,6 +234,7 @@ void SP2::getDM(dist_matrix::DistMatrix& submatM, // output getdm_tm_.stop(); } +#endif template <> void SP2::getDM(ReplicatedMatrix& submatM, // output diff --git a/src/computeHij.cc b/src/computeHij.cc index 5abd2a2c..0de557e0 100644 --- a/src/computeHij.cc +++ b/src/computeHij.cc @@ -11,7 +11,6 @@ #include #include "Control.h" -#include "DistMatrix.h" #include "Energy.h" #include "GridFuncVector.h" #include "Hamiltonian.h" @@ -26,7 +25,11 @@ #include "ProjectedMatricesInterface.h" #include "ProjectedMatricesSparse.h" #include "ReplicatedMatrix.h" + +#ifdef MGMOL_USE_SCALAPACK +#include "DistMatrix.h" #include "SquareSubMatrix2DistMatrix.h" +#endif template <> template <> @@ -125,6 +128,7 @@ void MGmol>::computeHij( } } +#ifdef MGMOL_USE_SCALAPACK template void MGmol::computeHij_private(OrbitalsType& orbitals_i, OrbitalsType& orbitals_j, const Ions& ions, @@ -203,6 +207,7 @@ void MGmol::computeHij_private(OrbitalsType& orbitals_i, // add local Hamiltonian part to phi^T*H*phi hamiltonian_->addHlocal2matrix(orbitals_i, orbitals_j, hij, false); } +#endif template void MGmol::computeHij(OrbitalsType& orbitals_i, diff --git a/src/jade.cc b/src/jade.cc index 6da9cac7..757e8125 100644 --- a/src/jade.cc +++ b/src/jade.cc @@ -6,7 +6,9 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE +#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" +#endif #include "MGmol_MPI.h" #include "MGmol_blas1.h" #include "fc_mangle.h" @@ -24,6 +26,7 @@ // The input matrices are given in rmat[k], k = 0,..,m-1 // the element a(i,j) of matrix a_k is in rmat[k][i+m_loc*j]; // the orthogonal transformation is returned in tmat[i+m_loc*j] +#ifdef MGMOL_USE_SCALAPACK double jade(std::vector*>& rmat, const int n_loc, const int m_loc, const int offset_, const int maxsweep, const double tol, MPI_Comm comm, std::vector& tmat, @@ -34,7 +37,7 @@ double jade(std::vector*>& rmat, const int m = 2 * NDIM; const int npecol = rmat[0]->npcol(); -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK MGmol_MPI& mmpi = *(MGmol_MPI::instance()); int mype = mmpi.mypeSpin(); @@ -279,7 +282,7 @@ double jade(std::vector*>& rmat, std::vector remote_actualv(n_loc); for (int pe = 0; pe < npecol; pe++) { -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK if (mype == pe) { v_small = u_small; @@ -320,7 +323,7 @@ double jade(std::vector*>& rmat, } // pe -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK for (int i = 0; i < 8; i++) req[i] = MPI_REQUEST_NULL; @@ -454,7 +457,7 @@ double jade(std::vector*>& rmat, } } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK double in = sum; MPI_Allreduce(&in, &sum, 1, MPI_DOUBLE, MPI_SUM, comm); in = sum_off; @@ -528,7 +531,7 @@ double jade(std::vector*>& rmat, for (int ip = 0; ip < n_loc; ip++) { int lip = actual[ip]; -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK mmpi.bcast(&lip, 1, root); #endif if (lip < m_loc) @@ -538,7 +541,7 @@ double jade(std::vector*>& rmat, memcpy(&tmat[m_loc * lip], &u_loc[m_loc * ip], m_loc * sizeof(double)); } -#ifdef SCALAPACK +#ifdef MGMOL_USE_SCALAPACK mmpi.bcast(&tmat[m_loc * lip], m_loc, root); #endif } @@ -555,3 +558,5 @@ double jade(std::vector*>& rmat, return delta; } + +#endif diff --git a/src/local_matrices/LocalVector.h b/src/local_matrices/LocalVector.h index 16ca3463..c8bfc84e 100644 --- a/src/local_matrices/LocalVector.h +++ b/src/local_matrices/LocalVector.h @@ -2,6 +2,7 @@ #define MGMOL_LOCALVECTOR #include "memory_space.h" +#include "mputils.h" #include diff --git a/src/md.cc b/src/md.cc index f376a8a3..79a8136d 100644 --- a/src/md.cc +++ b/src/md.cc @@ -436,7 +436,12 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ct.adaptiveLRs()) { assert(lrs_); - adaptLR(spreadf_.get(), nullptr); + adaptLR(spreadf_.get() +#ifdef MGMOL_USE_SCALAPACK + , + nullptr +#endif + ); last_move_is_small = lrs_->moveIsSmall(); diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index 92390e83..38b54a45 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -14,7 +14,7 @@ #include "Mesh.h" #include "PackedCommunicationBuffer.h" #include "ReplicatedWorkSpace.h" -#include "SparseDistMatrix.h" +// #include "SparseDistMatrix.h" #include "tools.h" #include @@ -147,8 +147,10 @@ void mgmol_finalize() if (!ct.short_sighted) { +#ifdef MGMOL_USE_SCALAPACK // need to destroy any MPI based object befor calling MPI_Finalize MatricesBlacsContext::instance().clear(); +#endif } // release memory for static arrays diff --git a/src/mlwf.cc b/src/mlwf.cc index dc920134..23805ffc 100644 --- a/src/mlwf.cc +++ b/src/mlwf.cc @@ -11,15 +11,18 @@ #include "LocGridOrbitals.h" #include "LocalizationRegions.h" #include "MGmol.h" -#include "MLWFTransform.h" #include "Mesh.h" -#include "NOLMOTransform.h" #include "ProjectedMatrices.h" #include "ReplicatedWorkSpace.h" #include "SinCosOps.h" #include "blas3_c.h" #include "mputils.h" +#ifdef MGMOL_USE_SCALAPACK +#include "MLWFTransform.h" +#include "NOLMOTransform.h" +#endif + #include Timer get_NOLMO_tm("get_NOLMO"); @@ -32,6 +35,7 @@ void dtrsm_c(const char side, const char uplo, const char transa, DTRSM(&side, &uplo, &transa, &diag, &m, &n, &alpha, a, &lda, b, &ldb); } +#ifdef MGMOL_USE_SCALAPACK void distributeColumns( std::vector& vmm, dist_matrix::DistMatrix& mat) { @@ -338,6 +342,7 @@ int MGmol::get_NOLMO(NOLMOTransform& noot, OrbitalsType& orbitals, return 0; } +#endif template class MGmol>; template class MGmol>; diff --git a/src/quench.cc b/src/quench.cc index 56305e59..81990c98 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -33,7 +33,6 @@ #include "MPIdata.h" #include "MasksSet.h" #include "Mesh.h" -#include "OrbitalsTransform.h" #include "PolakRibiereSolver.h" #include "Potentials.h" #include "ProjectedMatricesInterface.h" @@ -45,6 +44,10 @@ #include "SymmetricPair.h" #include "tools.h" +#ifdef MGMOL_USE_SCALAPACK +#include "OrbitalsTransform.h" +#endif + #define TEST_ENERGY 0 Timer quench_tm("quench"); @@ -53,8 +56,12 @@ Timer updateCenters_tm("MGmol::updateCenters"); template <> void MGmol>::adaptLR( - const SpreadsAndCenters>* /*spreadf*/, - const OrbitalsTransform* /*ot*/) + const SpreadsAndCenters>* /*spreadf*/ +#ifdef MGMOL_USE_SCALAPACK + , + const OrbitalsTransform* /*ot*/ +#endif +) { } @@ -64,8 +71,12 @@ void MGmol>::adaptLR( // 1 -> radius only // 2 -> center and radius template -void MGmol::adaptLR( - const SpreadsAndCenters* spreadf, const OrbitalsTransform* ot) +void MGmol::adaptLR(const SpreadsAndCenters* spreadf +#ifdef MGMOL_USE_SCALAPACK + , + const OrbitalsTransform* ot +#endif +) { assert(lrs_); @@ -105,6 +116,7 @@ void MGmol::adaptLR( } double avg; +#ifdef MGMOL_USE_SCALAPACK // if ct.lr_volume_calc true, calculate volume base on spreads if (ct.lr_volume_calc == 1) { @@ -130,6 +142,7 @@ void MGmol::adaptLR( avg = lrs_->updateRadii(ot, ratio); } else +#endif { if (onpe0) os_ << " Adapt with constant LR volume" << std::endl; avg = lrs_->updateRadiiConstVol(*spreadf); @@ -179,6 +192,7 @@ void MGmol::resetProjectedMatricesAndDM( dm_strategy_->initialize(orbitals); } +#ifdef MGMOL_USE_SCALAPACK // try to use some rotations to avoid degeneracies template bool MGmol::rotateStatesPairsCommonCenter( @@ -395,6 +409,7 @@ void MGmol::disentangleOrbitals(OrbitalsType& orbitals, ct.num_MD_steps--; } } +#endif template <> void MGmol>::applyAOMMprojection( @@ -510,6 +525,7 @@ int MGmol::outerSolve(OrbitalsType& orbitals, retval = solver.solve(orbitals, work_orbitals); } +#ifdef MGMOL_USE_SCALAPACK else { DavidsonSolver::outerSolve(OrbitalsType& orbitals, retval = solver.solve(orbitals, work_orbitals); } +#endif break; } @@ -567,7 +584,9 @@ int MGmol::quench(OrbitalsType& orbitals, Ions& ions, orbitals.setDataWithGhosts(); orbitals.trade_boundaries(); +#ifdef MGMOL_USE_SCALAPACK disentangleOrbitals(orbitals, work_orbitals, ions, max_steps); +#endif // setup "kernel" functions for AOMM algorithm if (ct.use_kernel_functions) @@ -656,6 +675,7 @@ int MGmol::quench(OrbitalsType& orbitals, Ions& ions, } } +#ifdef MGMOL_USE_SCALAPACK if (!ct.isLocMode()) { if (ct.wannier_transform_type >= 1) @@ -663,7 +683,7 @@ int MGmol::quench(OrbitalsType& orbitals, Ions& ions, wftransform(&orbitals, &work_orbitals, ions); } } - +#endif // delete hnl_orbitals_;hnl_orbitals_=0; quench_tm.stop(); diff --git a/src/read_config.cc b/src/read_config.cc index 8817b842..103517f6 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -185,7 +185,11 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(false), "print projected matrices in MM format")( "ProjectedMatrices.replicated", +#ifdef MGMOL_USE_SCALAPACK po::value()->default_value(false), +#else + po::value()->default_value(true), +#endif "use replicated projected matrices")("LocalizationRegions.radius", po::value()->default_value(1000.), "Localization regions radius")("LocalizationRegions.adaptive", diff --git a/src/setup.cc b/src/setup.cc index c7b67c60..32e9ef37 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -14,7 +14,9 @@ #include "MGmol.h" #include "Potentials.h" #include "ReplicatedWorkSpace.h" +#ifdef MGMOL_USE_SCALAPACK #include "SparseDistMatrix.h" +#endif #include "mgmol_run.h" @@ -84,6 +86,7 @@ int MGmol::setupFromInput(const std::string filename) { ReplicatedWorkSpace::instance().setup(ct.numst); +#ifdef MGMOL_USE_SCALAPACK if (!ct.rmatrices) { MatricesBlacsContext::instance().setup(mmpi.commSpin(), ct.numst); @@ -99,6 +102,7 @@ int MGmol::setupFromInput(const std::string filename) int npes = mmpi.size(); setSparseDistMatriConsolidationNumber(npes); } +#endif } if (ct.rmatrices) ReplicatedMatrix::setMPIcomm(mmpi.commSpin()); diff --git a/src/tools.cc b/src/tools.cc index aff34933..8dd0f403 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -12,9 +12,12 @@ #include "MGmol_MPI.h" #include "MPIdata.h" #include "Mesh.h" -#include "SparseDistMatrix.h" #include "Vector3D.h" +#ifdef MGMOL_USE_SCALAPACK +#include "SparseDistMatrix.h" +#endif + #include #include @@ -103,6 +106,7 @@ void read_comments(std::ifstream& tfile) } } +#ifdef MGMOL_USE_SCALAPACK void setSparseDistMatriConsolidationNumber(const int npes) { int consolidation_number = 9; @@ -131,6 +135,7 @@ void setSparseDistMatriConsolidationNumber(const int npes) dist_matrix::SparseDistMatrix::printConsolidationNumber( *MPIdata::sout); } +#endif void reduceBytes(std::vector& val, const MPI_Comm comm) { diff --git a/src/tools/SaveData.h b/src/tools/SaveData.h index ade177a5..5fb61503 100644 --- a/src/tools/SaveData.h +++ b/src/tools/SaveData.h @@ -10,9 +10,11 @@ #ifndef SAVEDATA_H_ #define SAVEDATA_H_ +#ifdef SCALAPACK #include "BlacsContext.h" #include "DistMatrix.h" #include "MatricesBlacsContext.h" +#endif #include #include @@ -69,10 +71,10 @@ void saveData(std::vector data, const char* filename) } } +#ifdef SCALAPACK template void saveData(dist_matrix::DistMatrix data, const char* filename) { - dist_matrix::BlacsContext bc(MPI_COMM_WORLD, 1, 1); std::ofstream outputFile(filename); @@ -81,9 +83,9 @@ void saveData(dist_matrix::DistMatrix data, const char* filename) if (outputFile.is_open()) { - data.print(outputFile); } } +#endif #endif /* SAVEDATA_H_ */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 15fb6ab4..f4ab50cb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,6 +8,7 @@ endif() add_executable(testMPI ${CMAKE_SOURCE_DIR}/tests/testMPI.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) +if(${MGMOL_WITH_SCALAPACK}) add_executable(testBlacsContext ${CMAKE_SOURCE_DIR}/tests/testBlacsContext.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc @@ -114,6 +115,7 @@ add_executable(testPowerDistMatrix ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) +endif(${MGMOL_WITH_SCALAPACK}) add_executable(testDirectionalReduce ${CMAKE_SOURCE_DIR}/tests/testDirectionalReduce.cc ${CMAKE_SOURCE_DIR}/src/sparse_linear_algebra/DirectionalReduce.cc @@ -237,6 +239,7 @@ add_executable(testMGkernels ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) add_executable(testIons ${CMAKE_SOURCE_DIR}/tests/testIons.cc) +if(${MGMOL_WITH_SCALAPACK}) add_executable(testGramMatrix ${CMAKE_SOURCE_DIR}/tests/testGramMatrix.cc ${CMAKE_SOURCE_DIR}/src/GramMatrix.cc @@ -282,6 +285,7 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/ReplicatedWorkSpace.cc ${CMAKE_SOURCE_DIR}/src/hdf_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_magma_main.cc) +endif(${MGMOL_WITH_SCALAPACK}) add_executable(testRhoVhRestart ${CMAKE_SOURCE_DIR}/tests/RhoVhRestart/testRhoVhRestart.cc) add_executable(testEnergyAndForces @@ -326,6 +330,7 @@ endif() add_test(NAME testMPI COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testMPI) +if(${MGMOL_WITH_SCALAPACK}) add_test(NAME testBlacsContext COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testBlacsContext) @@ -344,11 +349,12 @@ add_test(NAME testConditionDistMatrix add_test(NAME testConditionDistMatrixPower COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testConditionDistMatrixPower) -add_test(NAME testPower - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/testPower) add_test(NAME testPowerDistMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testPowerDistMatrix) +add_test(NAME testPower + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/testPower) +endif(${MGMOL_WITH_SCALAPACK}) add_test(NAME testDirectionalReduce COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testDirectionalReduce) @@ -378,12 +384,14 @@ add_test(NAME testIons COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testIons ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +if(${MGMOL_WITH_SCALAPACK}) add_test(NAME testGramMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testGramMatrix) add_test(NAME testDensityMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testDensityMatrix) +endif(${MGMOL_WITH_SCALAPACK}) add_test(NAME testEnergyAndForces COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/EnergyAndForces/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -517,6 +525,7 @@ add_test(NAME testSpinO2LDA ${CMAKE_CURRENT_SOURCE_DIR}/SpinO2LDA/mgmol.cfg ${CMAKE_CURRENT_SOURCE_DIR}/SpinO2LDA/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +if(${MGMOL_WITH_SCALAPACK}) add_test(NAME testMVP COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MVP/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -524,6 +533,7 @@ add_test(NAME testMVP ${CMAKE_CURRENT_SOURCE_DIR}/MVP/mvp.cfg ${CMAKE_CURRENT_SOURCE_DIR}/MVP/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +endif(${MGMOL_WITH_SCALAPACK}) add_test(NAME testMVPReplicated COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MVP/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -621,6 +631,7 @@ add_test(NAME testFIRE ${CMAKE_CURRENT_SOURCE_DIR}/FIRE/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/FIRE/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +if(${MGMOL_WITH_SCALAPACK}) add_test(NAME ChebyshevMVP COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/Chebyshev/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -629,6 +640,7 @@ add_test(NAME ChebyshevMVP ${CMAKE_CURRENT_SOURCE_DIR}/Chebyshev/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +endif(${MGMOL_WITH_SCALAPACK}) if(NOT ${MGMOL_WITH_MAGMA}) add_test(NAME testShortSighted COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ShortSighted/test.py @@ -649,6 +661,7 @@ endif() target_include_directories(testAndersonMix PRIVATE ${CMAKE_SOURCE_DIR}/tests/Anderson ${HDF5_INCLUDE_DIRS}) +if(${MGMOL_WITH_SCALAPACK}) target_include_directories(testDistVector PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testReplicated2DistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) @@ -658,13 +671,17 @@ target_include_directories(testPower PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testPowerDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) + +target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} + ${BLAS_LIBRARIES} MPI::MPI_CXX) +target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) +endif(${MGMOL_WITH_SCALAPACK}) + target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_include_directories(testRhoVhRestart PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(testMPI PRIVATE MPI::MPI_CXX) -target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} - ${BLAS_LIBRARIES} MPI::MPI_CXX) target_link_libraries(testSuperSampling PRIVATE MPI::MPI_CXX) target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) @@ -672,7 +689,6 @@ target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testRestartEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) -target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) target_link_libraries(testRhoVhRestart mgmol_src) if(${MAGMA_FOUND}) @@ -720,23 +736,29 @@ if(${MAGMA_FOUND}) MPI::MPI_CXX OpenMP::OpenMP_CXX PkgConfig::MAGMA) endif() else() - target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testReplicated2DistMatrix PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testDistMatrix PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testConditionDistMatrix PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testConditionDistMatrixPower PRIVATE - ${SCALAPACK_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} - MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testPower PRIVATE ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} - ${SCALAPACK_LIBRARIES} - MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testPowerDistMatrix PRIVATE ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} - ${SCALAPACK_LIBRARIES} MPI::MPI_CXX - OpenMP::OpenMP_CXX) + if(${MGMOL_WITH_SCALAPACK}) + target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testReplicated2DistMatrix PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testDistMatrix PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testConditionDistMatrix PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testConditionDistMatrixPower PRIVATE + ${SCALAPACK_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} + MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testPowerDistMatrix PRIVATE ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} + ${SCALAPACK_LIBRARIES} MPI::MPI_CXX + OpenMP::OpenMP_CXX) + target_link_libraries(testPower PRIVATE ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} + ${SCALAPACK_LIBRARIES} + MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testGramMatrix PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testDensityMatrix PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + endif(${MGMOL_WITH_SCALAPACK}) target_link_libraries(testAndersonMix PRIVATE ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) target_link_libraries(testVariableSizeMatrix PRIVATE ${BLAS_LIBRARIES} @@ -751,10 +773,6 @@ else() MPI::MPI_CXX OpenMP::OpenMP_CXX) target_link_libraries(testMGkernels PRIVATE ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testGramMatrix PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testDensityMatrix PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) endif() set_tests_properties(testSiH4 PROPERTIES REQUIRED_FILES diff --git a/tests/testDensityMatrix.cc b/tests/testDensityMatrix.cc index 6a65c995..5941e4e5 100644 --- a/tests/testDensityMatrix.cc +++ b/tests/testDensityMatrix.cc @@ -9,7 +9,7 @@ #include "DensityMatrix.h" #include "GramMatrix.h" -#ifndef HAVE_MAGMA +#ifdef MGMOL_USE_SCALAPACK #include "BlacsContext.h" #include "DistMatrix.h" #else @@ -23,10 +23,10 @@ TEST_CASE( "Check functionalities of class DensityMatrix", "[functions_DensityMatrix") { -#ifdef HAVE_MAGMA - typedef ReplicatedMatrix MatrixType; -#else +#ifdef MGMOL_USE_SCALAPACK typedef dist_matrix::DistMatrix MatrixType; +#else + typedef ReplicatedMatrix MatrixType; #endif int myrank; MPI_Comm_rank(MPI_COMM_WORLD, &myrank); @@ -36,7 +36,7 @@ TEST_CASE( MGmol_MPI::setup(MPI_COMM_WORLD, std::cout); -#ifndef HAVE_MAGMA +#ifdef MGMOL_USE_SCALAPACK INFO("This example to set up to use only 4 processes"); REQUIRE(npes == 4); diff --git a/tests/testGramMatrix.cc b/tests/testGramMatrix.cc index a25c39ef..6a356b0e 100644 --- a/tests/testGramMatrix.cc +++ b/tests/testGramMatrix.cc @@ -9,7 +9,7 @@ #include "GramMatrix.h" #include "ReplicatedMatrix.h" -#ifndef HAVE_MAGMA +#ifdef MGMOL_USE_SCALAPACK #include "BlacsContext.h" #include "DistMatrix.h" #endif @@ -20,10 +20,10 @@ TEST_CASE("Check functionalities of class GramMatrix", "[functions_GramMatrix") { -#ifdef HAVE_MAGMA - typedef ReplicatedMatrix MatrixType; -#else +#ifdef MGMOL_USE_SCALAPACK typedef dist_matrix::DistMatrix MatrixType; +#else + typedef ReplicatedMatrix MatrixType; #endif int myrank; MPI_Comm_rank(MPI_COMM_WORLD, &myrank); @@ -33,7 +33,7 @@ TEST_CASE("Check functionalities of class GramMatrix", "[functions_GramMatrix") MGmol_MPI::setup(MPI_COMM_WORLD, std::cout); -#ifndef HAVE_MAGMA +#ifdef MGMOL_USE_SCALAPACK INFO("This example to set up to use only 4 processes"); REQUIRE(npes == 4);