Refactor Parallel Codes

source/source_base/parallel_2d.h

@@ -74,7 +74,51 @@ class Parallel_2D
         return nb;
     };
 
+    /// number of processes in row dimension of the MPI Cartesian grid
+    int get_dim0() const
+    {
+        return dim0;
+    };
+
+    /// number of processes in column dimension of the MPI Cartesian grid
+    int get_dim1() const
+    {
+        return dim1;
+    };
+
+    /// row coordinate in the BLACS grid
+    int get_coord_row() const
+    {
+        return coord[0];
+    };
+
+    /// column coordinate in the BLACS grid
+    int get_coord_col() const
+    {
+        return coord[1];
+    };
+
+    /// check whether a given BLACS grid coordinate is this process
+    bool blacs_in_this_processor(const int iprow, const int ipcol) const
+    {
+        return iprow == coord[0] && ipcol == coord[1];
+    };
+
+    /// set process coordinate in the BLACS grid (intended for testing)
+    void set_coord(const int row, const int col)
+    {
+        coord[0] = row;
+        coord[1] = col;
+    };
+
 #ifdef __MPI
+
+    /// ScaLAPACK descriptor
+    const int* get_desc() const
+    {
+        return desc;
+    };
+
     /**
      * @brief Initialize a BLACS grid with the given MPI communicator
      * and set up the info of a block-cyclic distribution.
@@ -126,13 +170,14 @@ class Parallel_2D
     int dim0 = 0;
     int dim1 = 0;
 
+  private:
     /// process coordinate in the BLACS grid
     int coord[2] = {-1, -1};
-
     /// whether to use the serial mode
     bool is_serial = false;
 
   protected:
+
     /// map from global index to local index
     std::vector<int> global2local_row_;
     std::vector<int> global2local_col_;

source/source_base/parallel_comm.h

@@ -14,32 +14,32 @@ extern MPI_Comm DIAG_WORLD; // mohan add 2012-01-13
 class MPICommGroup
 {
 
-	public:
+    public:
 
-		MPICommGroup(MPI_Comm parent_comm);
-		~MPICommGroup();
+        MPICommGroup(MPI_Comm parent_comm);
+        ~MPICommGroup();
 
-		void divide_group_comm(const int& ngroup, const bool assert_even = true);
+        void divide_group_comm(const int& ngroup, const bool assert_even = true);
 
-		bool is_even = false; ///< whether the group is even
+        bool is_even = false; ///< whether the group is even
 
-		MPI_Comm parent_comm = MPI_COMM_NULL; ///< parent communicator
-		int gsize = 0;                        ///< size of parent communicator
-		int grank = 0;                        ///< rank of parent communicator
+        MPI_Comm parent_comm = MPI_COMM_NULL; ///< parent communicator
+        int gsize = 0;                        ///< size of parent communicator
+        int grank = 0;                        ///< rank of parent communicator
 
-		MPI_Comm group_comm = MPI_COMM_NULL; ///< group communicator
-		int ngroups = 0;                     ///< number of groups
-		int nprocs_in_group = 0;             ///<  number of processes in the group
-		int my_group = 0;                    ///< the group index
-		int rank_in_group = 0;               ///< the rank in the group
+        MPI_Comm group_comm = MPI_COMM_NULL; ///< group communicator
+        int ngroups = 0;                     ///< number of groups
+        int nprocs_in_group = 0;             ///<  number of processes in the group
+        int my_group = 0;                    ///< the group index
+        int rank_in_group = 0;               ///< the rank in the group
 
-		MPI_Comm inter_comm = MPI_COMM_NULL; ///< inter communicator
-		bool has_inter_comm = false;         ///< whether has inter communicator
-		int& nprocs_in_inter = ngroups;      ///< number of processes in the inter communicator
-		int& my_inter = rank_in_group;  ///< the rank in the inter communicator
-		int& rank_in_inter = my_group;  ///< the inter group index
+        MPI_Comm inter_comm = MPI_COMM_NULL; ///< inter communicator
+        bool has_inter_comm = false;         ///< whether has inter communicator
+        int& nprocs_in_inter = ngroups;      ///< number of processes in the inter communicator
+        int& my_inter = rank_in_group;  ///< the rank in the inter communicator
+        int& rank_in_inter = my_group;  ///< the inter group index
 };
 
 #endif
 
-#endif // PARALLEL_COMM_H
+#endif // PARALLEL_COMM_H

source/source_base/parallel_common.h

@@ -1,14 +1,16 @@
-#ifndef W_ABACUS_DEVELOP_ABACUS_DEVELOP_SOURCE_MODULE_BASE_PARALLEL_COMMON_H
-#define W_ABACUS_DEVELOP_ABACUS_DEVELOP_SOURCE_MODULE_BASE_PARALLEL_COMMON_H
+#ifndef PARALLEL_COMMON_H
+#define PARALLEL_COMMON_H
 
 #ifdef __MPI
 #include "mpi.h"
 #endif
+
 #include <complex>
 #include <string>
 
 namespace Parallel_Common
 {
+
 //(1) bcast array
 void bcast_complex_double(std::complex<double>* object, const int n);
 void bcast_string(std::string* object, const int n);

source/source_base/parallel_global.cpp

@@ -25,22 +25,6 @@ namespace Parallel_Global
 int mpi_number = 0;
 int omp_number = 0;
 } // namespace Parallel_Global
-
-void Parallel_Global::myProd(std::complex<double>* in, std::complex<double>* inout, int* len, MPI_Datatype* dptr)
-{
-    for (int i = 0; i < *len; i++)
-    {
-        //		(*inout).real()=(*inout).real()+(*in).real();
-        //		(*inout).imag()=(*inout).imag()+(*in).imag();
-
-        // mohan updat 2011-09-21
-        (*inout) = std::complex<double>((*inout).real() + (*in).real(), (*inout).imag() + (*in).imag());
-
-        in++;
-        inout++;
-    }
-    return;
-}
 #endif
 
 void Parallel_Global::split_diag_world(const int& diag_np,
@@ -108,8 +92,6 @@ void Parallel_Global::read_pal_param(int argc,
     MPI_Init(&argc, &argv); // Peize Lin change 2018-07-12
 #endif //_OPENMP
 
-    //  KPAR = atoi(argv[1]); // mohan abandon 2010-06-09
-
     // get world size --> NPROC
     // get global rank --> MY_RANK
     MPI_Comm_size(MPI_COMM_WORLD, &NPROC);
@@ -174,26 +156,10 @@ void Parallel_Global::read_pal_param(int argc,
 
     NTHREAD_PER_PROC = current_thread_num;
 
-    // for test
-    /*
-    for (int i=0; i<NPROC; i++)
-    {
-        if (MY_RANK == i)
-        {
-            std::cout << " PROCESSOR " << std::setw(4) << MY_RANK+1 << " IS READY." << std::endl;
-        }
-        MPI_Barrier(MPI_COMM_WORLD);
-    }
-    */
-
-    // This section can be chosen !!
-    // mohan 2011-03-15
     if (MY_RANK != 0)
     {
-        // std::cout.rdbuf(NULL);
-        std::cout.setstate(std::ios::failbit); // qianrui modify 2020-10-14
+        std::cout.setstate(std::ios::failbit);
     }
-    // end test
 #endif //__MPI
     return;
 }
@@ -229,6 +195,7 @@ void Parallel_Global::finalize_mpi()
 }
 #endif
 
+#ifdef __MPI
 void Parallel_Global::init_pools(const int& NPROC,
                                  const int& MY_RANK,
                                  const int& BNDPAR,
@@ -240,10 +207,6 @@ void Parallel_Global::init_pools(const int& NPROC,
                                  int& RANK_IN_POOL,
                                  int& MY_POOL)
 {
-#ifdef __MPI
-    //----------------------------------------------------------
-    // CALL Function : divide_pools
-    //----------------------------------------------------------
     Parallel_Global::divide_pools(NPROC,
                                   MY_RANK,
                                   BNDPAR,
@@ -254,42 +217,8 @@ void Parallel_Global::init_pools(const int& NPROC,
                                   NPROC_IN_POOL,
                                   RANK_IN_POOL,
                                   MY_POOL);
-
-    // for test
-    // turn on when you want to check the index of pools.
-    /*
-        if (GlobalV::MY_RANK==0)
-        {
-            std::cout << "\n     " << std::setw(8) << "MY_RANK"
-                 << std::setw(8) << "MY_POOL"
-                 << std::setw(13) << "RANK_IN_POOL"
-                 << std::setw(6) << "NPROC"
-                 << std::setw(6) << "KPAR"
-                 << std::setw(14) << "NPROC_IN_POOL" << std::endl;
-        }
-        for (int i=0; i<GlobalV::NPROC; i++)
-        {
-            if (GlobalV::MY_RANK == i)
-            {
-                std::cout << " I'm " << std::setw(8) << GlobalV::MY_RANK
-                     << std::setw(8) << GlobalV::MY_POOL
-                     << std::setw(13) << GlobalV::RANK_IN_POOL
-                     << std::setw(6) << GlobalV::NPROC
-                     << std::setw(6) << GlobalV::KPAR
-                     << std::setw(14) << GlobalV::NPROC_IN_POOL << std::endl;
-            }
-            MPI_Barrier(MPI_COMM_WORLD);
-        }
-
-        if (GlobalV::MY_RANK != 0 )
-        {
-            std::cout.rdbuf(NULL);
-        }
-    */
-
-    return;
-#endif
 }
+#endif
 
 #ifdef __MPI
 void Parallel_Global::divide_pools(const int& NPROC,

source/source_base/parallel_global.h

@@ -6,11 +6,8 @@
 #ifndef PARALLEL_GLOBAL_H
 #define PARALLEL_GLOBAL_H
 
-#include <complex>
 #include "parallel_comm.h"
 
-// void myProd(std::complex<double> *in,std::complex<double> *inout,int *len,MPI_Datatype *dptr);
-
 namespace Parallel_Global
 {
 extern int mpi_number;
@@ -21,9 +18,6 @@ extern int omp_number;
 
 // changed from read_mpi_parameters in 2024-1018
 void read_pal_param(int argc, char** argv, int& NPROC, int& NTHREAD_PER_PROC, int& MY_RANK);
-#ifdef __MPI
-void myProd(std::complex<double>* in, std::complex<double>* inout, int* len, MPI_Datatype* dptr);
-#endif
 
 /**-------------------------------------------
  * call to split the "diago world"