parallel_csr_matrix.hpp

Go to the documentation of this file.

#ifndef PARGEMSLR_PARALLEL_CSR_MATRIX_H
#define PARGEMSLR_PARALLEL_CSR_MATRIX_H
 
#include <iostream>
 
#include "../utils/utils.hpp"
#include "../utils/structs.hpp"
#include "matrix.hpp"
#include "csr_matrix.hpp"
 
#ifdef PARGEMSLR_CUDA
#include "cusparse.h"
#endif
 
namespace pargemslr
{
   
   class CommunicationHelperClass: public parallel_log
   {
   private:
      
      int                                 _location_send;
      
      int                                 _location_recv;
      
      int                                 _buff_unit_size;
      
      std::vector<void*>                  _send_buff_v2;
      
      std::vector<void*>                  _recv_buff_v2;
 
#ifdef PARGEMSLR_CUDA
 
      int                                 _buff_d_unit_size;
      
      std::vector<void*>                  _send_buff_v2_d;
      
      std::vector<void*>                  _recv_buff_v2_d;
 
#endif
 
   public:
      
      int                                 _n_in;
      
      int                                 _n_out;
      
      vector_int                          _send_to_v;
      
      vector_int                          _recv_from_v;
      
      std::vector<vector_int >            _send_idx_v2;
      
      std::vector<vector_int >            _recv_idx_v2;
      
      std::vector<vector_int >            _idx_helper_v2;
      
      vector<MPI_Request>                 _requests_v;
      
      bool                                _is_ready;
      
      bool                                _is_waiting;
      
      bool                                _is_nbhd_built;
      
      CommunicationHelperClass();
      
      CommunicationHelperClass(const CommunicationHelperClass &comm_helper);
      
      CommunicationHelperClass(CommunicationHelperClass &&comm_helper);
      
      CommunicationHelperClass& operator= (const CommunicationHelperClass &comm_helper);
      
      CommunicationHelperClass& operator= (CommunicationHelperClass &&comm_helper);
      
      ~CommunicationHelperClass();
      
      int                           Clear();
      
      int                           CreateHostBuffer(int unitsize);
 
#ifdef PARGEMSLR_CUDA
 
      int                           CreateDeviceBuffer(int unitsize);
 
#endif
 
      int                           MoveData(int location);
      
      int                           MoveSendData(int location);
      
      int                           MoveRecvData(int location);
      
      template <typename T>
      int                           DataTransfer(const VectorVirtualClass<T> &vec_in, VectorVirtualClass<T> &vec_out, int loc_in, int loc_out);
      
      template <typename T>
      int                           DataTransferStart(const VectorVirtualClass<T> &vec_in, int loc_in);
      
      template <typename T>
      int                           DataTransferOver(VectorVirtualClass<T> &vec_out, int loc_out);
      
      template <typename T, class VectorType>
      int                           DataTransfer(const std::vector<VectorType> &vec_in, std::vector<VectorType> &vec_out, int loc_in, int loc_out);
      
      template <typename T, class VectorType>
      int                           DataTransferStart(const std::vector<VectorType> &vec_in, int loc_in);
      
      template <typename T, class VectorType>
      int                           DataTransferOver(std::vector<VectorType> &vec_out, int loc_out);
      
      template <typename T>
      int                           DataTransferReverse(const VectorVirtualClass<T> &vec_in, VectorVirtualClass<T> &vec_out, int loc_in, int loc_out);
      
      template <typename T>
      int                           DataTransferStartReverse(const VectorVirtualClass<T> &vec_in, int loc_in);
      
      template <typename T>
      int                           DataTransferOverReverse(VectorVirtualClass<T> &vec_out, int loc_out);
      
   };
   
   template <typename T>
   class ParallelCsrMatrixClass: public MatrixClass<T>
   {
   private:
      
      /* variables */
      
      long int                   _nrow_global;
      
      long int                   _ncol_global;
      
      long int                   _nrow_start;
      
      long int                   _ncol_start;
      
      int                        _nrow_local;
      
      int                        _ncol_local;
      
      bool                       _is_offd_map_sorted;
      
      CsrMatrixClass<T>          _diag_mat;
      
      CsrMatrixClass<T>          _offd_mat;
      
      vector_long                _offd_map_v;
      
      SequentialVectorClass<T>   _matvec_working_vec;
      
      SequentialVectorClass<T>   _trans_matvec_working_vec;
      
      int                        _separator_ndom;
      
      vector_int                 _separator_domi;
      
   public:
      
      typename std::conditional<PargemslrIsDoublePrecision<T>::value, 
                                 double, 
                                 float>::type _diagonal_shift;
      
      CommunicationHelperClass   _comm_helper;
         
      CommunicationHelperClass   _trans_comm_helper;
      
      ParallelCsrMatrixClass();
      
      ParallelCsrMatrixClass(const ParallelCsrMatrixClass<T> &mat);
      
      ParallelCsrMatrixClass(ParallelCsrMatrixClass<T> &&mat);
      
      ParallelCsrMatrixClass<T>& operator= (const ParallelCsrMatrixClass<T> &mat);
      
      ParallelCsrMatrixClass<T>& operator= (ParallelCsrMatrixClass<T> &&mat);
      
      virtual ~ParallelCsrMatrixClass();
      
      int            Setup(int nrow_local, int ncol_local, parallel_log &parlog);
      
      int            Setup(int nrow_local, long int nrow_start, long int nrow_global, int ncol_local, long int ncol_start, long int ncol_global, parallel_log &parlog);
      
      virtual int    Clear();
      
      virtual int    GetDataLocation() const;
      
      virtual int    GetNumRowsLocal() const;
      
      virtual int    GetNumColsLocal() const;
      
      long int       GetNumRowsGlobal() const;
      
      long int       GetNumColsGlobal() const;
      
      long int       GetRowStartGlobal() const;
      
      long int       GetColStartGlobal() const;
      
      virtual long int  GetNumNonzeros() const;
      
      int            GetGraphArrays( vector_long &vtxdist, vector_long &xadj, vector_long &adjncy);
      
      CsrMatrixClass<T>&            GetDiagMat();
      
      CsrMatrixClass<T>&            GetOffdMat();
      
      vector_long&                  GetOffdMap();
      
      bool&          IsOffdMapSorted();
      
      int            SetOffdMatSorted(bool is_offd_map_sorted);
      
      int            SortOffdMap();
      
      int            SetupVectorPtrStr(ParallelVectorClass<T> &vec);
      
      virtual int    Eye();
      
      virtual int    Fill(const T &v);
      
      virtual int    MoveData( const int &location);
      
      virtual int    Scale(const T &alpha);
      
      int            GetDiagScale(T &scale);
      
      int            Transpose();
      
      
      int            Transpose(ParallelCsrMatrixClass<T> &AT);
      
      int            SetupMatvec();
      
      int            SetupMatvecStart();
      
      int            SetupMatvecOver();
      
      virtual int    MatVec( char trans, const T &alpha, const VectorClass<T> &x, const T &beta, VectorClass<T> &y);
      
      virtual int    MatVec( char trans, const T &alpha, const VectorClass<T> &x, const T &beta, const VectorClass<T> &y, VectorClass<T> &z);
      
      int            MatVecOffd( char trans, const T &alpha, const VectorClass<T> &x, const T &beta, VectorClass<T> &y);
      
      int            MatMat( const T &alpha, const ParallelCsrMatrixClass<T> &A, char transa, const ParallelCsrMatrixClass<T> &B, char transb, const T &beta);
      
      int            SubMatrix(vector_long &rows, vector_long &cols, int location, ParallelCsrMatrixClass<T> &parcsrmat_out);
      
      int            Laplacian(int nx, int ny, int nz, int dx, int dy, int dz, T alphax, T alphay, T alphaz, T shift, parallel_log &parlog, bool rand_perturb = false);
      
      int            Helmholtz(int n, int dx, int dy, int dz, T w, parallel_log &parlog);
      
      int            LaplacianWithPartition(int nx, int ny, int nz, int dx, int dy, int dz, int d2x, int d2y, int d2z, T alphax, T alphay, T alphaz, T shift, parallel_log &parlog, bool rand_perturb = false);
      
      int            HelmholtzWithPartition(int n, int dx, int dy, int dz, int d2x, int d2y, int d2z, T w, parallel_log &parlog);
      
      int            ReadFromSingleMMFile(const char *matfile, int idxin, parallel_log &parlog);
      
      int            PlotPatternGnuPlot( const char *datafilename, int pttype = 0);
      
      template <typename T1>
      int            SetComplexShift(T1 diagonal_shift)
      {
         this->_diag_mat.SetComplexShift(diagonal_shift);
         return PARGEMSLR_SUCCESS;
      }
      
      template <typename T1>
      int            GetComplexShift(T1 &diagonal_shift)
      {
         this->_diag_mat.GetComplexShift(diagonal_shift);
         return PARGEMSLR_SUCCESS;
      }
      
      int&           GetSeparatorNumSubdomains()
      {
         return this->_separator_ndom;
      }
      
      vector_int&    GetSeparatorDomi()
      {
         return this->_separator_domi;
      }
      
   };
   
   typedef ParallelCsrMatrixClass<float>     matrix_csr_par_float;
   typedef ParallelCsrMatrixClass<double>    matrix_csr_par_double;
   typedef ParallelCsrMatrixClass<complexs>  matrix_csr_par_complexs;
   typedef ParallelCsrMatrixClass<complexd>  matrix_csr_par_complexd;
   template<> struct PargemslrIsComplex<ParallelCsrMatrixClass<complexs> > : public std::true_type {};
   template<> struct PargemslrIsComplex<ParallelCsrMatrixClass<complexd> > : public std::true_type {};
   template<> struct PargemslrIsParallel<ParallelCsrMatrixClass<complexs> > : public std::true_type {};
   template<> struct PargemslrIsParallel<ParallelCsrMatrixClass<complexd> > : public std::true_type {};
   
}
 
#endif