Line data Source code
1 : /* +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 : Copyright (c) 2011-2023 The plumed team
3 : (see the PEOPLE file at the root of the distribution for a list of names)
4 :
5 : See http://www.plumed.org for more information.
6 :
7 : This file is part of plumed, version 2.
8 :
9 : plumed is free software: you can redistribute it and/or modify
10 : it under the terms of the GNU Lesser General Public License as published by
11 : the Free Software Foundation, either version 3 of the License, or
12 : (at your option) any later version.
13 :
14 : plumed is distributed in the hope that it will be useful,
15 : but WITHOUT ANY WARRANTY; without even the implied warranty of
16 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 : GNU Lesser General Public License for more details.
18 :
19 : You should have received a copy of the GNU Lesser General Public License
20 : along with plumed. If not, see <http://www.gnu.org/licenses/>.
21 : +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
22 : #ifndef __PLUMED_tools_Communicator_h
23 : #define __PLUMED_tools_Communicator_h
24 : #ifdef __PLUMED_HAS_MPI
25 : #include <mpi.h>
26 : #endif
27 : #include <cstdlib>
28 : #include "Exception.h"
29 : #include "TypesafePtr.h"
30 : #include <vector>
31 : #include <string>
32 : #include "Vector.h"
33 : #include "Tensor.h"
34 : #include "Matrix.h"
35 :
36 : namespace PLMD {
37 :
38 : #ifndef __PLUMED_HAS_MPI
39 : /// Surrogate of MPI_Comm when MPI library is not available
40 : class MPI_Comm {};
41 : /// Surrogate of MPI_Datatype when MPI library is not available
42 : class MPI_Datatype {};
43 : /// Surrogate of MPI_Status when MPI library is not available
44 : class MPI_Status {};
45 : /// Surrogate of MPI_Request when MPI library is not available
46 : class MPI_Request {};
47 : #endif
48 :
49 : /// \ingroup TOOLBOX
50 : /// Class containing wrappers to MPI.
51 : /// All the MPI related stuff is relegated here.
52 : class Communicator {
53 : /// Communicator
54 : MPI_Comm communicator;
55 : /// True if this process holds the communicator only as a stand-in.
56 : /// See setStandIn().
57 : bool standIn=false;
58 : /// In debug builds, raise an error if this communicator is a stand-in.
59 : void checkNotStandIn()const;
60 : /// Function returning the MPI type.
61 : /// You can use it to access to the MPI type of a C++ type, e.g.
62 : /// `MPI_Datatype type=getMPIType<double>();`
63 : template <class T>
64 : static MPI_Datatype getMPIType();
65 : /// Structure defining a buffer for MPI.
66 : /// It contains info on the pointed data and its type and size. It is useful to
67 : /// allow wrapper of MPI functions where the triplet (buffer,type,size)
68 : /// is grouped into a single object. It can be built starting from
69 : /// different kinds of data. To implement compatibility of MPI wrappers
70 : /// with e.g. vectors, add constructors here.
71 : struct Data {
72 : void*pointer;
73 : int size;
74 : int nbytes=0;
75 : MPI_Datatype type;
76 : /// Init from pointer and size
77 2954993 : template <typename T> Data(T*p,int s): pointer(p), size(s), nbytes(sizeof(T)), type(getMPIType<T>()) {}
78 : /// Init from reference
79 4249088 : template <typename T> explicit Data(T&p): pointer(&p), size(1), nbytes(sizeof(T)), type(getMPIType<T>()) {}
80 : /// Init from pointer to VectorGeneric
81 11179 : template <typename T, unsigned n> explicit Data(VectorTyped<T,n> *p,int s): pointer(p), size(n*s), nbytes(sizeof(T)), type(getMPIType<T>()) {}
82 : /// Init from reference to VectorGeneric
83 : template <typename T, unsigned n> explicit Data(VectorTyped<T,n> &p): pointer(&p), size(n), nbytes(sizeof(T)), type(getMPIType<T>()) {}
84 : /// Init from pointer to TensorTyped
85 16 : template <typename T, unsigned n,unsigned m> explicit Data(TensorTyped<T,n,m> *p,int s): pointer(p), size(n*m*s), nbytes(sizeof(T)), type(getMPIType<T>()) {}
86 : /// Init from reference to TensorTyped
87 21142 : template <typename T, unsigned n,unsigned m> explicit Data(TensorTyped<T,n,m> &p): pointer(&p), size(n*m), nbytes(sizeof(T)), type(getMPIType<T>()) {}
88 : /// Init from reference to std::vector
89 2774381 : template <typename T> explicit Data(std::vector<T>&v) {
90 2763381 : Data d(v.data(),v.size());
91 2774576 : pointer=d.pointer;
92 2774576 : size=d.size;
93 2774576 : type=d.type;
94 2774370 : }
95 : /// Init from reference to PLMD::Matrix
96 : template <typename T> explicit Data(Matrix<T>&m ) {
97 : if(m.nrows()*m.ncols()>0) {
98 : Data d(&m(0,0),m.nrows()*m.ncols());
99 : pointer=d.pointer;
100 : size=d.size;
101 : type=d.type;
102 : } else {
103 : pointer=NULL;
104 : size=0;
105 : }
106 : }
107 : /// Init from reference to std::string
108 188 : explicit Data(std::string&s) {
109 188 : if(s.size()>0) {
110 188 : Data d(&s[0],s.size());
111 188 : pointer=d.pointer;
112 188 : size=d.size;
113 188 : type=d.type;
114 : } else {
115 0 : pointer=NULL;
116 0 : size=0;
117 : }
118 188 : }
119 : };
120 : /// Const version of Communicator::Data
121 : /// See Communicator::Data documentation
122 : struct ConstData {
123 : const void*pointer;
124 : int size;
125 : int nbytes=0;
126 : MPI_Datatype type;
127 10597 : template <typename T> explicit ConstData(const T*p,int s): pointer(p), size(s), nbytes(sizeof(T)), type(getMPIType<T>()) {}
128 2486 : template <typename T> explicit ConstData(const T&p): pointer(&p), size(1), nbytes(sizeof(T)), type(getMPIType<T>()) {}
129 : template <typename T,unsigned n> explicit ConstData(const VectorTyped<T,n> *p,int s): pointer(p), size(n*s), nbytes(sizeof(T)), type(getMPIType<T>()) {}
130 : template <typename T,unsigned n> explicit ConstData(const VectorTyped<T,n> &p): pointer(&p), size(n), nbytes(sizeof(T)), type(getMPIType<T>()) {}
131 : template <typename T, unsigned n,unsigned m> explicit ConstData(const TensorTyped<T,n,m> *p,int s): pointer(p), size(n*m*s), nbytes(sizeof(T)), type(getMPIType<T>()) {}
132 : template <typename T, unsigned n,unsigned m> explicit ConstData(const TensorTyped<T,n,m> &p): pointer(&p), size(n*m), nbytes(sizeof(T)), type(getMPIType<T>()) {}
133 6349 : template <typename T> explicit ConstData(const std::vector<T>&v) {
134 6349 : ConstData d(v.data(),v.size());
135 6349 : pointer=d.pointer;
136 6349 : size=d.size;
137 6349 : type=d.type;
138 6349 : }
139 : template <typename T> explicit ConstData(const Matrix<T>&m ) {
140 : if(m.nrows()*m.ncols()>0) {
141 : ConstData d(&m(0,0),m.nrows()*m.ncols());
142 : pointer=d.pointer;
143 : size=d.size;
144 : type=d.type;
145 : } else {
146 : pointer=NULL;
147 : size=0;
148 : }
149 : }
150 69 : explicit ConstData(const std::string&s) {
151 69 : if(s.size()>0) {
152 69 : ConstData d(&s[0],s.size());
153 69 : pointer=d.pointer;
154 69 : size=d.size;
155 69 : type=d.type;
156 : } else {
157 0 : pointer=NULL;
158 0 : size=0;
159 : }
160 69 : }
161 : };
162 : public:
163 : ///Runtime acces to the __PLUMED_HAS_MPI definition
164 : static bool plumedHasMPI();
165 :
166 : /// Wrapper class for MPI_Status
167 : class Status {
168 : int Get_count(MPI_Datatype)const;
169 : public:
170 : MPI_Status s;
171 : template <class T>
172 8142 : int Get_count()const {
173 8142 : return Get_count(getMPIType<T>());
174 : }
175 : };
176 : /// Special status used when status should be ignored.
177 : /// E.g. `Recv(a,0,1,Communicator::StatusIgnore);`
178 : /// Notice that this is the default for Recv, so this is equivalent to
179 : /// `Recv(a,0,1);`
180 : static Status StatusIgnore;
181 : /// Wrapper class for MPI_Request
182 : class Request {
183 : public:
184 : MPI_Request r;
185 : void wait(Status&s=StatusIgnore);
186 : };
187 : /// Default constructor
188 : Communicator();
189 : /// Copy constructor.
190 : /// It effectively "clones" the communicator, providing a new one acting on the same group
191 : Communicator(const Communicator&);
192 : /// Assignment operator.
193 : /// It effectively "clones" the communicator, providing a new one acting on the same group
194 : Communicator& operator=(const Communicator&);
195 : /// Destructor
196 : virtual ~Communicator();
197 : /// Obtain the rank of the present process
198 : int Get_rank()const;
199 : /// Obtain the number of processes
200 : int Get_size()const;
201 : /// Set from a real MPI communicator.
202 : /// \param comm MPI communicator
203 : void Set_comm(MPI_Comm comm);
204 : /// Reference to MPI communicator
205 : MPI_Comm & Get_comm();
206 : /// Mark this communicator as a stand-in on this process.
207 : /// A stand-in is a communicator that other processes were given but this
208 : /// process was not, so it does not connect this process to the processes it
209 : /// is meant to connect. This is the case of PLMD::Action::multi_sim_comm on
210 : /// all the processes of a replica except the first one.
211 : /// In debug builds, using a stand-in (asking its rank or size, or
212 : /// communicating over it) raises an error. Set_comm() clears the mark.
213 : void setStandIn(bool s);
214 : /// True if this communicator has been marked as a stand-in with setStandIn().
215 : bool isStandIn()const;
216 : /// Set from a pointer to a real MPI communicator (C).
217 : /// \param comm Pointer to a C MPI communicator
218 : void Set_comm(const TypesafePtr & comm);
219 : /// Set from a pointer to a real MPI communicator (FORTRAN).
220 : /// \param comm Pointer to a FORTRAN MPI communicator (INTEGER)
221 : void Set_fcomm(const TypesafePtr & comm);
222 : /// Wrapper to MPI_Abort.
223 : /// \param code Error code
224 : void Abort(int code);
225 : /// Wrapper to MPI_Barrier
226 : void Barrier()const;
227 : /// Tests if MPI library is initialized
228 : static bool initialized();
229 : /// Wrapper for MPI_Allreduce with MPI_SUM (data struct)
230 : void Sum(Data);
231 : /// Wrapper for MPI_Allreduce with MPI_SUM (pointer)
232 75492 : template <class T> void Sum(T*buf,int count) {
233 75492 : Sum(Data(buf,count));
234 75492 : }
235 : /// Wrapper for MPI_Allreduce with MPI_SUM (reference)
236 4490692 : template <class T> void Sum(T&buf) {
237 4490692 : Sum(Data(buf));
238 4490692 : }
239 : /// Wrapper for MPI_Allreduce with MPI_PROD (data struct)
240 : void Prod(Data);
241 : /// Wrapper for MPI_Allreduce with MPI_PROD (pointer)
242 : template <class T> void Prod(T*buf,int count) {
243 : Prod(Data(buf,count));
244 : }
245 : /// Wrapper for MPI_Allreduce with MPI_PROD (reference)
246 : template <class T> void Prod(T&buf) {
247 : Prod(Data(buf));
248 : }
249 : /// Wrapper for MPI_Allreduce with MPI_MAX (data struct)
250 : void Max(Data);
251 : /// Wrapper for MPI_Allreduce with MPI_MAX (pointer)
252 4 : template <class T> void Max(T*buf,int count) {
253 4 : Max(Data(buf,count));
254 4 : }
255 : /// Wrapper for MPI_Allreduce with MPI_MAX (reference)
256 6386 : template <class T> void Max(T&buf) {
257 6386 : Max(Data(buf));
258 6386 : }
259 : /// Wrapper for MPI_Allreduce with MPI_MIN (data struct)
260 : void Min(Data);
261 : /// Wrapper for MPI_Allreduce with MPI_MIN (pointer)
262 : template <class T> void Min(T*buf,int count) {
263 : Min(Data(buf,count));
264 : }
265 : /// Wrapper for MPI_Allreduce with MPI_MIN (reference)
266 235 : template <class T> void Min(T&buf) {
267 235 : Min(Data(buf));
268 235 : }
269 :
270 : /// Wrapper for MPI_Bcast (data struct)
271 : void Bcast(Data,int);
272 : /// Wrapper for MPI_Bcast (pointer)
273 20 : template <class T> void Bcast(T*buf,int count,int root) {
274 20 : Bcast(Data(buf,count),root);
275 20 : }
276 : /// Wrapper for MPI_Bcast (reference)
277 403800 : template <class T> void Bcast(T&buf,int root) {
278 403800 : Bcast(Data(buf),root);
279 403800 : }
280 :
281 : /// Wrapper for MPI_Scatter (data struct)
282 : void Scatter(ConstData sendbuf,int sendcount,Data recvbuf,int recvcount,int root);
283 : /// Wrapper for MPI_Scatter (pointer)
284 8 : template <class T,class S> void Scatter(const T*sendbuf,int sendcount,S*recvbuf,int recvcount,int root) {
285 8 : Scatter(ConstData(sendbuf,sendcount*Get_size()),sendcount,Data(recvbuf,recvcount),recvcount,root);
286 8 : }
287 : /// Wrapper for MPI_Scatter (reference)
288 : template <class T,class S> void Scatter(const T&sendbuf,int sendcount,S&recvbuf,int recvcount,int root) {
289 : Scatter(ConstData(sendbuf),sendcount,Data(recvbuf),recvcount,root);
290 : }
291 :
292 : /// Wrapper for MPI_Gather (data struct)
293 : void Gather(ConstData sendbuf,int sendcount,Data recvbuf,int recvcount,int root);
294 : /// Wrapper for MPI_Gather (pointer)
295 8 : template <class T,class S> void Gather(const T*sendbuf,int sendcount,S*recvbuf,int recvcount,int root) {
296 16 : Gather(ConstData(sendbuf,sendcount),sendcount,Data(recvbuf,recvcount*Get_size()),recvcount,root);
297 8 : }
298 : /// Wrapper for MPI_Gather (reference)
299 : template <class T,class S> void Gather(const T&sendbuf,int sendcount,S&recvbuf,int recvcount,int root) {
300 : Gather(ConstData(sendbuf),sendcount,Data(recvbuf),recvcount,root);
301 : }
302 :
303 : /// Wrapper for MPI_Isend (data struct)
304 : Request Isend(ConstData,int,int);
305 : /// Wrapper for MPI_Isend (pointer)
306 16284 : template <class T> Request Isend(const T*buf,int count,int source,int tag) {
307 16284 : return Isend(ConstData(buf,count),source,tag);
308 : }
309 : /// Wrapper for MPI_Isend (reference)
310 207 : template <class T> Request Isend(const T&buf,int source,int tag) {
311 207 : return Isend(ConstData(buf),source,tag);
312 : }
313 :
314 : /// Wrapper for MPI_Allgatherv (data struct)
315 : void Allgatherv(ConstData in,Data out,const int*,const int*);
316 : /// Wrapper for MPI_Allgatherv (pointer)
317 3967 : template <class T,class S> void Allgatherv(const T*sendbuf,int sendcount,S*recvbuf,const int*recvcounts,const int*displs) {
318 3967 : Allgatherv(ConstData(sendbuf,sendcount),Data(recvbuf,0),recvcounts,displs);
319 3967 : }
320 : /// Wrapper for MPI_Allgatherv (reference)
321 414 : template <class T,class S> void Allgatherv(const T&sendbuf,S&recvbuf,const int*recvcounts,const int*displs) {
322 414 : Allgatherv(ConstData(sendbuf),Data(recvbuf),recvcounts,displs);
323 414 : }
324 :
325 : /// Wrapper for MPI_Allgather (data struct)
326 : void Allgather(ConstData in,Data out);
327 : /// Wrapper for MPI_Allgatherv (pointer)
328 204 : template <class T,class S> void Allgather(const T*sendbuf,int sendcount,S*recvbuf,int recvcount) {
329 408 : Allgather(ConstData(sendbuf,sendcount),Data(recvbuf,recvcount*Get_size()));
330 204 : }
331 : /// Wrapper for MPI_Allgatherv (reference)
332 8145 : template <class T,class S> void Allgather(const T&sendbuf,S&recvbuf) {
333 10355 : Allgather(ConstData(sendbuf),Data(recvbuf));
334 8145 : }
335 :
336 : /// Wrapper for MPI_Recv (data struct)
337 : void Recv(Data,int,int,Status&s=StatusIgnore);
338 : /// Wrapper for MPI_Recv (pointer)
339 16284 : template <class T> void Recv(T*buf,int count,int source,int tag,Status&s=StatusIgnore) {
340 16284 : Recv(Data(buf,count),source,tag,s);
341 16284 : }
342 : /// Wrapper for MPI_Recv (reference)
343 207 : template <class T> void Recv(T&buf,int source,int tag,Status&s=StatusIgnore) {
344 207 : Recv(Data(buf),source,tag,s);
345 207 : }
346 :
347 : /// Wrapper to MPI_Comm_split
348 : void Split(int,int,Communicator&)const;
349 : };
350 :
351 : }
352 :
353 : #endif
|
