Mallard 0.8.0
Unstructured, GPU accelerated, high-order finite volume CFD solver
Mallard website
Loading...
Searching...
No Matches
comm.h
Go to the documentation of this file.
1
13#ifndef COMM_H
14#define COMM_H
15
16#include <array>
17#include <cstddef>
18#include <cstdint>
19#include <span>
20#include <vector>
21#include <type_traits>
22
23#ifdef Mallard_HAS_MPI
24#include <mpi.h>
25#endif
26
27namespace comm {
28
34class Session {
35 public:
36 Session(int & argc, char **& argv);
37 ~Session();
38 Session(const Session &) = delete;
39 Session & operator=(const Session &) = delete;
40
41 private:
42 [[maybe_unused]] bool owns_mpi = false;
43};
44
46int rank();
47
49int size();
50
52inline bool is_root() { return rank() == 0; }
53
55void barrier();
56
58[[noreturn]] void abort(int code);
59
60enum class Op { SUM, MIN, MAX };
61
65template <typename T>
66void allreduce(std::span<T> data, Op op);
67
68template <typename T>
69T allreduce(T value, Op op) {
70 allreduce(std::span<T>(&value, 1), op);
71 return value;
72}
73
74template <typename T, std::size_t N>
75std::array<T, N> allreduce(std::array<T, N> values, Op op) {
76 allreduce(std::span<T>(values), op);
77 return values;
78}
79
84template <typename T>
85std::vector<std::vector<T>> alltoallv(const std::vector<std::vector<T>> & send);
86
91template <typename T>
92struct Received {
93 std::vector<T> data;
94 std::vector<uint64_t> offsets;
95
96 std::span<const T> from(size_t r) const { return {data.data() + offsets[r], data.data() + offsets[r + 1]}; }
97};
98
104template <typename T>
105Received<T> exchange(std::vector<std::vector<T>> && send);
106
110template <typename T>
111std::vector<T> allgatherv(const std::vector<T> & local);
112
113#ifdef Mallard_HAS_MPI
116#endif
117
118} // namespace comm
119
120#endif // COMM_H
Initializes MPI on construction (unless it already is) and finalizes it on destruction if this sessio...
Definition comm.h:34
~Session()
Definition comm.cpp:179
Session & operator=(const Session &)=delete
Session(const Session &)=delete
KOKKOS_INLINE_FUNCTION rtype dot(const rtype *a, const rtype *b)
Compute the dot product of two arrays.
Definition common_math.h:32
Definition comm.cpp:24
void allreduce(std::span< T >, Op)
In-place all-reduce over all ranks of the values in data.
Definition comm.cpp:188
Op
Definition comm.h:60
bool is_root()
Whether this is rank 0, which does all logging.
Definition comm.h:52
int rank()
Rank of this process in the world communicator.
Definition comm.cpp:181
std::vector< std::vector< T > > alltoallv(const std::vector< std::vector< T > > &send)
Personalized all-to-all: send[r] goes to rank r; returns what each rank sent to this one,...
Definition comm.cpp:196
Received< T > exchange(std::vector< std::vector< T > > &&send)
Personalized all-to-all like alltoallv, for large setup-time messages: frees each send list once pack...
Definition comm.cpp:202
int size()
Number of processes in the world communicator.
Definition comm.cpp:182
void abort(int code)
Ends every rank with the given exit code (MPI_Abort in parallel runs).
Definition comm.cpp:185
std::vector< T > allgatherv(const std::vector< T > &local)
Concatenation of every rank's local vector, in rank order.
Definition comm.cpp:191
void barrier()
Block until all ranks arrive.
Definition comm.cpp:183
What an exchange() received: the part from rank r is data[offsets[r], offsets[r + 1]).
Definition comm.h:92
std::vector< T > data
Definition comm.h:93
std::vector< uint64_t > offsets
Definition comm.h:94
std::span< const T > from(size_t r) const
Definition comm.h:96
uint8_t r
Definition teno_setup.cpp:687