ATLAS Offline Software
Loading...
Searching...
No Matches
MPIClusterSvc Class Reference

A service managing communications within a cluster using MPI. More...

#include <MPIClusterSvc.h>

Inheritance diagram for MPIClusterSvc:
Collaboration diagram for MPIClusterSvc:

Public Member Functions

 MPIClusterSvc (const std::string &name, ISvcLocator *svcLoc)
 Constructor.
virtual StatusCode initialize () override final
 Initialize.
virtual StatusCode finalize () override final
 Finalize.
virtual void handle (const Incident &inc) override
 IIncidentListener handle.
virtual int numRanks () const override final
 Return number of ranks.
virtual int rank () const override final
 Return our rank.
virtual void barrier () override final
 Insert a barrier No rank will continue until all ranks reach this point.
virtual void abort () override final
 Abort the MPI run.
virtual void sendMessage (int destRank, ClusterMessage message, ClusterComm communicator=ClusterComm::Default) override final
 Send an MPI message.
virtual ClusterMessage waitReceiveMessage (ClusterComm communicator=ClusterComm::Default) override final
 Block until we receive an MPI message.
virtual mpi3::communicator & data_communicator () override final
 Return the data communicator.
virtual void log_addEvent (int eventIdx, std::int64_t run_number, std::int64_t event_number, std::int64_t request_time_ns, std::size_t slot) override final
 Add (begin) an event in the log.
virtual void log_completeEvent (int eventIdx, std::int64_t run_number, std::int64_t event_number, std::int64_t status) override final
 Complete an event in the log.

Private Attributes

std::unique_ptr< mpi3::environment > m_env
mpi3::communicator m_world
mpi3::communicator m_datacom
int m_rank = -1
ServiceHandle< ISQLiteDBSvc > m_mpiLog
SQLite::Statement m_mpiLog_addEvent
SQLite::Statement m_mpiLog_completeEvent
SQLite::Statement m_mpiLog_addFile
std::int64_t m_lastInputFileHash {}
std::map< std::size_t, std::int64_t > m_inputFileSlotMap {}

Detailed Description

A service managing communications within a cluster using MPI.

Definition at line 24 of file MPIClusterSvc.h.

Constructor & Destructor Documentation

◆ MPIClusterSvc()

MPIClusterSvc::MPIClusterSvc ( const std::string & name,
ISvcLocator * svcLoc )
inline

Constructor.

Definition at line 28 of file MPIClusterSvc.h.

29 : extends(name, svcLoc) {}

Member Function Documentation

◆ abort()

void MPIClusterSvc::abort ( )
finaloverridevirtual

Abort the MPI run.

Definition at line 149 of file MPIClusterSvc.cxx.

149 {
150 m_world.abort();
151}
mpi3::communicator m_world

◆ barrier()

void MPIClusterSvc::barrier ( )
finaloverridevirtual

Insert a barrier No rank will continue until all ranks reach this point.

Definition at line 144 of file MPIClusterSvc.cxx.

144 {
145 ATH_MSG_DEBUG("Barrier on rank {} of {}", rank(), numRanks());
146 m_world.barrier();
147}
#define ATH_MSG_DEBUG(x,...)
virtual int rank() const override final
Return our rank.
virtual int numRanks() const override final
Return number of ranks.

◆ data_communicator()

virtual mpi3::communicator & MPIClusterSvc::data_communicator ( )
inlinefinaloverridevirtual

Return the data communicator.

Definition at line 63 of file MPIClusterSvc.h.

63 {
64 return m_datacom;
65 }
mpi3::communicator m_datacom

◆ finalize()

StatusCode MPIClusterSvc::finalize ( )
finaloverridevirtual

Finalize.

Definition at line 98 of file MPIClusterSvc.cxx.

98 {
100 ->createStatement(
101 "UPDATE ranks SET end_time = julianday('now') WHERE rank = ?1")
102 .run(m_rank);
103 // Gaudi/Python references can keep this service alive past finalization.
104 // Release communicators and mpi3's attributes before finalizing MPI, while
105 // all ranks are still here, rather than relying on process-exit destruction.
106 m_datacom = mpi3::communicator{};
107 m_world = mpi3::communicator{};
108 m_env.reset();
109 return StatusCode::SUCCESS;
110}
std::unique_ptr< mpi3::environment > m_env
ServiceHandle< ISQLiteDBSvc > m_mpiLog

◆ handle()

void MPIClusterSvc::handle ( const Incident & inc)
overridevirtual

IIncidentListener handle.

Handles BeginInputFile to keep track of which input file an event came from.

Definition at line 113 of file MPIClusterSvc.cxx.

113 {
114 // Fill in slot map at start of every event
115 if (inc.type() == IncidentType::BeginProcessing) {
116 const std::size_t slot = Gaudi::Hive::currentContext().slot();
118 }
119
120 // Cache new input filename on start of every file
121 if (inc.type() == IncidentType::BeginInputFile) {
122 const FileIncident* fileInc = dynamic_cast<const FileIncident*>(&inc);
123 if (fileInc == nullptr) {
124 ATH_MSG_ERROR("BeginInputFile does not have a file name attached");
125 return;
126 }
127
128 const std::string fileName = fileInc->fileName();
129 // Convert the hash into a signed int64. Just a hash so this doesn't matter.
130 m_lastInputFileHash = static_cast<std::int64_t>(xxh3::hash64(fileName));
131 m_mpiLog_addFile.run(m_lastInputFileHash, std::move(fileName));
132 }
133 return;
134}
#define ATH_MSG_ERROR(x,...)
std::int64_t m_lastInputFileHash
SQLite::Statement m_mpiLog_addFile
std::map< std::size_t, std::int64_t > m_inputFileSlotMap
std::uint64_t hash64(const void *data, std::size_t size)
Passthrough to XXH3_64bits.
Definition XXH.cxx:9

◆ initialize()

StatusCode MPIClusterSvc::initialize ( )
finaloverridevirtual

Initialize.

Definition at line 13 of file MPIClusterSvc.cxx.

13 {
14 ATH_MSG_DEBUG("Initializing MPI");
15 m_env = std::make_unique<mpi3::environment>(mpi3::thread_level::multiple);
16 // Suggestion from Codex (GPT 5.6-sol)
17 if (m_env->thread_support() != mpi3::thread_level::multiple) {
18 ATH_MSG_ERROR("MPI_THREAD_MULTIPLE is required but unavailable");
19 return StatusCode::FAILURE;
20 }
21 ATH_MSG_DEBUG("Created MPI environment");
22
23 // Print version information
24 char version_string[MPI_MAX_LIBRARY_VERSION_STRING];
25 int version_string_len;
26 MPI_Get_library_version(version_string, &version_string_len);
27 ATH_MSG_INFO("Running on {}", version_string);
28 m_world = m_env->world();
29
30 if (m_world.size() < 2) {
31 ATH_MSG_ERROR("Only have {} ranks! This is insufficient!",
32 m_world.size());
33 return StatusCode::FAILURE;
34 }
35 m_datacom =
36 m_world.duplicate(); // make a duplicate communicator for event data
37 ATH_MSG_DEBUG("Got MPI_COMM_WORLD");
38 m_rank = m_world.rank();
39 ATH_MSG_INFO("On MPI rank {}", m_rank);
40 if (const char* env_rank = std::getenv("RANK");
41 env_rank && std::to_string(m_rank) != env_rank) {
42 ATH_MSG_WARNING("MPI rank ({}) does not match $RANK = {}",
43 m_rank, env_rank);
44 }
45
46 ATH_CHECK(m_mpiLog.retrieve());
47 m_mpiLog->createStatement("PRAGMA foreign_keys = ON").run();
48
50 ->createStatement(
51 "CREATE TABLE ranks (rank INTEGER PRIMARY KEY, "
52 "node TEXT, start_time FLOAT, end_time FLOAT)")
53 .run();
55 ->createStatement(
56 "INSERT INTO ranks (rank, node, start_time) "
57 "VALUES(?1, ?2, julianday('now'))")
58 .run(m_rank, m_env->processor_name());
60 ->createStatement(
61 "CREATE TABLE files (fileId INTEGER PRIMARY KEY, fileName TEXT)")
62 .run();
64 ->createStatement(
65 "CREATE TABLE event_log (rank INTEGER, id INTEGER UNIQUE,"
66 "inputFileId INTEGER,"
67 "runNumber INTEGER, eventNumber INTEGER, complete INTEGER,"
68 "status INTEGER, request_time_ns INTEGER, start_time FLOAT,"
69 "end_time FLOAT, PRIMARY KEY (runNumber, eventNumber, id), "
70 "FOREIGN KEY (rank) REFERENCES ranks(rank),"
71 "FOREIGN KEY (inputFileId) REFERENCES files(fileId))")
72 .run();
73 m_mpiLog_addEvent = m_mpiLog->createStatement(
74 "INSERT INTO event_log(id, rank, inputFileId, runNumber, eventNumber, "
75 "complete, "
76 "start_time, request_time_ns) "
77 // inputFileId stays NULL until the first BeginInputFile (hash 0)
78 "VALUES(?1, ?4, NULLIF(?6, 0), ?2, ?3, 0, julianday('now'), ?5)");
79 m_mpiLog_completeEvent = m_mpiLog->createStatement(
80 "UPDATE event_log SET complete = 1, status = ?4, end_time = "
81 "julianday('now') WHERE runNumber = ?2 "
82 "AND eventNumber = ?3 AND id = ?1");
83 m_mpiLog_addFile = m_mpiLog->createStatement(
84 "INSERT INTO files (fileId, fileName) VALUES(?1, ?2)");
85
86 // Set up incident listener
87 ServiceHandle<IIncidentSvc> incsvc("IncidentSvc", this->name());
88 if (!incsvc.retrieve().isSuccess()) {
89 ATH_MSG_FATAL("Cannot get IncidentSvc.");
90 return (StatusCode::FAILURE);
91 }
92 incsvc->addListener(this, IncidentType::BeginInputFile, 100);
93 incsvc->addListener(this, IncidentType::BeginProcessing, 100);
94
95 return StatusCode::SUCCESS;
96}
#define ATH_CHECK
Evaluate an expression and check for errors.
#define ATH_MSG_WARNING(x,...)
#define ATH_MSG_INFO(x,...)
#define ATH_MSG_FATAL(x,...)
SQLite::Statement m_mpiLog_addEvent
SQLite::Statement m_mpiLog_completeEvent

◆ log_addEvent()

void MPIClusterSvc::log_addEvent ( int eventIdx,
std::int64_t run_number,
std::int64_t event_number,
std::int64_t request_time_ns,
std::size_t slot )
finaloverridevirtual

Add (begin) an event in the log.

Definition at line 243 of file MPIClusterSvc.cxx.

246 {
247 m_mpiLog_addEvent.run(eventIdx, run_number, event_number, m_rank,
248 request_time_ns, m_inputFileSlotMap[slot]);
249}

◆ log_completeEvent()

void MPIClusterSvc::log_completeEvent ( int eventIdx,
std::int64_t run_number,
std::int64_t event_number,
std::int64_t status )
finaloverridevirtual

Complete an event in the log.

Definition at line 251 of file MPIClusterSvc.cxx.

253 {
254 m_mpiLog_completeEvent.run(eventIdx, run_number, event_number, status);
255}

◆ numRanks()

int MPIClusterSvc::numRanks ( ) const
finaloverridevirtual

Return number of ranks.

Definition at line 136 of file MPIClusterSvc.cxx.

136 {
137 return m_world.size();
138}

◆ rank()

int MPIClusterSvc::rank ( ) const
finaloverridevirtual

Return our rank.

Definition at line 140 of file MPIClusterSvc.cxx.

140 {
141 return m_rank;
142}

◆ sendMessage()

void MPIClusterSvc::sendMessage ( int destRank,
ClusterMessage message,
ClusterComm communicator = ClusterComm::Default )
finaloverridevirtual

Send an MPI message.

Definition at line 153 of file MPIClusterSvc.cxx.

154 {
155 ATH_MSG_DEBUG("Sending message from rank {} to {}", rank(), destRank);
156 // Don't send event request message if we're not the master *and* we have a
157 // message waiting.
158 // Probably an emergency stop message
159 if (m_rank != 0 && message.messageType == ClusterMessageType::RequestEvent &&
160 m_world.iprobe().has_value()) {
161 return;
162 }
163
164 // Select correct communicator
165 mpi3::communicator& comm =
166 (communicator == ClusterComm::EventData) ? m_datacom : m_world;
167 if (message.messageType == ClusterMessageType::Data &&
168 communicator != ClusterComm::EventData) {
170 "Event data should be sent with EventData communicator. "
171 "Dropping message");
172 return;
173 }
174
175 message.source = m_rank;
176 const auto& [header, body] = message.wire_msg();
177 comm.send_n(header.begin(), header.size(), destRank, 0);
178 if (body.has_value()) {
179 comm.send_n(body->begin(), body->size(), destRank, header[2]);
180 if (message.messageType == ClusterMessageType::Data) {
181 const ClusterMessage::WireMsgBody& bdy = *body;
182 // Decode the body to figure out what to send
183 char* ptr = reinterpret_cast<char*>((std::uint64_t(bdy[0]) << 32) +
184 std::uint64_t(bdy[1]));
185 std::size_t len = (std::uint64_t(bdy[2]) << 32) + std::uint64_t(bdy[3]);
186
187 // Offset the tag by 16384 to minimize chance of conflict
188 // (max tag in MPI spec is 32767)
189 constexpr int tag_offset = 16384;
190 comm.send_n(ptr, len, destRank, header[2] + tag_offset);
191 }
192 }
193}
void * ptr(T *p)
Definition SGImplSvc.cxx:75
std::array< std::uint32_t, 10 > WireMsgBody

◆ waitReceiveMessage()

ClusterMessage MPIClusterSvc::waitReceiveMessage ( ClusterComm communicator = ClusterComm::Default)
finaloverridevirtual

Block until we receive an MPI message.

Definition at line 195 of file MPIClusterSvc.cxx.

195 {
196 // Same offset as line 114
197 constexpr int tag_offset = 16384;
198 constexpr std::uint64_t thirtytwo_ones = 0xFFFFFFFF;
199
200 // Select correct communicator
201 mpi3::communicator& comm =
202 (communicator == ClusterComm::EventData) ? m_datacom : m_world;
204 auto&& [head, body] = msg;
205 comm.receive_n(head.begin(), head.size());
206 // Only time we need to figure out ourselves whether there's a body
207 if (head[0] == int(ClusterMessageType::FinalWorkerStatus) ||
208 head[0] == int(ClusterMessageType::WorkerError) ||
209 head[0] == int(ClusterMessageType::Data)) {
211 comm.receive_n(body->begin(), body->size(), head[1], head[2]);
212 if (head[0] == int(ClusterMessageType::Data)) {
213 ClusterMessage::WireMsgBody& bdy = *body;
214 // Decode the body to figure out what to recieve
215 std::size_t len = (std::uint64_t(bdy[2]) << 32) + std::uint64_t(bdy[3]);
216 std::size_t align = (std::uint64_t(bdy[4]) << 32) + std::uint64_t(bdy[5]);
217 std::size_t alloc_size = len;
218 if (!std::has_single_bit(align)) {
219 ATH_MSG_WARNING("Alignment {} is not a power of two!", align);
220 align = std::bit_ceil(align);
221 }
222 if (len % align != 0) {
223 ATH_MSG_WARNING("Length {} is not a multiple of alignment {}!",
224 len, align);
225 // Convert to next multiple by adding align - 1, then zeroing out those
226 // final bits
227 alloc_size = (len + align - 1) & ~(align - 1);
228 }
229
230 char* ptr = static_cast<char*>(std::aligned_alloc(align, alloc_size));
231 comm.receive_n(ptr, len, head[1], head[2] + tag_offset);
232
233 // update the pointer in the WireMsgBody
234 bdy[0] = int(std::uint64_t(ptr) >> 32);
235 bdy[1] = int(std::uint64_t(ptr) & thirtytwo_ones);
236 }
237 }
238 ClusterMessage message(msg);
239 ATH_MSG_DEBUG("Rank {} received message from {}", rank(), message.source);
240 return message;
241}
std::string head(std::string s, const std::string &pattern)
head of a string
std::tuple< WireMsgHdr, std::optional< WireMsgBody > > WireMsg
MsgStream & msg
Definition testRead.cxx:32

Member Data Documentation

◆ m_datacom

mpi3::communicator MPIClusterSvc::m_datacom
private

Definition at line 81 of file MPIClusterSvc.h.

◆ m_env

std::unique_ptr<mpi3::environment> MPIClusterSvc::m_env
private

Definition at line 78 of file MPIClusterSvc.h.

◆ m_inputFileSlotMap

std::map<std::size_t, std::int64_t> MPIClusterSvc::m_inputFileSlotMap {}
private

Definition at line 93 of file MPIClusterSvc.h.

93{};

◆ m_lastInputFileHash

std::int64_t MPIClusterSvc::m_lastInputFileHash {}
private

Definition at line 92 of file MPIClusterSvc.h.

92{};

◆ m_mpiLog

ServiceHandle<ISQLiteDBSvc> MPIClusterSvc::m_mpiLog
private
Initial value:
{this, "LogDatabaseSvc", "",
"SQLiteDBSvc for the MPI event log"}

Definition at line 85 of file MPIClusterSvc.h.

85 {this, "LogDatabaseSvc", "",
86 "SQLiteDBSvc for the MPI event log"};

◆ m_mpiLog_addEvent

SQLite::Statement MPIClusterSvc::m_mpiLog_addEvent
private

Definition at line 87 of file MPIClusterSvc.h.

◆ m_mpiLog_addFile

SQLite::Statement MPIClusterSvc::m_mpiLog_addFile
private

Definition at line 89 of file MPIClusterSvc.h.

◆ m_mpiLog_completeEvent

SQLite::Statement MPIClusterSvc::m_mpiLog_completeEvent
private

Definition at line 88 of file MPIClusterSvc.h.

◆ m_rank

int MPIClusterSvc::m_rank = -1
private

Definition at line 82 of file MPIClusterSvc.h.

◆ m_world

mpi3::communicator MPIClusterSvc::m_world
private

Definition at line 79 of file MPIClusterSvc.h.


The documentation for this class was generated from the following files: