ATLAS Offline Software
Loading...
Searching...
No Matches
MPIClusterSvc Class Reference

A service managing communications within a cluster using MPI. More...

#include <MPIClusterSvc.h>

Inheritance diagram for MPIClusterSvc:
Collaboration diagram for MPIClusterSvc:

Public Member Functions

 MPIClusterSvc (const std::string &name, ISvcLocator *svcLoc)
 Constructor.
virtual StatusCode initialize () override final
 Initialize.
virtual StatusCode finalize () override final
 Finalize.
virtual void handle (const Incident &inc) override
 IIncidentListener handle.
virtual int numRanks () const override final
 Return number of ranks.
virtual int rank () const override final
 Return our rank.
virtual void barrier () override final
 Insert a barrier No rank will continue until all ranks reach this point.
virtual void abort () override final
 Abort the MPI run.
virtual void sendMessage (int destRank, ClusterMessage message, ClusterComm communicator=ClusterComm::Default) override final
 Send an MPI message.
virtual ClusterMessage waitReceiveMessage (ClusterComm communicator=ClusterComm::Default) override final
 Block until we receive an MPI message.
virtual mpi3::communicator & data_communicator () override final
 Return the data communicator.
virtual void log_addEvent (int eventIdx, std::int64_t run_number, std::int64_t event_number, std::int64_t request_time_ns, std::size_t slot) override final
 Add (begin) an event in the log.
virtual void log_completeEvent (int eventIdx, std::int64_t run_number, std::int64_t event_number, std::int64_t status) override final
 Complete an event in the log.

Private Attributes

std::unique_ptr< mpi3::environment > m_env
mpi3::communicator m_world
mpi3::communicator m_datacom
int m_rank = -1
ServiceHandle< ISQLiteDBSvcm_mpiLog
SQLite::Statement m_mpiLog_addEvent
SQLite::Statement m_mpiLog_completeEvent
SQLite::Statement m_mpiLog_addFile
std::int64_t m_lastInputFileHash {}
std::map< std::size_t, std::int64_t > m_inputFileSlotMap {}

Detailed Description

A service managing communications within a cluster using MPI.

Definition at line 24 of file MPIClusterSvc.h.

Constructor & Destructor Documentation

◆ MPIClusterSvc()

MPIClusterSvc::MPIClusterSvc ( const std::string & name,
ISvcLocator * svcLoc )
inline

Constructor.

Definition at line 28 of file MPIClusterSvc.h.

29 : extends(name, svcLoc) {}

Member Function Documentation

◆ abort()

void MPIClusterSvc::abort ( )
finaloverridevirtual

Abort the MPI run.

Definition at line 142 of file MPIClusterSvc.cxx.

142 {
143 m_world.abort();
144}
mpi3::communicator m_world

◆ barrier()

void MPIClusterSvc::barrier ( )
finaloverridevirtual

Insert a barrier No rank will continue until all ranks reach this point.

Definition at line 137 of file MPIClusterSvc.cxx.

137 {
138 ATH_MSG_DEBUG("Barrier on rank " << rank() << " of " << numRanks());
139 m_world.barrier();
140}
#define ATH_MSG_DEBUG(x)
virtual int rank() const override final
Return our rank.
virtual int numRanks() const override final
Return number of ranks.

◆ data_communicator()

virtual mpi3::communicator & MPIClusterSvc::data_communicator ( )
inlinefinaloverridevirtual

Return the data communicator.

Definition at line 63 of file MPIClusterSvc.h.

63 {
64 return m_datacom;
65 }
mpi3::communicator m_datacom

◆ finalize()

StatusCode MPIClusterSvc::finalize ( )
finaloverridevirtual

Finalize.

Definition at line 97 of file MPIClusterSvc.cxx.

97 {
99 ->createStatement(
100 "UPDATE ranks SET end_time = julianday('now') WHERE rank = ?1")
101 .run(m_rank);
102 return StatusCode::SUCCESS;
103}
ServiceHandle< ISQLiteDBSvc > m_mpiLog

◆ handle()

void MPIClusterSvc::handle ( const Incident & inc)
overridevirtual

IIncidentListener handle.

Handles BeginInputFile to keep track of which input file an event came from.

Definition at line 106 of file MPIClusterSvc.cxx.

106 {
107 // Fill in slot map at start of every event
108 if (inc.type() == IncidentType::BeginProcessing) {
109 const std::size_t slot = Gaudi::Hive::currentContext().slot();
111 }
112
113 // Cache new input filename on start of every file
114 if (inc.type() == IncidentType::BeginInputFile) {
115 const FileIncident* fileInc = dynamic_cast<const FileIncident*>(&inc);
116 if (fileInc == nullptr) {
117 ATH_MSG_ERROR("BeginInputFile does not have a file name attached");
118 return;
119 }
120
121 const std::string fileName = fileInc->fileName();
122 // Convert the hash into a signed int64. Just a hash so this doesn't matter.
123 m_lastInputFileHash = static_cast<std::int64_t>(xxh3::hash64(fileName));
124 m_mpiLog_addFile.run(m_lastInputFileHash, std::move(fileName));
125 }
126 return;
127}
#define ATH_MSG_ERROR(x)
std::int64_t m_lastInputFileHash
SQLite::Statement m_mpiLog_addFile
std::map< std::size_t, std::int64_t > m_inputFileSlotMap
std::uint64_t hash64(const void *data, std::size_t size)
Passthrough to XXH3_64bits.
Definition XXH.cxx:9

◆ initialize()

StatusCode MPIClusterSvc::initialize ( )
finaloverridevirtual

Initialize.

Definition at line 13 of file MPIClusterSvc.cxx.

13 {
14 ATH_MSG_DEBUG("Initializing MPI");
15 m_env = std::make_unique<mpi3::environment>(mpi3::thread_level::multiple);
16 // Suggestion from Codex (GPT 5.6-sol)
17 if (m_env->thread_support() != mpi3::thread_level::multiple) {
18 ATH_MSG_ERROR("MPI_THREAD_MULTIPLE is required but unavailable");
19 return StatusCode::FAILURE;
20 }
21 ATH_MSG_DEBUG("Created MPI environment");
22
23 // Print version information
24 char version_string[MPI_MAX_LIBRARY_VERSION_STRING];
25 int version_string_len;
26 MPI_Get_library_version(version_string, &version_string_len);
27 ATH_MSG_INFO("Running on " << version_string);
28 m_world = m_env->world();
29
30 if (m_world.size() < 2) {
31 ATH_MSG_ERROR("Only have " << m_world.size()
32 << " ranks! This is insufficient!");
33 return StatusCode::FAILURE;
34 }
35 m_datacom =
36 m_world.duplicate(); // make a duplicate communicator for event data
37 ATH_MSG_DEBUG("Got MPI_COMM_WORLD");
38 m_rank = m_world.rank();
39 ATH_MSG_INFO("On MPI rank " << m_rank);
40 if (std::getenv("RANK") != std::to_string(m_rank)) {
41 const char* env_rank = std::getenv("RANK");
42 ATH_MSG_WARNING("MPI rank (" << m_rank
43 << ") does not match $RANK = " << env_rank);
44 }
45
46 ATH_CHECK(m_mpiLog.retrieve());
47 m_mpiLog->createStatement("PRAGMA foreign_keys = ON").run();
48
50 ->createStatement(
51 "CREATE TABLE ranks (rank INTEGER PRIMARY KEY, "
52 "node TEXT, start_time FLOAT, end_time FLOAT)")
53 .run();
55 ->createStatement(
56 "INSERT INTO ranks (rank, node, start_time) "
57 "VALUES(?1, ?2, julianday('now'))")
58 .run(m_rank, m_env->processor_name());
60 ->createStatement(
61 "CREATE TABLE files (fileId INTEGER PRIMARY KEY, fileName TEXT)")
62 .run();
64 ->createStatement(
65 "CREATE TABLE event_log (rank INTEGER, id INTEGER UNIQUE,"
66 "inputFileId INTEGER,"
67 "runNumber INTEGER, eventNumber INTEGER, complete INTEGER,"
68 "status INTEGER, request_time_ns INTEGER, start_time FLOAT,"
69 "end_time FLOAT, PRIMARY KEY (runNumber, eventNumber, id), "
70 "FOREIGN KEY (rank) REFERENCES ranks(rank),"
71 "FOREIGN KEY (inputFileId) REFERENCES files(fileId))")
72 .run();
73 m_mpiLog_addEvent = m_mpiLog->createStatement(
74 "INSERT INTO event_log(id, rank, inputFileId, runNumber, eventNumber, "
75 "complete, "
76 "start_time, request_time_ns) "
77 "VALUES(?1, ?4, ?6, ?2, ?3, 0, julianday('now'), ?5)");
78 m_mpiLog_completeEvent = m_mpiLog->createStatement(
79 "UPDATE event_log SET complete = 1, status = ?4, end_time = "
80 "julianday('now') WHERE runNumber = ?2 "
81 "AND eventNumber = ?3 AND id = ?1");
82 m_mpiLog_addFile = m_mpiLog->createStatement(
83 "INSERT INTO files (fileId, fileName) VALUES(?1, ?2)");
84
85 // Set up incident listener
86 ServiceHandle<IIncidentSvc> incsvc("IncidentSvc", this->name());
87 if (!incsvc.retrieve().isSuccess()) {
88 ATH_MSG_FATAL("Cannot get IncidentSvc.");
89 return (StatusCode::FAILURE);
90 }
91 incsvc->addListener(this, IncidentType::BeginInputFile, 100);
92 incsvc->addListener(this, IncidentType::BeginProcessing, 100);
93
94 return StatusCode::SUCCESS;
95}
#define ATH_CHECK
Evaluate an expression and check for errors.
#define ATH_MSG_FATAL(x)
#define ATH_MSG_INFO(x)
#define ATH_MSG_WARNING(x)
std::unique_ptr< mpi3::environment > m_env
SQLite::Statement m_mpiLog_addEvent
SQLite::Statement m_mpiLog_completeEvent

◆ log_addEvent()

void MPIClusterSvc::log_addEvent ( int eventIdx,
std::int64_t run_number,
std::int64_t event_number,
std::int64_t request_time_ns,
std::size_t slot )
finaloverridevirtual

Add (begin) an event in the log.

Definition at line 237 of file MPIClusterSvc.cxx.

240 {
241 m_mpiLog_addEvent.run(eventIdx, run_number, event_number, m_rank,
242 request_time_ns, m_inputFileSlotMap[slot]);
243}

◆ log_completeEvent()

void MPIClusterSvc::log_completeEvent ( int eventIdx,
std::int64_t run_number,
std::int64_t event_number,
std::int64_t status )
finaloverridevirtual

Complete an event in the log.

Definition at line 245 of file MPIClusterSvc.cxx.

247 {
248 m_mpiLog_completeEvent.run(eventIdx, run_number, event_number, status);
249}

◆ numRanks()

int MPIClusterSvc::numRanks ( ) const
finaloverridevirtual

Return number of ranks.

Definition at line 129 of file MPIClusterSvc.cxx.

129 {
130 return m_world.size();
131}

◆ rank()

int MPIClusterSvc::rank ( ) const
finaloverridevirtual

Return our rank.

Definition at line 133 of file MPIClusterSvc.cxx.

133 {
134 return m_rank;
135}

◆ sendMessage()

void MPIClusterSvc::sendMessage ( int destRank,
ClusterMessage message,
ClusterComm communicator = ClusterComm::Default )
finaloverridevirtual

Send an MPI message.

Definition at line 146 of file MPIClusterSvc.cxx.

147 {
148 ATH_MSG_DEBUG("Sending message from rank " << rank() << " to " << destRank);
149 // Don't send event request message if we're not the master *and* we have a
150 // message waiting.
151 // Probably an emergency stop message
152 if (m_rank != 0 && message.messageType == ClusterMessageType::RequestEvent &&
153 m_world.iprobe().has_value()) {
154 return;
155 }
156
157 // Select correct communicator
158 mpi3::communicator& comm =
159 (communicator == ClusterComm::EventData) ? m_datacom : m_world;
160 if (message.messageType == ClusterMessageType::Data &&
161 communicator != ClusterComm::EventData) {
163 "Event data should be sent with EventData communicator. "
164 "Dropping message");
165 return;
166 }
167
168 message.source = m_rank;
169 const auto& [header, body] = message.wire_msg();
170 comm.send_n(header.begin(), header.size(), destRank, 0);
171 if (body.has_value()) {
172 comm.send_n(body->begin(), body->size(), destRank, header[2]);
173 if (message.messageType == ClusterMessageType::Data) {
174 const ClusterMessage::WireMsgBody& bdy = *body;
175 // Decode the body to figure out what to send
176 char* ptr = reinterpret_cast<char*>((std::uint64_t(bdy[0]) << 32) +
177 std::uint64_t(bdy[1]));
178 std::size_t len = (std::uint64_t(bdy[2]) << 32) + std::uint64_t(bdy[3]);
179
180 // Offset the tag by 16384 to minimize chance of conflict
181 // (max tag in MPI spec is 32767)
182 constexpr int tag_offset = 16384;
183 comm.send_n(ptr, len, destRank, header[2] + tag_offset);
184 }
185 }
186}
void * ptr(T *p)
Definition SGImplSvc.cxx:74
std::array< std::uint32_t, 10 > WireMsgBody

◆ waitReceiveMessage()

ClusterMessage MPIClusterSvc::waitReceiveMessage ( ClusterComm communicator = ClusterComm::Default)
finaloverridevirtual

Block until we receive an MPI message.

Definition at line 188 of file MPIClusterSvc.cxx.

188 {
189 // Same offset as line 114
190 constexpr int tag_offset = 16384;
191 constexpr std::uint64_t thirtytwo_ones = 0xFFFFFFFF;
192
193 // Select correct communicator
194 mpi3::communicator& comm =
195 (communicator == ClusterComm::EventData) ? m_datacom : m_world;
197 auto&& [head, body] = msg;
198 comm.receive_n(head.begin(), head.size());
199 // Only time we need to figure out ourselves whether there's a body
200 if (head[0] == int(ClusterMessageType::FinalWorkerStatus) ||
201 head[0] == int(ClusterMessageType::WorkerError) ||
202 head[0] == int(ClusterMessageType::Data)) {
204 comm.receive_n(body->begin(), body->size(), head[1], head[2]);
205 if (head[0] == int(ClusterMessageType::Data)) {
206 ClusterMessage::WireMsgBody& bdy = *body;
207 // Decode the body to figure out what to recieve
208 std::size_t len = (std::uint64_t(bdy[2]) << 32) + std::uint64_t(bdy[3]);
209 std::size_t align = (std::uint64_t(bdy[4]) << 32) + std::uint64_t(bdy[5]);
210 std::size_t alloc_size = len;
211 if (!std::has_single_bit(align)) {
212 ATH_MSG_WARNING("Alignment " << align << " is not a power of two!");
213 align = std::bit_ceil(align);
214 }
215 if (len % align != 0) {
216 ATH_MSG_WARNING("Length " << len << " is not a multiple of alignment "
217 << align << "!");
218 // Convert to next multiple by adding align - 1, then zeroing out those
219 // final bits
220 alloc_size = (len + align - 1) & ~(align - 1);
221 }
222
223 char* ptr = static_cast<char*>(std::aligned_alloc(align, alloc_size));
224 comm.receive_n(ptr, len, head[1], head[2] + tag_offset);
225
226 // update the pointer in the WireMsgBody
227 bdy[0] = int(std::uint64_t(ptr) >> 32);
228 bdy[1] = int(std::uint64_t(ptr) & thirtytwo_ones);
229 }
230 }
231 ClusterMessage message(msg);
232 ATH_MSG_DEBUG("Rank " << rank() << " received message from "
233 << message.source);
234 return message;
235}
std::string head(std::string s, const std::string &pattern)
head of a string
std::tuple< WireMsgHdr, std::optional< WireMsgBody > > WireMsg
MsgStream & msg
Definition testRead.cxx:32

Member Data Documentation

◆ m_datacom

mpi3::communicator MPIClusterSvc::m_datacom
private

Definition at line 81 of file MPIClusterSvc.h.

◆ m_env

std::unique_ptr<mpi3::environment> MPIClusterSvc::m_env
private

Definition at line 78 of file MPIClusterSvc.h.

◆ m_inputFileSlotMap

std::map<std::size_t, std::int64_t> MPIClusterSvc::m_inputFileSlotMap {}
private

Definition at line 93 of file MPIClusterSvc.h.

93{};

◆ m_lastInputFileHash

std::int64_t MPIClusterSvc::m_lastInputFileHash {}
private

Definition at line 92 of file MPIClusterSvc.h.

92{};

◆ m_mpiLog

ServiceHandle<ISQLiteDBSvc> MPIClusterSvc::m_mpiLog
private
Initial value:
{this, "LogDatabaseSvc", "",
"SQLiteDBSvc for the MPI event log"}

Definition at line 85 of file MPIClusterSvc.h.

85 {this, "LogDatabaseSvc", "",
86 "SQLiteDBSvc for the MPI event log"};

◆ m_mpiLog_addEvent

SQLite::Statement MPIClusterSvc::m_mpiLog_addEvent
private

Definition at line 87 of file MPIClusterSvc.h.

◆ m_mpiLog_addFile

SQLite::Statement MPIClusterSvc::m_mpiLog_addFile
private

Definition at line 89 of file MPIClusterSvc.h.

◆ m_mpiLog_completeEvent

SQLite::Statement MPIClusterSvc::m_mpiLog_completeEvent
private

Definition at line 88 of file MPIClusterSvc.h.

◆ m_rank

int MPIClusterSvc::m_rank = -1
private

Definition at line 82 of file MPIClusterSvc.h.

◆ m_world

mpi3::communicator MPIClusterSvc::m_world
private

Definition at line 79 of file MPIClusterSvc.h.


The documentation for this class was generated from the following files: