ATLAS Offline Software
Loading...
Searching...
No Matches
MPIHiveEventLoopMgr.cxx
Go to the documentation of this file.
1/*
2 Copyright (C) 2002-2026 CERN for the benefit of the ATLAS collaboration
3*/
5
6// Gaudi includes
7#include "GaudiKernel/AppReturnCode.h"
8
9// Utilities
12
13// Standard Library
14#include <chrono>
15#include <string>
16
17using Clock = std::chrono::high_resolution_clock;
18
19// Standard Constructor
21 ISvcLocator* svcLoc)
22 : AthenaHiveEventLoopMgr(name, svcLoc) {}
23
24// Standard Destructor
26
30 // Initialize cluster svc
31 ATH_CHECK(m_clusterSvc.retrieve());
33}
34
40
43StatusCode MPIHiveEventLoopMgr::nextEvent(int maxevt) {
44 // make nextEvent(0) a dummy call
45 if (0 == maxevt) {
46 return StatusCode::SUCCESS;
47 }
48
49 // Reset the application return code.
50 Gaudi::setAppReturnCode(m_appMgrProperty, Gaudi::ReturnCode::Success, true)
51 .ignore();
52 ATH_MSG_INFO("Starting loop on events");
53
54 if (m_clusterSvc->rank() == 0) {
55 return masterEventLoop(maxevt);
56 }
57 return workerEventLoop();
58}
59
62 ATH_MSG_INFO("Running with " << m_clusterSvc->numRanks() << " ranks");
63 // Determine number of events to process
64 int skipEvts = int(m_firstEventIndex.value());
65 if (m_evtSelector != nullptr) {
66 int evt = size();
67 if (evt == -1) {
68 m_clusterSvc->abort();
69 return StatusCode::FAILURE;
70 }
71 if (maxEvt < 0 || (skipEvts + maxEvt) > evt) {
72 maxEvt = evt - skipEvts;
73 }
74 ATH_MSG_INFO("Will be processing " << maxEvt << " events");
75 }
76
77 // Setup worker status DB (Spare one at start)
78 std::vector<bool> workers_done(m_clusterSvc->numRanks(), false);
79 workers_done[0] =
80 true; // Set 0 to true because it doesn't correspond to a worker
81 int num_workers_done = 1; // Init to 1 so we can compare to numRanks
82 std::vector<ClusterMessage::WorkerStatus> statuses(m_clusterSvc->numRanks());
83
84 // Entering event loop
85 m_clusterSvc->barrier();
86 // Note: no ++evt. This is because this is really a message loop, and we don't
87 // want to increment evt if we haven't provided an event
88 auto start = Clock::now();
89 for (int evt = skipEvts; evt < skipEvts + maxEvt;) {
90 ClusterMessage msg = m_clusterSvc->waitReceiveMessage();
91 // Messages we can get are RequestEvent, FinalWorkerStatus, or WorkerError
92 if (msg.messageType == ClusterMessageType::RequestEvent) {
93 ATH_MSG_DEBUG("Starting event " << evt << " on " << msg.source);
94 m_clusterSvc->sendMessage(
95 msg.source, ClusterMessage(ClusterMessageType::ProvideEvent, evt));
96 ++evt;
97 continue;
98 }
99
100 if (msg.messageType == ClusterMessageType::WorkerError) {
101 ATH_MSG_ERROR("Received WorkerError message from " << msg.source);
102 statuses.at(msg.source) = get<ClusterMessage::WorkerStatus>(msg.payload);
103 workers_done.at(msg.source) =
104 true; // If a worker hits an error, it's done
105 ++num_workers_done;
106 for (int i = 1; i < m_clusterSvc->numRanks(); ++i) {
107 if (!workers_done.at(i)) {
108 // Tell workers that aren't done to emergency stop
109 m_clusterSvc->sendMessage(
110 i, ClusterMessage(ClusterMessageType::EmergencyStop));
111 workers_done[i] = true;
112 ++num_workers_done;
113 }
114 }
115 break;
116 }
117
118 if (msg.messageType == ClusterMessageType::FinalWorkerStatus) {
119 ATH_MSG_INFO("Received FinalWorkerStatus from " << msg.source);
120 statuses.at(msg.source) = get<ClusterMessage::WorkerStatus>(msg.payload);
121 workers_done.at(msg.source) = true; // Worker hit end of stream
122 ++num_workers_done;
123 continue;
124 }
125
126 // Other message types are an error
127 ATH_MSG_ERROR("Received unexpected message {} from {}",
128 msg.messageType, msg.source);
129 }
130 auto all_provided = Clock::now() - start;
131 ATH_MSG_INFO("Provided all events to workers, waiting for them to complete.");
132 // Event loop done, tell remaining workers
133 while (num_workers_done < m_clusterSvc->numRanks()) {
134 ClusterMessage msg = m_clusterSvc->waitReceiveMessage();
135 // Messages we can get are RequestEvent, FinalWorkerStatus, or WorkerError
136 if (msg.messageType == ClusterMessageType::RequestEvent) {
137 m_clusterSvc->sendMessage(msg.source,
138 ClusterMessage(ClusterMessageType::EventsDone));
139 continue;
140 }
141
142 if (msg.messageType == ClusterMessageType::WorkerError) {
143 ATH_MSG_ERROR("Received WorkerError message from {}", msg.source);
144 statuses.at(msg.source) = get<ClusterMessage::WorkerStatus>(msg.payload);
145 workers_done.at(msg.source) =
146 true; // If a worker hits an error, it's done
147 ++num_workers_done;
148 for (int i = 1; i < m_clusterSvc->numRanks(); ++i) {
149 if (!workers_done.at(i)) {
150 // Tell workers that aren't done to emergency stop
151 m_clusterSvc->sendMessage(
152 i, ClusterMessage(ClusterMessageType::EmergencyStop));
153 workers_done[i] = true;
154 ++num_workers_done;
155 }
156 }
157 break;
158 }
159
160 if (msg.messageType == ClusterMessageType::FinalWorkerStatus) {
161 ATH_MSG_INFO("Received FinalWorkerStatus from {}", msg.source);
162 statuses.at(msg.source) = get<ClusterMessage::WorkerStatus>(msg.payload);
163 workers_done.at(msg.source) = true; // Told worker we're done
164 ++num_workers_done;
165 continue;
166 }
167
168 // Other message types are an error
169 ATH_MSG_ERROR("Received unexpected message {} from {}",
170 msg.messageType, msg.source);
171 }
172 auto all_done = Clock::now() - start;
173 // Collate status
174 int n_created = 0;
175 int n_skipped = 0;
176 int n_finished = 0;
177
178 StatusCode sc = StatusCode::SUCCESS;
179 int worker_idx = 0;
180 for (const auto& worker_status : statuses) {
181 if (worker_status.status.isFailure() &&
182 worker_status.status != StatusCode(9999)) {
183 sc = worker_status.status;
184 }
185 n_created += worker_status.createdEvents;
186 n_skipped += worker_status.skippedEvents;
187 n_finished += worker_status.finishedEvents;
188
189 if ((worker_idx++) != 0) {
190 ATH_MSG_INFO("Worker " << worker_idx << ": SC " << worker_status.status
191 << ", created " << worker_status.createdEvents
192 << ", skipped " << worker_status.skippedEvents
193 << ", finished " << worker_status.finishedEvents);
194 }
195 }
196
197 ATH_MSG_INFO("Overall: SC " << sc << ", created " << n_created << ", skipped "
198 << n_skipped << ", finished " << n_finished);
199 ATH_MSG_INFO("MASTER: Took {} to provide all events.",
200 std::chrono::hh_mm_ss(all_provided));
201 ATH_MSG_INFO("MASTER: Took {} to complete all events.",
202 std::chrono::hh_mm_ss(all_done));
203 return sc;
204}
205
208 bool end_of_stream = false;
209 // barrier so all ranks enter message loop together
210 m_clusterSvc->barrier();
211 auto start = Clock::now();
213 while (true) {
214 // Drain the scheduler (wait for at least one event to complete, then free
215 // up completed slots) in two circumstances
216 // 1. Have created exactly one event, so the first event runs to completion
217 // before any more are scheduled
218 // 2. There are no free slots left
219 bool haveFreeSlots =
220 m_schedulerSvc->freeSlots() > 0 && m_whiteboard->freeSlots() > 0;
221 if (!haveFreeSlots || m_nLocalCreatedEvts == 1) {
222 StatusCode sc = drainLocalScheduler();
223 if (sc.isFailure()) {
225 status.status = sc;
226 status.createdEvents = m_nLocalCreatedEvts;
227 status.skippedEvents = m_nLocalSkippedEvts;
228 status.finishedEvents = m_nLocalFinishedEvts;
229 m_clusterSvc->sendMessage(
230 0, ClusterMessage(ClusterMessageType::WorkerError, status));
231 return sc;
232 }
233 }
234
235 auto start_time = Clock::now();
236 m_clusterSvc->sendMessage(0,
237 ClusterMessage(ClusterMessageType::RequestEvent));
238 ClusterMessage msg = m_clusterSvc->waitReceiveMessage();
239 auto request_time = Clock::now() - start_time;
240 if (msg.messageType == ClusterMessageType::EmergencyStop) {
241 // Emergency stop, return FAILURE after fully draining the scheduler to
242 // prevent segfault
243 std::size_t numSlots = m_whiteboard->getNumberOfStores();
244 while (m_schedulerSvc->freeSlots() < numSlots) {
245 // Ignore StatusCode, going to return FAILURE anyway
246 (void)(drainLocalScheduler());
247 }
248 ATH_MSG_ERROR("Received EmergencyStop message!");
249 return StatusCode::FAILURE;
250 }
251
252 if (msg.messageType == ClusterMessageType::EventsDone) {
253 auto loop_time = Clock::now() - start;
254 ATH_MSG_INFO("Worker {} DONE. Loop took {} to process {} events.",
255 m_clusterSvc->rank(),
256 std::chrono::hh_mm_ss(loop_time),
258 // Been told we've reached end
259 // Provide status to master
261 // At end of stream, we need to *fully* drain the scheduler
262 StatusCode sc = StatusCode::SUCCESS;
263 std::size_t numSlots = m_whiteboard->getNumberOfStores();
264 while (sc.isSuccess() && m_schedulerSvc->freeSlots() < numSlots) {
266 }
267 status.status = sc;
268 status.createdEvents = m_nLocalCreatedEvts;
269 status.skippedEvents = m_nLocalSkippedEvts;
270 status.finishedEvents = m_nLocalFinishedEvts;
271 m_clusterSvc->sendMessage(
272 0, ClusterMessage(ClusterMessageType::FinalWorkerStatus, status));
273 return sc;
274 }
275
276 // Any other message other than ProvideEvent would now be an error
277 if (msg.messageType != ClusterMessageType::ProvideEvent ||
278 msg.source != 0) {
279 ATH_MSG_ERROR("Received unexpected message {} from {}",
280 msg.messageType, msg.source);
282 status.status = StatusCode::FAILURE;
283 status.createdEvents = m_nLocalCreatedEvts;
284 status.skippedEvents = m_nLocalSkippedEvts;
285 status.finishedEvents = m_nLocalFinishedEvts;
286 m_clusterSvc->sendMessage(
287 0, ClusterMessage(ClusterMessageType::WorkerError, status));
288 return StatusCode::FAILURE;
289 }
290
291 int evt = get<int>(msg.payload);
292 ATH_MSG_INFO("Starting event {}", evt);
293 StatusCode sc = insertEvent(
294 evt, end_of_stream,
295 std::chrono::duration_cast<std::chrono::nanoseconds>(request_time)
296 .count());
297 if (sc.isFailure() && !sc.isRecoverable()) {
299 status.status = sc;
300 status.createdEvents = m_nLocalCreatedEvts;
301 status.skippedEvents = m_nLocalSkippedEvts;
302 status.finishedEvents = m_nLocalFinishedEvts;
303 m_clusterSvc->sendMessage(
304 0, ClusterMessage(ClusterMessageType::WorkerError, status));
305 return sc;
306 }
307 if (end_of_stream || m_terminateLoop) {
308 auto loop_time = Clock::now() - start;
309 ATH_MSG_INFO("Worker {} DONE. Loop took to process events.",
310 m_clusterSvc->rank(),
311 std::chrono::hh_mm_ss(loop_time),
313 // reached end of stream, drain scheduler
315 // At end of stream, we need to *fully* drain the scheduler
316 StatusCode sc = StatusCode::SUCCESS;
317 std::size_t numSlots = m_whiteboard->getNumberOfStores();
318 while (sc.isSuccess() && m_schedulerSvc->freeSlots() < numSlots) {
320 }
321 status.status = sc;
322 status.createdEvents = m_nLocalCreatedEvts;
323 status.skippedEvents = m_nLocalSkippedEvts;
324 status.finishedEvents = m_nLocalFinishedEvts;
325 m_clusterSvc->sendMessage(
326 0, ClusterMessage(ClusterMessageType::FinalWorkerStatus, status));
327 return sc;
328 }
329 }
330}
331
333StatusCode MPIHiveEventLoopMgr::insertEvent(int eventIdx, bool& endOfStream,
334 std::int64_t requestTime_ns) {
335 // fast-forward to event
336 // Create the event context now so next writes into the next slot when
337 // skipping, not the one that's being used
338 endOfStream = false;
339 auto ctx = createEventContext();
340 Gaudi::Hive::setCurrentContext(ctx);
341 ctx.setEvt(
342 eventIdx); // Make the event numbers in the log actually make sense
343 if (!ctx.valid()) {
344 endOfStream = true; // BUG: Doesn't actually mean end of stream. Remove
345 // after making sure!
346 return StatusCode::FAILURE;
347 }
348
349 const std::size_t slot = ctx.slot(); // Need this for later
350 // Codex (GPT 5.6-sol) identified use of seek should be reverted since that
351 // function isn't implemented properly
352 if (m_evtSelector != nullptr) {
353 const int nToJump = (eventIdx - 1) - m_evtSelectorCurrentPos;
354 if (nToJump < 0) {
355 ATH_MSG_ERROR("Cannot jump backwards by {} events", nToJump);
356 return StatusCode::FAILURE;
357 }
358 if (nToJump > 0) {
359 ATH_CHECK(m_evtSelector->next(*m_evtContext, nToJump));
360 }
361 m_evtSelectorCurrentPos = eventIdx;
362 }
363 // execute event
364 StatusCode sc = executeEvent(std::move(ctx));
365 const auto evtID =
366 m_lastEventContext.eventID(); // Set in AthenaHiveEventLoopMgr
367 m_clusterSvc->log_addEvent(eventIdx, evtID.run_number(), evtID.event_number(),
368 requestTime_ns, slot);
369
370 if (sc.isRecoverable()) {
372 } else if (sc.isSuccess()) {
374 }
375 return sc;
376}
377
382 ? StatusCode::FAILURE : StatusCode::SUCCESS;
383}
384
385StatusCode MPIHiveEventLoopMgr::eventFinished(const EventContext& ctx) {
386 const auto status = m_aess->eventStatus(ctx);
387 m_clusterSvc->log_completeEvent(ctx.evt(), ctx.eventID().run_number(),
388 ctx.eventID().event_number(), status);
389
390 if (status == EventStatus::Success) {
392 return StatusCode::SUCCESS;
393 }
394
395 ATH_MSG_ERROR("Failed event detected on " << ctx
396 << " w/ fail mode: " << status);
399 // Preserve the MPI policy: tolerate isolated failures, stop after three
400 // consecutive failures or ten in total. Even at the threshold, finishEvent()
401 // still balances EndProcessing and releases this completed event's slot.
402 if (m_contiguousFailedEvts >= 3 || m_totalFailedEvts >= 10) {
403 return StatusCode::FAILURE;
404 }
405 return StatusCode::SUCCESS;
406}
#define ATH_CHECK
Evaluate an expression and check for errors.
#define ATH_MSG_DEBUG(x,...)
#define ATH_MSG_ERROR(x,...)
#define ATH_MSG_INFO(x,...)
static Double_t sc
std::chrono::high_resolution_clock Clock
The MPI event loop manager.
IEvtSelector * m_evtSelector
Reference to the Event Selector.
virtual StatusCode initialize() override
implementation of IAppMgrUI::initalize
SmartIF< IHiveWhiteBoard > m_whiteboard
Reference to the Whiteboard interface.
int drainScheduler(int &finishedEvents)
Drain the scheduler from all actions that may be queued.
virtual int size() override
Return the size of the collection.
SmartIF< IAlgExecStateSvc > m_aess
Reference to the Algorithm Execution State Svc.
virtual const std::string & name() const override
SmartIF< IProperty > m_appMgrProperty
Property interface of ApplicationMgr.
MsgStream & msg() const
The standard message stream.
EvtContext * m_evtContext
Gaudi event selector Context (may be used as a cursor by the evt selector).
AthenaHiveEventLoopMgr(const std::string &nam, ISvcLocator *svcLoc)
Standard Constructor.
virtual StatusCode finalize() override
implementation of IAppMgrUI::finalize
virtual StatusCode executeEvent(EventContext &&ctx) override
implementation of IEventProcessor::executeEvent(void* par)
virtual EventContext createEventContext() override
Create event context.
SmartIF< IScheduler > m_schedulerSvc
A shortcut for the scheduler.
StatusCode workerEventLoop()
Worker event loop (runs on worker, requests events over MPI).
StatusCode drainLocalScheduler()
Drain the local scheduler of any (at least one) completed events.
ServiceHandle< IMPIClusterSvc > m_clusterSvc
Reference to the MPIClusterSvc.
MPIHiveEventLoopMgr(const std::string &name, ISvcLocator *svcLoc)
Standard Constructor.
virtual StatusCode finalize() override
implementation of IAppMgrUI::finalize
virtual ~MPIHiveEventLoopMgr()
Standard Destructor.
virtual StatusCode nextEvent(int maxevt) override
implementation of IAppMgrUI::nextEvent. maxevt==0 returns immediately
StatusCode eventFinished(const EventContext &ctx) override
Record completion and apply MPI's tolerated-failure thresholds.
StatusCode masterEventLoop(int maxEvt)
Master event loop (runs on master, provides events over MPI).
virtual StatusCode initialize() override
implementation of IAppMgrUI::initalize
StatusCode insertEvent(int eventIdx, bool &endOfStream, std::int64_t requestTime_ns)
Insert an event into the local scheduler.
UnsignedIntegerProperty m_firstEventIndex
T * get(TKey *tobj)
get a TObject* from a TKey* (why can't a TObject be a TKey?)
Definition hcg.cxx:132
int count(std::string s, const std::string &regx)
count how many occurances of a regx are in a string
Definition hcg.cxx:148
A class describing a message sent between nodes in a cluster.