7#include "GaudiKernel/AppReturnCode.h"
17using Clock = std::chrono::high_resolution_clock;
46 return StatusCode::SUCCESS;
69 return StatusCode::FAILURE;
71 if (maxEvt < 0 || (skipEvts + maxEvt) > evt) {
72 maxEvt = evt - skipEvts;
74 ATH_MSG_INFO(
"Will be processing " << maxEvt <<
" events");
78 std::vector<bool> workers_done(
m_clusterSvc->numRanks(),
false);
81 int num_workers_done = 1;
82 std::vector<ClusterMessage::WorkerStatus> statuses(
m_clusterSvc->numRanks());
88 auto start = Clock::now();
89 for (
int evt = skipEvts; evt < skipEvts + maxEvt;) {
92 if (
msg.messageType == ClusterMessageType::RequestEvent) {
100 if (
msg.messageType == ClusterMessageType::WorkerError) {
103 workers_done.at(
msg.source) =
107 if (!workers_done.at(i)) {
111 workers_done[i] =
true;
118 if (
msg.messageType == ClusterMessageType::FinalWorkerStatus) {
121 workers_done.at(
msg.source) =
true;
128 msg.messageType,
msg.source);
130 auto all_provided = Clock::now() - start;
131 ATH_MSG_INFO(
"Provided all events to workers, waiting for them to complete.");
133 while (num_workers_done < m_clusterSvc->numRanks()) {
136 if (
msg.messageType == ClusterMessageType::RequestEvent) {
142 if (
msg.messageType == ClusterMessageType::WorkerError) {
145 workers_done.at(
msg.source) =
149 if (!workers_done.at(i)) {
153 workers_done[i] =
true;
160 if (
msg.messageType == ClusterMessageType::FinalWorkerStatus) {
163 workers_done.at(
msg.source) =
true;
170 msg.messageType,
msg.source);
172 auto all_done = Clock::now() - start;
178 StatusCode
sc = StatusCode::SUCCESS;
180 for (
const auto& worker_status : statuses) {
181 if (worker_status.status.isFailure() &&
182 worker_status.status != StatusCode(9999)) {
183 sc = worker_status.status;
185 n_created += worker_status.createdEvents;
186 n_skipped += worker_status.skippedEvents;
187 n_finished += worker_status.finishedEvents;
189 if ((worker_idx++) != 0) {
190 ATH_MSG_INFO(
"Worker " << worker_idx <<
": SC " << worker_status.status
191 <<
", created " << worker_status.createdEvents
192 <<
", skipped " << worker_status.skippedEvents
193 <<
", finished " << worker_status.finishedEvents);
197 ATH_MSG_INFO(
"Overall: SC " <<
sc <<
", created " << n_created <<
", skipped "
198 << n_skipped <<
", finished " << n_finished);
200 std::chrono::hh_mm_ss(all_provided));
202 std::chrono::hh_mm_ss(all_done));
208 bool end_of_stream =
false;
211 auto start = Clock::now();
223 if (
sc.isFailure()) {
235 auto start_time = Clock::now();
239 auto request_time = Clock::now() - start_time;
240 if (
msg.messageType == ClusterMessageType::EmergencyStop) {
243 std::size_t numSlots =
m_whiteboard->getNumberOfStores();
249 return StatusCode::FAILURE;
252 if (
msg.messageType == ClusterMessageType::EventsDone) {
253 auto loop_time = Clock::now() - start;
254 ATH_MSG_INFO(
"Worker {} DONE. Loop took {} to process {} events.",
256 std::chrono::hh_mm_ss(loop_time),
262 StatusCode
sc = StatusCode::SUCCESS;
263 std::size_t numSlots =
m_whiteboard->getNumberOfStores();
272 0,
ClusterMessage(ClusterMessageType::FinalWorkerStatus, status));
277 if (
msg.messageType != ClusterMessageType::ProvideEvent ||
280 msg.messageType,
msg.source);
282 status.status = StatusCode::FAILURE;
288 return StatusCode::FAILURE;
295 std::chrono::duration_cast<std::chrono::nanoseconds>(request_time)
297 if (
sc.isFailure() && !
sc.isRecoverable()) {
308 auto loop_time = Clock::now() - start;
309 ATH_MSG_INFO(
"Worker {} DONE. Loop took to process events.",
311 std::chrono::hh_mm_ss(loop_time),
316 StatusCode
sc = StatusCode::SUCCESS;
317 std::size_t numSlots =
m_whiteboard->getNumberOfStores();
326 0,
ClusterMessage(ClusterMessageType::FinalWorkerStatus, status));
334 std::int64_t requestTime_ns) {
340 Gaudi::Hive::setCurrentContext(ctx);
346 return StatusCode::FAILURE;
349 const std::size_t slot = ctx.slot();
355 ATH_MSG_ERROR(
"Cannot jump backwards by {} events", nToJump);
356 return StatusCode::FAILURE;
367 m_clusterSvc->log_addEvent(eventIdx, evtID.run_number(), evtID.event_number(),
368 requestTime_ns, slot);
370 if (
sc.isRecoverable()) {
372 }
else if (
sc.isSuccess()) {
382 ? StatusCode::FAILURE : StatusCode::SUCCESS;
386 const auto status =
m_aess->eventStatus(ctx);
387 m_clusterSvc->log_completeEvent(ctx.evt(), ctx.eventID().run_number(),
388 ctx.eventID().event_number(), status);
390 if (status == EventStatus::Success) {
392 return StatusCode::SUCCESS;
396 <<
" w/ fail mode: " << status);
403 return StatusCode::FAILURE;
405 return StatusCode::SUCCESS;
#define ATH_CHECK
Evaluate an expression and check for errors.
#define ATH_MSG_DEBUG(x,...)
#define ATH_MSG_ERROR(x,...)
#define ATH_MSG_INFO(x,...)
std::chrono::high_resolution_clock Clock
The MPI event loop manager.
IEvtSelector * m_evtSelector
Reference to the Event Selector.
virtual StatusCode initialize() override
implementation of IAppMgrUI::initalize
SmartIF< IHiveWhiteBoard > m_whiteboard
Reference to the Whiteboard interface.
int drainScheduler(int &finishedEvents)
Drain the scheduler from all actions that may be queued.
virtual int size() override
Return the size of the collection.
SmartIF< IAlgExecStateSvc > m_aess
Reference to the Algorithm Execution State Svc.
virtual const std::string & name() const override
SmartIF< IProperty > m_appMgrProperty
Property interface of ApplicationMgr.
MsgStream & msg() const
The standard message stream.
EvtContext * m_evtContext
Gaudi event selector Context (may be used as a cursor by the evt selector).
AthenaHiveEventLoopMgr(const std::string &nam, ISvcLocator *svcLoc)
Standard Constructor.
virtual StatusCode finalize() override
implementation of IAppMgrUI::finalize
virtual StatusCode executeEvent(EventContext &&ctx) override
implementation of IEventProcessor::executeEvent(void* par)
EventContext m_lastEventContext
virtual EventContext createEventContext() override
Create event context.
SmartIF< IScheduler > m_schedulerSvc
A shortcut for the scheduler.
StatusCode workerEventLoop()
Worker event loop (runs on worker, requests events over MPI).
StatusCode drainLocalScheduler()
Drain the local scheduler of any (at least one) completed events.
int m_contiguousFailedEvts
ServiceHandle< IMPIClusterSvc > m_clusterSvc
Reference to the MPIClusterSvc.
MPIHiveEventLoopMgr(const std::string &name, ISvcLocator *svcLoc)
Standard Constructor.
virtual StatusCode finalize() override
implementation of IAppMgrUI::finalize
virtual ~MPIHiveEventLoopMgr()
Standard Destructor.
virtual StatusCode nextEvent(int maxevt) override
implementation of IAppMgrUI::nextEvent. maxevt==0 returns immediately
StatusCode eventFinished(const EventContext &ctx) override
Record completion and apply MPI's tolerated-failure thresholds.
StatusCode masterEventLoop(int maxEvt)
Master event loop (runs on master, provides events over MPI).
virtual StatusCode initialize() override
implementation of IAppMgrUI::initalize
StatusCode insertEvent(int eventIdx, bool &endOfStream, std::int64_t requestTime_ns)
Insert an event into the local scheduler.
UnsignedIntegerProperty m_firstEventIndex
int m_evtSelectorCurrentPos
T * get(TKey *tobj)
get a TObject* from a TKey* (why can't a TObject be a TKey?)
int count(std::string s, const std::string ®x)
count how many occurances of a regx are in a string
A class describing a message sent between nodes in a cluster.