Algorithm to monitor event timeouts. More...

#include <TimeoutAlg.h>

Inheritance diagram for TimeoutAlg:

Collaboration diagram for TimeoutAlg:

Public Member Functions
virtual StatusCode	initialize () override
	Algorithm to monitor event timeouts. More...

virtual StatusCode	execute (const EventContext &ctx) const override

virtual StatusCode	stop () override

virtual void	handle (const Incident &inc) override

Protected Member Functions
void	setTimeout (Timeout &instance)
	Set timeout. More...

void	resetTimeout (Timeout &instance)
	Reset timeout. More...

Private Types
using	clock_t = std::chrono::steady_clock

Private Member Functions
void	timeoutThread ()
	Watchdog thread. More...

void	handleTimeout (EventContext::ContextID_t slot)
	Handle timeout. More...

Private Attributes
std::chrono::nanoseconds	m_timeout
	Timeout property as duration. More...

SG::SlotSpecificObj< clock_t::time_point > m_eventStartTime	ATLAS_THREAD_SAFE
	Start time of each event per slot. More...

std::thread m_thread	ATLAS_THREAD_SAFE
	Watchdog thread. More...

std::promise< void >	m_stop_thread
	Signal to stop watchdog thread. More...

std::atomic< bool >	m_stopped {false}
	Has watchdog thread already been stopped? (to avoid setting future twice) More...

std::mutex	m_handleMutex
	Mutex for handleTimeout. More...

Properties
Gaudi::Property< unsigned long long >	m_timeoutProp

Gaudi::Property< unsigned long long >	m_checkInterval

Gaudi::Property< bool >	m_dumpState

Gaudi::Property< bool >	m_abort

Detailed Description

Algorithm to monitor event timeouts.

Author: Frank Winklmeier

Date: Sep, 2025

Algorithm providing a watchdog thread for event timeouts.

This algorithm should run early on (ideally first) in the event sequence. It records the event start time and launches a watchdog thread that checks periodically if an event has timed out.

See the algorithm properties for possible actions on an event timeout.

Definition at line 35 of file TimeoutAlg.h.

Member Typedef Documentation

◆ clock_t

using TimeoutAlg::clock_t = std::chrono::steady_clock

private

Definition at line 48 of file TimeoutAlg.h.

Member Function Documentation

◆ execute()

StatusCode TimeoutAlg::execute ( const EventContext & ctx ) const

overridevirtual

Definition at line 33 of file TimeoutAlg.cxx.

 {
   // Timeout thread is started on first event to make sure this also works
   // in athenaMP (threads usually don't survive forking).
   [[maybe_unused]] static const bool initThread = [&](){
     if (m_timeoutProp > 0) {
       const auto nc_this ATLAS_THREAD_SAFE = const_cast<TimeoutAlg*>(this);
       m_thread = std::thread(&TimeoutAlg::timeoutThread, nc_this);
     }
     return true;
   }();
  
   // Set event start time for current slot
   *m_eventStartTime.get(ctx) = clock_t::now();
  
   return StatusCode::SUCCESS;
 }

◆ handle()

void TimeoutAlg::handle ( const Incident & inc )

overridevirtual

Definition at line 52 of file TimeoutAlg.cxx.

 {
   if (inc.type() == "EndAlgorithms") {
     ATH_MSG_DEBUG("Resetting event timeout for slot " << inc.context().slot());
     // Reset start time for slot to zero
     *m_eventStartTime.get(inc.context()) = {};
   }
 }

◆ handleTimeout()

void TimeoutAlg::handleTimeout ( EventContext::ContextID_t slot )

private

Handle timeout.

Definition at line 103 of file TimeoutAlg.cxx.

 {
   // To avoid getting another timeout while handling this one
   std::scoped_lock lock(m_handleMutex);
  
   // Create minimal context with slot number
   const EventContext ctx(0, slot);
  
   // Don't duplicate the actions if the timeout was already reached for this slot
   if (Athena::Timeout::instance(ctx).reached()) return;
  
   // Print ERROR message
   const std::string msg = std::format("Event timeout ({}) in slot {} reached",
                                       std::chrono::duration<double>(m_timeout), slot);
   ATH_MSG_ERROR(msg);
  
   // Set timeout flag
   setTimeout(Athena::Timeout::instance(ctx));
  
   // Dump scheduler state if requested
   if (m_dumpState) {
     ServiceHandle<IScheduler> schedulerSvc("AvalancheSchedulerSvc", name());
     if (schedulerSvc.retrieve().isSuccess()) {
       schedulerSvc->dumpState();
     }
   }
  
   // Abort job if requested
   if (m_abort) {
     // Stop the timeout thread to avoid additional triggers
     stop().ignore();
  
     // Tell CoreDumpSvc about the reason for the abort
     ServiceHandle<ICoreDumpSvc> coreDumpSvc("CoreDumpSvc", name());
     if ( coreDumpSvc.retrieve().isSuccess() ) {
       coreDumpSvc->setCoreDumpInfo(ctx, "Reason", msg);
     }
     else {
       std::cerr << msg << std::endl;
     }
     // Abort job (and let CoreDumpSvc handle SIGABRT)
     std::abort();
   }
  
 }

◆ initialize()

StatusCode TimeoutAlg::initialize ( )

overridevirtual

Algorithm to monitor event timeouts.

Author: Frank Winklmeier

Date: Sep, 2025

Definition at line 20 of file TimeoutAlg.cxx.

 {
   m_timeout = std::chrono::nanoseconds(m_timeoutProp);
  
   // Subscribe to EndAlgorithms (includes output sequence)
   ServiceHandle<IIncidentSvc> incSvc("IncidentSvc/IncidentSvc", name());
   ATH_CHECK(incSvc.retrieve());
   incSvc->addListener(this, "EndAlgorithms", /*priority*/ 0);
  
   return StatusCode::SUCCESS;
 }

◆ resetTimeout()

void Athena::TimeoutMaster::resetTimeout ( Timeout & instance )

inlineprotectedinherited

Reset timeout.

Definition at line 83 of file Timeout.h.

83 { instance.reset(); }

◆ setTimeout()

void Athena::TimeoutMaster::setTimeout ( Timeout & instance )

inlineprotectedinherited

Set timeout.

Definition at line 80 of file Timeout.h.

80 { instance.set(); }

◆ stop()

StatusCode TimeoutAlg::stop ( )

overridevirtual

Definition at line 62 of file TimeoutAlg.cxx.

 {
   if (m_thread.joinable() && !m_stopped.exchange(true)) {
     // Signal timeout thread to stop
     ATH_MSG_DEBUG("Stopping timeout thread");
     m_stop_thread.set_value();
     m_thread.join();
   }
  
   return StatusCode::SUCCESS;
 }

◆ timeoutThread()

void TimeoutAlg::timeoutThread ( )

private

Watchdog thread.

Definition at line 75 of file TimeoutAlg.cxx.

 {
   ATH_MSG_INFO(std::format("Setting per-event timeout of {}",
                            std::chrono::duration<double>(m_timeout)));
  
   // Wakeup at regular intervals (with a minimum frequency, useful for long timeouts)
   const std::chrono::nanoseconds wakeup_interval =
     std::min(m_timeout, std::chrono::nanoseconds(m_checkInterval));
  
   // Loop until we have received stop signal
   auto stop_signal = m_stop_thread.get_future();
   while ( stop_signal.wait_for(wakeup_interval) == std::future_status::timeout ) {
  
     // Loop over all slots and check if event has reached timeout
     const auto now = clock_t::now();
     for (EventContext::ContextID_t slot = 0;
          const auto& startTime : m_eventStartTime) {
  
       if (startTime.time_since_epoch().count() > 0 &&  now > startTime + m_timeout) {
         handleTimeout(slot);
       }
  
       ++slot;
     }
   }
 }

Member Data Documentation

◆ ATLAS_THREAD_SAFE [1/2]

SG::SlotSpecificObj<clock_t::time_point> m_eventStartTime TimeoutAlg::ATLAS_THREAD_SAFE

mutableprivate

Start time of each event per slot.

Definition at line 76 of file TimeoutAlg.h.

◆ ATLAS_THREAD_SAFE [2/2]

std::thread m_thread TimeoutAlg::ATLAS_THREAD_SAFE

mutableprivate

Watchdog thread.

Definition at line 79 of file TimeoutAlg.h.

◆ m_abort

Gaudi::Property<bool> TimeoutAlg::m_abort

private

Initial value:

{
    this, "AbortJob", false, "Abort job on timeout"
  }

Definition at line 67 of file TimeoutAlg.h.

◆ m_checkInterval

Gaudi::Property<unsigned long long> TimeoutAlg::m_checkInterval

private

Initial value:

{
    this, "MaxCheckInterval", 10*1e9, "Maximum time (ns) between timeout checks"
  }

Definition at line 61 of file TimeoutAlg.h.

◆ m_dumpState

Gaudi::Property<bool> TimeoutAlg::m_dumpState

private

Initial value:

{
    this, "DumpSchedulerState", false, "Print scheduler state on timeout"
  }

Definition at line 64 of file TimeoutAlg.h.

◆ m_handleMutex

std::mutex TimeoutAlg::m_handleMutex

private

Mutex for handleTimeout.

Definition at line 88 of file TimeoutAlg.h.

◆ m_stop_thread

std::promise<void> TimeoutAlg::m_stop_thread

private

Signal to stop watchdog thread.

Definition at line 82 of file TimeoutAlg.h.

◆ m_stopped

std::atomic<bool> TimeoutAlg::m_stopped {false}

private

Has watchdog thread already been stopped? (to avoid setting future twice)

Definition at line 85 of file TimeoutAlg.h.

◆ m_timeout

std::chrono::nanoseconds TimeoutAlg::m_timeout

private

Timeout property as duration.

Definition at line 73 of file TimeoutAlg.h.

◆ m_timeoutProp

Gaudi::Property<unsigned long long> TimeoutAlg::m_timeoutProp

private

Initial value:

{
    this, "Timeout", 0, "Timeout in nanoseconds (0 means disabled)"
  }

Definition at line 58 of file TimeoutAlg.h.

The documentation for this class was generated from the following files:

Public Member Functions

Protected Member Functions

Private Types

Private Member Functions

Private Attributes

Detailed Description

Member Typedef Documentation

◆ clock_t

Member Function Documentation

◆ execute()

◆ handle()

◆ handleTimeout()

◆ initialize()

◆ resetTimeout()

◆ setTimeout()

◆ stop()

◆ timeoutThread()

Member Data Documentation

◆ ATLAS_THREAD_SAFE [1/2]

◆ ATLAS_THREAD_SAFE [2/2]

◆ m_abort

◆ m_checkInterval

◆ m_dumpState

◆ m_handleMutex

◆ m_stop_thread

◆ m_stopped

◆ m_timeout

◆ m_timeoutProp