TimeoutAlg Class Reference

Algorithm to monitor event timeouts. More...

#include <TimeoutAlg.h>

Inheritance diagram for TimeoutAlg:

Collaboration diagram for TimeoutAlg:

Public Member Functions
virtual StatusCode	initialize () override
	Algorithm to monitor event timeouts.
virtual StatusCode	execute (const EventContext &ctx) const override
virtual StatusCode	stop () override
virtual void	handle (const Incident &inc) override

Protected Member Functions
void	setTimeout (Timeout &instance)
	Set timeout.
void	resetTimeout (Timeout &instance)
	Reset timeout.

Private Types
using	clock_t = std::chrono::steady_clock

Private Member Functions
void	timeoutThread ()
	Watchdog thread.
void	handleTimeout (EventContext::ContextID_t slot)
	Handle timeout.

Private Attributes
std::chrono::nanoseconds	m_timeout
	Timeout property as duration.
SG::SlotSpecificObj< clock_t::time_point > m_eventStartTime	ATLAS_THREAD_SAFE
	Start time of each event per slot.
std::thread m_thread	ATLAS_THREAD_SAFE
	Watchdog thread.
std::promise< void >	m_stop_thread
	Signal to stop watchdog thread.
std::atomic< bool >	m_stopped {false}
	Has watchdog thread already been stopped? (to avoid setting future twice)
std::mutex	m_handleMutex
	Mutex for handleTimeout.
Properties
Gaudi::Property< unsigned long long >	m_timeoutProp
Gaudi::Property< unsigned long long >	m_checkInterval
Gaudi::Property< bool >	m_dumpState
Gaudi::Property< bool >	m_abort

Detailed Description

Algorithm to monitor event timeouts.

Author

Frank Winklmeier

Date

Sep, 2025

Algorithm providing a watchdog thread for event timeouts.

This algorithm should run early on (ideally first) in the event sequence. It records the event start time and launches a watchdog thread that checks periodically if an event has timed out.

See the algorithm properties for possible actions on an event timeout.

Definition at line 35 of file TimeoutAlg.h.

Member Typedef Documentation

clock_t

using TimeoutAlg::clock_t = std::chrono::steady_clock

private

Definition at line 48 of file TimeoutAlg.h.

Member Function Documentation

execute()

StatusCode TimeoutAlg::execute ( const EventContext & ctx ) const

overridevirtual

Definition at line 39 of file TimeoutAlg.cxx.

{
  // Timeout thread is started on first event to make sure this also works
  // in athenaMP (threads usually don't survive forking).
  [[maybe_unused]] static const bool initThread = [&](){
    if (m_timeoutProp > 0) {
      const auto nc_this ATLAS_THREAD_SAFE = const_cast<TimeoutAlg*>(this);
      m_thread = std::thread(&TimeoutAlg::timeoutThread, nc_this);
    }
    return true;
  }();
 
  // Set event start time for current slot
  *m_eventStartTime.get(ctx) = clock_t::now();
 
  return StatusCode::SUCCESS;
}

handle()

void TimeoutAlg::handle ( const Incident & inc )

overridevirtual

Definition at line 58 of file TimeoutAlg.cxx.

{
  if (inc.type() == "EndAlgorithms") {
    ATH_MSG_DEBUG("Resetting event timeout for slot " << inc.context().slot());
    // Reset start time for slot to zero
    *m_eventStartTime.get(inc.context()) = {};
  }
}

handleTimeout()

void TimeoutAlg::handleTimeout ( EventContext::ContextID_t slot )

private

Handle timeout.

Definition at line 109 of file TimeoutAlg.cxx.

{
  // To avoid getting another timeout while handling this one
  std::scoped_lock lock(m_handleMutex);
 
  // Create minimal context with slot number
  const EventContext ctx(0, slot);
 
  // Don't duplicate the actions if the timeout was already reached for this slot
  if (Athena::Timeout::instance(ctx).reached()) return;
 
  // Print ERROR message
  const std::string msg = std::format("Event timeout ({}) in slot {} reached",
                                      std::chrono::duration<double>(m_timeout), slot);
  ATH_MSG_ERROR(msg);
 
  // Set timeout flag
  setTimeout(Athena::Timeout::instance(ctx));
 
  // Dump scheduler state if requested
  if (m_dumpState) {
    ServiceHandle<IScheduler> schedulerSvc("AvalancheSchedulerSvc", name());
    if (schedulerSvc.retrieve().isSuccess()) {
      schedulerSvc->dumpState();
    }
  }
 
  // Abort job if requested
  if (m_abort) {
    // Stop the timeout thread to avoid additional triggers
    stop().ignore();
 
    // Tell CoreDumpSvc about the reason for the abort
    ServiceHandle<ICoreDumpSvc> coreDumpSvc("CoreDumpSvc", name());
    if ( coreDumpSvc.retrieve().isSuccess() ) {
      coreDumpSvc->setCoreDumpInfo(ctx, "Reason", msg);
    }
    else {
      std::cerr << msg << std::endl;
    }
    // Abort job (and let CoreDumpSvc handle SIGABRT)
    std::abort();
  }
 
}

initialize()

StatusCode TimeoutAlg::initialize ( )

overridevirtual

Algorithm to monitor event timeouts.

Author

Frank Winklmeier

Date

Sep, 2025

Definition at line 21 of file TimeoutAlg.cxx.

{
  if (RUNNING_ON_VALGRIND) {
    ATH_MSG_INFO("Detected running on valgrind. Disabling algorithm timeout");
    m_timeoutProp=0;
    return StatusCode::SUCCESS;
  }
  m_timeout = std::chrono::nanoseconds(m_timeoutProp);
 
  // Subscribe to EndAlgorithms (includes output sequence)
  ServiceHandle<IIncidentSvc> incSvc("IncidentSvc/IncidentSvc", name());
  ATH_CHECK(incSvc.retrieve());
  incSvc->addListener(this, "EndAlgorithms", /*priority*/ 0);
 
  return StatusCode::SUCCESS;
}

resetTimeout()

void Athena::TimeoutMaster::resetTimeout ( Timeout & instance )

inlineprotectedinherited

Reset timeout.

Definition at line 83 of file Timeout.h.

{ instance.reset(); }

setTimeout()

void Athena::TimeoutMaster::setTimeout ( Timeout & instance )

inlineprotectedinherited

Set timeout.

Definition at line 80 of file Timeout.h.

{ instance.set(); }

stop()

StatusCode TimeoutAlg::stop ( )

overridevirtual

Definition at line 68 of file TimeoutAlg.cxx.

{
  if (m_thread.joinable() && !m_stopped.exchange(true)) {
    // Signal timeout thread to stop
    ATH_MSG_DEBUG("Stopping timeout thread");
    m_stop_thread.set_value();
    m_thread.join();
  }
 
  return StatusCode::SUCCESS;
}

timeoutThread()

void TimeoutAlg::timeoutThread ( )

private

Watchdog thread.

Definition at line 81 of file TimeoutAlg.cxx.

{
  ATH_MSG_INFO(std::format("Setting per-event timeout of {}",
                           std::chrono::duration<double>(m_timeout)));
 
  // Wakeup at regular intervals (with a minimum frequency, useful for long timeouts)
  const std::chrono::nanoseconds wakeup_interval =
    std::min(m_timeout, std::chrono::nanoseconds(m_checkInterval));
 
  // Loop until we have received stop signal
  auto stop_signal = m_stop_thread.get_future();
  while ( stop_signal.wait_for(wakeup_interval) == std::future_status::timeout ) {
 
    // Loop over all slots and check if event has reached timeout
    const auto now = clock_t::now();
    for (EventContext::ContextID_t slot = 0;
         const auto& startTime : m_eventStartTime) {
 
      if (startTime.time_since_epoch().count() > 0 &&  now > startTime + m_timeout) {
        handleTimeout(slot);
      }
 
      ++slot;
    }
  }
}

Member Data Documentation

ATLAS_THREAD_SAFE [1/2]

SG::SlotSpecificObj<clock_t::time_point> m_eventStartTime TimeoutAlg::ATLAS_THREAD_SAFE

mutableprivate

Start time of each event per slot.

Definition at line 76 of file TimeoutAlg.h.

ATLAS_THREAD_SAFE [2/2]

std::thread m_thread TimeoutAlg::ATLAS_THREAD_SAFE

mutableprivate

Watchdog thread.

Definition at line 79 of file TimeoutAlg.h.

m_abort

Gaudi::Property<bool> TimeoutAlg::m_abort

private

Initial value:

{
    this, "AbortJob", false, "Abort job on timeout"
  }

Definition at line 67 of file TimeoutAlg.h.

                             {
    this, "AbortJob", false, "Abort job on timeout"
  };

m_checkInterval

Gaudi::Property<unsigned long long> TimeoutAlg::m_checkInterval

private

Initial value:

{
    this, "MaxCheckInterval", 10*1e9, "Maximum time (ns) between timeout checks"
  }

Definition at line 61 of file TimeoutAlg.h.

                                                   {
    this, "MaxCheckInterval", 10*1e9, "Maximum time (ns) between timeout checks"
  };

m_dumpState

Gaudi::Property<bool> TimeoutAlg::m_dumpState

private

Initial value:

{
    this, "DumpSchedulerState", false, "Print scheduler state on timeout"
  }

Definition at line 64 of file TimeoutAlg.h.

                                 {
    this, "DumpSchedulerState", false, "Print scheduler state on timeout"
  };

m_handleMutex

std::mutex TimeoutAlg::m_handleMutex

private

Mutex for handleTimeout.

Definition at line 88 of file TimeoutAlg.h.

m_stop_thread

std::promise<void> TimeoutAlg::m_stop_thread

private

Signal to stop watchdog thread.

Definition at line 82 of file TimeoutAlg.h.

m_stopped

std::atomic<bool> TimeoutAlg::m_stopped {false}

private

Has watchdog thread already been stopped? (to avoid setting future twice)

Definition at line 85 of file TimeoutAlg.h.

{false};

m_timeout

std::chrono::nanoseconds TimeoutAlg::m_timeout

private

Timeout property as duration.

Definition at line 73 of file TimeoutAlg.h.

m_timeoutProp

Gaudi::Property<unsigned long long> TimeoutAlg::m_timeoutProp

private

Initial value:

{
    this, "Timeout", 0, "Timeout in nanoseconds (0 means disabled)"
  }

Definition at line 58 of file TimeoutAlg.h.

                                                 {
    this, "Timeout", 0, "Timeout in nanoseconds (0 means disabled)"
  };

---

The documentation for this class was generated from the following files:

• TimeoutAlg.h
• TimeoutAlg.cxx