SharedEvtQueueConsumer Class Reference

#include <SharedEvtQueueConsumer.h>

Inheritance diagram for SharedEvtQueueConsumer:

Collaboration diagram for SharedEvtQueueConsumer:

Public Member Functions
	SharedEvtQueueConsumer (const std::string &type, const std::string &name, const IInterface *parent)
virtual	~SharedEvtQueueConsumer () override
virtual StatusCode	initialize () override
virtual StatusCode	finalize () override
virtual int makePool	ATLAS_NOT_THREAD_SAFE (int maxevt, int nprocs, const std::string &topdir) override
virtual StatusCode exec	ATLAS_NOT_THREAD_SAFE () override
virtual StatusCode wait_once	ATLAS_NOT_THREAD_SAFE (pid_t &pid) override
virtual void	reportSubprocessStatuses () override
virtual void	subProcessLogs (std::vector< std::string > &) override
virtual std::unique_ptr< AthenaInterprocess::ScheduledWork >	bootstrap_func () override
virtual std::unique_ptr< AthenaInterprocess::ScheduledWork >	exec_func () override
virtual std::unique_ptr< AthenaInterprocess::ScheduledWork >	fin_func () override
virtual AthenaMP::AllWorkerOutputs_ptr	generateOutputReport () override
virtual void	useFdsRegistry (std::shared_ptr< AthenaInterprocess::FdsRegistry >) override
virtual void	setRandString (const std::string &randStr) override
virtual void	killChildren () override
virtual std::unique_ptr< AthenaInterprocess::ScheduledWork > virtual operator() ATLAS_NOT_THREAD_SAFE(const AthenaInterprocess std::unique_ptr< AthenaInterprocess::ScheduledWork >	bootstrap_func ()=0
ServiceHandle< StoreGateSvc > &	evtStore ()
	The standard StoreGateSvc (event store) Returns (kind of) a pointer to the StoreGateSvc. More...
const ServiceHandle< StoreGateSvc > &	evtStore () const
	The standard StoreGateSvc (event store) Returns (kind of) a pointer to the StoreGateSvc. More...
const ServiceHandle< StoreGateSvc > &	detStore () const
	The standard StoreGateSvc/DetectorStore Returns (kind of) a pointer to the StoreGateSvc. More...
virtual StatusCode	sysInitialize () override
	Perform system initialization for an algorithm. More...
virtual StatusCode	sysStart () override
	Handle START transition. More...
virtual std::vector< Gaudi::DataHandle * >	inputHandles () const override
	Return this algorithm's input handles. More...
virtual std::vector< Gaudi::DataHandle * >	outputHandles () const override
	Return this algorithm's output handles. More...
Gaudi::Details::PropertyBase &	declareProperty (Gaudi::Property< T > &t)
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, SG::VarHandleKey &hndl, const std::string &doc, const SG::VarHandleKeyType &)
	Declare a new Gaudi property. More...
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, SG::VarHandleBase &hndl, const std::string &doc, const SG::VarHandleType &)
	Declare a new Gaudi property. More...
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, SG::VarHandleKeyArray &hndArr, const std::string &doc, const SG::VarHandleKeyArrayType &)
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, T &property, const std::string &doc, const SG::NotHandleType &)
	Declare a new Gaudi property. More...
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, T &property, const std::string &doc="none")
	Declare a new Gaudi property. More...
void	updateVHKA (Gaudi::Details::PropertyBase &)
MsgStream &	msg () const
MsgStream &	msg (const MSG::Level lvl) const
bool	msgLvl (const MSG::Level lvl) const
virtual std::unique_ptr< ScheduledWork >	operator () ATLAS_NOT_THREAD_SAFE(const ScheduledWork &)=0

Static Public Member Functions
static const InterfaceID &	interfaceID ()

Protected Types
enum	ESRange_Status {   ESRANGE_SUCCESS, ESRANGE_NOTFOUND, ESRANGE_SEEKFAILED, ESRANGE_PROCFAILED,   ESRANGE_FILENOTMADE, ESRANGE_BADINPFILE }
enum	Func_Flag { FUNC_BOOTSTRAP, FUNC_EXEC, FUNC_FIN }

Protected Member Functions
int mapAsyncFlag	ATLAS_NOT_THREAD_SAFE (Func_Flag flag, pid_t pid=0)
int	redirectLog (const std::string &rundir, bool addTimeStamp=true)
int	updateIoReg (const std::string &rundir)
std::string	fmterror (int errnum)
int	reopenFds ()
int	handleSavedPfc (const std::filesystem::path &dest_path)
void	waitForSignal ()
IEvtSelector *	evtSelector ()
void	renounceArray (SG::VarHandleKeyArray &handlesArray)
	remove all handles from I/O resolution More...
std::enable_if_t< std::is_void_v< std::result_of_t< decltype(&T::renounce)(T)> > &&!std::is_base_of_v< SG::VarHandleKeyArray, T > &&std::is_base_of_v< Gaudi::DataHandle, T >, void >	renounce (T &h)
void	extraDeps_update_handler (Gaudi::Details::PropertyBase &ExtraDeps)
	Add StoreName to extra input/output deps as needed. More...

Protected Attributes
int	m_nprocs
std::string	m_subprocTopDir
std::string	m_subprocDirPrefix
std::string	m_evtSelName
AthenaInterprocess::ProcessGroup *	m_processGroup
ServiceHandle< IEventProcessor >	m_evtProcessor
ServiceHandle< IAppMgrUI >	m_appMgr
ServiceHandle< IFileMgr >	m_fileMgr
ServiceHandle< IIoComponentMgr >	m_ioMgr
IEvtSelector *	m_evtSelector
std::string	m_fileMgrLog
std::shared_ptr< AthenaInterprocess::FdsRegistry >	m_fdsRegistry
std::string	m_randStr
Gaudi::Property< bool >	m_isPileup {this, "IsPileup", false, "Flag for configuring PileUpEventLoopMgr"}

Private Types
typedef System::ProcessTime::TimeValueType	TimeValType
typedef ServiceHandle< StoreGateSvc >	StoreGateSvc_t

Private Member Functions
	SharedEvtQueueConsumer ()
	SharedEvtQueueConsumer (const SharedEvtQueueConsumer &)
SharedEvtQueueConsumer &	operator= (const SharedEvtQueueConsumer &)
int decodeProcessResult	ATLAS_NOT_THREAD_SAFE (const AthenaInterprocess::ProcessResult *presult, bool doFinalize)
int	reopenFd (int fd, const std::string &name)
Gaudi::Details::PropertyBase &	declareGaudiProperty (Gaudi::Property< T > &hndl, const SG::VarHandleKeyType &)
	specialization for handling Gaudi::Property<SG::VarHandleKey> More...
Gaudi::Details::PropertyBase &	declareGaudiProperty (Gaudi::Property< T > &hndl, const SG::VarHandleKeyArrayType &)
	specialization for handling Gaudi::Property<SG::VarHandleKeyArray> More...
Gaudi::Details::PropertyBase &	declareGaudiProperty (Gaudi::Property< T > &hndl, const SG::VarHandleType &)
	specialization for handling Gaudi::Property<SG::VarHandleBase> More...
Gaudi::Details::PropertyBase &	declareGaudiProperty (Gaudi::Property< T > &t, const SG::NotHandleType &)
	specialization for handling everything that's not a Gaudi::Property<SG::VarHandleKey> or a <SG::VarHandleKeyArray> More...

Private Attributes
bool	m_useSharedReader
bool	m_useSharedWriter
bool	m_isRoundRobin
int	m_nEventsBeforeFork
int	m_nSkipEvents
bool	m_debug
int	m_rankId
ServiceHandle< IChronoStatSvc >	m_chronoStatSvc
IEventSeek *	m_evtSeek
IEvtSelectorSeek *	m_evtSelSeek
IEvtSelector::Context *	m_evtContext
IEventShare *	m_evtShare
IDataShare *	m_dataShare
AthenaInterprocess::SharedQueue *	m_sharedEventQueue
AthenaInterprocess::SharedQueue *	m_sharedRankQueue
std::map< pid_t, std::pair< int, TimeValType > >	m_eventStat
std::queue< pid_t >	m_finQueue
bool	m_readEventOrders
std::string	m_eventOrdersFile
std::vector< int >	m_eventOrders
pid_t	m_masterPid
StoreGateSvc_t	m_evtStore
	Pointer to StoreGate (event store by default) More...
StoreGateSvc_t	m_detStore
	Pointer to StoreGate (detector store by default) More...
std::vector< SG::VarHandleKeyArray * >	m_vhka
bool	m_varHandleArraysDeclared

Detailed Description

Definition at line 21 of file SharedEvtQueueConsumer.h.

Member Typedef Documentation

StoreGateSvc_t

typedef ServiceHandle<StoreGateSvc> AthCommonDataStore< AthCommonMsg< AlgTool > >::StoreGateSvc_t

privateinherited

Definition at line 388 of file AthCommonDataStore.h.

TimeValType

typedef System::ProcessTime::TimeValueType SharedEvtQueueConsumer::TimeValType

private

Definition at line 76 of file SharedEvtQueueConsumer.h.

Member Enumeration Documentation

ESRange_Status

enum AthenaMPToolBase::ESRange_Status

protectedinherited

Enumerator
ESRANGE_SUCCESS
ESRANGE_NOTFOUND
ESRANGE_SEEKFAILED
ESRANGE_PROCFAILED
ESRANGE_FILENOTMADE
ESRANGE_BADINPFILE

Definition at line 56 of file AthenaMPToolBase.h.

                      {
    ESRANGE_SUCCESS
    , ESRANGE_NOTFOUND
    , ESRANGE_SEEKFAILED
    , ESRANGE_PROCFAILED
    , ESRANGE_FILENOTMADE
    , ESRANGE_BADINPFILE
  };

Func_Flag

enum AthenaMPToolBase::Func_Flag

protectedinherited

Enumerator
FUNC_BOOTSTRAP
FUNC_EXEC
FUNC_FIN

Definition at line 65 of file AthenaMPToolBase.h.

                 {
    FUNC_BOOTSTRAP
    , FUNC_EXEC
    , FUNC_FIN
  };

Constructor & Destructor Documentation

SharedEvtQueueConsumer() [1/3]

SharedEvtQueueConsumer::SharedEvtQueueConsumer	(	const std::string &	type,
		const std::string &	name,
		const IInterface *	parent
	)

Definition at line 33 of file SharedEvtQueueConsumer.cxx.

  : AthenaMPToolBase(type,name,parent)
  , m_useSharedReader(false)
  , m_useSharedWriter(false)
  , m_isRoundRobin(false)
  , m_nEventsBeforeFork(0)
  , m_nSkipEvents(0)
  , m_debug(false)
  , m_rankId(-1)
  , m_chronoStatSvc("ChronoStatSvc", name)
  , m_evtSeek(nullptr)
  , m_evtSelSeek(nullptr)
  , m_evtContext(nullptr)
  , m_evtShare(nullptr)
  , m_dataShare(nullptr)
  , m_sharedEventQueue(nullptr)
  , m_sharedRankQueue(nullptr)
  , m_readEventOrders(false)
  , m_eventOrdersFile("athenamp_eventorders.txt")
  , m_masterPid(getpid())
{
  declareInterface<IAthenaMPTool>(this);
 
  declareProperty("UseSharedReader",m_useSharedReader);
  declareProperty("UseSharedWriter",m_useSharedWriter);
  declareProperty("IsRoundRobin",m_isRoundRobin);
  declareProperty("EventsBeforeFork",m_nEventsBeforeFork);
  declareProperty("Debug", m_debug);
  declareProperty("ReadEventOrders",m_readEventOrders);
  declareProperty("EventOrdersFile",m_eventOrdersFile);
 
  m_subprocDirPrefix = "worker_";
}

~SharedEvtQueueConsumer()

SharedEvtQueueConsumer::~SharedEvtQueueConsumer ( )

overridevirtual

Definition at line 69 of file SharedEvtQueueConsumer.cxx.

{
}

SharedEvtQueueConsumer() [2/3]

SharedEvtQueueConsumer::SharedEvtQueueConsumer ( )

private

SharedEvtQueueConsumer() [3/3]

SharedEvtQueueConsumer::SharedEvtQueueConsumer ( const SharedEvtQueueConsumer &  )

private

Member Function Documentation

ATLAS_NOT_THREAD_SAFE() [1/5]

virtual StatusCode exec SharedEvtQueueConsumer::ATLAS_NOT_THREAD_SAFE ( )

overridevirtual

Implements IAthenaMPTool.

ATLAS_NOT_THREAD_SAFE() [2/5]

int decodeProcessResult SharedEvtQueueConsumer::ATLAS_NOT_THREAD_SAFE ( const AthenaInterprocess::ProcessResult *  presult, bool  doFinalize  )

private

ATLAS_NOT_THREAD_SAFE() [3/5]

int mapAsyncFlag AthenaMPToolBase::ATLAS_NOT_THREAD_SAFE ( Func_Flag  flag, pid_t  pid = 0  )

protectedinherited

ATLAS_NOT_THREAD_SAFE() [4/5]

virtual int makePool SharedEvtQueueConsumer::ATLAS_NOT_THREAD_SAFE ( int  maxevt, int  nprocs, const std::string &  topdir  )

overridevirtual

Implements IAthenaMPTool.

ATLAS_NOT_THREAD_SAFE() [5/5]

virtual StatusCode wait_once SharedEvtQueueConsumer::ATLAS_NOT_THREAD_SAFE ( pid_t &  pid )

overridevirtual

Reimplemented from AthenaMPToolBase.

bootstrap_func() [1/2]

std::unique_ptr< AthenaInterprocess::ScheduledWork > SharedEvtQueueConsumer::bootstrap_func ( )

overridevirtual

Definition at line 285 of file SharedEvtQueueConsumer.cxx.

{
  if(m_debug) waitForSignal();
 
  std::unique_ptr<AthenaInterprocess::ScheduledWork> outwork(new AthenaInterprocess::ScheduledWork);
  outwork->data = malloc(sizeof(int));
  *(int*)(outwork->data) = 1; // Error code: for now use 0 success, 1 failure
  outwork->size = sizeof(int);
 
  // For PileUp Digi Fork-After-N-Events >>>>
  // Retrieve cuEvent-s for all background event selectors, if we forked after N events
  std::map<IService*,int> bkgEvtSelectors;
 
  if(m_isPileup) {
    for(IService* ptrSvc : serviceLocator()->getServices()) {
      IEvtSelector* evtsel = dynamic_cast<IEvtSelector*>(ptrSvc);
      if(evtsel && (evtsel != m_evtSelector)) {
    if(m_nEventsBeforeFork>0) {
      IEvtSelectorSeek* evtselseek = dynamic_cast<IEvtSelectorSeek*>(evtsel);
      if(evtselseek) {
        bkgEvtSelectors.emplace(ptrSvc,evtselseek->curEvent(*m_evtContext));
      }
      else {
        ATH_MSG_ERROR("Failed to cast IEvtSelector* onto IEvtSelectorSeek* for " << (ptrSvc)->name());
        return outwork;
      }
    }
    else {
      bkgEvtSelectors.emplace(ptrSvc,0);
    }
      }
    }
  }
  // <<<< For PileUp Digi Fork-After-N-Events
 
  // ...
  // (possible) TODO: extend outwork with some error message, which will be eventually
  // reported in the master proces
  // ...
 
  // ________________________ Get IncidentSvc and fire PostFork ________________________
  IIncidentSvc* p_incidentSvc(0);
  if(!serviceLocator()->service("IncidentSvc", p_incidentSvc).isSuccess()) {
    ATH_MSG_ERROR("Unable to retrieve IncidentSvc");
    return outwork;
  }
  p_incidentSvc->fireIncident(Incident(name(),"PostFork"));
 
 
  // ________________________ Get RankID ________________________
  //
  if(!m_sharedRankQueue->receive_basic<int>(m_rankId)) {
    ATH_MSG_ERROR("Unable to get rank ID!");
    return outwork;
  }
  std::ostringstream workindex;
  workindex<<m_rankId;
 
  // ________________________ Worker dir: mkdir ________________________
  std::filesystem::path worker_rundir(m_subprocTopDir);
  worker_rundir /= std::filesystem::path(m_subprocDirPrefix+workindex.str());
  // TODO: this "worker_" can be made configurable too
 
  if(mkdir(worker_rundir.string().c_str(),S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH)==-1) {
    ATH_MSG_ERROR("Unable to make worker run directory: " << worker_rundir.string() << ". " << fmterror(errno));
    return outwork;
  }
 
  // __________ Redirect logs unless we want to attach debugger ____________
  if(!m_debug) {
    if(redirectLog(worker_rundir.string()))
      return outwork;
 
    ATH_MSG_INFO("Logs redirected in the AthenaMP event worker PID=" << getpid());
  }
 
  // ________________________ Update Io Registry ____________________________
  if(updateIoReg(worker_rundir.string()))
    return outwork;
 
  ATH_MSG_INFO("Io registry updated in the AthenaMP event worker PID=" << getpid());
 
  // ________________________ SimParams & DigiParams & PDGTABLE.MeV ____________________________
  std::filesystem::path abs_worker_rundir = std::filesystem::absolute(worker_rundir);
  if(std::filesystem::is_regular_file("SimParams.db"))
    COPY_FILE_HACK("SimParams.db", abs_worker_rundir.string()+"/SimParams.db");
  if(std::filesystem::is_regular_file("DigitParams.db"))
    COPY_FILE_HACK("DigitParams.db", abs_worker_rundir.string()+"/DigitParams.db");
  if(std::filesystem::is_regular_file("PDGTABLE.MeV"))
    COPY_FILE_HACK("PDGTABLE.MeV", abs_worker_rundir.string()+"/PDGTABLE.MeV");
 
  // _______________________ Handle saved PFC (if any) ______________________
  if(handleSavedPfc(abs_worker_rundir))
    return outwork;
 
  // ________________________  reopen descriptors ____________________________
  if(reopenFds())
    return outwork;
 
  ATH_MSG_INFO("File descriptors re-opened in the AthenaMP event worker PID=" << getpid());
 
  
  // ________________________ Make Shared Reader/Writer Client ________________________
  if(m_useSharedReader && m_evtShare) {
    if(!m_evtShare->makeClient(m_rankId).isSuccess()) {
      ATH_MSG_ERROR("Failed to make the event selector a share client");
      return outwork;
    } 
    else {
      ATH_MSG_DEBUG("Successfully made the event selector a share client");
    }
  }
 
  if(m_useSharedWriter && m_dataShare) {
    IProperty* propertyServer = dynamic_cast<IProperty*>(m_dataShare);
    if (propertyServer==0 || propertyServer->setProperty("MakeStreamingToolClient", m_rankId + 1).isFailure()) {
      ATH_MSG_ERROR("Could not change AthenaPoolCnvSvc MakeClient Property");
      return outwork;
    } 
    else {
      ATH_MSG_DEBUG("Successfully made the conversion service a share client");
    }
  }
 
  // ________________________ I/O reinit ________________________
  if(!m_ioMgr->io_reinitialize().isSuccess()) {
    ATH_MSG_ERROR("Failed to reinitialize I/O");
    return outwork;
  } 
  else {
    ATH_MSG_DEBUG("Successfully reinitialized I/O");
  }
 
  // _______________ Get the value of SkipEvent ________________________
  if(m_evtSelector) {
    IProperty* propertyServer = dynamic_cast<IProperty*>(m_evtSelector);
    if(!propertyServer) {
      ATH_MSG_ERROR("Unable to cast event selector to IProperty");
      return outwork;
    }
    else {
      std::string propertyName("SkipEvents");
      IntegerProperty skipEventsProp(propertyName,m_nSkipEvents);
      if(propertyServer->getProperty(&skipEventsProp).isFailure()) {
    ATH_MSG_INFO("Event Selector does not have SkipEvents property");
      }
      else {
    m_nSkipEvents = skipEventsProp.value();
      }
    }
 
 
    // ________________________ Event selector restart ________________________
    IService* evtSelSvc = dynamic_cast<IService*>(m_evtSelector);
    if(!evtSelSvc) {
      ATH_MSG_ERROR("Failed to dyncast event selector to IService");
      return outwork;
    }
    if(!evtSelSvc->start().isSuccess()) {
      ATH_MSG_ERROR("Failed to restart the event selector");
      return outwork;
    } 
    else {
      ATH_MSG_DEBUG("Successfully restarted the event selector");
    }
  } 
  // For PileUp jobs >>>>
  // Main event selector: advance it if we either forked after N events, or skipEvents!=0
  // Background event selectors: restart, and advance if we forked after N events
  if(m_isPileup) {
    // Deal with the main event selector first
    if(serviceLocator()->service(m_evtSelName,m_evtSelSeek).isFailure() || !m_evtSelSeek) {
      ATH_MSG_ERROR("Error retrieving Event Selector with IEvtSelectorSeek interface for PileUp job");
      return outwork;
    }
 
    if((m_nEventsBeforeFork + m_nSkipEvents)
       && m_evtSelSeek->seek(*m_evtContext, m_nEventsBeforeFork+m_nSkipEvents).isFailure()) {
      ATH_MSG_ERROR("Failed to seek to " << m_nEventsBeforeFork+m_nSkipEvents);
      return outwork;
    }
 
    // Deal with background event selectors
    for(auto [evtsel,curEvt] : bkgEvtSelectors) {
      if(evtsel->start().isSuccess()) {
    if (m_nEventsBeforeFork>0) {
      IEvtSelectorSeek* evtselseek = dynamic_cast<IEvtSelectorSeek*>(evtsel);
      if(evtselseek->seek(*m_evtContext,curEvt).isFailure()) {
        ATH_MSG_ERROR("Failed to seek to " << curEvt << " in the BKG Event Selector " << evtsel->name());
        return outwork;
      }
    }
      }
      else {
    ATH_MSG_ERROR("Failed to restart BKG Event Selector " << evtsel->name());
    return outwork;
      }
    }
  }
  // <<<< For PileUp jobs
 
  // _______________________ Event orders for debugging ________________________________
  if(m_readEventOrders) {
    std::fstream fs(m_eventOrdersFile.c_str(),std::fstream::in);
    if(fs.good()) {
      ATH_MSG_INFO("Reading predefined event orders from " << m_eventOrdersFile);
      while(fs.good()){
    std::string line;
    std::getline(fs,line);
    if(line.empty())continue;
 
    // Parse the string
    size_t idx(0);
    int rank = std::stoi(line,&idx);
    if(rank==m_rankId) {
      msg(MSG::INFO) << "This worker will proces the following events #";
      while(idx<line.size()-1) {
        line = line.substr(idx+1);
        int evtnum = std::stoi(line,&idx);
        m_eventOrders.push_back(evtnum);
        msg(MSG::INFO) << " " << evtnum;
      }
      msg(MSG::INFO) << endmsg;
    }
      }
      if(m_eventOrders.empty()) {
    ATH_MSG_ERROR("Could not read event orders for the rank " << m_rankId);
    return outwork;
      }
      fs.close();
    }
    else {
      ATH_MSG_ERROR("Unable to read predefined event orders from " << m_eventOrdersFile);
      return outwork;
    }
  }
 
  // ________________________ Worker dir: chdir ________________________
  if(chdir(worker_rundir.string().c_str())==-1) {
    ATH_MSG_ERROR("Failed to chdir to " << worker_rundir.string());
    return outwork;
  }
 
  // ___________________ Fire UpdateAfterFork incident _________________
  p_incidentSvc->fireIncident(AthenaInterprocess::UpdateAfterFork(m_rankId,getpid(),name()));
 
  // Declare success and return
  *(int*)(outwork->data) = 0;
  return outwork;
}

bootstrap_func() [2/2]

virtual std::unique_ptr<AthenaInterprocess::ScheduledWork> virtual operator () ATLAS_NOT_THREAD_SAFE ( const AthenaInterprocess std::unique_ptr<AthenaInterprocess::ScheduledWork> AthenaMPToolBase::bootstrap_func ( )

pure virtualinherited

declareGaudiProperty() [1/4]

Gaudi::Details::PropertyBase& AthCommonDataStore< AthCommonMsg< AlgTool > >::declareGaudiProperty ( Gaudi::Property< T > &  hndl, const SG::VarHandleKeyArrayType &    )

inlineprivateinherited

specialization for handling Gaudi::Property<SG::VarHandleKeyArray>

Definition at line 170 of file AthCommonDataStore.h.

  {
    return *AthCommonDataStore<PBASE>::declareProperty(hndl.name(),
                                                       hndl.value(), 
                                                       hndl.documentation());
 
  }

declareGaudiProperty() [2/4]

Gaudi::Details::PropertyBase& AthCommonDataStore< AthCommonMsg< AlgTool > >::declareGaudiProperty ( Gaudi::Property< T > &  hndl, const SG::VarHandleKeyType &    )

inlineprivateinherited

specialization for handling Gaudi::Property<SG::VarHandleKey>

Definition at line 156 of file AthCommonDataStore.h.

  {
    return *AthCommonDataStore<PBASE>::declareProperty(hndl.name(),
                                                       hndl.value(), 
                                                       hndl.documentation());
 
  }

declareGaudiProperty() [3/4]

Gaudi::Details::PropertyBase& AthCommonDataStore< AthCommonMsg< AlgTool > >::declareGaudiProperty ( Gaudi::Property< T > &  hndl, const SG::VarHandleType &    )

inlineprivateinherited

specialization for handling Gaudi::Property<SG::VarHandleBase>

Definition at line 184 of file AthCommonDataStore.h.

  {
    return *AthCommonDataStore<PBASE>::declareProperty(hndl.name(),
                                                       hndl.value(), 
                                                       hndl.documentation());
  }

declareGaudiProperty() [4/4]

Gaudi::Details::PropertyBase& AthCommonDataStore< AthCommonMsg< AlgTool > >::declareGaudiProperty ( Gaudi::Property< T > &  t, const SG::NotHandleType &    )

inlineprivateinherited

specialization for handling everything that's not a Gaudi::Property<SG::VarHandleKey> or a <SG::VarHandleKeyArray>

Definition at line 199 of file AthCommonDataStore.h.

  {
    return PBASE::declareProperty(t);
  }

declareProperty() [1/6]

Gaudi::Details::PropertyBase* AthCommonDataStore< AthCommonMsg< AlgTool > >::declareProperty ( const std::string &  name, SG::VarHandleBase &  hndl, const std::string &  doc, const SG::VarHandleType &    )

inlineinherited

Declare a new Gaudi property.

Parameters

name	Name of the property.
hndl	Object holding the property value.
doc	Documentation string for the property.

This is the version for types that derive from SG::VarHandleBase. The property value object is put on the input and output lists as appropriate; then we forward to the base class.

Definition at line 245 of file AthCommonDataStore.h.

  {
    this->declare(hndl.vhKey());
    hndl.vhKey().setOwner(this);
 
    return PBASE::declareProperty(name,hndl,doc);
  }

declareProperty() [2/6]

Gaudi::Details::PropertyBase* AthCommonDataStore< AthCommonMsg< AlgTool > >::declareProperty ( const std::string &  name, SG::VarHandleKey &  hndl, const std::string &  doc, const SG::VarHandleKeyType &    )

inlineinherited

Declare a new Gaudi property.

Parameters

name	Name of the property.
hndl	Object holding the property value.
doc	Documentation string for the property.

This is the version for types that derive from SG::VarHandleKey. The property value object is put on the input and output lists as appropriate; then we forward to the base class.

Definition at line 221 of file AthCommonDataStore.h.

  {
    this->declare(hndl);
    hndl.setOwner(this);
 
    return PBASE::declareProperty(name,hndl,doc);
  }

declareProperty() [3/6]

Gaudi::Details::PropertyBase* AthCommonDataStore< AthCommonMsg< AlgTool > >::declareProperty ( const std::string &  name, SG::VarHandleKeyArray &  hndArr, const std::string &  doc, const SG::VarHandleKeyArrayType &    )

inlineinherited

Definition at line 259 of file AthCommonDataStore.h.

  {
 
    // std::ostringstream ost;
    // ost << Algorithm::name() << " VHKA declareProp: " << name 
    //     << " size: " << hndArr.keys().size() 
    //     << " mode: " << hndArr.mode() 
    //     << "  vhka size: " << m_vhka.size()
    //     << "\n";
    // debug() << ost.str() << endmsg;
 
    hndArr.setOwner(this);
    m_vhka.push_back(&hndArr);
 
    Gaudi::Details::PropertyBase* p =  PBASE::declareProperty(name, hndArr, doc);
    if (p != 0) {
      p->declareUpdateHandler(&AthCommonDataStore<PBASE>::updateVHKA, this);
    } else {
      ATH_MSG_ERROR("unable to call declareProperty on VarHandleKeyArray " 
                    << name);
    }
 
    return p;
 
  }

declareProperty() [4/6]

Gaudi::Details::PropertyBase* AthCommonDataStore< AthCommonMsg< AlgTool > >::declareProperty ( const std::string &  name, T &  property, const std::string &  doc, const SG::NotHandleType &    )

inlineinherited

Declare a new Gaudi property.

Parameters

name	Name of the property.
property	Object holding the property value.
doc	Documentation string for the property.

This is the generic version, for types that do not derive from SG::VarHandleKey. It just forwards to the base class version of declareProperty.

Definition at line 333 of file AthCommonDataStore.h.

  {
    return PBASE::declareProperty(name, property, doc);
  }

declareProperty() [5/6]

Gaudi::Details::PropertyBase* AthCommonDataStore< AthCommonMsg< AlgTool > >::declareProperty ( const std::string &  name, T &  property, const std::string &  doc = "none"  )

inlineinherited

Declare a new Gaudi property.

Parameters

name	Name of the property.
property	Object holding the property value.
doc	Documentation string for the property.

This dispatches to either the generic declareProperty or the one for VarHandle/Key/KeyArray.

Definition at line 352 of file AthCommonDataStore.h.

  {
    typedef typename SG::HandleClassifier<T>::type htype;
    return declareProperty (name, property, doc, htype());
  }

declareProperty() [6/6]

Gaudi::Details::PropertyBase& AthCommonDataStore< AthCommonMsg< AlgTool > >::declareProperty ( Gaudi::Property< T > &  t )

inlineinherited

Definition at line 145 of file AthCommonDataStore.h.

                                                                 {
    typedef typename SG::HandleClassifier<T>::type htype;
    return AthCommonDataStore<PBASE>::declareGaudiProperty(t, htype());
  }

detStore()

const ServiceHandle<StoreGateSvc>& AthCommonDataStore< AthCommonMsg< AlgTool > >::detStore ( ) const

inlineinherited

The standard StoreGateSvc/DetectorStore Returns (kind of) a pointer to the StoreGateSvc.

Definition at line 95 of file AthCommonDataStore.h.

{ return m_detStore; }

evtSelector()

IEvtSelector* AthenaMPToolBase::evtSelector ( )

inlineprotectedinherited

Definition at line 81 of file AthenaMPToolBase.h.

{ return m_evtSelector; }

evtStore() [1/2]

ServiceHandle<StoreGateSvc>& AthCommonDataStore< AthCommonMsg< AlgTool > >::evtStore ( )

inlineinherited

The standard StoreGateSvc (event store) Returns (kind of) a pointer to the StoreGateSvc.

Definition at line 85 of file AthCommonDataStore.h.

{ return m_evtStore; }

evtStore() [2/2]

const ServiceHandle<StoreGateSvc>& AthCommonDataStore< AthCommonMsg< AlgTool > >::evtStore ( ) const

inlineinherited

The standard StoreGateSvc (event store) Returns (kind of) a pointer to the StoreGateSvc.

Definition at line 90 of file AthCommonDataStore.h.

{ return m_evtStore; }

exec_func()

std::unique_ptr< AthenaInterprocess::ScheduledWork > SharedEvtQueueConsumer::exec_func ( )

overridevirtual

Implements AthenaMPToolBase.

Definition at line 536 of file SharedEvtQueueConsumer.cxx.

{
  ATH_MSG_INFO("Exec function in the AthenaMP worker PID=" << getpid());
 
  bool all_ok(true);
 
  long intmask = pow(0x100,sizeof(int))-1; // Mask for decoding event number from the value posted to the queue
  int nEvt(m_nEventsBeforeFork);
  int nEventsProcessed(0);
  long evtnumAndChunk(0);
 
  unsigned evtCounter(0);
  int evtnum(0), chunkSize(1);
  auto predefinedEvt = m_eventOrders.cbegin();
 
  // If the event orders file already exists in worker's run directory, then it's an unexpected error!
  std::filesystem::path ordersFile(m_eventOrdersFile);
  if(std::filesystem::exists(ordersFile)) {
    ATH_MSG_ERROR(m_eventOrdersFile << " already exists in the worker's run directory!");
    all_ok = false;
  }
  
  // For the round robin we need to know the maximum number of events for this job
  if(m_isRoundRobin) {
    evtnumAndChunk = 1;
    while(evtnumAndChunk>0) {
      if(!m_sharedEventQueue->try_receive_basic<long>(evtnumAndChunk)) {
    usleep(1000);
      }
    }
    evtnumAndChunk *= -1;
  }
 
  System::ProcessTime time_start = System::getProcessTime();
  if(all_ok) {
    std::fstream fs(m_eventOrdersFile.c_str(),std::fstream::out);
    fs << m_rankId;
    bool firstOrder(true);
    while(true) {
      if(m_isRoundRobin) {
    evtnum = m_nSkipEvents + m_nprocs*evtCounter + m_rankId; 
    if(evtnum>=evtnumAndChunk+m_nSkipEvents) { 
      break;
    }
    evtCounter++;
      }
      else {
    if(m_readEventOrders) {
      if(predefinedEvt==m_eventOrders.cend()) break;
      evtnum = *predefinedEvt;
      predefinedEvt++;
      fs << (firstOrder?":":",") << evtnum;
      fs.flush();
      firstOrder=false;
      ATH_MSG_INFO("Read event number from the orders file: " << evtnum);
    }
    else {
      if(!m_sharedEventQueue->try_receive_basic<long>(evtnumAndChunk)) {
        // The event queue is empty, but we should check whether there are more events to come or not
        ATH_MSG_DEBUG("Event queue is empty"); 
        usleep(1000);
        continue;
      }
      if(evtnumAndChunk<=0) {
        evtnumAndChunk *= -1;
        ATH_MSG_DEBUG("No more events are expected. The total number of events for this job = " << evtnumAndChunk);
        break;
      }
      ATH_MSG_DEBUG("Received value from the queue 0x" << std::hex << evtnumAndChunk << std::dec);
      chunkSize = evtnumAndChunk >> (sizeof(int)*8);
      evtnum = evtnumAndChunk & intmask;
      ATH_MSG_INFO("Received from the queue: event num=" << evtnum << " chunk size=" << chunkSize);
 
      // Save event order
      for(int i(0);i<chunkSize;++i) {
        fs << (firstOrder?":":",") << evtnum+i;
        firstOrder=false;
      }
      fs.flush();
    } // Get event numbers from the shared queue
      } // Not RoundRobin
      nEvt+=chunkSize;
      StatusCode sc;
      if(m_useSharedReader) {
    sc = m_evtShare->share(evtnum);
    if(sc.isFailure()){
      ATH_MSG_ERROR("Unable to share " << evtnum);
      all_ok=false;
      break;
    }
    else {
      ATH_MSG_INFO("Share of " << evtnum << " succeeded");
    }
      }
      else if(m_evtSelector) {
    m_chronoStatSvc->chronoStart("AthenaMP_seek");
        if (m_evtSeek) {
          sc=m_evtSeek->seek(evtnum);
        }
        else {
          sc=m_evtSelSeek->seek(*m_evtContext, evtnum);
        }
    if(sc.isFailure()){
      ATH_MSG_ERROR("Unable to seek to " << evtnum);
      all_ok=false;
      break;
    }
    else {
      ATH_MSG_INFO("Seek to " << evtnum << " succeeded");
    }
    m_chronoStatSvc->chronoStop("AthenaMP_seek");
      }
      m_chronoStatSvc->chronoStart("AthenaMP_nextEvent");
      sc = m_evtProcessor->nextEvent(nEvt);
      nEventsProcessed += chunkSize;
      if(sc.isFailure()){
    if(chunkSize==1) {
      ATH_MSG_ERROR("Unable to process event " << evtnum);
    }
    else {
      ATH_MSG_ERROR("Unable to process the chunk (" << evtnum << "," << evtnum+chunkSize-1 << ")");
    }
    all_ok=false;
    break;
      }
      m_chronoStatSvc->chronoStop("AthenaMP_nextEvent"); 
    }
    fs.close();
  }
  System::ProcessTime time_delta = System::getProcessTime() - time_start;
  TimeValType elapsedTime = time_delta.elapsedTime<System::Sec>();
 
  if(all_ok) {
    if(m_evtProcessor->executeRun(0).isFailure()) {
      ATH_MSG_ERROR("Could not finalize the Run");
      all_ok=false;
    }
    else if(!m_useSharedReader && m_evtSelector) {
      StatusCode sc;
      if (m_evtSeek) {
        sc = m_evtSeek->seek(evtnumAndChunk+m_nSkipEvents);
      }
      else {
        sc = m_evtSelSeek->seek(*m_evtContext, evtnumAndChunk+m_nSkipEvents);
      }
      if(sc.isFailure()) {
    ATH_MSG_WARNING("Seek past maxevt to " << evtnumAndChunk+m_nSkipEvents << " returned failure.");
      }
    }
  }
 
  std::unique_ptr<AthenaInterprocess::ScheduledWork> outwork(new AthenaInterprocess::ScheduledWork);
 
  // Return value: "ERRCODE|Func_Flag|NEvt|EvtLoopTime"
  int outsize = 2*sizeof(int)+sizeof(AthenaMPToolBase::Func_Flag)+sizeof(elapsedTime);
  void* outdata = malloc(outsize);
  *(int*)(outdata) = (all_ok?0:1); // Error code: for now use 0 success, 1 failure
  AthenaMPToolBase::Func_Flag func = AthenaMPToolBase::FUNC_EXEC;
  memcpy((char*)outdata+sizeof(int),&func,sizeof(func));
  memcpy((char*)outdata+sizeof(int)+sizeof(func),&nEventsProcessed,sizeof(int));
  memcpy((char*)outdata+2*sizeof(int)+sizeof(func),&elapsedTime,sizeof(elapsedTime));
  outwork->data = outdata;
  outwork->size = outsize;
  // ...
  // (possible) TODO: extend outwork with some error message, which will be eventually
  // reported in the master proces
  // ...
  return outwork;
}

extraDeps_update_handler()

void AthCommonDataStore< AthCommonMsg< AlgTool > >::extraDeps_update_handler ( Gaudi::Details::PropertyBase &  ExtraDeps )

protectedinherited

Add StoreName to extra input/output deps as needed.

use the logic of the VarHandleKey to parse the DataObjID keys supplied via the ExtraInputs and ExtraOuputs Properties to add the StoreName if it's not explicitly given

fin_func()

std::unique_ptr< AthenaInterprocess::ScheduledWork > SharedEvtQueueConsumer::fin_func ( )

overridevirtual

Implements AthenaMPToolBase.

Definition at line 706 of file SharedEvtQueueConsumer.cxx.

{
  ATH_MSG_INFO("Fin function in the AthenaMP worker PID=" << getpid());
 
  bool all_ok(true);
 
  if(m_appMgr->stop().isFailure()) {
    ATH_MSG_ERROR("Unable to stop AppMgr"); 
    all_ok=false;
  }
  else { 
    if(m_appMgr->finalize().isFailure()) {
      std::cerr << "Unable to finalize AppMgr" << std::endl;
      all_ok=false;
    }
  }
 
  std::unique_ptr<AthenaInterprocess::ScheduledWork> outwork(new AthenaInterprocess::ScheduledWork);
 
  // Return value: "ERRCODE|Func_Flag|NEvt|EvtLoopTime"  (Here NEvt=-1 and EvtLoopTime=-1)
  int outsize = 2*sizeof(int)+sizeof(AthenaMPToolBase::Func_Flag)+sizeof(TimeValType);
  void* outdata = malloc(outsize);
  *(int*)(outdata) = (all_ok?0:1); // Error code: for now use 0 success, 1 failure
  AthenaMPToolBase::Func_Flag func = AthenaMPToolBase::FUNC_FIN;
  memcpy((char*)outdata+sizeof(int),&func,sizeof(func));
  int nEvt = -1;
  memcpy((char*)outdata+sizeof(int)+sizeof(func),&nEvt,sizeof(int));
  TimeValType elapsed = -1;
  memcpy((char*)outdata+2*sizeof(int)+sizeof(func),&elapsed,sizeof(elapsed));
 
  outwork->data = outdata;
  outwork->size = outsize;
 
  return outwork;
}

finalize()

StatusCode SharedEvtQueueConsumer::finalize ( )

overridevirtual

Reimplemented from AthenaMPToolBase.

Definition at line 121 of file SharedEvtQueueConsumer.cxx.

{
  if(getpid()==m_masterPid) {
    ATH_MSG_INFO("finalize() in the master process");
    // Merge saved event orders into one in the master run directory
 
    // 1. Check if master run directory already contains a file with saved orders
    // If so, then rename it with random suffix
    std::filesystem::path ordersFile(m_eventOrdersFile);
    if(std::filesystem::exists(ordersFile)) {
      srand((unsigned)time(0));
      std::ostringstream randname;
      randname << rand();
      std::string ordersFileBak = m_eventOrdersFile+std::string("-bak-")+randname.str();
      ATH_MSG_WARNING("File " << m_eventOrdersFile << " already exists in the master run directory!");
      ATH_MSG_WARNING("Saving a backup with new name " << ordersFileBak);
 
      std::filesystem::path ordersFileBakpath(ordersFileBak);
      std::filesystem::rename(ordersFile,ordersFileBakpath);
    }
 
    // 2. Merge workers event orders into the master file
    std::fstream fs(m_eventOrdersFile.c_str(),std::fstream::out);
    for(int i=0; i<m_nprocs; ++i) {
      std::ostringstream workerIndex;
      workerIndex << i;
      std::filesystem::path worker_rundir(m_subprocTopDir);
      worker_rundir /= std::filesystem::path(m_subprocDirPrefix+workerIndex.str());
      std::string ordersFileWorker(worker_rundir.string()+std::string("/")+m_eventOrdersFile);
      ATH_MSG_INFO("Processing " << ordersFileWorker << " ...");
      std::fstream fs_worker(ordersFileWorker.c_str(),std::fstream::in);
      std::string line;
      while(fs_worker.good()) {
    std::getline(fs_worker,line);
    fs << line << std::endl;
      }
      fs_worker.close();
    }
    fs.close();
  } // if(getpid()==m_masterPid)
 
  if (m_evtContext) {
    ATH_CHECK( m_evtSelector->releaseContext (m_evtContext) );
    m_evtContext = nullptr;
  }
 
  delete m_sharedRankQueue;
  return StatusCode::SUCCESS;
}

fmterror()

std::string AthenaMPToolBase::fmterror ( int  errnum )

protectedinherited

Definition at line 362 of file AthenaMPToolBase.cxx.

{
  char buf[256];
  strerror_r(errnum, buf, sizeof(buf));
  return std::string(buf);
}

generateOutputReport()

AthenaMP::AllWorkerOutputs_ptr AthenaMPToolBase::generateOutputReport ( )

overridevirtualinherited

Implements IAthenaMPTool.

Reimplemented in EvtRangeProcessor, EvtRangeScatterer, SharedEvtQueueProvider, and SharedWriterTool.

Definition at line 128 of file AthenaMPToolBase.cxx.

{
  AthenaMP::AllWorkerOutputs_ptr jobOutputs(new AthenaMP::AllWorkerOutputs());
 
  if(m_fileMgrLog.empty()) {
    ATH_MSG_WARNING(name() << " cannot make output report because FileMgr has not been configured to write log file!");
  }
  else {
    // Collect output files made by the workers
    std::string line;
    for(int i=0;i<m_nprocs;++i) {
      // Get the name of FileMgr log
      std::ostringstream workindex;
      workindex<<i;
      std::filesystem::path logFilePath(m_subprocTopDir);
      logFilePath /= std::filesystem::path(m_subprocDirPrefix+workindex.str());
      std::filesystem::path logFile(logFilePath);
      logFile /= std::filesystem::path(m_fileMgrLog);
      if(!(std::filesystem::exists(logFile)&&std::filesystem::is_regular_file(logFile))) {
        ATH_MSG_WARNING(logFile.string() << " either does not exist or is not a regular file. Skipping");
        continue;
      }
 
      ATH_MSG_DEBUG("FileMgr log file (" << i << ") " << logFile);
 
      std::ifstream inpStream(logFile.string().c_str());
      std::set<std::string> reportedFiles; // Don't report the same file twice
      while(!inpStream.eof()) {
    std::getline(inpStream,line);
        if(line.find("WRITE")!=std::string::npos) {
          // Parse the entry
          size_t startpos(0);
      std::vector<std::string> entries;
          while(startpos<line.size()) {
            while(line[startpos]==' ')
              startpos++;
 
            size_t endpos = line.find(' ',startpos);
            if(endpos==std::string::npos) endpos = line.size();
            entries.push_back(line.substr(startpos,endpos-startpos));
            startpos=endpos+1;
          }
 
          // enties[0] is filename
      std::filesystem::path filenamePath(entries[0]);
      std::filesystem::path basename = filenamePath.filename();
          if(reportedFiles.find(basename.string())==reportedFiles.end())
            reportedFiles.insert(basename.string());
          else
            continue;
      std::filesystem::path absolutename = basename.is_absolute() ? basename : std::filesystem::absolute(std::filesystem::path(logFilePath)/=basename);
      AthenaMP::AllWorkerOutputsIterator it1 = jobOutputs->find(basename.string());
          if(it1==jobOutputs->end()) {
            (*jobOutputs)[basename.string()] = AthenaMP::SingleWorkerOutputs();
            (*jobOutputs)[basename.string()].reserve(m_nprocs);
          }
 
      AthenaMP::WorkerOutput newOutput;
          newOutput.filename = absolutename.string();
          newOutput.technology = entries[1];
          newOutput.description = entries[2];
          newOutput.access_mode = entries[3];
          newOutput.shared = (line.find("SHARED")!=std::string::npos);
 
          (*jobOutputs)[basename.string()].push_back(newOutput);
        }
      }
    }
  }
  return jobOutputs;
}

handleSavedPfc()

int AthenaMPToolBase::handleSavedPfc ( const std::filesystem::path &  dest_path )

protectedinherited

Definition at line 425 of file AthenaMPToolBase.cxx.

{
  if(std::filesystem::is_regular_file("PoolFileCatalog.xml.AthenaMP-saved"))
    COPY_FILE_HACK("PoolFileCatalog.xml.AthenaMP-saved",dest_path.string()+"/PoolFileCatalog.xml");
  return 0;
}

initialize()

StatusCode SharedEvtQueueConsumer::initialize ( )

overridevirtual

Reimplemented from AthenaMPToolBase.

Definition at line 73 of file SharedEvtQueueConsumer.cxx.

{
  ATH_MSG_DEBUG("In initialize");
 
  ATH_CHECK(AthenaMPToolBase::initialize());
 
  // For pile-up jobs use event loop manager for seeking
  // otherwise use event selector
  if(m_isPileup) {
    m_evtSeek = dynamic_cast<IEventSeek*>(m_evtProcessor.operator->());
    if(!m_evtSeek) {
      ATH_MSG_ERROR("Unable to dyn-cast PileUpEventLoopMgr to IEventSeek");
      return StatusCode::FAILURE;
    }
  }
  else if(m_evtSelector) {
    ATH_CHECK(serviceLocator()->service(m_evtSelName,m_evtSelSeek));
  }
 
  if(m_evtSelector) {
    ATH_CHECK( m_evtSelector->createContext (m_evtContext) );
 
    StatusCode sc = serviceLocator()->service(m_evtSelName,m_evtShare);
    if(sc.isFailure() || m_evtShare==0) {
      if(m_useSharedReader) {
    ATH_MSG_ERROR("Error retrieving IEventShare");
    return StatusCode::FAILURE;
      }
      ATH_MSG_INFO("Could not retrieve IEventShare");
    }
 
    //FIXME: AthenaPool dependent for now
    IConversionSvc* cnvSvc{nullptr};
    sc = serviceLocator()->service("AthenaPoolCnvSvc",cnvSvc);
    m_dataShare = dynamic_cast<IDataShare*>(cnvSvc);
    if(sc.isFailure() || m_dataShare==0) {
      if(m_useSharedWriter) {
    ATH_MSG_ERROR("Error retrieving AthenaPoolCnvSvc " << cnvSvc);
    return StatusCode::FAILURE;
      }
    }
  }
 
  ATH_CHECK(m_chronoStatSvc.retrieve());
  
  return StatusCode::SUCCESS;
}

inputHandles()

virtual std::vector<Gaudi::DataHandle*> AthCommonDataStore< AthCommonMsg< AlgTool > >::inputHandles ( ) const

overridevirtualinherited

Return this algorithm's input handles.

We override this to include handle instances from key arrays if they have not yet been declared. See comments on updateVHKA.

interfaceID()

static const InterfaceID& IAthenaMPTool::interfaceID ( )

inlinestaticinherited

Definition at line 36 of file IAthenaMPTool.h.

{ return IID_IAthenaMPTool; }

killChildren()

void AthenaMPToolBase::killChildren ( )

overridevirtualinherited

Implements IAthenaMPTool.

Definition at line 210 of file AthenaMPToolBase.cxx.

{
  for(const AthenaInterprocess::Process& child : m_processGroup->getChildren()) {
    kill(child.getProcessID(),SIGKILL);
  }
}

msg() [1/2]

MsgStream& AthCommonMsg< AlgTool >::msg ( ) const

inlineinherited

Definition at line 24 of file AthCommonMsg.h.

                                {
    return this->msgStream();
  }

msg() [2/2]

MsgStream& AthCommonMsg< AlgTool >::msg ( const MSG::Level  lvl ) const

inlineinherited

Definition at line 27 of file AthCommonMsg.h.

                                                  {
    return this->msgStream(lvl);
  }

msgLvl()

bool AthCommonMsg< AlgTool >::msgLvl ( const MSG::Level  lvl ) const

inlineinherited

Definition at line 30 of file AthCommonMsg.h.

                                               {
    return this->msgLevel(lvl);
  }

operator()

virtual std::unique_ptr<ScheduledWork> AthenaInterprocess::IMessageDecoder::operator ( ) const &

pure virtualinherited

operator=()

SharedEvtQueueConsumer& SharedEvtQueueConsumer::operator= ( const SharedEvtQueueConsumer &  )

private

outputHandles()

virtual std::vector<Gaudi::DataHandle*> AthCommonDataStore< AthCommonMsg< AlgTool > >::outputHandles ( ) const

overridevirtualinherited

Return this algorithm's output handles.

We override this to include handle instances from key arrays if they have not yet been declared. See comments on updateVHKA.

redirectLog()

int AthenaMPToolBase::redirectLog ( const std::string &  rundir, bool  addTimeStamp = true  )

protectedinherited

Definition at line 278 of file AthenaMPToolBase.cxx.

{
  // Redirect both stdout and stderr to the same file AthenaMP.log
  int dup2result1(0), dup2result2(0);
 
  int newout = open(std::string(rundir+"/AthenaMP.log").c_str(),O_CREAT | O_RDWR, S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH);
  if(newout==-1) {
    ATH_MSG_ERROR("Unable to open log file in the run directory. " << fmterror(errno));
    return -1;
  }
  dup2result1 = dup2(newout, STDOUT_FILENO);
  dup2result2 = dup2(newout, STDERR_FILENO);
  TEMP_FAILURE_RETRY(close(newout));
  if(dup2result1==-1) {
    ATH_MSG_ERROR("Unable to redirect standard output. " << fmterror(errno));
    return -1;
  }
  if(dup2result2==-1) {
    ATH_MSG_ERROR("Unable to redirect standard error. " << fmterror(errno));
    return -1;
  }
 
  if(addTimeStamp) {
    IMessageSvc* messageSvc(0);
    StatusCode sc = serviceLocator()->service("MessageSvc",messageSvc);
    if(sc.isFailure()) {
      ATH_MSG_ERROR("Error retrieving IMessageSvc");
      return -1;
    }
 
    IProperty* propertyServer = dynamic_cast<IProperty*>(messageSvc);
    if(propertyServer==0) {
      ATH_MSG_ERROR("Unable to cast message svc to IProperty");
      return -1;
    }
 
    std::string propertyName("Format");
    std::string oldFormat("");
    StringProperty formatProp(propertyName,oldFormat);
    sc = propertyServer->getProperty(&formatProp);
    if(sc.isFailure()) {
      ATH_MSG_WARNING("Message Service does not have Format property");
    }
    else {
      oldFormat = formatProp.value();
      if(oldFormat.find("%t")==std::string::npos) {
    // Add time stamps
    std::string newFormat("%t " + oldFormat);
    StringProperty newFormatProp(propertyName,newFormat);
    if(propertyServer->setProperty(newFormatProp).isFailure()) {
      ATH_MSG_ERROR("Unable to set new Format property on the Message Service");
      return -1;
    }
      }
      else {
    ATH_MSG_DEBUG("MsgSvc format already contains timestamps. Nothing to be done");
      }
    }
  }
 
  return 0;
}

renounce()

std::enable_if_t<std::is_void_v<std::result_of_t<decltype(&T::renounce)(T)> > && !std::is_base_of_v<SG::VarHandleKeyArray, T> && std::is_base_of_v<Gaudi::DataHandle, T>, void> AthCommonDataStore< AthCommonMsg< AlgTool > >::renounce ( T &  h )

inlineprotectedinherited

Definition at line 380 of file AthCommonDataStore.h.

  {
    h.renounce();
    PBASE::renounce (h);
  }

renounceArray()

void AthCommonDataStore< AthCommonMsg< AlgTool > >::renounceArray ( SG::VarHandleKeyArray &  handlesArray )

inlineprotectedinherited

remove all handles from I/O resolution

Definition at line 364 of file AthCommonDataStore.h.

                                                          {
    handlesArray.renounce();
  }

reopenFd()

int AthenaMPToolBase::reopenFd ( int  fd, const std::string &  name  )

privateinherited

Definition at line 448 of file AthenaMPToolBase.cxx.

{
  ATH_MSG_DEBUG("Attempting to reopen descriptor for " << name);
  int old_openflags = fcntl(fd,F_GETFL,0);
  switch(old_openflags & O_ACCMODE) {
  case O_RDONLY: {
    ATH_MSG_DEBUG("The File Access Mode is RDONLY");
    break;
  }
  case O_WRONLY: {
    ATH_MSG_DEBUG("The File Access Mode is WRONLY");
    break;
  }
  case O_RDWR: {
    ATH_MSG_DEBUG("The File Access Mode is RDWR");
    break;
  }
  }
   
  int old_descflags = fcntl(fd,F_GETFD,0);
  off_t oldpos = lseek(fd,0,SEEK_CUR);
  if(oldpos==-1) {
    if(errno==ESPIPE) {
      ATH_MSG_WARNING("Dealing with PIPE. Skipping ... (FIXME!)");
    }
    else {
      ATH_MSG_ERROR("When re-opening file descriptors lseek failed on " << name << ". " << fmterror(errno));
      return -1;
    }
  }
  else {
    Io::Fd newfd = open(name.c_str(),old_openflags);
    if(newfd==-1) {
      ATH_MSG_ERROR("When re-opening file descriptors unable to open " << name << " for reading. " << fmterror(errno));
      return -1;
    }
    if(lseek(newfd,oldpos,SEEK_SET)==-1){
      ATH_MSG_ERROR("When re-opening file descriptors lseek failed on the newly opened " << name << ". " << fmterror(errno));
      TEMP_FAILURE_RETRY(close(newfd));
      return -1;
    }
    TEMP_FAILURE_RETRY(close(fd));
    if(dup2(newfd,fd)==-1) {
      ATH_MSG_ERROR("When re-opening file descriptors unable to duplicate descriptor for " << name << ". " << fmterror(errno));
      TEMP_FAILURE_RETRY(close(newfd));
      return -1;
    }
    if(fcntl(fd,F_SETFD,old_descflags)==-1) {
      ATH_MSG_ERROR("When re-opening file descriptors unable to set descriptor flags for " << name << ". " << fmterror(errno));
      TEMP_FAILURE_RETRY(close(newfd));
      return -1;
    }
    TEMP_FAILURE_RETRY(close(newfd));
  }
  return 0;
}

reopenFds()

int AthenaMPToolBase::reopenFds ( )

protectedinherited

Definition at line 369 of file AthenaMPToolBase.cxx.

{
  // Reopen file descriptors.
  // First go over all open files, which have been registered with the FileMgr
  // Then also check the FdsRegistry, in case it contains some files not registered with the FileMgr
  std::set<int> fdLog;
  
  // Query the FileMgr contents
  std::vector<const Io::FileAttr*> filemgrFiles;
  std::vector<const Io::FileAttr*>::const_iterator itFile;
  unsigned filenum = m_fileMgr->getFiles(filemgrFiles); // Get attributes for open files only. We don't care about closed ones at this point
  if(filenum!=filemgrFiles.size())
    ATH_MSG_WARNING("getFiles returned " << filenum << " while vector size is " << filemgrFiles.size());
  
  for(itFile=filemgrFiles.begin();itFile!=filemgrFiles.end();++itFile) {
    ATH_MSG_DEBUG("* " << **itFile);
    const std::string& filename = (**itFile).name();
    Io::Fd fd = (**itFile).fd();
 
    if(fd==-1) {
      // It is legal to have fd=-1 for remote inputs
      // On the other hand, these inputs should not remain open after fork. The issue being tracked at ATEAM-434.
      // So, this hopefully is a temporary patch
      ATH_MSG_WARNING("FD=-1 detected on an open file retrieved from FileMgr. Skip FD reopening. File name: " << filename);
      continue;
    }
    
    if(reopenFd(fd,filename))
      return -1;
    
    fdLog.insert(fd);
  }
  
  // Check the FdsRegistry
  for(const AthenaInterprocess::FdsRegistryEntry& regEntry : *m_fdsRegistry) {
    if(fdLog.find(regEntry.fd)!=fdLog.end()) {
      ATH_MSG_DEBUG("The file from FdsRegistry " << regEntry.name << " was registered with FileMgr. Skip reopening");
    }
    else {
      ATH_MSG_WARNING("The file " << regEntry.name << " has not been registered with the FileMgr!");
      
      if(regEntry.fd==-1) {
      // Same protection as the one above
    ATH_MSG_WARNING("FD=-1 detected on an open file retrieved from FD Registry. Skip FD reopening. File name: " << regEntry.name);
    continue;
      }
 
      if(reopenFd(regEntry.fd,regEntry.name))
        return -1;
      
      fdLog.insert(regEntry.fd);
    }
  }
  return 0;
}

reportSubprocessStatuses()

void SharedEvtQueueConsumer::reportSubprocessStatuses ( )

overridevirtual

Reimplemented from AthenaMPToolBase.

Definition at line 254 of file SharedEvtQueueConsumer.cxx.

{
  ATH_MSG_INFO("Statuses of event processors");
  const std::vector<AthenaInterprocess::ProcessStatus>& statuses = m_processGroup->getStatuses();
  for(size_t i=0; i<statuses.size(); ++i) {
    // Get the number of events processed by this worker
    auto it = m_eventStat.find(statuses[i].pid);
    msg(MSG::INFO) << "*** Process PID=" << statuses[i].pid 
           << ". Status " << ((statuses[i].exitcode)?"FAILURE":"SUCCESS") 
           << ". Number of events processed: ";
    if(it==m_eventStat.end())
      msg(MSG::INFO) << "N/A" << endmsg;
    else
      msg(MSG::INFO) << it->second.first
             << ", Event Loop Time: " << it->second.second << "sec."
             << endmsg;
  }
}

setRandString()

void AthenaMPToolBase::setRandString ( const std::string &  randStr )

overridevirtualinherited

Implements IAthenaMPTool.

Definition at line 205 of file AthenaMPToolBase.cxx.

{
  m_randStr = randStr;
}

subProcessLogs()

void SharedEvtQueueConsumer::subProcessLogs ( std::vector< std::string > &  filenames )

overridevirtual

Implements IAthenaMPTool.

Definition at line 273 of file SharedEvtQueueConsumer.cxx.

{
  filenames.clear();
  for(int i=0; i<m_nprocs; ++i) {
    std::ostringstream workerIndex;
    workerIndex << i;
    std::filesystem::path worker_rundir(m_subprocTopDir);
    worker_rundir /= std::filesystem::path(m_subprocDirPrefix+workerIndex.str());
    filenames.push_back(worker_rundir.string()+std::string("/AthenaMP.log"));
  }
}

sysInitialize()

virtual StatusCode AthCommonDataStore< AthCommonMsg< AlgTool > >::sysInitialize ( )

overridevirtualinherited

Perform system initialization for an algorithm.

We override this to declare all the elements of handle key arrays at the end of initialization. See comments on updateVHKA.

Reimplemented in DerivationFramework::CfAthAlgTool, AthCheckedComponent< AthAlgTool >, AthCheckedComponent<::AthAlgTool >, and asg::AsgMetadataTool.

sysStart()

virtual StatusCode AthCommonDataStore< AthCommonMsg< AlgTool > >::sysStart ( )

overridevirtualinherited

Handle START transition.

We override this in order to make sure that conditions handle keys can cache a pointer to the conditions container.

updateIoReg()

int AthenaMPToolBase::updateIoReg ( const std::string &  rundir )

protectedinherited

Definition at line 341 of file AthenaMPToolBase.cxx.

{
  if (!m_ioMgr.retrieve().isSuccess()) {
    ATH_MSG_ERROR("Error retrieving IoComponentMgr");
    return -1;
  } else {
    ATH_MSG_DEBUG("Successfully retrieved IoComponentMgr");
  }
 
  // update the IoRegistry for the new workdir - make sure we use absolute path
  std::filesystem::path abs_rundir = std::filesystem::absolute(rundir);
  if(!m_ioMgr->io_update_all(abs_rundir.string()).isSuccess()) {
    ATH_MSG_ERROR("Error updating IoRegistry");
    return -1;
  } else {
    ATH_MSG_DEBUG("Successfully updated IoRegistry");
  }
 
  return 0;
}

updateVHKA()

void AthCommonDataStore< AthCommonMsg< AlgTool > >::updateVHKA ( Gaudi::Details::PropertyBase &  )

inlineinherited

Definition at line 308 of file AthCommonDataStore.h.

                                               {
    // debug() << "updateVHKA for property " << p.name() << " " << p.toString() 
    //         << "  size: " << m_vhka.size() << endmsg;
    for (auto &a : m_vhka) {
      std::vector<SG::VarHandleKey*> keys = a->keys();
      for (auto k : keys) {
        k->setOwner(this);
      }
    }
  }

useFdsRegistry()

void AthenaMPToolBase::useFdsRegistry ( std::shared_ptr< AthenaInterprocess::FdsRegistry >  registry )

overridevirtualinherited

Implements IAthenaMPTool.

Definition at line 200 of file AthenaMPToolBase.cxx.

{
  m_fdsRegistry = registry;
}

waitForSignal()

void AthenaMPToolBase::waitForSignal ( )

protectedinherited

Definition at line 432 of file AthenaMPToolBase.cxx.

{
  ATH_MSG_INFO("Bootstrap worker PID " << getpid() << " - waiting for SIGUSR1");
  sigset_t mask, oldmask;
  
  signal(SIGUSR1, AthenaMPToolBase_d::pauseForDebug);
  
  sigemptyset (&mask);
  sigaddset (&mask, SIGUSR1);
  
  sigprocmask (SIG_BLOCK, &mask, &oldmask);
  while (!AthenaMPToolBase_d::sig_done)
    sigsuspend (&oldmask);
  sigprocmask (SIG_UNBLOCK, &mask, NULL);
}

Member Data Documentation

m_appMgr

ServiceHandle<IAppMgrUI> AthenaMPToolBase::m_appMgr

protectedinherited

Definition at line 91 of file AthenaMPToolBase.h.

m_chronoStatSvc

ServiceHandle<IChronoStatSvc> SharedEvtQueueConsumer::m_chronoStatSvc

private

Definition at line 66 of file SharedEvtQueueConsumer.h.

m_dataShare

IDataShare* SharedEvtQueueConsumer::m_dataShare

private

Definition at line 71 of file SharedEvtQueueConsumer.h.

m_debug

bool SharedEvtQueueConsumer::m_debug

private

Definition at line 62 of file SharedEvtQueueConsumer.h.

m_detStore

StoreGateSvc_t AthCommonDataStore< AthCommonMsg< AlgTool > >::m_detStore

privateinherited

Pointer to StoreGate (detector store by default)

Definition at line 393 of file AthCommonDataStore.h.

m_eventOrders

std::vector<int> SharedEvtQueueConsumer::m_eventOrders

private

Definition at line 83 of file SharedEvtQueueConsumer.h.

m_eventOrdersFile

std::string SharedEvtQueueConsumer::m_eventOrdersFile

private

Definition at line 82 of file SharedEvtQueueConsumer.h.

m_eventStat

std::map<pid_t,std::pair<int,TimeValType> > SharedEvtQueueConsumer::m_eventStat

private

Definition at line 77 of file SharedEvtQueueConsumer.h.

m_evtContext

IEvtSelector::Context* SharedEvtQueueConsumer::m_evtContext

private

Definition at line 69 of file SharedEvtQueueConsumer.h.

m_evtProcessor

ServiceHandle<IEventProcessor> AthenaMPToolBase::m_evtProcessor

protectedinherited

Definition at line 90 of file AthenaMPToolBase.h.

m_evtSeek

IEventSeek* SharedEvtQueueConsumer::m_evtSeek

private

Definition at line 67 of file SharedEvtQueueConsumer.h.

m_evtSelector

IEvtSelector* AthenaMPToolBase::m_evtSelector

protectedinherited

Definition at line 94 of file AthenaMPToolBase.h.

m_evtSelName

std::string AthenaMPToolBase::m_evtSelName

protectedinherited

Definition at line 86 of file AthenaMPToolBase.h.

m_evtSelSeek

IEvtSelectorSeek* SharedEvtQueueConsumer::m_evtSelSeek

private

Definition at line 68 of file SharedEvtQueueConsumer.h.

m_evtShare

IEventShare* SharedEvtQueueConsumer::m_evtShare

private

Definition at line 70 of file SharedEvtQueueConsumer.h.

m_evtStore

StoreGateSvc_t AthCommonDataStore< AthCommonMsg< AlgTool > >::m_evtStore

privateinherited

Pointer to StoreGate (event store by default)

Definition at line 390 of file AthCommonDataStore.h.

m_fdsRegistry

std::shared_ptr<AthenaInterprocess::FdsRegistry> AthenaMPToolBase::m_fdsRegistry

protectedinherited

Definition at line 96 of file AthenaMPToolBase.h.

m_fileMgr

ServiceHandle<IFileMgr> AthenaMPToolBase::m_fileMgr

protectedinherited

Definition at line 92 of file AthenaMPToolBase.h.

m_fileMgrLog

std::string AthenaMPToolBase::m_fileMgrLog

protectedinherited

Definition at line 95 of file AthenaMPToolBase.h.

m_finQueue

std::queue<pid_t> SharedEvtQueueConsumer::m_finQueue

private

Definition at line 78 of file SharedEvtQueueConsumer.h.

m_ioMgr

ServiceHandle<IIoComponentMgr> AthenaMPToolBase::m_ioMgr

protectedinherited

Definition at line 93 of file AthenaMPToolBase.h.

m_isPileup

Gaudi::Property<bool> AthenaMPToolBase::m_isPileup {this, "IsPileup", false, "Flag for configuring PileUpEventLoopMgr"}

protectedinherited

Definition at line 99 of file AthenaMPToolBase.h.

m_isRoundRobin

bool SharedEvtQueueConsumer::m_isRoundRobin

private

Definition at line 59 of file SharedEvtQueueConsumer.h.

m_masterPid

pid_t SharedEvtQueueConsumer::m_masterPid

private

Definition at line 84 of file SharedEvtQueueConsumer.h.

m_nEventsBeforeFork

int SharedEvtQueueConsumer::m_nEventsBeforeFork

private

Definition at line 60 of file SharedEvtQueueConsumer.h.

m_nprocs

int AthenaMPToolBase::m_nprocs

protectedinherited

Definition at line 83 of file AthenaMPToolBase.h.

m_nSkipEvents

int SharedEvtQueueConsumer::m_nSkipEvents

private

Definition at line 61 of file SharedEvtQueueConsumer.h.

m_processGroup

AthenaInterprocess::ProcessGroup* AthenaMPToolBase::m_processGroup

protectedinherited

Definition at line 88 of file AthenaMPToolBase.h.

m_randStr

std::string AthenaMPToolBase::m_randStr

protectedinherited

Definition at line 97 of file AthenaMPToolBase.h.

m_rankId

int SharedEvtQueueConsumer::m_rankId

private

Definition at line 64 of file SharedEvtQueueConsumer.h.

m_readEventOrders

bool SharedEvtQueueConsumer::m_readEventOrders

private

Definition at line 81 of file SharedEvtQueueConsumer.h.

m_sharedEventQueue

AthenaInterprocess::SharedQueue* SharedEvtQueueConsumer::m_sharedEventQueue

private

Definition at line 73 of file SharedEvtQueueConsumer.h.

m_sharedRankQueue

AthenaInterprocess::SharedQueue* SharedEvtQueueConsumer::m_sharedRankQueue

private

Definition at line 74 of file SharedEvtQueueConsumer.h.

m_subprocDirPrefix

std::string AthenaMPToolBase::m_subprocDirPrefix

protectedinherited

Definition at line 85 of file AthenaMPToolBase.h.

m_subprocTopDir

std::string AthenaMPToolBase::m_subprocTopDir

protectedinherited

Definition at line 84 of file AthenaMPToolBase.h.

m_useSharedReader

bool SharedEvtQueueConsumer::m_useSharedReader

private

Definition at line 57 of file SharedEvtQueueConsumer.h.

m_useSharedWriter

bool SharedEvtQueueConsumer::m_useSharedWriter

private

Definition at line 58 of file SharedEvtQueueConsumer.h.

m_varHandleArraysDeclared

bool AthCommonDataStore< AthCommonMsg< AlgTool > >::m_varHandleArraysDeclared

privateinherited

Definition at line 399 of file AthCommonDataStore.h.

m_vhka

std::vector<SG::VarHandleKeyArray*> AthCommonDataStore< AthCommonMsg< AlgTool > >::m_vhka

privateinherited

Definition at line 398 of file AthCommonDataStore.h.

---

The documentation for this class was generated from the following files:

• SharedEvtQueueConsumer.h
• SharedEvtQueueConsumer.cxx