ATLAS Offline Software
AthenaOutputStream.h
Go to the documentation of this file.
1 // Dear emacs, this is -*- C++ -*-
2 
3 /*
4  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
5 */
6 
7 #ifndef ATHENASERVICES_ATHENAOUTPUTSTREAM_H
8 #define ATHENASERVICES_ATHENAOUTPUTSTREAM_H
9 
10 // STL include files
11 #include <map>
12 #include <memory>
13 #include <mutex>
14 #include <set>
15 #include <string>
16 #include <vector>
17 
18 // Required for inheritance
19 #include "Gaudi/Property.h"
20 #include "GaudiKernel/IDataSelector.h"
21 #include "GaudiKernel/ServiceHandle.h"
22 #include "GaudiKernel/ToolHandle.h"
23 
24 // Framework include files
29 #include "AthenaKernel/ITPCnvSvc.h"
30 #include "GaudiKernel/IClassIDSvc.h"
31 #include "GaudiKernel/IIncidentListener.h"
32 #include "GaudiKernel/IIoComponent.h"
34 
35 // Local include files
36 #include "CompressionInfo.h"
37 #include "MetaDataSvc.h"
39 #include "SelectionVetoes.h"
40 
41 // forward declarations
42 namespace SG {
43  class DataProxy;
44  class IFolder;
45  class IAuxStoreIO;
46  class FolderItem;
47 }
48 
53 class AthenaOutputStream : public extends<FilteredAlgorithm,
54  IIncidentListener, IIoComponent> {
55 
56 public:
58  typedef std::vector<SG::DataProxy*> Items;
59  typedef std::vector<std::pair<std::string, std::string> > TypeKeyPairs;
60  typedef std::recursive_mutex mutex_t;
61 
63  AthenaOutputStream(const std::string& name, ISvcLocator* pSvcLocator);
64 
66  virtual ~AthenaOutputStream();
67 
69 
70  virtual StatusCode initialize() override;
71  virtual StatusCode finalize() override;
72  virtual StatusCode execute() override;
74 
76  virtual StatusCode write();
77 
79  void clearSelection();
80 
83 
85  IDataSelector* selectedObjects() {
86  return &m_objects;
87  }
88 
90  virtual void handle(const Incident& incident) override;
91 
93  virtual StatusCode io_reinit() override;
94  virtual StatusCode io_finalize() override;
95 
96 protected:
99  ServiceHandle<StoreGateSvc> m_dataStore{this, "Store", "StoreGateSvc/StoreGateSvc", "Handle to event store"};
100  ServiceHandle<StoreGateSvc> m_metadataStore{this, "MetadataStore", "StoreGateSvc/MetaDataStore", "Handle to metadata store"};
102 
104  ServiceHandle<MetaDataSvc> m_metaDataSvc{this, "MetaDataSvc", "MetaDataSvc"};
105  ServiceHandle<IDictLoaderSvc> m_dictLoader{this, "AthDictLoaderSvc", "AthDictLoaderSvc"};
106  ServiceHandle<ITPCnvSvc> m_tpCnvSvc{this, "AthTPCnvSvc", "AthTPCnvSvc"};
107  ServiceHandle<IIncidentSvc> m_incidentSvc{this, "IncidentSvc", "IncidentSvc"};
108  ServiceHandle<IClassIDSvc> m_pCLIDSvc{this, "ClassIDSvc", "ClassIDSvc"};
109  ServiceHandle<OutputStreamSequencerSvc> m_outSeqSvc{this, "OutputStreamSequencerSvc", "OutputStreamSequencerSvc"};
110 
112  StringProperty m_streamName{this, "StreamName", "", "Name of the output stream"};
113 
115  StringArrayProperty m_itemList{this, "ItemList", {}, "List of items to write", "OutputStreamItemList"};
116 
118  StringArrayProperty m_metadataItemList{this, "MetadataItemList", {}, "List of metadata items to write","OutputStreamItemList"};
119 
121  StringProperty m_keepProvenances {this, "KeepProvenanceTagsRegEx", {".*"},
122  "RegEx pattern to select processing tags for which DataHeader should retain provenances"};
123 
125  StringArrayProperty m_compressionListHigh{this, "CompressionListHigh", {}, "Lossy float compression list (high)"};
126 
128  StringArrayProperty m_compressionListLow{this, "CompressionListLow", {}, "Lossy float compression list (low)"};
129 
131  UnsignedIntegerProperty m_compressionBitsHigh{this, "CompressionBitsHigh", 7, "Lossy float compression bits (high)"};
132 
134  UnsignedIntegerProperty m_compressionBitsLow{this, "CompressionBitsLow", 15, "Lossy float compression bits (low)"};
135 
138  StringArrayProperty m_transientItems{this, "TransientItems", {}, "Transient item list"};
139 
141  StringProperty m_outputName{this, "OutputFile", "DidNotNameOutput.root", "Name of the output file"};
142 
144  StringProperty m_persName{this, "EvtConversionSvc", "EventPersistencySvc", "Name of the persistency service writing data"};
145 
147  BooleanProperty m_forceRead{this, "ForceRead", true, "Force read data objects in ItemList"};
148 
151  BooleanProperty m_extendProvenanceRecord{this, "ExtendProvenanceRecord", true, "Extend provenance record"};
152 
154  BooleanProperty m_itemListFromTool{this, "TakeItemsFromInput", false, "Write everything in input DataHeader to output"};
155 
157  ToolHandle<SG::IFolder> m_p2BWritten;
158 
160  ToolHandle<SG::IFolder> m_compressionDecoderHigh;
161 
163  ToolHandle<SG::IFolder> m_compressionDecoderLow;
164 
166  ToolHandle<SG::IFolder> m_transient;
167 
169  std::multimap<CLID,std::string> m_CLIDKeyPairs;
170 
172  IDataSelector m_objects;
173 
175  IDataSelector m_altObjects;
176 
180  std::vector<std::unique_ptr<DataObject> > m_ownedObjects;
181 
183  ToolHandle<IAthenaOutputStreamTool> m_streamer;
184 
186  ToolHandleArray<IAthenaOutputTool> m_helperTools{this, "HelperTools", {}, "List of AlgTools used by this stream"};
187 
188  // flag set by MetaDataStop if OutputSequencer is used with EndEvent
190 
192  std::atomic<int> m_events{0};
193 
194  // ------- Event Ranges handling in MT -------
196  std::map< unsigned, std::string > m_slotRangeMap;
197 
199  std::map< std::string, std::string > m_rangeIDforRangeFN;
200 
202  std::map< std::string, std::unique_ptr<IAthenaOutputStreamTool> > m_streamerMap;
203 
206 
208  void itemListHandler(Gaudi::Details::PropertyBase& /* theProp */);
209 
211  void excludeListHandler(Gaudi::Details::PropertyBase& /* theProp */);
212 
214  void compressionListHandlerHigh(Gaudi::Details::PropertyBase& /* theProp */);
215 
217  void compressionListHandlerLow(Gaudi::Details::PropertyBase& /* theProp */);
218 
219 private:
223  { this, "SelVetoesKey", "" };
224 
228  { this, "CompInfoKey", "" };
229 
231  std::string m_outputAttributes;
232 
235 
236  void handleVariableSelection (const SG::IConstAuxStore& auxstore,
237  SG::DataProxy& itemProxy,
238  const std::string& aux_attr,
239  SG::SelectionVetoes& vetoes) const;
240 
242  void writeMetaData( const std::string& outputFN="" );
243 
245  std::set<std::string> buildCompressionSet (const ToolHandle<SG::IFolder>& handle,
246  const CLID& item_id,
247  const std::string& item_key) const;
248 
249  // close an EventService substream that was writing to 'rangeFN' output
250  void finalizeRange( const std::string & rangeFN );
251 
257  void loadDict (CLID clid);
258 
260  bool simpleMatch(const std::string& pattern, const std::string& text);
261 };
262 
263 #endif // ATHENASERVICES_OUTPUTSTREAM_H
mergePhysValFiles.pattern
pattern
Definition: DataQuality/DataQualityUtils/scripts/mergePhysValFiles.py:25
AthenaOutputStream::finalize
virtual StatusCode finalize() override
Definition: AthenaOutputStream.cxx:378
AthenaOutputStream::m_rangeIDforRangeFN
std::map< std::string, std::string > m_rangeIDforRangeFN
map of RangeIDs (as used by the Sequencer) for each Range filename generated
Definition: AthenaOutputStream.h:199
AthenaOutputStream::writeMetaData
void writeMetaData(const std::string &outputFN="")
Write MetaData for this stream (by default) or for a substream outputFN (in ES mode)
Definition: AthenaOutputStream.cxx:318
AthenaOutputStream::m_outputAttributes
std::string m_outputAttributes
Output attributes.
Definition: AthenaOutputStream.h:231
FilteredAlgorithm.h
SG
Forward declaration.
Definition: CaloCellPacker_400_500.h:32
AthenaOutputStream::addItemObjects
StatusCode addItemObjects(const SG::FolderItem &, SG::SelectionVetoes &vetoes, SG::CompressionInfo &compInfo)
Add item data objects to output streamer list.
Definition: AthenaOutputStream.cxx:567
AthenaOutputStream::m_CLIDKeyPairs
std::multimap< CLID, std::string > m_CLIDKeyPairs
Map of (clid,key) pairs to be excluded (comes from m_excludeList)
Definition: AthenaOutputStream.h:169
ITPCnvSvc.h
IAthenaOutputTool.h
This file contains the class definition for the IAthenaOutputTool class.
AthenaOutputStream::m_itemList
StringArrayProperty m_itemList
Vector of item names.
Definition: AthenaOutputStream.h:115
AthenaOutputStream::buildCompressionSet
std::set< std::string > buildCompressionSet(const ToolHandle< SG::IFolder > &handle, const CLID &item_id, const std::string &item_key) const
Helper function for building the compression lists.
Definition: AthenaOutputStream.cxx:774
AthenaOutputStream::m_dataStore
ServiceHandle< StoreGateSvc > m_dataStore
Handle to the StoreGateSvc store where the data we want to write out resides.
Definition: AthenaOutputStream.h:99
AthenaOutputStream::m_compressionListLow
StringArrayProperty m_compressionListLow
Vector of item names.
Definition: AthenaOutputStream.h:128
AthenaOutputStream::mutex_t
std::recursive_mutex mutex_t
Definition: AthenaOutputStream.h:60
SG::SelectionVetoes
std::unordered_map< std::string, SG::auxid_set_t > SelectionVetoes
Map of vetoed variables.
Definition: SelectionVetoes.h:50
OutputStreamSequencerSvc.h
This file contains the class definition for the OutputStreamSequencerSvc class.
AthenaOutputStream::initialize
virtual StatusCode initialize() override
Definition: AthenaOutputStream.cxx:73
AthenaOutputStream::m_forceRead
BooleanProperty m_forceRead
set to true to force read of data objects in item list
Definition: AthenaOutputStream.h:147
AthenaOutputStream::Items
std::vector< SG::DataProxy * > Items
typedefs
Definition: AthenaOutputStream.h:58
AthenaOutputStream::selectedObjects
IDataSelector * selectedObjects()
Return the list of selected objects.
Definition: AthenaOutputStream.h:85
AthenaOutputStream::loadDict
void loadDict(CLID clid)
Helper function to load dictionaries (both transient and persistent) for a given type.
Definition: AthenaOutputStream.cxx:928
AthenaOutputStream::m_transientItems
StringArrayProperty m_transientItems
List of items that are known to be present in the transient store (and hence we can make input depend...
Definition: AthenaOutputStream.h:138
AthenaOutputStream::m_objects
IDataSelector m_objects
Collection of objects being selected.
Definition: AthenaOutputStream.h:172
AthenaOutputStream::m_metadataItemList
StringArrayProperty m_metadataItemList
Vector of item names.
Definition: AthenaOutputStream.h:118
AthenaOutputStream::m_currentStore
ServiceHandle< StoreGateSvc > * m_currentStore
Definition: AthenaOutputStream.h:101
AthenaOutputStream::collectAllObjects
StatusCode collectAllObjects()
Collect data objects for output streamer list.
Definition: AthenaOutputStream.cxx:533
IAthenaOutputStreamTool.h
Interface to an output stream tool.
AthenaOutputStream::m_compressionListHigh
StringArrayProperty m_compressionListHigh
Vector of item names.
Definition: AthenaOutputStream.h:125
AthenaOutputStream::m_streamer
ToolHandle< IAthenaOutputStreamTool > m_streamer
pointer to AthenaOutputStreamTool
Definition: AthenaOutputStream.h:183
AthenaOutputStream::m_outSeqSvc
ServiceHandle< OutputStreamSequencerSvc > m_outSeqSvc
Definition: AthenaOutputStream.h:109
SG::WriteHandleKey< SG::SelectionVetoes >
AthenaOutputStream::AthenaOutputStream
AthenaOutputStream(const std::string &name, ISvcLocator *pSvcLocator)
Standard algorithm Constructor.
Definition: AthenaOutputStream.cxx:43
AthenaOutputStream::m_compressionBitsHigh
UnsignedIntegerProperty m_compressionBitsHigh
Number of mantissa bits in the float compression.
Definition: AthenaOutputStream.h:131
AthenaOutputStream::m_slotRangeMap
std::map< unsigned, std::string > m_slotRangeMap
map of filenames assigned to active slots
Definition: AthenaOutputStream.h:196
EL::StatusCode
::StatusCode StatusCode
StatusCode definition for legacy code.
Definition: PhysicsAnalysis/D3PDTools/EventLoop/EventLoop/StatusCode.h:22
AthenaOutputStream::m_outputName
StringProperty m_outputName
Name of the output file.
Definition: AthenaOutputStream.h:141
AthenaOutputStream::compressionListHandlerLow
void compressionListHandlerLow(Gaudi::Details::PropertyBase &)
Handler for ItemNames Property.
Definition: AthenaOutputStream.cxx:882
DataProxy
DataProxy provides the registry services for StoreGate.
Definition: DataProxy.h:32
AthenaOutputStream::m_helperTools
ToolHandleArray< IAthenaOutputTool > m_helperTools
vector of AlgTools that that are executed by this stream
Definition: AthenaOutputStream.h:186
AthenaOutputStream::m_ownedObjects
std::vector< std::unique_ptr< DataObject > > m_ownedObjects
Collection of DataObject instances owned by this service.
Definition: AthenaOutputStream.h:180
AthenaOutputStream::excludeListHandler
void excludeListHandler(Gaudi::Details::PropertyBase &)
Handler for ItemNames Property.
AthenaOutputStream::compressionListHandlerHigh
void compressionListHandlerHigh(Gaudi::Details::PropertyBase &)
Handler for ItemNames Property.
Definition: AthenaOutputStream.cxx:873
CLID
uint32_t CLID
The Class ID type.
Definition: Event/xAOD/xAODCore/xAODCore/ClassID_traits.h:47
AthenaOutputStream::io_finalize
virtual StatusCode io_finalize() override
Definition: AthenaOutputStream.cxx:907
AthenaOutputStream::m_persName
StringProperty m_persName
Name of the persistency service capable to write data from the store.
Definition: AthenaOutputStream.h:144
AthenaOutputStream::m_writeMetadataAndDisconnect
bool m_writeMetadataAndDisconnect
Definition: AthenaOutputStream.h:189
AthenaOutputStream::m_extendProvenanceRecord
BooleanProperty m_extendProvenanceRecord
Set to false to omit adding the current DataHeader into the DataHeader history This will cause the in...
Definition: AthenaOutputStream.h:151
AthenaOutputStream::m_itemListFromTool
BooleanProperty m_itemListFromTool
Set to write out everything in input DataHeader.
Definition: AthenaOutputStream.h:154
AthenaOutputStream::m_transient
ToolHandle< SG::IFolder > m_transient
Decoded list of transient ids.
Definition: AthenaOutputStream.h:166
AthenaOutputStream::finalizeRange
void finalizeRange(const std::string &rangeFN)
Definition: AthenaOutputStream.cxx:298
AthenaOutputStream::m_incidentSvc
ServiceHandle< IIncidentSvc > m_incidentSvc
Definition: AthenaOutputStream.h:107
SG::CompressionInfo
std::unordered_map< std::string, SG::ThinningInfo::compression_map_t > CompressionInfo
Map of compressed variables and their compression levels.
Definition: CompressionInfo.h:37
AthenaOutputStream::itemListHandler
void itemListHandler(Gaudi::Details::PropertyBase &)
Handler for ItemNames Property.
Definition: AthenaOutputStream.cxx:863
AthenaOutputStream::TypeKeyPairs
std::vector< std::pair< std::string, std::string > > TypeKeyPairs
Definition: AthenaOutputStream.h:59
WriteHandleKey.h
Property holding a SG store/key/clid from which a WriteHandle is made.
AthenaOutputStream::handleVariableSelection
void handleVariableSelection(const SG::IConstAuxStore &auxstore, SG::DataProxy &itemProxy, const std::string &aux_attr, SG::SelectionVetoes &vetoes) const
Here we build the vetoed AuxIDs.
Definition: AthenaOutputStream.cxx:817
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
AthenaOutputStream::m_metadataStore
ServiceHandle< StoreGateSvc > m_metadataStore
Definition: AthenaOutputStream.h:100
AthenaOutputStream::io_reinit
virtual StatusCode io_reinit() override
Callback method to reinitialize the internal state of the component for I/O purposes (e....
Definition: AthenaOutputStream.cxx:893
AthenaOutputStream::m_tpCnvSvc
ServiceHandle< ITPCnvSvc > m_tpCnvSvc
Definition: AthenaOutputStream.h:106
AthenaOutputStream::m_streamName
StringProperty m_streamName
Stream name (defaults to algorithm name)
Definition: AthenaOutputStream.h:112
AthenaOutputStream::m_compressionBitsLow
UnsignedIntegerProperty m_compressionBitsLow
Number of mantissa bits in the float compression.
Definition: AthenaOutputStream.h:134
AthenaOutputStream::m_altObjects
IDataSelector m_altObjects
Objects overridden by ‘exact’ handling.
Definition: AthenaOutputStream.h:175
AthenaOutputStream::m_dictLoader
ServiceHandle< IDictLoaderSvc > m_dictLoader
Definition: AthenaOutputStream.h:105
AthenaOutputStream::m_metaDataSvc
ServiceHandle< MetaDataSvc > m_metaDataSvc
Handles to all the necessary services.
Definition: AthenaOutputStream.h:104
MetaDataSvc.h
This file contains the class definition for the MetaDataSvc class.
IDictLoaderSvc.h
AthenaOutputStream::m_events
std::atomic< int > m_events
Number of events written to this output stream.
Definition: AthenaOutputStream.h:192
AthenaOutputStream::simpleMatch
bool simpleMatch(const std::string &pattern, const std::string &text)
Glob-style matcher, where the only meta-character is '*'.
Definition: AthenaOutputStream.cxx:940
AthenaOutputStream::m_compressionDecoderHigh
ToolHandle< SG::IFolder > m_compressionDecoderHigh
The top-level folder with items to be compressed high.
Definition: AthenaOutputStream.h:160
AthenaOutputStream::m_mutex
mutex_t m_mutex
mutex for this Stream write() and handle() methods
Definition: AthenaOutputStream.h:205
AthenaOutputStream
algorithm that marks for write data objects in SG
Definition: AthenaOutputStream.h:54
AthenaOutputStream::execute
virtual StatusCode execute() override
Definition: AthenaOutputStream.cxx:406
AthenaOutputStream::m_streamerMap
std::map< std::string, std::unique_ptr< IAthenaOutputStreamTool > > m_streamerMap
map of streamerTools handling event ranges in MT
Definition: AthenaOutputStream.h:202
AthenaOutputStream::m_keepProvenances
StringProperty m_keepProvenances
Provenance record selection.
Definition: AthenaOutputStream.h:121
makeTransCanvas.text
text
Definition: makeTransCanvas.py:11
SelectionVetoes.h
CompressionInfo.h
AthenaOutputStream::write
virtual StatusCode write()
Stream the data.
Definition: AthenaOutputStream.cxx:442
AthenaOutputStream::m_pCLIDSvc
ServiceHandle< IClassIDSvc > m_pCLIDSvc
Definition: AthenaOutputStream.h:108
AthenaOutputStream::m_selVetoesKey
SG::WriteHandleKey< SG::SelectionVetoes > m_selVetoesKey
Key used for recording selected dynamic variable information to the event store.
Definition: AthenaOutputStream.h:223
SG::IConstAuxStore
Interface for const operations on an auxiliary store.
Definition: IConstAuxStore.h:64
GetLBsToIgnore.outputFN
outputFN
Definition: GetLBsToIgnore.py:224
SG::DataProxy
Definition: DataProxy.h:45
SG::FolderItem
a Folder item (data object) is identified by the clid/key pair
Definition: SGFolderItem.h:24
AthenaOutputStream::handle
virtual void handle(const Incident &incident) override
Incident service handle listening for MetaDataStop.
Definition: AthenaOutputStream.cxx:223
AthenaOutputStream::m_p2BWritten
ToolHandle< SG::IFolder > m_p2BWritten
The top-level folder with items to be written.
Definition: AthenaOutputStream.h:157
AthenaOutputStream::clearSelection
void clearSelection()
Clear list of selected objects.
Definition: AthenaOutputStream.cxx:526
AthenaOutputStream::m_compressionDecoderLow
ToolHandle< SG::IFolder > m_compressionDecoderLow
The top-level folder with items to be compressed low.
Definition: AthenaOutputStream.h:163
ServiceHandle< StoreGateSvc >
AthenaOutputStream::~AthenaOutputStream
virtual ~AthenaOutputStream()
Standard Destructor.
Definition: AthenaOutputStream.cxx:67
AthenaOutputStream::m_compInfoKey
SG::WriteHandleKey< SG::CompressionInfo > m_compInfoKey
Key used for recording lossy float compressed variable information to the event store.
Definition: AthenaOutputStream.h:228