ATLAS Offline Software
AthenaOutputStream.h
Go to the documentation of this file.
1 // Dear emacs, this is -*- C++ -*-
2 
3 /*
4  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
5 */
6 
7 #ifndef ATHENASERVICES_ATHENAOUTPUTSTREAM_H
8 #define ATHENASERVICES_ATHENAOUTPUTSTREAM_H
9 
10 // STL include files
11 #include <map>
12 #include <memory>
13 #include <mutex>
14 #include <set>
15 #include <string>
16 #include <vector>
17 
18 // Required for inheritance
19 #include "Gaudi/Property.h"
20 #include "GaudiKernel/IDataSelector.h"
21 #include "GaudiKernel/ServiceHandle.h"
22 #include "GaudiKernel/ToolHandle.h"
23 
24 // Framework include files
30 #include "AthenaKernel/ITPCnvSvc.h"
31 #include "GaudiKernel/IClassIDSvc.h"
32 #include "GaudiKernel/IIncidentListener.h"
33 #include "GaudiKernel/IIoComponent.h"
35 
36 // Local include files
37 #include "CompressionInfo.h"
38 #include "MetaDataSvc.h"
40 #include "SelectionVetoes.h"
41 
42 // forward declarations
43 namespace SG {
44  class DataProxy;
45  class IFolder;
46  class IAuxStoreIO;
47  class FolderItem;
48 }
49 
54 class AthenaOutputStream : public extends<FilteredAlgorithm,
55  IIncidentListener, IIoComponent> {
56 
57 public:
59  typedef std::vector<SG::DataProxy*> Items;
60  typedef std::vector<std::pair<std::string, std::string> > TypeKeyPairs;
61  typedef std::recursive_mutex mutex_t;
62 
64  AthenaOutputStream(const std::string& name, ISvcLocator* pSvcLocator);
65 
67  virtual ~AthenaOutputStream();
68 
70 
71  virtual StatusCode initialize() override;
72  virtual StatusCode finalize() override;
73  virtual StatusCode execute() override;
75 
77  virtual StatusCode write();
78 
80  void clearSelection();
81 
84 
86  IDataSelector* selectedObjects() {
87  return &m_objects;
88  }
89 
91  virtual void handle(const Incident& incident) override;
92 
94  virtual StatusCode io_reinit() override;
95  virtual StatusCode io_finalize() override;
96 
97 protected:
100  ServiceHandle<StoreGateSvc> m_dataStore{this, "Store", "StoreGateSvc/StoreGateSvc", "Handle to event store"};
101  ServiceHandle<StoreGateSvc> m_metadataStore{this, "MetadataStore", "StoreGateSvc/MetaDataStore", "Handle to metadata store"};
103 
105  ServiceHandle<IItemListSvc> m_itemSvc{this, "ItemListSvc", "ItemListSvc"};
106  ServiceHandle<MetaDataSvc> m_metaDataSvc{this, "MetaDataSvc", "MetaDataSvc"};
107  ServiceHandle<IDictLoaderSvc> m_dictLoader{this, "AthDictLoaderSvc", "AthDictLoaderSvc"};
108  ServiceHandle<ITPCnvSvc> m_tpCnvSvc{this, "AthTPCnvSvc", "AthTPCnvSvc"};
109  ServiceHandle<IIncidentSvc> m_incidentSvc{this, "IncidentSvc", "IncidentSvc"};
110  ServiceHandle<IClassIDSvc> m_pCLIDSvc{this, "ClassIDSvc", "ClassIDSvc"};
111  ServiceHandle<OutputStreamSequencerSvc> m_outSeqSvc{this, "OutputStreamSequencerSvc", "OutputStreamSequencerSvc"};
112 
114  StringProperty m_streamName{this, "StreamName", "", "Name of the output stream"};
115 
117  StringArrayProperty m_itemList{this, "ItemList", {}, "List of items to write", "OutputStreamItemList"};
118 
120  StringArrayProperty m_metadataItemList{this, "MetadataItemList", {}, "List of metadata items to write","OutputStreamItemList"};
121 
123  StringProperty m_keepProvenances {this, "KeepProvenanceTagsRegEx", {".*"},
124  "RegEx pattern to select processing tags for which DataHeader should retain provenances"};
125 
127  StringArrayProperty m_compressionListHigh{this, "CompressionListHigh", {}, "Lossy float compression list (high)"};
128 
130  StringArrayProperty m_compressionListLow{this, "CompressionListLow", {}, "Lossy float compression list (low)"};
131 
133  UnsignedIntegerProperty m_compressionBitsHigh{this, "CompressionBitsHigh", 7, "Lossy float compression bits (high)"};
134 
136  UnsignedIntegerProperty m_compressionBitsLow{this, "CompressionBitsLow", 15, "Lossy float compression bits (low)"};
137 
140  StringArrayProperty m_transientItems{this, "TransientItems", {}, "Transient item list"};
141 
143  StringProperty m_outputName{this, "OutputFile", "DidNotNameOutput.root", "Name of the output file"};
144 
146  StringProperty m_persName{this, "EvtConversionSvc", "EventPersistencySvc", "Name of the persistency service writing data"};
147 
149  BooleanProperty m_forceRead{this, "ForceRead", true, "Force read data objects in ItemList"};
150 
153  BooleanProperty m_extendProvenanceRecord{this, "ExtendProvenanceRecord", true, "Extend provenance record"};
154 
156  BooleanProperty m_itemListFromTool{this, "TakeItemsFromInput", false, "Write everything in input DataHeader to output"};
157 
159  ToolHandle<SG::IFolder> m_p2BWritten;
160 
162  ToolHandle<SG::IFolder> m_compressionDecoderHigh;
163 
165  ToolHandle<SG::IFolder> m_compressionDecoderLow;
166 
168  ToolHandle<SG::IFolder> m_transient;
169 
171  std::multimap<CLID,std::string> m_CLIDKeyPairs;
172 
174  IDataSelector m_objects;
175 
177  IDataSelector m_altObjects;
178 
182  std::vector<std::unique_ptr<DataObject> > m_ownedObjects;
183 
185  ToolHandle<IAthenaOutputStreamTool> m_streamer;
186 
188  ToolHandleArray<IAthenaOutputTool> m_helperTools{this, "HelperTools", {}, "List of AlgTools used by this stream"};
189 
190  // flag set by MetaDataStop if OutputSequencer is used with EndEvent
192 
194  std::atomic<int> m_events{0};
195 
196  // ------- Event Ranges handling in MT -------
198  std::map< unsigned, std::string > m_slotRangeMap;
199 
201  std::map< std::string, std::string > m_rangeIDforRangeFN;
202 
204  std::map< std::string, std::unique_ptr<IAthenaOutputStreamTool> > m_streamerMap;
205 
208 
210  void itemListHandler(Gaudi::Details::PropertyBase& /* theProp */);
211 
213  void excludeListHandler(Gaudi::Details::PropertyBase& /* theProp */);
214 
216  void compressionListHandlerHigh(Gaudi::Details::PropertyBase& /* theProp */);
217 
219  void compressionListHandlerLow(Gaudi::Details::PropertyBase& /* theProp */);
220 
221 private:
225  { this, "SelVetoesKey", "" };
226 
230  { this, "CompInfoKey", "" };
231 
233  std::string m_outputAttributes;
234 
237 
238  void handleVariableSelection (const SG::IConstAuxStore& auxstore,
239  SG::DataProxy& itemProxy,
240  const std::string& tns,
241  const std::string& aux_attr,
242  SG::SelectionVetoes& vetoes) const;
243 
245  void writeMetaData( const std::string& outputFN="" );
246 
248  std::set<std::string> buildCompressionSet (const ToolHandle<SG::IFolder>& handle,
249  const CLID& item_id,
250  const std::string& item_key) const;
251 
252  // close an EventService substream that was writing to 'rangeFN' output
253  void finalizeRange( const std::string & rangeFN );
254 
260  void loadDict (CLID clid);
261 
263  bool simpleMatch(const std::string& pattern, const std::string& text);
264 };
265 
266 #endif // ATHENASERVICES_OUTPUTSTREAM_H
mergePhysValFiles.pattern
pattern
Definition: DataQuality/DataQualityUtils/scripts/mergePhysValFiles.py:25
AthenaOutputStream::finalize
virtual StatusCode finalize() override
Definition: AthenaOutputStream.cxx:379
AthenaOutputStream::m_rangeIDforRangeFN
std::map< std::string, std::string > m_rangeIDforRangeFN
map of RangeIDs (as used by the Sequencer) for each Range filename generated
Definition: AthenaOutputStream.h:201
AthenaOutputStream::writeMetaData
void writeMetaData(const std::string &outputFN="")
Write MetaData for this stream (by default) or for a substream outputFN (in ES mode)
Definition: AthenaOutputStream.cxx:319
AthenaOutputStream::m_outputAttributes
std::string m_outputAttributes
Output attributes.
Definition: AthenaOutputStream.h:233
FilteredAlgorithm.h
SG
Forward declaration.
Definition: CaloCellPacker_400_500.h:32
AthenaOutputStream::addItemObjects
StatusCode addItemObjects(const SG::FolderItem &, SG::SelectionVetoes &vetoes, SG::CompressionInfo &compInfo)
Add item data objects to output streamer list.
Definition: AthenaOutputStream.cxx:564
AthenaOutputStream::m_CLIDKeyPairs
std::multimap< CLID, std::string > m_CLIDKeyPairs
Map of (clid,key) pairs to be excluded (comes from m_excludeList)
Definition: AthenaOutputStream.h:171
ITPCnvSvc.h
IAthenaOutputTool.h
This file contains the class definition for the IAthenaOutputTool class.
AthenaOutputStream::m_itemList
StringArrayProperty m_itemList
Vector of item names.
Definition: AthenaOutputStream.h:117
AthenaOutputStream::buildCompressionSet
std::set< std::string > buildCompressionSet(const ToolHandle< SG::IFolder > &handle, const CLID &item_id, const std::string &item_key) const
Helper function for building the compression lists.
Definition: AthenaOutputStream.cxx:786
AthenaOutputStream::m_dataStore
ServiceHandle< StoreGateSvc > m_dataStore
Handle to the StoreGateSvc store where the data we want to write out resides.
Definition: AthenaOutputStream.h:100
AthenaOutputStream::m_compressionListLow
StringArrayProperty m_compressionListLow
Vector of item names.
Definition: AthenaOutputStream.h:130
AthenaOutputStream::mutex_t
std::recursive_mutex mutex_t
Definition: AthenaOutputStream.h:61
SG::SelectionVetoes
std::unordered_map< std::string, SG::auxid_set_t > SelectionVetoes
Map of vetoed variables.
Definition: SelectionVetoes.h:50
OutputStreamSequencerSvc.h
This file contains the class definition for the OutputStreamSequencerSvc class.
AthenaOutputStream::initialize
virtual StatusCode initialize() override
Definition: AthenaOutputStream.cxx:73
AthenaOutputStream::m_forceRead
BooleanProperty m_forceRead
set to true to force read of data objects in item list
Definition: AthenaOutputStream.h:149
AthenaOutputStream::Items
std::vector< SG::DataProxy * > Items
typedefs
Definition: AthenaOutputStream.h:59
AthenaOutputStream::selectedObjects
IDataSelector * selectedObjects()
Return the list of selected objects.
Definition: AthenaOutputStream.h:86
AthenaOutputStream::loadDict
void loadDict(CLID clid)
Helper function to load dictionaries (both transient and persistent) for a given type.
Definition: AthenaOutputStream.cxx:946
AthenaOutputStream::m_transientItems
StringArrayProperty m_transientItems
List of items that are known to be present in the transient store (and hence we can make input depend...
Definition: AthenaOutputStream.h:140
AthenaOutputStream::m_objects
IDataSelector m_objects
Collection of objects being selected.
Definition: AthenaOutputStream.h:174
AthenaOutputStream::m_metadataItemList
StringArrayProperty m_metadataItemList
Vector of item names.
Definition: AthenaOutputStream.h:120
AthenaOutputStream::m_currentStore
ServiceHandle< StoreGateSvc > * m_currentStore
Definition: AthenaOutputStream.h:102
AthenaOutputStream::collectAllObjects
StatusCode collectAllObjects()
Collect data objects for output streamer list.
Definition: AthenaOutputStream.cxx:530
IAthenaOutputStreamTool.h
Interface to an output stream tool.
AthenaOutputStream::m_compressionListHigh
StringArrayProperty m_compressionListHigh
Vector of item names.
Definition: AthenaOutputStream.h:127
AthenaOutputStream::m_streamer
ToolHandle< IAthenaOutputStreamTool > m_streamer
pointer to AthenaOutputStreamTool
Definition: AthenaOutputStream.h:185
AthenaOutputStream::m_outSeqSvc
ServiceHandle< OutputStreamSequencerSvc > m_outSeqSvc
Definition: AthenaOutputStream.h:111
SG::WriteHandleKey< SG::SelectionVetoes >
IItemListSvc.h
AthenaOutputStream::AthenaOutputStream
AthenaOutputStream(const std::string &name, ISvcLocator *pSvcLocator)
Standard algorithm Constructor.
Definition: AthenaOutputStream.cxx:43
AthenaOutputStream::m_compressionBitsHigh
UnsignedIntegerProperty m_compressionBitsHigh
Number of mantissa bits in the float compression.
Definition: AthenaOutputStream.h:133
AthenaOutputStream::m_itemSvc
ServiceHandle< IItemListSvc > m_itemSvc
Handles to all the necessary services.
Definition: AthenaOutputStream.h:105
AthenaOutputStream::m_slotRangeMap
std::map< unsigned, std::string > m_slotRangeMap
map of filenames assigned to active slots
Definition: AthenaOutputStream.h:198
EL::StatusCode
::StatusCode StatusCode
StatusCode definition for legacy code.
Definition: PhysicsAnalysis/D3PDTools/EventLoop/EventLoop/StatusCode.h:22
AthenaOutputStream::m_outputName
StringProperty m_outputName
Name of the output file.
Definition: AthenaOutputStream.h:143
AthenaOutputStream::compressionListHandlerLow
void compressionListHandlerLow(Gaudi::Details::PropertyBase &)
Handler for ItemNames Property.
Definition: AthenaOutputStream.cxx:900
DataProxy
DataProxy provides the registry services for StoreGate.
Definition: DataProxy.h:32
AthenaOutputStream::m_helperTools
ToolHandleArray< IAthenaOutputTool > m_helperTools
vector of AlgTools that that are executed by this stream
Definition: AthenaOutputStream.h:188
AthenaOutputStream::m_ownedObjects
std::vector< std::unique_ptr< DataObject > > m_ownedObjects
Collection of DataObject instances owned by this service.
Definition: AthenaOutputStream.h:182
AthenaOutputStream::excludeListHandler
void excludeListHandler(Gaudi::Details::PropertyBase &)
Handler for ItemNames Property.
AthenaOutputStream::compressionListHandlerHigh
void compressionListHandlerHigh(Gaudi::Details::PropertyBase &)
Handler for ItemNames Property.
Definition: AthenaOutputStream.cxx:891
AthenaOutputStream::handleVariableSelection
void handleVariableSelection(const SG::IConstAuxStore &auxstore, SG::DataProxy &itemProxy, const std::string &tns, const std::string &aux_attr, SG::SelectionVetoes &vetoes) const
Here we build the vetoed AuxIDs.
Definition: AthenaOutputStream.cxx:829
CLID
uint32_t CLID
The Class ID type.
Definition: Event/xAOD/xAODCore/xAODCore/ClassID_traits.h:47
AthenaOutputStream::io_finalize
virtual StatusCode io_finalize() override
Definition: AthenaOutputStream.cxx:925
AthenaOutputStream::m_persName
StringProperty m_persName
Name of the persistency service capable to write data from the store.
Definition: AthenaOutputStream.h:146
AthenaOutputStream::m_writeMetadataAndDisconnect
bool m_writeMetadataAndDisconnect
Definition: AthenaOutputStream.h:191
AthenaOutputStream::m_extendProvenanceRecord
BooleanProperty m_extendProvenanceRecord
Set to false to omit adding the current DataHeader into the DataHeader history This will cause the in...
Definition: AthenaOutputStream.h:153
AthenaOutputStream::m_itemListFromTool
BooleanProperty m_itemListFromTool
Set to write out everything in input DataHeader.
Definition: AthenaOutputStream.h:156
AthenaOutputStream::m_transient
ToolHandle< SG::IFolder > m_transient
Decoded list of transient ids.
Definition: AthenaOutputStream.h:168
AthenaOutputStream::finalizeRange
void finalizeRange(const std::string &rangeFN)
Definition: AthenaOutputStream.cxx:299
AthenaOutputStream::m_incidentSvc
ServiceHandle< IIncidentSvc > m_incidentSvc
Definition: AthenaOutputStream.h:109
SG::CompressionInfo
std::unordered_map< std::string, SG::ThinningInfo::compression_map_t > CompressionInfo
Map of compressed variables and their compression levels.
Definition: CompressionInfo.h:37
AthenaOutputStream::itemListHandler
void itemListHandler(Gaudi::Details::PropertyBase &)
Handler for ItemNames Property.
Definition: AthenaOutputStream.cxx:881
AthenaOutputStream::TypeKeyPairs
std::vector< std::pair< std::string, std::string > > TypeKeyPairs
Definition: AthenaOutputStream.h:60
WriteHandleKey.h
Property holding a SG store/key/clid from which a WriteHandle is made.
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
AthenaOutputStream::m_metadataStore
ServiceHandle< StoreGateSvc > m_metadataStore
Definition: AthenaOutputStream.h:101
AthenaOutputStream::io_reinit
virtual StatusCode io_reinit() override
Callback method to reinitialize the internal state of the component for I/O purposes (e....
Definition: AthenaOutputStream.cxx:911
AthenaOutputStream::m_tpCnvSvc
ServiceHandle< ITPCnvSvc > m_tpCnvSvc
Definition: AthenaOutputStream.h:108
AthenaOutputStream::m_streamName
StringProperty m_streamName
Stream name (defaults to algorithm name)
Definition: AthenaOutputStream.h:114
AthenaOutputStream::m_compressionBitsLow
UnsignedIntegerProperty m_compressionBitsLow
Number of mantissa bits in the float compression.
Definition: AthenaOutputStream.h:136
AthenaOutputStream::m_altObjects
IDataSelector m_altObjects
Objects overridden by ‘exact’ handling.
Definition: AthenaOutputStream.h:177
AthenaOutputStream::m_dictLoader
ServiceHandle< IDictLoaderSvc > m_dictLoader
Definition: AthenaOutputStream.h:107
AthenaOutputStream::m_metaDataSvc
ServiceHandle< MetaDataSvc > m_metaDataSvc
Definition: AthenaOutputStream.h:106
MetaDataSvc.h
This file contains the class definition for the MetaDataSvc class.
IDictLoaderSvc.h
AthenaOutputStream::m_events
std::atomic< int > m_events
Number of events written to this output stream.
Definition: AthenaOutputStream.h:194
AthenaOutputStream::simpleMatch
bool simpleMatch(const std::string &pattern, const std::string &text)
Glob-style matcher, where the only meta-character is '*'.
Definition: AthenaOutputStream.cxx:958
AthenaOutputStream::m_compressionDecoderHigh
ToolHandle< SG::IFolder > m_compressionDecoderHigh
The top-level folder with items to be compressed high.
Definition: AthenaOutputStream.h:162
AthenaOutputStream::m_mutex
mutex_t m_mutex
mutex for this Stream write() and handle() methods
Definition: AthenaOutputStream.h:207
AthenaOutputStream
algorithm that marks for write data objects in SG
Definition: AthenaOutputStream.h:55
AthenaOutputStream::execute
virtual StatusCode execute() override
Definition: AthenaOutputStream.cxx:407
AthenaOutputStream::m_streamerMap
std::map< std::string, std::unique_ptr< IAthenaOutputStreamTool > > m_streamerMap
map of streamerTools handling event ranges in MT
Definition: AthenaOutputStream.h:204
AthenaOutputStream::m_keepProvenances
StringProperty m_keepProvenances
Provenance record selection.
Definition: AthenaOutputStream.h:123
makeTransCanvas.text
text
Definition: makeTransCanvas.py:11
SelectionVetoes.h
CompressionInfo.h
AthenaOutputStream::write
virtual StatusCode write()
Stream the data.
Definition: AthenaOutputStream.cxx:443
AthenaOutputStream::m_pCLIDSvc
ServiceHandle< IClassIDSvc > m_pCLIDSvc
Definition: AthenaOutputStream.h:110
AthenaOutputStream::m_selVetoesKey
SG::WriteHandleKey< SG::SelectionVetoes > m_selVetoesKey
Key used for recording selected dynamic variable information to the event store.
Definition: AthenaOutputStream.h:225
SG::IConstAuxStore
Interface for const operations on an auxiliary store.
Definition: IConstAuxStore.h:64
GetLBsToIgnore.outputFN
outputFN
Definition: GetLBsToIgnore.py:224
SG::DataProxy
Definition: DataProxy.h:45
SG::FolderItem
a Folder item (data object) is identified by the clid/key pair
Definition: SGFolderItem.h:24
AthenaOutputStream::handle
virtual void handle(const Incident &incident) override
Incident service handle listening for MetaDataStop.
Definition: AthenaOutputStream.cxx:224
AthenaOutputStream::m_p2BWritten
ToolHandle< SG::IFolder > m_p2BWritten
The top-level folder with items to be written.
Definition: AthenaOutputStream.h:159
AthenaOutputStream::clearSelection
void clearSelection()
Clear list of selected objects.
Definition: AthenaOutputStream.cxx:523
AthenaOutputStream::m_compressionDecoderLow
ToolHandle< SG::IFolder > m_compressionDecoderLow
The top-level folder with items to be compressed low.
Definition: AthenaOutputStream.h:165
ServiceHandle< StoreGateSvc >
AthenaOutputStream::~AthenaOutputStream
virtual ~AthenaOutputStream()
Standard Destructor.
Definition: AthenaOutputStream.cxx:67
AthenaOutputStream::m_compInfoKey
SG::WriteHandleKey< SG::CompressionInfo > m_compInfoKey
Key used for recording lossy float compressed variable information to the event store.
Definition: AthenaOutputStream.h:230