ATLAS Offline Software
Loading...
Searching...
No Matches
python.PoolWriteConfig Namespace Reference

Functions

 _overrideTreeAutoFlush (logger, flags, stream, value)
 _getStreamsFromFlags (flags)
 PoolWriteCfg (flags)

Detailed Description

Configuration for POOL file writing

Copyright (C) 2002-2026 CERN for the benefit of the ATLAS collaboration

Function Documentation

◆ _getStreamsFromFlags()

_getStreamsFromFlags ( flags)
protected
Helper to get all the streams from configuration flags
For each stream that's configured to be written out
we have two flags w/ the following convention:
    + Output.{STREAM}FileName
    + Output.doWrite{STREAM}

Definition at line 25 of file PoolWriteConfig.py.

25def _getStreamsFromFlags(flags):
26 """
27 Helper to get all the streams from configuration flags
28 For each stream that's configured to be written out
29 we have two flags w/ the following convention:
30 + Output.{STREAM}FileName
31 + Output.doWrite{STREAM}
32 """
33 result = []
34 for key, value in flags._flagdict.items():
35 if key.startswith("Output.") and key.endswith("FileName") and value.get():
36 stream = key.removeprefix("Output.").removesuffix("FileName")
37 if stream not in ["HIST"]: # AthenaPool is not responsible for HIST storage settings
38 result.append(stream)
39 return result
40
41
42@AccumulatorCache

◆ _overrideTreeAutoFlush()

_overrideTreeAutoFlush ( logger,
flags,
stream,
value )
protected
Helper function to override TreeAutoFlush from flags.

Definition at line 10 of file PoolWriteConfig.py.

10def _overrideTreeAutoFlush(logger, flags, stream, value):
11 """Helper function to override TreeAutoFlush from flags."""
12 if not flags.Output.TreeAutoFlush or not isinstance(flags.Output.TreeAutoFlush, dict):
13 return value
14
15 if stream not in flags.Output.TreeAutoFlush:
16 return value
17
18 override = flags.Output.TreeAutoFlush[stream]
19 if override is not None:
20 logger.info('Overriding TreeAutoFlush value for stream "%s" from %d to %d', stream, value, override)
21 return override
22
23 return value
24

◆ PoolWriteCfg()

PoolWriteCfg ( flags)
Return ComponentAccumulator configured to Write POOL files

Definition at line 43 of file PoolWriteConfig.py.

43def PoolWriteCfg(flags):
44 """Return ComponentAccumulator configured to Write POOL files"""
45 # based on WriteAthenaPool._configureWriteAthenaPool
46
47 from AthenaCommon.Logging import logging
48 logger = logging.getLogger( 'PoolWriteCfg' )
49
50 PoolAttributes = []
51 # Switch off splitting by setting default SplitLevel to 0
52 PoolAttributes += ["DEFAULT_SPLITLEVEL ='0'"]
53
54 # Set as default the member-wise streaming, ROOT default
55 PoolAttributes += ["STREAM_MEMBER_WISE = '1'"]
56
57 # Set the default container type
58 PoolAttributes += [f"DEFAULT_CONTAINER_TYPE = '{ROOT.DbType.getType(flags.Output.DefaultContainerType).type()}'"]
59
60 # Increase default BasketSize to 32K, ROOT default (but overwritten by POOL)
61 PoolAttributes += ["DEFAULT_BUFFERSIZE = '32000'"]
62
63 # Set POOLContainerForm(DataHeaderForm) split level to 0
64 PoolAttributes += ["ContainerName = 'TTree=POOLContainerForm(DataHeaderForm)'; CONTAINER_SPLITLEVEL = '0'"]
65 PoolAttributes += ["TREE_BRANCH_OFFSETTAB_LEN ='100'"]
66
67 oneDHForm = flags.Output.OneDataHeaderForm
68
69 # Kept in sync with RecoUtils.py
70 from AthenaPoolCnvSvc import PoolAttributeHelper as pah
71
72 # Defaults for common formats
73 # Stream : [compression algorithm, compression level, auto flush, split level, dyn split level]
74 defaults = {
75 "EVNT" : [2, 1, 500, 0, 0],
76 "EVNT_TR" : [2, 1, 1, 0, 0],
77 "HITS" : [2, 1, 10, 0, 0],
78 "RDO" : [2, 1, 10, 0, 0],
79 "ESD" : [2, 1, 10, 0, 0],
80 "AOD" : [2, 1, 100, 0, 0],
81 "DAOD_PHYSVAL" : [5, 5, 100, 0, 1],
82 "DAOD_PHYS" : [5, 5, 500, 0, 1],
83 "DAOD_PHYSLITE" : [5, 5, 500, 1, 1],
84 "DAOD_TRUTH3" : [5, 5, 500, 1, 1],
85 "D2AOD_PHYSLITE" : [5, 5, 500, 1, 1],
86 }
87
88 # RNTuple output: different compression for some formats, TTree keeps the defaults above
89 # Stream : [compression algorithm, compression level]
90 rntupleCompression = {
91 "EVNT" : [5, 9],
92 "HITS" : [5, 9],
93 "RDO" : [5, 9],
94 "AOD" : [5, 9],
95 "DAOD_PHYS" : [5, 9],
96 "DAOD_PHYSLITE" : [5, 9],
97 }
98
99 # Metadata containers needed for augmentations
100 OutputMetadataContainers = []
101
102 # Loop over all streams and set the appropriate attributes
103 fileFlushSetting = {}
104 maxAutoFlush = -1
105 for stream in _getStreamsFromFlags(flags):
106
107 # Get the file name - Guaranteed to exist at this point
108 fileName = getattr(flags.Output, f"{stream}FileName")
109
110 # Get the ROOT settings to be applied
111 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = 2, 1, 10, 0, 0 # Defaults: LZMA, Level 1, AutoFlush 10, No Splitting
112 if stream in defaults:
113 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = defaults[stream]
114 elif "DAOD" in stream:
115 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = 5, 5, 100, 0, 1 # Change the defaults for DAODs
116 elif "D2AOD" in stream:
117 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = 5, 5, 500, 1, 1 # Change the defaults for D2AODs
118 if "RNTUPLE" in flags.Output.DefaultContainerType and stream in rntupleCompression:
119 compAlg, compLvl = rntupleCompression[stream]
120
121 # For temporary streams/files we use either ZLIB or ZSTD for the compression algorithm to save CPU cycles
122 # Temporary in this context might mean one of three things:
123 # a) Outputs of intermediate steps of chained workflows (file name begins with tmp.),
124 # b) Outputs of workers in AthenaMP jobs that are to be merged (file name ends with _000), and
125 # c) Any output stream that is marked by the user as being temporary (via the CA flag Output.TemporaryStreams)
126 # The ultimate goal is to reconcile all three cases and propagate the information between the job transform
127 # and the job configuration (CA) so that we don't need to rely on the file names here...
128 isTemporaryStream = fileName.endswith('_000') or fileName.startswith('tmp.') or stream in flags.Output.TemporaryStreams
129 tempFileCompressionSetting = (5,1) # ZSTD at level 1
130 if isTemporaryStream:
131 # Outputs created in certain workflows are read with older ROOT versions.
132 # E.g., temporary RDO files that are used in Run-2 simulation.
133 # For those, we have to use ZLIB
134 from AthenaConfiguration.Enums import LHCPeriod
135 if "RDO" in stream and flags.hasCategory("GeoModel") and flags.GeoModel.Run < LHCPeriod.Run3:
136 tempFileCompressionSetting = (1,1) # ZLIB at level 1
137 logger.info(f"Stream {stream} is marked as temporary, overwriting the compression settings to {tempFileCompressionSetting}")
138 compAlg, compLvl = tempFileCompressionSetting if isTemporaryStream else (compAlg, compLvl)
139
140 # See if the user asked for the AutoFlush to be overwritten
141 autoFlush = _overrideTreeAutoFlush(logger, flags, stream, autoFlush)
142
143 # Print some debugging information
144 logger.debug(f"{fileName=} {stream=} {compAlg=} {compLvl=} {autoFlush=} {splitLvl=} {dynSplitLvl=}")
145
146 # Set the Collection/Container prefixes (make configurable?)
147 outputCollection = "POOLContainer"
148 poolContainerPrefix = "CollectionTree"
149
150 # Check to see if this stream is an augmentation
151 # Only set file-level attributes for the owning stream
152 isAugmentation = flags.hasFlag(f"Output.{stream}ParentStream")
153 if not isAugmentation:
154 # Set the Compression attributes
155 PoolAttributes += [ pah.setFileCompAlg( fileName, compAlg ) ]
156 PoolAttributes += [ pah.setFileCompLvl( fileName, compLvl ) ]
157
158 # By default use a maximum basket buffer size of 128k and minimum buffer entries of 10 for (D)AODs
159 if "AOD" in stream:
160 PoolAttributes += [ pah.setMaxBufferSize( fileName, "131072" ) ]
161 PoolAttributes += [ pah.setMinBufferEntries( fileName, "10" ) ]
162 else:
163 # Changes in this else block need to be coordinated w/ OutputStreamConfig!
164 # Set the master index
165 PoolAttributes += [ f"DatabaseName = '{fileName}'; INDEX_MASTER = 'POOLContainer(DataHeader)'" ]
166
167 # Set the Collection/Container prefixes
168 outputCollection += f"_{stream}"
169 poolContainerPrefix += f"_{stream}"
170 OutputMetadataContainers += [f"MetaData_{stream}"]
171
172 # Set the AutoFlush & Maximum Size attributes
173 PoolAttributes += [ pah.setTreeAutoFlush( fileName, poolContainerPrefix, autoFlush ) ]
174 PoolAttributes += [ pah.setTreeAutoFlush( fileName, outputCollection, autoFlush ) ]
175 PoolAttributes += [ pah.setTreeAutoFlush( fileName, "POOLContainerForm", autoFlush ) ]
176 PoolAttributes += [ pah.setTreeMaxSize( fileName, "*", "1099511627776L" ) ] # 1 TB
177 if flags.MP.UseSharedWriter and flags.MP.UseParallelCompression:
178 fileFlushSetting[fileName] = ( flags.MP.SharedWriter.FileFlushSetting.get(fileName, autoFlush) )
179 logger.info(f"Setting auto write for {fileName} to {fileFlushSetting[fileName]} events")
180
181 # Set the Spit Level attributes
182 PoolAttributes += [ pah.setContainerSplitLevel( fileName, poolContainerPrefix, splitLvl ) ]
183 PoolAttributes += [ pah.setContainerSplitLevel( fileName, "Aux.", splitLvl ) ]
184 PoolAttributes += [ pah.setContainerSplitLevel( fileName, "Dyn.", dynSplitLvl ) ]
185
186 # ROOT inadvertently broke forward compatibility in v6.30+ (see root/issues/15964)
187 # This workaround is needed so that older releases can read files created by the new ones
188 # For more information see ATEAM-1001
189 if "EVNT" in stream or "RDO" in stream:
190 PoolAttributes += [ f"DatabaseName = '{fileName}'; FILEFORWARD_COMPATIBILITY = '1'" ]
191 # also for compatibility with rel21 disable single DataHeaderForm
192 oneDHForm = False
193
194 # Find the maximum AutoFlush across all formats
195 maxAutoFlush = max(maxAutoFlush, autoFlush)
196
197 # If we don't have "enough" events, disable parallelCompression if we're using SharedWriter
198 # In this context, "enough" means each worker has a chance to make at least one flush to the disk
199 useParallelCompression = flags.MP.UseSharedWriter and flags.MP.UseParallelCompression
200 if useParallelCompression:
201 # Now compute the total number of events this job will process
202 requestedEvents = flags.Exec.MaxEvents
203 availableEvents = flags.Input.FileNentries - flags.Exec.SkipEvents
204 totalEntries = availableEvents if requestedEvents == -1 else min( availableEvents, requestedEvents )
205 if ( totalEntries > 0 ) and ( maxAutoFlush > 0 ) and ( maxAutoFlush * flags.Concurrency.NumProcs > totalEntries ):
206 logger.info( "Not enough events to process, disabling parallel compression for SharedWriter!" )
207 logger.info( f"Processing {totalEntries} events in {flags.Concurrency.NumProcs} workers "
208 f"and a maximum (across all outputs) AutoFlush of {maxAutoFlush}")
209 useParallelCompression = False
210
211 if flags.MP.UseSharedReader or flags.MP.UseSharedWriter:
212 from AthenaPoolCnvSvc.PoolCommonConfig import AthenaPoolSharedIOCnvSvcCfg
213 return AthenaPoolSharedIOCnvSvcCfg(flags,
214 PoolAttributes=PoolAttributes,
215 ParallelCompression=useParallelCompression,
216 OutputMetadataContainers=OutputMetadataContainers,
217 OneDataHeaderForm=oneDHForm,
218 FileFlushSetting=fileFlushSetting,
219 PoolContainerNamingScheme=("Canonical" if "RNTUPLE" in flags.Output.DefaultContainerType else "Historical"))
220 else:
221 from AthenaPoolCnvSvc.PoolCommonConfig import AthenaPoolCnvSvcCfg
222 return AthenaPoolCnvSvcCfg(flags,
223 PoolAttributes=PoolAttributes,
224 OneDataHeaderForm=oneDHForm,
225 PoolContainerNamingScheme=("Canonical" if "RNTUPLE" in flags.Output.DefaultContainerType else "Historical"))
#define min(a, b)
Definition cfImp.cxx:40
#define max(a, b)
Definition cfImp.cxx:41