44 """Return ComponentAccumulator configured to Write POOL files"""
47 from AthenaCommon.Logging
import logging
48 logger = logging.getLogger(
'PoolWriteCfg' )
52 PoolAttributes += [
"DEFAULT_SPLITLEVEL ='0'"]
55 PoolAttributes += [
"STREAM_MEMBER_WISE = '1'"]
58 PoolAttributes += [f
"DEFAULT_CONTAINER_TYPE = '{ROOT.DbType.getType(flags.Output.DefaultContainerType).type()}'"]
61 PoolAttributes += [
"DEFAULT_BUFFERSIZE = '32000'"]
64 PoolAttributes += [
"ContainerName = 'TTree=POOLContainerForm(DataHeaderForm)'; CONTAINER_SPLITLEVEL = '0'"]
65 PoolAttributes += [
"TREE_BRANCH_OFFSETTAB_LEN ='100'"]
67 oneDHForm = flags.Output.OneDataHeaderForm
70 from AthenaPoolCnvSvc
import PoolAttributeHelper
as pah
75 "EVNT" : [2, 1, 500, 0, 0],
76 "EVNT_TR" : [2, 1, 1, 0, 0],
77 "HITS" : [2, 1, 10, 0, 0],
78 "RDO" : [2, 1, 10, 0, 0],
79 "ESD" : [2, 1, 10, 0, 0],
80 "AOD" : [2, 1, 100, 0, 0],
81 "DAOD_PHYSVAL" : [5, 5, 100, 0, 1],
82 "DAOD_PHYS" : [5, 5, 500, 0, 1],
83 "DAOD_PHYSLITE" : [5, 5, 500, 1, 1],
84 "DAOD_TRUTH3" : [5, 5, 500, 1, 1],
85 "D2AOD_PHYSLITE" : [5, 5, 500, 1, 1],
90 rntupleCompression = {
96 "DAOD_PHYSLITE" : [5, 9],
100 OutputMetadataContainers = []
103 fileFlushSetting = {}
108 fileName = getattr(flags.Output, f
"{stream}FileName")
111 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = 2, 1, 10, 0, 0
112 if stream
in defaults:
113 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = defaults[stream]
114 elif "DAOD" in stream:
115 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = 5, 5, 100, 0, 1
116 elif "D2AOD" in stream:
117 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = 5, 5, 500, 1, 1
118 if "RNTUPLE" in flags.Output.DefaultContainerType
and stream
in rntupleCompression:
119 compAlg, compLvl = rntupleCompression[stream]
128 isTemporaryStream = fileName.endswith(
'_000')
or fileName.startswith(
'tmp.')
or stream
in flags.Output.TemporaryStreams
129 tempFileCompressionSetting = (5,1)
130 if isTemporaryStream:
134 from AthenaConfiguration.Enums
import LHCPeriod
135 if "RDO" in stream
and flags.hasCategory(
"GeoModel")
and flags.GeoModel.Run < LHCPeriod.Run3:
136 tempFileCompressionSetting = (1,1)
137 logger.info(f
"Stream {stream} is marked as temporary, overwriting the compression settings to {tempFileCompressionSetting}")
138 compAlg, compLvl = tempFileCompressionSetting
if isTemporaryStream
else (compAlg, compLvl)
144 logger.debug(f
"{fileName=} {stream=} {compAlg=} {compLvl=} {autoFlush=} {splitLvl=} {dynSplitLvl=}")
147 outputCollection =
"POOLContainer"
148 poolContainerPrefix =
"CollectionTree"
152 isAugmentation = flags.hasFlag(f
"Output.{stream}ParentStream")
153 if not isAugmentation:
155 PoolAttributes += [ pah.setFileCompAlg( fileName, compAlg ) ]
156 PoolAttributes += [ pah.setFileCompLvl( fileName, compLvl ) ]
160 PoolAttributes += [ pah.setMaxBufferSize( fileName,
"131072" ) ]
161 PoolAttributes += [ pah.setMinBufferEntries( fileName,
"10" ) ]
165 PoolAttributes += [ f
"DatabaseName = '{fileName}'; INDEX_MASTER = 'POOLContainer(DataHeader)'" ]
168 outputCollection += f
"_{stream}"
169 poolContainerPrefix += f
"_{stream}"
170 OutputMetadataContainers += [f
"MetaData_{stream}"]
173 PoolAttributes += [ pah.setTreeAutoFlush( fileName, poolContainerPrefix, autoFlush ) ]
174 PoolAttributes += [ pah.setTreeAutoFlush( fileName, outputCollection, autoFlush ) ]
175 PoolAttributes += [ pah.setTreeAutoFlush( fileName,
"POOLContainerForm", autoFlush ) ]
176 PoolAttributes += [ pah.setTreeMaxSize( fileName,
"*",
"1099511627776L" ) ]
177 if flags.MP.UseSharedWriter
and flags.MP.UseParallelCompression:
178 fileFlushSetting[fileName] = ( flags.MP.SharedWriter.FileFlushSetting.get(fileName, autoFlush) )
179 logger.info(f
"Setting auto write for {fileName} to {fileFlushSetting[fileName]} events")
182 PoolAttributes += [ pah.setContainerSplitLevel( fileName, poolContainerPrefix, splitLvl ) ]
183 PoolAttributes += [ pah.setContainerSplitLevel( fileName,
"Aux.", splitLvl ) ]
184 PoolAttributes += [ pah.setContainerSplitLevel( fileName,
"Dyn.", dynSplitLvl ) ]
189 if "EVNT" in stream
or "RDO" in stream:
190 PoolAttributes += [ f
"DatabaseName = '{fileName}'; FILEFORWARD_COMPATIBILITY = '1'" ]
195 maxAutoFlush =
max(maxAutoFlush, autoFlush)
199 useParallelCompression = flags.MP.UseSharedWriter
and flags.MP.UseParallelCompression
200 if useParallelCompression:
202 requestedEvents = flags.Exec.MaxEvents
203 availableEvents = flags.Input.FileNentries - flags.Exec.SkipEvents
204 totalEntries = availableEvents
if requestedEvents == -1
else min( availableEvents, requestedEvents )
205 if ( totalEntries > 0 )
and ( maxAutoFlush > 0 )
and ( maxAutoFlush * flags.Concurrency.NumProcs > totalEntries ):
206 logger.info(
"Not enough events to process, disabling parallel compression for SharedWriter!" )
207 logger.info( f
"Processing {totalEntries} events in {flags.Concurrency.NumProcs} workers "
208 f
"and a maximum (across all outputs) AutoFlush of {maxAutoFlush}")
209 useParallelCompression =
False
211 if flags.MP.UseSharedReader
or flags.MP.UseSharedWriter:
212 from AthenaPoolCnvSvc.PoolCommonConfig
import AthenaPoolSharedIOCnvSvcCfg
213 return AthenaPoolSharedIOCnvSvcCfg(flags,
214 PoolAttributes=PoolAttributes,
215 ParallelCompression=useParallelCompression,
216 OutputMetadataContainers=OutputMetadataContainers,
217 OneDataHeaderForm=oneDHForm,
218 FileFlushSetting=fileFlushSetting,
219 PoolContainerNamingScheme=(
"Canonical" if "RNTUPLE" in flags.Output.DefaultContainerType
else "Historical"))
221 from AthenaPoolCnvSvc.PoolCommonConfig
import AthenaPoolCnvSvcCfg
222 return AthenaPoolCnvSvcCfg(flags,
223 PoolAttributes=PoolAttributes,
224 OneDataHeaderForm=oneDHForm,
225 PoolContainerNamingScheme=(
"Canonical" if "RNTUPLE" in flags.Output.DefaultContainerType
else "Historical"))