43def PoolWriteCfg(flags):
44 """Return ComponentAccumulator configured to Write POOL files"""
45
46
47 from AthenaCommon.Logging import logging
48 logger = logging.getLogger( 'PoolWriteCfg' )
49
50 PoolAttributes = []
51
52 PoolAttributes += ["DEFAULT_SPLITLEVEL ='0'"]
53
54
55 PoolAttributes += ["STREAM_MEMBER_WISE = '1'"]
56
57
58 PoolAttributes += [f"DEFAULT_CONTAINER_TYPE = '{ROOT.DbType.getType(flags.Output.DefaultContainerType).type()}'"]
59
60
61 PoolAttributes += ["DEFAULT_BUFFERSIZE = '32000'"]
62
63
64 PoolAttributes += ["ContainerName = 'TTree=POOLContainerForm(DataHeaderForm)'; CONTAINER_SPLITLEVEL = '0'"]
65 PoolAttributes += ["TREE_BRANCH_OFFSETTAB_LEN ='100'"]
66
67 oneDHForm = flags.Output.OneDataHeaderForm
68
69
70 from AthenaPoolCnvSvc import PoolAttributeHelper as pah
71
72
73
74 defaults = {
75 "EVNT" : [2, 1, 500, 0, 0],
76 "EVNT_TR" : [2, 1, 1, 0, 0],
77 "HITS" : [2, 1, 10, 0, 0],
78 "RDO" : [2, 1, 10, 0, 0],
79 "ESD" : [2, 1, 10, 0, 0],
80 "AOD" : [2, 1, 100, 0, 0],
81 "DAOD_PHYSVAL" : [5, 5, 100, 0, 1],
82 "DAOD_PHYS" : [5, 5, 500, 0, 1],
83 "DAOD_PHYSLITE" : [5, 5, 500, 1, 1],
84 "DAOD_TRUTH3" : [5, 5, 500, 1, 1],
85 "D2AOD_PHYSLITE" : [5, 5, 500, 1, 1],
86 }
87
88
89
90 rntupleCompression = {
91 "EVNT" : [5, 9],
92 "HITS" : [5, 9],
93 "RDO" : [5, 9],
94 "AOD" : [5, 9],
95 "DAOD_PHYS" : [5, 9],
96 "DAOD_PHYSLITE" : [5, 9],
97 }
98
99
100 OutputMetadataContainers = []
101
102
103 fileFlushSetting = {}
104 maxAutoFlush = -1
105 for stream in _getStreamsFromFlags(flags):
106
107
108 fileName = getattr(flags.Output, f"{stream}FileName")
109
110
111 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = 2, 1, 10, 0, 0
112 if stream in defaults:
113 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = defaults[stream]
114 elif "DAOD" in stream:
115 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = 5, 5, 100, 0, 1
116 elif "D2AOD" in stream:
117 compAlg, compLvl, autoFlush, splitLvl, dynSplitLvl = 5, 5, 500, 1, 1
118 if "RNTUPLE" in flags.Output.DefaultContainerType and stream in rntupleCompression:
119 compAlg, compLvl = rntupleCompression[stream]
120
121
122
123
124
125
126
127
128 isTemporaryStream = fileName.endswith('_000') or fileName.startswith('tmp.') or stream in flags.Output.TemporaryStreams
129 tempFileCompressionSetting = (5,1)
130 if isTemporaryStream:
131
132
133
134 from AthenaConfiguration.Enums import LHCPeriod
135 if "RDO" in stream and flags.hasCategory("GeoModel") and flags.GeoModel.Run < LHCPeriod.Run3:
136 tempFileCompressionSetting = (1,1)
137 logger.info(f"Stream {stream} is marked as temporary, overwriting the compression settings to {tempFileCompressionSetting}")
138 compAlg, compLvl = tempFileCompressionSetting if isTemporaryStream else (compAlg, compLvl)
139
140
141 autoFlush = _overrideTreeAutoFlush(logger, flags, stream, autoFlush)
142
143
144 logger.debug(f"{fileName=} {stream=} {compAlg=} {compLvl=} {autoFlush=} {splitLvl=} {dynSplitLvl=}")
145
146
147 outputCollection = "POOLContainer"
148 poolContainerPrefix = "CollectionTree"
149
150
151
152 isAugmentation = flags.hasFlag(f"Output.{stream}ParentStream")
153 if not isAugmentation:
154
155 PoolAttributes += [ pah.setFileCompAlg( fileName, compAlg ) ]
156 PoolAttributes += [ pah.setFileCompLvl( fileName, compLvl ) ]
157
158
159 if "AOD" in stream:
160 PoolAttributes += [ pah.setMaxBufferSize( fileName, "131072" ) ]
161 PoolAttributes += [ pah.setMinBufferEntries( fileName, "10" ) ]
162 else:
163
164
165 PoolAttributes += [ f"DatabaseName = '{fileName}'; INDEX_MASTER = 'POOLContainer(DataHeader)'" ]
166
167
168 outputCollection += f"_{stream}"
169 poolContainerPrefix += f"_{stream}"
170 OutputMetadataContainers += [f"MetaData_{stream}"]
171
172
173 PoolAttributes += [ pah.setTreeAutoFlush( fileName, poolContainerPrefix, autoFlush ) ]
174 PoolAttributes += [ pah.setTreeAutoFlush( fileName, outputCollection, autoFlush ) ]
175 PoolAttributes += [ pah.setTreeAutoFlush( fileName, "POOLContainerForm", autoFlush ) ]
176 PoolAttributes += [ pah.setTreeMaxSize( fileName, "*", "1099511627776L" ) ]
177 if flags.MP.UseSharedWriter and flags.MP.UseParallelCompression:
178 fileFlushSetting[fileName] = ( flags.MP.SharedWriter.FileFlushSetting.get(fileName, autoFlush) )
179 logger.info(f"Setting auto write for {fileName} to {fileFlushSetting[fileName]} events")
180
181
182 PoolAttributes += [ pah.setContainerSplitLevel( fileName, poolContainerPrefix, splitLvl ) ]
183 PoolAttributes += [ pah.setContainerSplitLevel( fileName, "Aux.", splitLvl ) ]
184 PoolAttributes += [ pah.setContainerSplitLevel( fileName, "Dyn.", dynSplitLvl ) ]
185
186
187
188
189 if "EVNT" in stream or "RDO" in stream:
190 PoolAttributes += [ f"DatabaseName = '{fileName}'; FILEFORWARD_COMPATIBILITY = '1'" ]
191
192 oneDHForm = False
193
194
195 maxAutoFlush =
max(maxAutoFlush, autoFlush)
196
197
198
199 useParallelCompression = flags.MP.UseSharedWriter and flags.MP.UseParallelCompression
200 if useParallelCompression:
201
202 requestedEvents = flags.Exec.MaxEvents
203 availableEvents = flags.Input.FileNentries - flags.Exec.SkipEvents
204 totalEntries = availableEvents
if requestedEvents == -1
else min( availableEvents, requestedEvents )
205 if ( totalEntries > 0 ) and ( maxAutoFlush > 0 ) and ( maxAutoFlush * flags.Concurrency.NumProcs > totalEntries ):
206 logger.info( "Not enough events to process, disabling parallel compression for SharedWriter!" )
207 logger.info( f"Processing {totalEntries} events in {flags.Concurrency.NumProcs} workers "
208 f"and a maximum (across all outputs) AutoFlush of {maxAutoFlush}")
209 useParallelCompression = False
210
211 if flags.MP.UseSharedReader or flags.MP.UseSharedWriter:
212 from AthenaPoolCnvSvc.PoolCommonConfig import AthenaPoolSharedIOCnvSvcCfg
213 return AthenaPoolSharedIOCnvSvcCfg(flags,
214 PoolAttributes=PoolAttributes,
215 ParallelCompression=useParallelCompression,
216 OutputMetadataContainers=OutputMetadataContainers,
217 OneDataHeaderForm=oneDHForm,
218 FileFlushSetting=fileFlushSetting,
219 PoolContainerNamingScheme=("Canonical" if "RNTUPLE" in flags.Output.DefaultContainerType else "Historical"))
220 else:
221 from AthenaPoolCnvSvc.PoolCommonConfig import AthenaPoolCnvSvcCfg
222 return AthenaPoolCnvSvcCfg(flags,
223 PoolAttributes=PoolAttributes,
224 OneDataHeaderForm=oneDHForm,
225 PoolContainerNamingScheme=("Canonical" if "RNTUPLE" in flags.Output.DefaultContainerType else "Historical"))