Perform an event count check.
1102 def decide(self):
1103
1104
1105 for inData, neventsInData in self._inEventDict.items():
1106 if not isinstance(neventsInData, int):
1107 msg.warning('File size metadata for {inData} was not countable, found {neventsInData}. No event checks possible for this input data.'.format(inData=inData, neventsInData=neventsInData))
1108 continue
1109 if inData in self._eventCountConf:
1110 inDataKey = inData
1111 else:
1112
1113 matchedInData = False
1114 for inDataKey in self._eventCountConf:
1115 if fnmatch.fnmatch(inData, inDataKey):
1116 msg.info("Matched input data type {inData} to {inDataKey} by globbing".format(inData=inData, inDataKey=inDataKey))
1117 matchedInData = True
1118 break
1119 if not matchedInData:
1120 msg.warning('No defined event count match for {inData} -> {outData}, so no check(s) possible in this case.'.format(inData=inData, outData=list(self._outEventDict)))
1121 continue
1122
1123
1124 expectedEvents = neventsInData
1125 if self._skipEvents is not None and self._skipEvents > 0:
1126 expectedEvents -= self._skipEvents
1127 if expectedEvents < 0:
1128 msg.warning('skipEvents was set higher than the input events in {inData}: {skipEvents} > {neventsInData}. This is not an error, but it is not a normal configuration. Expected events is now 0.'.format(inData=inData, skipEvents=self._skipEvents, neventsInData=neventsInData))
1129 expectedEvents = 0
1130 if self._maxEvents is not None:
1131 if expectedEvents < self._maxEvents:
1132 if self._skipEvents is not None:
1133 msg.warning('maxEvents was set higher than inputEvents-skipEvents for {inData}: {maxEvents} > {neventsInData}-{skipEvents}. This is not an error, but it is not a normal configuration. Expected events remains {expectedEvents}.'.format(inData=inData, maxEvents=self._maxEvents, neventsInData=neventsInData, skipEvents=self._skipEvents, expectedEvents=expectedEvents))
1134 else:
1135 msg.warning('maxEvents was set higher than inputEvents for {inData}: {maxEvents} > {neventsInData}. This is not an error, but it is not a normal configuration. Expected events remains {expectedEvents}.'.format(inData=inData, maxEvents=self._maxEvents, neventsInData=neventsInData, expectedEvents=expectedEvents))
1136 else:
1137 expectedEvents = self._maxEvents
1138 msg.debug('Expected number of processed events for {0} is {1}'.format(inData, expectedEvents))
1139
1140
1141 for outData, neventsOutData in self._outEventDict.items():
1142 if not isinstance(neventsOutData, int):
1143 msg.warning('File size metadata for {outData} was not countable, found "{neventsOutData}". No event checks possible for this output data.'.format(outData=outData, neventsOutData=neventsOutData))
1144 continue
1145 if outData in self._eventCountConf[inDataKey]:
1146 checkConf = self._eventCountConf[inDataKey][outData]
1147 outDataKey = outData
1148 else:
1149
1150 checkConf = None
1151 for outDataKey, outDataConf in self._eventCountConf[inDataKey].items():
1152 if fnmatch.fnmatch(outData, outDataKey):
1153 msg.info('Matched output data type {outData} to {outDatakey} by globbing'.format(outData=outData, outDatakey=outDataKey))
1154 outDataKey = outData
1155 checkConf = outDataConf
1156 break
1157 if not checkConf:
1158 msg.warning('No defined event count match for {inData} -> {outData}, so no check possible in this case.'.format(inData=inData, outData=outData))
1159 continue
1160 msg.debug('Event count check for {inData} to {outData} is {checkConf}'.format(inData=inData, outData=outData, checkConf=checkConf))
1161
1162
1163 if checkConf == 'match':
1164
1165 if neventsOutData == expectedEvents:
1166 msg.info("Event count check for {inData} to {outData} passed: all processed events found ({neventsOutData} output events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData))
1167 else:
1168 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1169 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1170 elif checkConf == 'filter':
1171 if neventsOutData <= expectedEvents and neventsOutData >= 0:
1172 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1173 else:
1174 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1175 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from 0 to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1176 elif checkConf == 'minEff':
1177 if neventsOutData >= int(expectedEvents * self._evAccEff) and neventsOutData <= expectedEvents:
1178 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1179 else:
1180 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1181 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from {minEvents} to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData,
1182 minEvents=int(expectedEvents * self._evAccEff), expectedEvents=expectedEvents))
1183 elif isinstance(checkConf, (float, int)):
1184 checkConf = float(checkConf)
1185 if checkConf < 0.0 or checkConf > 1.0:
1186 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1187 'Event count check for {inData} to {outData} is misconfigured: the efficiency factor of {eff} is not between 0 and 1.'.format(inData=inData, outData=outData, eff=checkConf))
1188 if neventsOutData >= int(expectedEvents * checkConf) and neventsOutData <= expectedEvents:
1189 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1190 else:
1191 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1192 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from {minEvents} to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData,
1193 minEvents=int(expectedEvents * checkConf), expectedEvents=expectedEvents))
1194 else:
1195 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1196 'Unrecognised event count configuration for {inData} to {outData}: "{conf}" is not known'.format(inData=inData, outData=outData, conf=checkConf))
1197 self._eventCount = expectedEvents
1198 return True
1.16.1