Perform an event count check.
1099 def decide(self):
1100
1101
1102 for inData, neventsInData in self._inEventDict.items():
1103 if not isinstance(neventsInData, int):
1104 msg.warning('File size metadata for {inData} was not countable, found {neventsInData}. No event checks possible for this input data.'.format(inData=inData, neventsInData=neventsInData))
1105 continue
1106 if inData in self._eventCountConf:
1107 inDataKey = inData
1108 else:
1109
1110 matchedInData = False
1111 for inDataKey in self._eventCountConf:
1112 if fnmatch.fnmatch(inData, inDataKey):
1113 msg.info("Matched input data type {inData} to {inDataKey} by globbing".format(inData=inData, inDataKey=inDataKey))
1114 matchedInData = True
1115 break
1116 if not matchedInData:
1117 msg.warning('No defined event count match for {inData} -> {outData}, so no check(s) possible in this case.'.format(inData=inData, outData=list(self._outEventDict)))
1118 continue
1119
1120
1121 expectedEvents = neventsInData
1122 if self._skipEvents is not None and self._skipEvents > 0:
1123 expectedEvents -= self._skipEvents
1124 if expectedEvents < 0:
1125 msg.warning('skipEvents was set higher than the input events in {inData}: {skipEvents} > {neventsInData}. This is not an error, but it is not a normal configuration. Expected events is now 0.'.format(inData=inData, skipEvents=self._skipEvents, neventsInData=neventsInData))
1126 expectedEvents = 0
1127 if self._maxEvents is not None:
1128 if expectedEvents < self._maxEvents:
1129 if self._skipEvents is not None:
1130 msg.warning('maxEvents was set higher than inputEvents-skipEvents for {inData}: {maxEvents} > {neventsInData}-{skipEvents}. This is not an error, but it is not a normal configuration. Expected events remains {expectedEvents}.'.format(inData=inData, maxEvents=self._maxEvents, neventsInData=neventsInData, skipEvents=self._skipEvents, expectedEvents=expectedEvents))
1131 else:
1132 msg.warning('maxEvents was set higher than inputEvents for {inData}: {maxEvents} > {neventsInData}. This is not an error, but it is not a normal configuration. Expected events remains {expectedEvents}.'.format(inData=inData, maxEvents=self._maxEvents, neventsInData=neventsInData, expectedEvents=expectedEvents))
1133 else:
1134 expectedEvents = self._maxEvents
1135 msg.debug('Expected number of processed events for {0} is {1}'.format(inData, expectedEvents))
1136
1137
1138 for outData, neventsOutData in self._outEventDict.items():
1139 if not isinstance(neventsOutData, int):
1140 msg.warning('File size metadata for {outData} was not countable, found "{neventsOutData}". No event checks possible for this output data.'.format(outData=outData, neventsOutData=neventsOutData))
1141 continue
1142 if outData in self._eventCountConf[inDataKey]:
1143 checkConf = self._eventCountConf[inDataKey][outData]
1144 outDataKey = outData
1145 else:
1146
1147 checkConf = None
1148 for outDataKey, outDataConf in self._eventCountConf[inDataKey].items():
1149 if fnmatch.fnmatch(outData, outDataKey):
1150 msg.info('Matched output data type {outData} to {outDatakey} by globbing'.format(outData=outData, outDatakey=outDataKey))
1151 outDataKey = outData
1152 checkConf = outDataConf
1153 break
1154 if not checkConf:
1155 msg.warning('No defined event count match for {inData} -> {outData}, so no check possible in this case.'.format(inData=inData, outData=outData))
1156 continue
1157 msg.debug('Event count check for {inData} to {outData} is {checkConf}'.format(inData=inData, outData=outData, checkConf=checkConf))
1158
1159
1160 if checkConf == 'match':
1161
1162 if neventsOutData == expectedEvents:
1163 msg.info("Event count check for {inData} to {outData} passed: all processed events found ({neventsOutData} output events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData))
1164 else:
1165 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1166 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1167 elif checkConf == 'filter':
1168 if neventsOutData <= expectedEvents and neventsOutData >= 0:
1169 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1170 else:
1171 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1172 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from 0 to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1173 elif checkConf == 'minEff':
1174 if neventsOutData >= int(expectedEvents * self._evAccEff) and neventsOutData <= expectedEvents:
1175 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1176 else:
1177 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1178 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from {minEvents} to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData,
1179 minEvents=int(expectedEvents * self._evAccEff), expectedEvents=expectedEvents))
1180 elif isinstance(checkConf, (float, int)):
1181 checkConf = float(checkConf)
1182 if checkConf < 0.0 or checkConf > 1.0:
1183 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1184 'Event count check for {inData} to {outData} is misconfigured: the efficiency factor of {eff} is not between 0 and 1.'.format(inData=inData, outData=outData, eff=checkConf))
1185 if neventsOutData >= int(expectedEvents * checkConf) and neventsOutData <= expectedEvents:
1186 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1187 else:
1188 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1189 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from {minEvents} to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData,
1190 minEvents=int(expectedEvents * checkConf), expectedEvents=expectedEvents))
1191 else:
1192 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1193 'Unrecognised event count configuration for {inData} to {outData}: "{conf}" is not known'.format(inData=inData, outData=outData, conf=checkConf))
1194 self._eventCount = expectedEvents
1195 return True
1.14.0