Perform an event count check.
1067 def decide(self):
1068
1069
1070 for inData, neventsInData in self._inEventDict.items():
1071 if not isinstance(neventsInData, int):
1072 msg.warning('File size metadata for {inData} was not countable, found {neventsInData}. No event checks possible for this input data.'.format(inData=inData, neventsInData=neventsInData))
1073 continue
1074 if inData in self._eventCountConf:
1075 inDataKey = inData
1076 else:
1077
1078 matchedInData = False
1079 for inDataKey in self._eventCountConf:
1080 if fnmatch.fnmatch(inData, inDataKey):
1081 msg.info("Matched input data type {inData} to {inDataKey} by globbing".format(inData=inData, inDataKey=inDataKey))
1082 matchedInData = True
1083 break
1084 if not matchedInData:
1085 msg.warning('No defined event count match for {inData} -> {outData}, so no check(s) possible in this case.'.format(inData=inData, outData=list(self._outEventDict)))
1086 continue
1087
1088
1089 expectedEvents = neventsInData
1090 if self._skipEvents is not None and self._skipEvents > 0:
1091 expectedEvents -= self._skipEvents
1092 if expectedEvents < 0:
1093 msg.warning('skipEvents was set higher than the input events in {inData}: {skipEvents} > {neventsInData}. This is not an error, but it is not a normal configuration. Expected events is now 0.'.format(inData=inData, skipEvents=self._skipEvents, neventsInData=neventsInData))
1094 expectedEvents = 0
1095 if self._maxEvents is not None:
1096 if expectedEvents < self._maxEvents:
1097 if self._skipEvents is not None:
1098 msg.warning('maxEvents was set higher than inputEvents-skipEvents for {inData}: {maxEvents} > {neventsInData}-{skipEvents}. This is not an error, but it is not a normal configuration. Expected events remains {expectedEvents}.'.format(inData=inData, maxEvents=self._maxEvents, neventsInData=neventsInData, skipEvents=self._skipEvents, expectedEvents=expectedEvents))
1099 else:
1100 msg.warning('maxEvents was set higher than inputEvents for {inData}: {maxEvents} > {neventsInData}. This is not an error, but it is not a normal configuration. Expected events remains {expectedEvents}.'.format(inData=inData, maxEvents=self._maxEvents, neventsInData=neventsInData, expectedEvents=expectedEvents))
1101 else:
1102 expectedEvents = self._maxEvents
1103 msg.debug('Expected number of processed events for {0} is {1}'.format(inData, expectedEvents))
1104
1105
1106 for outData, neventsOutData in self._outEventDict.items():
1107 if not isinstance(neventsOutData, int):
1108 msg.warning('File size metadata for {outData} was not countable, found "{neventsOutData}". No event checks possible for this output data.'.format(outData=outData, neventsOutData=neventsOutData))
1109 continue
1110 if outData in self._eventCountConf[inDataKey]:
1111 checkConf = self._eventCountConf[inDataKey][outData]
1112 outDataKey = outData
1113 else:
1114
1115 checkConf = None
1116 for outDataKey, outDataConf in self._eventCountConf[inDataKey].items():
1117 if fnmatch.fnmatch(outData, outDataKey):
1118 msg.info('Matched output data type {outData} to {outDatakey} by globbing'.format(outData=outData, outDatakey=outDataKey))
1119 outDataKey = outData
1120 checkConf = outDataConf
1121 break
1122 if not checkConf:
1123 msg.warning('No defined event count match for {inData} -> {outData}, so no check possible in this case.'.format(inData=inData, outData=outData))
1124 continue
1125 msg.debug('Event count check for {inData} to {outData} is {checkConf}'.format(inData=inData, outData=outData, checkConf=checkConf))
1126
1127
1128 if checkConf == 'match':
1129
1130 if neventsOutData == expectedEvents:
1131 msg.info("Event count check for {inData} to {outData} passed: all processed events found ({neventsOutData} output events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData))
1132 else:
1133 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1134 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1135 elif checkConf == 'filter':
1136 if neventsOutData <= expectedEvents and neventsOutData >= 0:
1137 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1138 else:
1139 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1140 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from 0 to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1141 elif checkConf == 'minEff':
1142 if neventsOutData >= int(expectedEvents * self._evAccEff) and neventsOutData <= expectedEvents:
1143 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1144 else:
1145 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1146 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from {minEvents} to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData,
1147 minEvents=int(expectedEvents * self._evAccEff), expectedEvents=expectedEvents))
1148 elif isinstance(checkConf, (float, int)):
1149 checkConf = float(checkConf)
1150 if checkConf < 0.0 or checkConf > 1.0:
1151 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1152 'Event count check for {inData} to {outData} is misconfigured: the efficiency factor of {eff} is not between 0 and 1.'.format(inData=inData, outData=outData, eff=checkConf))
1153 if neventsOutData >= int(expectedEvents * checkConf) and neventsOutData <= expectedEvents:
1154 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1155 else:
1156 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1157 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from {minEvents} to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData,
1158 minEvents=int(expectedEvents * checkConf), expectedEvents=expectedEvents))
1159 else:
1160 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1161 'Unrecognised event count configuration for {inData} to {outData}: "{conf}" is not known'.format(inData=inData, outData=outData, conf=checkConf))
1162 self._eventCount = expectedEvents
1163 return True
1.14.0