Perform an event count check.
1032 def decide(self):
1033
1034
1035 for inData, neventsInData in self._inEventDict.items():
1036 if not isinstance(neventsInData, int):
1037 msg.warning('File size metadata for {inData} was not countable, found {neventsInData}. No event checks possible for this input data.'.format(inData=inData, neventsInData=neventsInData))
1038 continue
1039 if inData in self._eventCountConf:
1040 inDataKey = inData
1041 else:
1042
1043 matchedInData = False
1044 for inDataKey in self._eventCountConf:
1045 if fnmatch.fnmatch(inData, inDataKey):
1046 msg.info("Matched input data type {inData} to {inDataKey} by globbing".format(inData=inData, inDataKey=inDataKey))
1047 matchedInData = True
1048 break
1049 if not matchedInData:
1050 msg.warning('No defined event count match for {inData} -> {outData}, so no check(s) possible in this case.'.format(inData=inData, outData=list(self._outEventDict)))
1051 continue
1052
1053
1054 expectedEvents = neventsInData
1055 if self._skipEvents is not None and self._skipEvents > 0:
1056 expectedEvents -= self._skipEvents
1057 if expectedEvents < 0:
1058 msg.warning('skipEvents was set higher than the input events in {inData}: {skipEvents} > {neventsInData}. This is not an error, but it is not a normal configuration. Expected events is now 0.'.format(inData=inData, skipEvents=self._skipEvents, neventsInData=neventsInData))
1059 expectedEvents = 0
1060 if self._maxEvents is not None:
1061 if expectedEvents < self._maxEvents:
1062 if self._skipEvents is not None:
1063 msg.warning('maxEvents was set higher than inputEvents-skipEvents for {inData}: {maxEvents} > {neventsInData}-{skipEvents}. This is not an error, but it is not a normal configuration. Expected events remains {expectedEvents}.'.format(inData=inData, maxEvents=self._maxEvents, neventsInData=neventsInData, skipEvents=self._skipEvents, expectedEvents=expectedEvents))
1064 else:
1065 msg.warning('maxEvents was set higher than inputEvents for {inData}: {maxEvents} > {neventsInData}. This is not an error, but it is not a normal configuration. Expected events remains {expectedEvents}.'.format(inData=inData, maxEvents=self._maxEvents, neventsInData=neventsInData, expectedEvents=expectedEvents))
1066 else:
1067 expectedEvents = self._maxEvents
1068 msg.debug('Expected number of processed events for {0} is {1}'.format(inData, expectedEvents))
1069
1070
1071 for outData, neventsOutData in self._outEventDict.items():
1072 if not isinstance(neventsOutData, int):
1073 msg.warning('File size metadata for {outData} was not countable, found "{neventsOutData}". No event checks possible for this output data.'.format(outData=outData, neventsOutData=neventsOutData))
1074 continue
1075 if outData in self._eventCountConf[inDataKey]:
1076 checkConf = self._eventCountConf[inDataKey][outData]
1077 outDataKey = outData
1078 else:
1079
1080 checkConf = None
1081 for outDataKey, outDataConf in self._eventCountConf[inDataKey].items():
1082 if fnmatch.fnmatch(outData, outDataKey):
1083 msg.info('Matched output data type {outData} to {outDatakey} by globbing'.format(outData=outData, outDatakey=outDataKey))
1084 outDataKey = outData
1085 checkConf = outDataConf
1086 break
1087 if not checkConf:
1088 msg.warning('No defined event count match for {inData} -> {outData}, so no check possible in this case.'.format(inData=inData, outData=outData))
1089 continue
1090 msg.debug('Event count check for {inData} to {outData} is {checkConf}'.format(inData=inData, outData=outData, checkConf=checkConf))
1091
1092
1093 if checkConf == 'match':
1094
1095 if neventsOutData == expectedEvents:
1096 msg.info("Event count check for {inData} to {outData} passed: all processed events found ({neventsOutData} output events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData))
1097 else:
1098 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1099 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1100 elif checkConf == 'filter':
1101 if neventsOutData <= expectedEvents and neventsOutData >= 0:
1102 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1103 else:
1104 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1105 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from 0 to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1106 elif checkConf == 'minEff':
1107 if neventsOutData >= int(expectedEvents * self._evAccEff) and neventsOutData <= expectedEvents:
1108 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1109 else:
1110 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1111 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from {minEvents} to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData,
1112 minEvents=int(expectedEvents * self._evAccEff), expectedEvents=expectedEvents))
1113 elif isinstance(checkConf, (float, int)):
1114 checkConf = float(checkConf)
1115 if checkConf < 0.0 or checkConf > 1.0:
1116 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1117 'Event count check for {inData} to {outData} is misconfigured: the efficiency factor of {eff} is not between 0 and 1.'.format(inData=inData, outData=outData, eff=checkConf))
1118 if neventsOutData >= int(expectedEvents * checkConf) and neventsOutData <= expectedEvents:
1119 msg.info("Event count check for {inData} to {outData} passed: found ({neventsOutData} output events selected from {expectedEvents} processed events)".format(inData=inData, outData=outData, neventsOutData=neventsOutData, expectedEvents=expectedEvents))
1120 else:
1121 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1122 'Event count check for {inData} to {outData} failed: found {neventsOutData} events, expected from {minEvents} to {expectedEvents}'.format(inData=inData, outData=outData, neventsOutData=neventsOutData,
1123 minEvents=int(expectedEvents * checkConf), expectedEvents=expectedEvents))
1124 else:
1125 raise trfExceptions.TransformValidationException(trfExit.nameToCode('TRF_EXEC_VALIDATION_EVENTCOUNT'),
1126 'Unrecognised event count configuration for {inData} to {outData}: "{conf}" is not known'.format(inData=inData, outData=outData, conf=checkConf))
1127 self._eventCount = expectedEvents
1128 return True
1.17.0