13 from ROOT
import gSystem
16 gSystem.Load(
'libDataQualityUtils')
17 from ROOT
import dqutils
22 defect_val = collections.namedtuple(
'defect_val',
23 'defect, comment, recoverable')
24 defect_iov = collections.namedtuple(
'defect_iov',
25 'defect, comment, recoverable, since, until')
27 logger = logging.getLogger(
'hancoolmod')
28 logger.addHandler(logging.NullHandler())
50 from .
import detmaskmod
51 runNumber = re.match(
r'run_(\d+)_.*han.root', name).
group(1)
52 max_hi_limit = detmaskmod.getNumLumiBlocks(
int(runNumber))+1
53 if (name.find(
'minutes10_') > -1):
54 t = name.split(
'minutes10_')
56 low_limit =
int((digit-1.0)*10.0/LBlength+1)-1
57 hi_limit =
int(digit*10.0/LBlength)
58 elif (name.find(
'minutes30_') > -1):
59 t = name.split(
'minutes30_')
61 low_limit =
int((digit-1.0)*30.0/LBlength+1)-1
62 hi_limit =
int(digit*30.0/LBlength)
63 elif 'lowStat_' in name:
64 t = name.split(
'lowStat_LB')[-1]
65 t = t.split(
'_han.root')[0]
67 low_limit =
int(digits[0])
68 hi_limit =
min(
int(digits[1])+1, max_hi_limit)
69 elif 'medStat_' in name:
70 t = name.split(
'medStat_LB')[-1]
71 t = t.split(
'_han.root')[0]
73 low_limit =
int(digits[0])
74 hi_limit =
min(
int(digits[1])+1, max_hi_limit)
77 hi_limit = max_hi_limit
78 except Exception
as e:
79 logging.warning(
'Could not determine limits because: %s', e)
83 return (low_limit, hi_limit)
89 rootFolder = file+
":"+rootFolder
92 result = of.getStringName(rootFolder)
104 filePath="/afs/cern.ch/user/a/atlasdqm/dqmdisk/han_results/tier0/FDR2/NoStream/",
105 dbConnection="sqlite://;schema=MyCOOL.db;dbname=CONDBR2
", isESn=True):
107 logger.info('====> Running hancool_defects')
109 logger.info(
'<==== Done with hancool_defects')
113 from .
import detmaskmod
114 blacks = detmaskmod.decodeBlack(detmaskmod.getRunMask(runNumber),
116 nlbs = detmaskmod.getNumLumiBlocks(runNumber)
118 for defect
in blacks:
123 comment=
'Automatically added by hancool')
129 mapping = {1:
'TRIG_L1_CTP_CTP_ROD_bcid',
130 2:
'TRIG_L1_CTP_bcid',
131 3:
'TRIG_L1_CTP_CTP_MuCTPI_bcid',
132 4:
'TRIG_L1_CTP_candnumber',
133 5:
'TRIG_L1_CTP_multpt',
134 6:
'TRIG_L1_CTP_roiNum',
135 7:
'TRIG_L1_CTP_roiCand',
136 8:
'TRIG_L1_CTP_bcidrange',
137 9:
'TRIG_L1_CTP_lumiblockrange',
138 10:
'TRIG_L1_CTP_lumiblocktime',
139 11:
'TRIG_L1_CTP_nanosectime',
140 12:
'TRIG_L1_CTP_TAPnoTBP',
141 13:
'TRIG_L1_CTP_TAVnoTAP',
142 14:
'TRIG_L1_CTP_CTPsim',
143 15:
'TRIG_L1_CTP_incompletefragment',
148 when = d.Get(
'CentralTrigger/ErrorSummary/errorSummaryPerLumiBlock')
152 overflow_bad_lbs = {}
157 for errorBin
in mapping:
158 bad_lbs[errorBin] = [bin
for bin
in range(1, when.GetNbinsX(
159 )+1)
if when.GetBinContent(bin, errorBin) > 0]
160 overflow_bad_lbs[errorBin] = (
161 when.GetBinContent(when.GetNbinsX()+1, errorBin) > 0)
163 message =
'Automatically set'
165 for defect
in mapping:
166 for lb
in bad_lbs[defect]:
167 rv.append(
defect_iov(mapping[defect], message,
False, lb, lb+1))
168 if overflow_bad_lbs[defect]:
169 message +=
'; defect occurred past end of monitoring histogram, marking end of run as bad'
170 from .
import detmaskmod
171 nlbs = detmaskmod.getNumLumiBlocks(runNumber)
173 False, when.GetNbinsX(), nlbs+1))
180 histogram = d.Get(
'InnerDetector/SCT/Summary/tracksPerRegion')
183 if histogram.GetEntries() < 200:
184 return [
defect_val(
'SCT_GLOBAL_LOWSTAT',
'Low statistics',
False)]
190 def sct_conf_defects_core(d, i, runNumber, histname, mapping):
192 histogram = d.Get(histname)
197 if mapping[bin] ==
'SCT_MOD_OUT_GT40':
199 if histogram.GetBinContent(bin) > threshold:
201 mapping[bin],
'%.1d modules affected' % histogram.GetBinContent(bin),
False))
204 rv1 = sct_conf_defects_core(d, i, runNumber,
'InnerDetector/SCT/Summary/SCTConfOutM',
205 {1:
'SCT_MOD_OUT_GT40'})
206 rv2 = sct_conf_defects_core(d, i, runNumber,
'InnerDetector/SCT/Summary/SCTConfNew',
207 {3:
'SCT_MOD_ERR_GT40',
208 5:
'SCT_MOD_NOISE_GT40'})
209 if rv1
is None and rv2
is None:
212 return (rv1
if rv1
is not None else [])+(rv2
if rv2
is not None else [])
216 pairs = [(
'InnerDetector/SCT/Summary/SCT_LinksWithLinkLevelErrorsVsLbs',
217 'SCT_PERIOD_ERR_GT40',
lambda _: _ > 80),
218 (
'InnerDetector/SCT/Summary/SCT_LinksWithRODLevelErrorsVsLbs',
219 'SCT_ROD_OUT',
lambda _: _ >= 1)]
223 overflow_bad_lbs = {}
224 message =
'Automatically set'
227 for hname, dname, policy
in pairs:
234 bad_lbs[dname] = [bin
for bin
in range(
235 1, when.GetNbinsX()+1)
if policy(when.GetBinContent(bin))]
236 overflow_bad_lbs[dname] = policy(
237 when.GetBinContent(when.GetNbinsX()+1))
239 for lb
in bad_lbs[dname]:
240 rv.append(
defect_iov(dname, message,
False, lb, lb+1))
241 if overflow_bad_lbs[dname]:
242 message +=
'; defect occurred past end of monitoring histogram, marking end of run as bad'
243 from .
import detmaskmod
244 nlbs = detmaskmod.getNumLumiBlocks(runNumber)
246 False, when.GetNbinsX(), nlbs+1))
254 l.sort(key=
lambda x: x.since)
262 until = previous.until
264 if l[i].since != until
or previous.comment != l[i].comment:
266 rl.append(previous._replace(until=until))
268 until = previous.until
273 rl.append(previous._replace(until=until))
278 h1 = d.Get(
'InnerDetector/SCT/Summary/SctTotalEffBCID_/Results/Status')
279 h2 = d.Get(
'InnerDetector/SCT/Summary/SctTotalEff_/Results/Status')
282 badstatuses = {
'Yellow',
'Red'}
285 status =
set(x.GetName()
for x
in h.GetListOfKeys())
286 if len(badstatuses & status) > 0:
288 status) == 1,
'Status must be length one or the file is corrupt'
289 statuscheck.append(
True)
291 statuscheck.append(
False)
293 return [
defect_val(
'SCT_EFF_LT99',
'Automatically set for whole run',
False)]
299 badstatuses =
set(badstatuses)
301 def dqmf_node_defect_core(d, i, runNumber):
302 filenode = d.Get(node +
'_/Results/Status')
305 status =
set(x.GetName()
for x
in filenode.GetListOfKeys())
306 if len(badstatuses & status) > 0:
308 status) == 1,
'Status must be length one or the file is corrupt'
309 return [
defect_val(defect, node +
' DQMF color ' + status.pop(),
False)]
312 return dqmf_node_defect_core
316 from .
import pix_defect
320 analyzers += [ctp_defects]
322 analyzers += [sct_eff_defect,
328 if (len(filePath) == 0
or filePath[-1] !=
'/'):
330 if (len(dbConnection) < 1):
331 dbConnection =
"/afs/cern.ch/user/a/atlasdqm/dqmdisk1/cherrypy-devel/defectstest.db/COMP200"
335 defects_by_function = {}
339 fnames = ([[filePath+
"run_"+
str(runNumber)+
"_han.root"], []]
340 + [glob.glob(os.path.join(filePath,
'run_%s%s*_han.root' % (runNumber, intervalType[i])))
for i
in [2, 3]])
342 for i, itype
in enumerate(fnames):
343 ldefects_by_function = {}
344 for globname
in itype:
345 filename = os.path.basename(globname)
349 default_iov = default_iov._replace(since=since, until=until)
351 fobj = ROOT.TFile.Open(globname)
352 for func
in analyzers:
353 rv = func(fobj, i, runNumber)
355 rvt = [default_iov._replace(**(i._asdict()))
for i
in rv]
356 if func
not in ldefects_by_function:
357 ldefects_by_function[func] = rvt
359 ldefects_by_function[func] += rvt
360 defects_by_function.update(ldefects_by_function)
361 defects =
sum(defects_by_function.values(), [])
364 globname = fnames[0][0]
365 filename = os.path.basename(globname)
368 defects += pix_defect.execute(runNumber, globname, until-1)
369 except Exception
as e:
370 logging.warning(
'Unable to execute pixel hancool code')
371 logging.warning(
'--> %s: %s',
type(e).__name__, e)
373 from DQDefects
import DefectsDB, DEFECT_IOV
374 from DQUtils.sugar
import RunLumi
376 ddb =
DefectsDB(dbConnection, read_only=
False)
378 logging.info(
'Running detmask_defects')
382 for defect
in dm_defects +
iovs_merge(defects):
383 logger.debug(
'Working with %s', defect)
385 RunLumi(runNumber, defect.until),
386 channel=defect.defect,
387 comment=defect.comment,
389 recoverable=defect.recoverable,
391 secret_path=os.environ.get(
'COOLFLASK_SECRET',
'/afs/cern.ch/user/a/atlasdqm/private/coolflask_secret/coolflask_secret.json')
392 auth = json.loads(
open(secret_path).
read())
393 logger.debug(
'Flask upload')
394 ddb.insert_multiple(defectlist, use_flask=(
'sqlite' not in dbConnection),