ATLAS Offline Software
GetLBsToIgnore.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
2 
3 
4 import sys, re
5 import xmlrpc.client
6 from PyCool import cool
7 from CoolConvUtilities.AtlCoolLib import indirectOpen
8 from DQDefects import DefectsDB
9 
10 try:
11  serverfile=open("/afs/cern.ch/user/l/larmon/public/atlasdqmpass.txt")
12  password=serverfile.readline().strip()
13  serverfile.close()
14 except Exception as e:
15  print ("Failed to read xmlrpc server connection details from AFS location")
16  print (e)
17  sys.exit(-1)
18 
19 
20 def getLBsToIgnore(runnum,burstsFromCosmic=True,bulkProcessing=False, dropNonReady=True):
21 
22  badLBs=set()
23 
24  # 1. Get LB range for this run and LBs without "ATLAS-READY"
25  nReadyLBs=0
26  nNotReadyLBs=0
27  tdaqdb=indirectOpen('COOLONL_TDAQ/CONDBR2')
28  if (tdaqdb is None):
29  print ("ERROR: Can't access COOLONL_TDAQ/CONDBR2")
30  sys.exit(-1)
31 
32  fmode=tdaqdb.getFolder("/TDAQ/RunCtrl/DataTakingMode")
33 
34  since=(runnum<<32)+1
35  until=((1+runnum)<<32)-1
36  maxLb=0
37  minLb=1
38  itr=fmode.browseObjects(since,until,cool.ChannelSelection.all())
39  while itr.goToNext():
40  obj=itr.currentRef()
41  pl=obj.payload()
42  isReady=pl["ReadyForPhysics"]
43  lb1=max(since,obj.since()) & 0xFFFFFFFF
44  ts2=obj.until()
45  if ts2<until: #ignore the IOV beyond the end of the run
46  lb2=ts2 & 0xFFFFFFFF
47  if lb2>maxLb:
48  maxLb=lb2
49  if not isReady:
50  if dropNonReady:
51  print ("Ignoring LumiBlocks %i - %i not ATLAS READY" % (lb1,lb2))
52  badLBs.update(range(lb1,lb2))
53  nNotReadyLBs+=(lb2-lb1)
54  else:
55  nReadyLBs+=(lb2-lb1)
56  pass
57  pass
58  pass
59  pass
60  itr.close()
61  tdaqdb.closeDatabase()
62 
63  print ("Run %i goes up to LB %i" % (runnum,maxLb))
64 
65  #2. Get problematic LBs
66  #2.1 Look for collisions in empty bunches - Fetch from DQ Web Server
67  source = 'tier0'
68  stream = 'physics_CosmicCalo'
69 
70  serverstring="https://%s@atlasdqm.cern.ch" % password
71  server = xmlrpc.client.ServerProxy(serverstring)
72  multicall = xmlrpc.client.MultiCall(server)
73 
74  # Look for the highest(latest) processing version of CosmicCalo by retrieving amitag
75  run_spec = {'source': source, 'high_run': runnum, 'low_run': runnum}
76  multicall.get_procpass_amitag_mapping(run_spec)
77  results = multicall()
78  if len(results[0])==0:
79  print ("Nothing found about run",runnum,"on DQM server")
80  proc = 0
81  try:
82  list = results[0][str(runnum)]
83  for item in list:
84  if ("f" in item[2] and bulkProcessing and "CosmicCalo" in item[1] and item[0]>proc):
85  proc = 2
86  if ("x" in item[2] and (not bulkProcessing) and "CosmicCalo" in item[1] and item[0]>proc):
87  print (item)
88  proc = 1
89  pass
90  pass
91  except Exception as e:
92  print ("ERROR: can't retrieve the AMI Tag")
93  print (e)
94 
95  if (proc == 0):
96  print ("I haven't found any processing version for CosmicCalo. Assume express processing")
97  proc=1
98 
99 
100  try:
101  multicall = xmlrpc.client.MultiCall(server)
102  run_spec = {'source': source, 'high_run': runnum, 'stream': stream, 'proc_ver': proc, 'low_run': runnum}
103  multicall.get_timestamp(run_spec)
104  results=multicall()
105  timestamp=results[0][str(runnum)]
106  from time import asctime,localtime
107  print ("DQM server timestamp:", asctime(localtime(timestamp)))
108  print ("Now: ",asctime())
109  except Exception as e:
110  print ("ERROR: can't get timestamp from DQM server")
111  print (e)
112 
113 
114  multicall = xmlrpc.client.MultiCall(server)
115  run_spec = {'source': source, 'high_run': runnum, 'stream': stream, 'proc_ver': proc, 'low_run': runnum}
116  multicall.get_dqmf_all_results(run_spec,'LAr/LAR_GLOBAL/Collisions-Bkg/LArCollTimeLumiBlockTimeCut')
117  results = multicall()
118  RE = re.compile(r'\‍((?P<lb>\S+)\.0*\‍)')
119 
120  try:
121  list = results[0][str(runnum)]
122  for item in list:
123  if 'NBins' in item:
124  continue
125  m = RE.search(item).groupdict()
126  lb=int(m['lb'])
127  ncollisions=int(results[0][str(runnum)][item])
128  if ncollisions > 50:
129  badLBs.add(lb)
130  print ("LumiBlock %i ignored because it is empty bunches are polluted with collisions" % lb)
131  pass
132  pass
133  except Exception as e:
134  print ("ERROR: can't get LArCollTimeLumiBlockTimeCut from DQM server")
135  print (e)
136 
137  if (burstsFromCosmic):# CosmicCalo stream : from the DQ web
138  histoName = {'EMBC':'BarrelC','EMBA':'BarrelA','EMECC':'EMECC','EMECA':'EMECA'}
139  for iPart in histoName.keys():
140  multicall = xmlrpc.client.MultiCall(server)
141  #multicall.get_dqmf_all_results(run_spec,'LAr/%s/Noise/Partition/NoisyEvent_TimeVeto_%s'%(iPart,histoName[iPart]))
142  multicall.get_dqmf_all_results(run_spec,'/LAr/%s/Occupancy-Noise/Noise_Burst/NoisyEvent_TimeVeto_%s'%(iPart,iPart))
143 
144  results = multicall()
145  try:
146  resultlist = results[0][str(runnum)]
147  #print ("Got %i items for NoisyEvent_TimeVeto_%s" % (len(list),histoName[iPart]))
148  for item in resultlist:
149  if 'NBins' in item:
150 
151  continue
152  m = RE.search(item).groupdict()
153  lb=int(m['lb'])
154  yieldbursts=float(results[0][str(runnum)][item])
155  if yieldbursts > 0:
156  badLBs.add(lb)
157  print ("LumiBlock %i ignored because it contains bursts in CosmicCalo stream in %s" % (lb,iPart))
158  pass
159  pass
160  except Exception as e:
161  print ("ERROR: can't get NoisyEvent from DQM server")
162  print (e)
163 
164 
165  del multicall
166  del server
167 
168  #3.2 Get defects from Defects DB
169  db = DefectsDB()
170  lar_defects = [d for d in (db.defect_names | db.virtual_defect_names) if d.startswith("LAR")]
171  defects = db.retrieve((runnum, minLb), (runnum, maxLb), lar_defects)
172  for defect in defects:
173  part=defect.channel.split("_")[1]
174  #3.2.1 Check for HV trip
175  if "HVTRIP" in defect.channel and defect.present:
176  for lb in range(defect.since.lumi,defect.until.lumi):
177  badLBs.add(lb)
178  print ("LumiBlock %i ignored because of a HV trip in partition %s" % (lb,part))
179  pass
180  pass
181  #3.2.2 Check for Noise Bursts from the defects
182  if (not burstsFromCosmic):
183  if not bulkProcessing:
184  if "NOISEBURST" in defect.channel and defect.present:
185  for lb in range(defect.since.lumi,defect.until.lumi):
186  badLBs.add(lb)
187  print ("LumiBlock %i ignored because of a noise burst in partition %s" % (lb,part))
188  pass
189  pass
190  else: #not bulk processing
191  if "SEVNOISEBURST" in defect.channel and defect.present:
192  for lb in range(defect.since.lumi,defect.until.lumi):
193  badLBs.add(lb)
194  print ("LumiBlock %i ignored because of a severe noise burst in partition %s" % (lb,part))
195  pass
196  pass
197 
198  del db #Close Defects DB
199 
200  nBadLBs=len(badLBs)
201  if dropNonReady:
202  nBadLBs=nBadLBs-nNotReadyLBs
203 
204  print ("Found %i not-ready LBs, %i atlas-ready LBs and %i bad LBs" % (nNotReadyLBs,nReadyLBs,nBadLBs))
205 
206  return badLBs
207 
208 
209 
210 
211 
212 if __name__ == "__main__":
213  import getopt
214  if len(sys.argv) == 1 :
215  print ()
216  print ("usage: python %s <options> <runnumber> "%(sys.argv[0]))
217  print ()
218  sys.exit(1)
219 
220  burstsFromCosmic=True
221  bulkProcessing=False
222  dropNonReady=True
223 
224  outputFN=None
225 
226  opts,args=getopt.getopt(sys.argv[1:],"brco:",[])
227  for o,a in opts:
228  if (o=='-c'):
229  burstsFromCosmics=False
230  if (o=='-b'):
231  bulkProcessing=True
232  if (o=='-r'):
233  dropNonReady=False
234  if (o=='-o'):
235  outputFN=a
236 
237  if len(args)<0:
238  print ("No run number found")
239  sys.exit(-1)
240 
241  if len(args)>1:
242  print ("Too many arguments")
243  sys.exit(-1)
244 
245  run=int(args[0])
246 
247  if (bulkProcessing):
248  print ("Searching for bad lumi blocks in run %d for bulk processing"%run)
249  else:
250  print ("Searching for bad lumi blocks in run %d for express processing"%run)
251 
252  if (dropNonReady):
253  print ("LB not marked as AtlasReady will be considered bad")
254  else:
255  print ("LB not marked as AtlasReady will be considered good")
256 
257 
258 
259  badLBset=getLBsToIgnore(run,burstsFromCosmic,bulkProcessing, dropNonReady)
260 
261  badLBsorted=sorted(badLBset)
262  print ("LBs to ignore:",badLBsorted)
263 
264  if outputFN is not None:
265  out=open(outputFN,"w")
266  out.write(', '.join([ str(i) for i in badLBsorted ]))
267  out.write("\n")
268  out.close()
269 
270 
271 # badLBset2=getLBsToIgnore(run)
272 # print ("LBs to ignore:",sorted(badLBset))
DerivationFramework::TriggerMatchingUtils::sorted
std::vector< typename R::value_type > sorted(const R &r, PROJ proj={})
Helper function to create a sorted vector from an unsorted range.
max
constexpr double max()
Definition: ap_fixedTest.cxx:33
GetLBsToIgnore.getLBsToIgnore
def getLBsToIgnore(runnum, burstsFromCosmic=True, bulkProcessing=False, dropNonReady=True)
Definition: GetLBsToIgnore.py:20
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:194
CxxUtils::set
constexpr std::enable_if_t< is_bitmask_v< E >, E & > set(E &lhs, E rhs)
Convenience function to set bits in a class enum bitmask.
Definition: bitmask.h:232
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
Trk::open
@ open
Definition: BinningType.h:40
python.CaloAddPedShiftConfig.int
int
Definition: CaloAddPedShiftConfig.py:45
str
Definition: BTagTrackIpAccessor.cxx:11
python.AtlCoolLib.indirectOpen
def indirectOpen(coolstr, readOnly=True, debug=False)
Definition: AtlCoolLib.py:129
python.LArMinBiasAlgConfig.float
float
Definition: LArMinBiasAlgConfig.py:65