ATLAS Offline Software
isDSinFAX.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
4 
5 import subprocess, threading, os, sys
6 import urllib2
7 import json
8 
9 import argparse
10 
11 parser = argparse.ArgumentParser(description='Checks if dataset is accessible through FAX.')
12 parser.add_argument('dataset', type=str, help='Dataset name')
13 parser.add_argument('-af','--accessfile', action='store_const', const='1', help='try to open the first root file of the dataset using root. ')
14 parser.add_argument('-aa','--accessall', action='store_const', const='1', help='try to open all the root files of the dataset using root. ')
15 
16 args = vars(parser.parse_args())
17 
18 try:
19  import dq2.clientapi.cli.cliutil
20  from dq2.common.cli.DQDashboardTool import DQDashboardTool
21  from dq2.clientapi.cli.cliutil import getDQ2
22  from dq2.filecatalog.lfc.lfcconventions import to_native_lfn
23 except ImportError:
24  print ("Environment not set [error importing DQ2 dependencies]!")
25  sys.exit(1)
26 
27 
28 class Command(object):
29 
30  def __init__(self, cmd):
31  self.cmd = cmd
32  self.process = None
33 
34  def run(self, timeout):
35  def target():
36 # print ('command started: ', self.cmd)
37  self.process = subprocess.Popen(self.cmd, shell=True)
38  self.process.communicate()
39 
40  thread = threading.Thread(target=target)
41  thread.start()
42 
43  thread.join(timeout)
44  if thread.is_alive():
45  print ('Terminating process')
46  self.process.terminate()
47  thread.join()
48  return self.process.returncode
49 
50 
51 #print ('Geting data from AGIS ...')
52 
53 class site:
54  name=''
55  host=''
56  port=1094
57 
58  def __init__(self, na, ho):
59  self.name=na
60  ho=ho.replace("root://","")
61  self.host=ho.split(":")[0]
62  if ho.count(":"):
63  self.port=ho.split(":")[1]
64 
65  def prnt(self, what):
66  if (what>=0 and self.redirector!=what): return
67  print ( 'name:', self.name, '\thost:', self.host, '\tport:', self.port )
68 
69 sites=[];
70 
71 try:
72  req = urllib2.Request("http://atlas-agis-api-0.cern.ch/request/service/query/get_se_services/?json&state=ACTIVE&flavour=XROOTD", None)
73  opener = urllib2.build_opener()
74  f = opener.open(req)
75  res=json.load(f)
76  for s in res:
77 # print (s["name"], s["rc_site"], s["endpoint"])
78  ns = site( s["rc_site"], s["endpoint"] )
79  sites.append(ns)
80 # print (res)
81 # print (' got FAX SEs from AGIS.')
82 
83 except:
84  print ("Unexpected error:", sys.exc_info()[0] )
85 
86 #for s in sites: s.prnt(-1)
87 
88 allddms=set()
89 
90 try:
91  req = urllib2.Request("http://atlas-agis-api-0.cern.ch/request/ddmendpoint/query/list/?json&state=ACTIVE", None)
92  opener = urllib2.build_opener()
93  f = opener.open(req)
94  res=json.load(f)
95  for s in res:
96  for c in sites:
97  if s["rc_site"]==c.name:
98  # print (s["rc_site"], s["name"])
99  allddms.add(s["name"])
100  break
101 # print (' got related ddmendpoints from agis.')
102 
103 except:
104  print ("Unexpected error:", sys.exc_info()[0] )
105 
106 
107 DS=args['dataset']
108 
109 com=Command('dq2-ls -r '+ DS + ' > fax.tmp' )
110 com.run(300)
111 dsets={}
112 cds=''
113 f = open('fax.tmp', 'r')
114 for line in f:
115  if line.startswith('Multiple'): break
116  line=line.strip()
117  if line.count(':')==0:continue
118  line=line.split(":")
119  # print (line)
120 
121  if line[0]=='INCOMPLETE':
122  if len(line[1])==0: continue
123  rep=line[1].split(',')
124  for r in rep:
125  if r in allddms:
126  dsets[cds][0]+=1
127  continue
128 
129  if line[0]=='COMPLETE':
130  if len(line[1])==0: continue
131  rep=line[1].split(',')
132  for r in rep:
133  if r in allddms:
134  dsets[cds][1]+=1
135  continue
136 
137  cds=line[0]
138  dsets[cds]=[0,0]
139 
140 for d in dsets.keys():
141  print (d,'\tcomplete replicas:',dsets[d][1],'\tincomplete:',dsets[d][0])
isDSinFAX.Command.run
def run(self, timeout)
Definition: isDSinFAX.py:34
isDSinFAX.Command.__init__
def __init__(self, cmd)
Definition: isDSinFAX.py:30
isDSinFAX.site.prnt
def prnt(self, what)
Definition: isDSinFAX.py:65
isDSinFAX.site.host
host
Definition: isDSinFAX.py:55
isDSinFAX.site.port
port
Definition: isDSinFAX.py:56
isDSinFAX.site.__init__
def __init__(self, na, ho)
Definition: isDSinFAX.py:58
isDSinFAX.site.name
name
Definition: isDSinFAX.py:54
isDSinFAX.Command
Definition: isDSinFAX.py:28
CxxUtils::set
constexpr std::enable_if_t< is_bitmask_v< E >, E & > set(E &lhs, E rhs)
Convenience function to set bits in a class enum bitmask.
Definition: bitmask.h:232
Trk::open
@ open
Definition: BinningType.h:40
copySelective.target
string target
Definition: copySelective.py:36
isDSinFAX.site
Definition: isDSinFAX.py:53
isDSinFAX.Command.process
process
Definition: isDSinFAX.py:32
pickleTool.object
object
Definition: pickleTool.py:29
isDSinFAX.Command.cmd
cmd
Definition: isDSinFAX.py:31
Trk::split
@ split
Definition: LayerMaterialProperties.h:38