ATLAS Offline Software
Classes | Functions | Variables
python.scripts.cmake_depends Namespace Reference

Classes

class  AthGraph
 

Functions

def read_package_list (package_file)
 
def externals_name (lib)
 
def lrstrip (s, prefix, postfix)
 
def traverse (graph, root, reverse=False, maxdepth=None, nodegetter=lambda n:n)
 
def subgraph (graph, sources, reverse=False, maxdepth=None, nodegetter=lambda n :n.attr.get('label'))
 
def add_legend (graph)
 
def copy_graph (source, dest)
 
def create_dep_graph (target, deps, pydeps, args)
 
def print_dep_graph (graph, args, package_paths={})
 
def run (args)
 
def main (args)
 

Variables

 py_style
 

Function Documentation

◆ add_legend()

def python.scripts.cmake_depends.add_legend (   graph)
Add legend to graph

Definition at line 99 of file cmake_depends.py.

99 def add_legend(graph):
100  """Add legend to graph"""
101  graph.add_subgraph(name='clusterLegend', label='Legend')
102  l = graph.subgraphs()[-1]
103  for n in 'abcd':
104  l.add_node(n, shape='point', style='invis')
105  l.add_edge('a','b', label='C++', constraint=False)
106  l.add_edge('c','d', label='Python', style=py_style, constraint=False)
107 
108 

◆ copy_graph()

def python.scripts.cmake_depends.copy_graph (   source,
  dest 
)
Copy graph nodes and edges from source to dest including attributes

Definition at line 109 of file cmake_depends.py.

109 def copy_graph(source, dest):
110  """Copy graph nodes and edges from source to dest including attributes"""
111  for e in source.edges_iter():
112  dest.add_edge(e, **e.attr)
113  for n in source.nodes_iter():
114  dest.add_node(n, **n.attr)
115 
116 

◆ create_dep_graph()

def python.scripts.cmake_depends.create_dep_graph (   target,
  deps,
  pydeps,
  args 
)
Create and return dependency graph.

@param target  name of target
@param deps    AthGraph cmake dependencies
@param pydeps  python dependencies
@param args    command line arguments

Definition at line 171 of file cmake_depends.py.

171 def create_dep_graph(target, deps, pydeps, args):
172  """Create and return dependency graph.
173 
174  @param target name of target
175  @param deps AthGraph cmake dependencies
176  @param pydeps python dependencies
177  @param args command line arguments
178  """
179  # Helper for graph traversal below:
180  def getnode(node):
181  if not args.all and deps.ignore_target(node): return None
182  if args.externals or not node.attr['external']:
183  a = 'label' if args.target else 'package'
184  return node.attr[a]
185 
186  target = target.split('/')[-1] # in case of full package path
187 
188  # In package mode we have one extra level due to the Package_ target:
189  depth = args.recursive
190  if not args.target and not args.clients and args.recursive is not None:
191  depth += 1
192 
193  # With regex, find all matching targets:
194  if args.regex:
195  r = re.compile(target)
196  targets = [getnode(n) for n in deps.graph.nodes_iter() if r.match(n.attr['label'])]
197  if args.py:
198  targets += [n for n in pydeps.nodes_iter() if r.match(n)]
199  targets = sorted(set(filter(lambda t : t is not None, targets)))
200  else:
201  targets = [target]
202 
203  # Find the nodes from which graph traversal starts:
204  sources = []
205  for l in targets:
206  if not args.target:
207  l = 'Package_'+l
208  try:
209  if deps.get_node(l).attr['external'] and not args.externals:
210  raise RuntimeError(f"{l} is an external target. Run with -e/--externals.")
211 
212  # To find clients of a package means finding clients of the targets
213  # within that package. First find all targets within the package:
214  if args.clients and not args.target:
215  sources.extend([b for a,b in traverse(deps.graph, deps.get_node(l), maxdepth=1)])
216  else:
217  sources.extend([deps.get_node(l)])
218  except KeyError:
219  raise RuntimeError(f"Target with name {l} does not exist.")
220 
221  # Extract the dependency subgraph:
222  g = subgraph(deps.graph, sources, reverse=args.clients,
223  maxdepth=depth, nodegetter=getnode)
224 
225  graph = pygraphviz.AGraph(name=target, directed=True, strict=False)
226  graph.add_edges_from(g)
227 
228  # Add python dependencies:
229  if args.py:
230  # Here the nodes are the actual package names:
231  pysources = [pydeps.get_node(t) for t in targets if pydeps.has_node(t)]
232  g = subgraph(pydeps, pysources, reverse=args.clients,
233  maxdepth=args.recursive, nodegetter=lambda n : n.name)
234 
235  graph.add_edges_from(g, style=py_style)
236 
237  # Change style of nodes that have only Python dependencies:
238  for n in graph.nodes_iter():
239  if all(e.attr['style']==py_style for e in graph.edges_iter(n)):
240  n.attr['style'] = py_style
241 
242  return graph
243 
244 

◆ externals_name()

def python.scripts.cmake_depends.externals_name (   lib)
Return a short name for an external library

Definition at line 32 of file cmake_depends.py.

32 def externals_name(lib):
33  """Return a short name for an external library"""
34  if '/LCG_' in lib:
35  dirs = lib.split('/')
36  lcg = next(d for d in dirs if d.startswith('LCG_'))
37  return '%s::%s' % (dirs[dirs.index(lcg)+1], dirs[-1])
38  elif lib.startswith('Gaudi'):
39  return 'Gaudi::%s' % lib
40  else:
41  return os.path.basename(lib)
42 
43 

◆ lrstrip()

def python.scripts.cmake_depends.lrstrip (   s,
  prefix,
  postfix 
)
Strip `prefix` and `postfix` from string `s`

Definition at line 44 of file cmake_depends.py.

44 def lrstrip(s, prefix, postfix):
45  """Strip `prefix` and `postfix` from string `s`"""
46  if s.startswith(prefix): s = s[len(prefix):]
47  if s.endswith(postfix): s = s[:-len(postfix)]
48  return s
49 
50 

◆ main()

def python.scripts.cmake_depends.main (   args)

Definition at line 378 of file cmake_depends.py.

378 def main(args):
379  try:
380  run(args)
381  except RuntimeError as e:
382  print(e)
383  return 1

◆ print_dep_graph()

def python.scripts.cmake_depends.print_dep_graph (   graph,
  args,
  package_paths = {} 
)
Output final graph

Definition at line 245 of file cmake_depends.py.

245 def print_dep_graph(graph, args, package_paths={}):
246  """Output final graph"""
247 
248  # txt output
249  if args.batch or not args.dot:
250  f = open(graph.name+'.txt', 'w') if args.batch else sys.stdout
251  nodes = [e[0] for e in graph.in_edges_iter()] if args.clients \
252  else [e[1] for e in graph.out_edges_iter()]
253 
254  output = []
255  for p in set(nodes):
256  suffix = ':py' if p.attr['style']==py_style else ''
257  output.append('%s%s' % (package_paths.get(p,p), suffix))
258  print('\n'.join(sorted(output)), file=f)
259 
260  # dot output
261  if args.batch or args.dot:
262  f = open(graph.name+'.dot', 'w') if args.batch else sys.stdout
263  if args.legend:
264  add_legend(graph)
265  print(graph, file=f)
266 
267 
268 #
269 # Main function and command line arguments
270 #
271 @acmdlib.command(name='cmake.depends',
272  description=__doc__)
273 
274 @acmdlib.argument('names', nargs='+', metavar='NAME',
275  help='package/target name or regular expression')
276 
277 @acmdlib.argument('-t', '--target', action='store_true',
278  help='treat NAME as target instead of package name')
279 
280 @acmdlib.argument('-c', '--clients', action='store_true',
281  help='show clients (instead of dependencies)')
282 
283 @acmdlib.argument('-e', '--externals', action='store_true',
284  help='include external dependencies')
285 
286 @acmdlib.argument('-l', '--long', action='store_true',
287  help='show full package names (only for txt output)')
288 
289 @acmdlib.argument('-r', '--recursive', nargs='?', metavar='DEPTH',
290  type=int, default=1, const=None,
291  help='recursively resolve dependencies up to DEPTH (default: unlimited)')
292 
293 @acmdlib.argument('--py', action='store_true',
294  help=f'add Python dependencies (marked with ":py" in printout, {py_style} in graph)')
295 
296 @acmdlib.argument('--regex', action='store_true',
297  help='treat NAME as regular expression')
298 
299 @acmdlib.argument('--all', action='store_true',
300  help='do not apply any target filter (e.g. custom targets)')
301 
302 @acmdlib.argument('-d', '--dot', action='store_true',
303  help='print DOT graph')
304 
305 @acmdlib.argument('--legend', action='store_true',
306  help='add legend to graph')
307 
308 @acmdlib.argument('--batch', nargs='?', metavar='N', type=int, const=1,
309  help='Batch mode using N jobs (default: 1). Create dot and txt dependencies '
310  'for all NAMEs and store them in separate files.')
311 
312 
313 # Debugging/expert options:
314 @acmdlib.argument('--cmakedot', help=argparse.SUPPRESS)
315 @acmdlib.argument('--pydot', help=argparse.SUPPRESS)
316 
317 

◆ read_package_list()

def python.scripts.cmake_depends.read_package_list (   package_file)
Read packages.txt as a source for the full package path

Definition at line 23 of file cmake_depends.py.

23 def read_package_list(package_file):
24  """Read packages.txt as a source for the full package path"""
25 
26  with open(package_file) as f:
27  packages = [line.rstrip() for line in f if not line.startswith('#')]
28 
29  return dict([(p.split('/')[-1],p) for p in packages])
30 
31 

◆ run()

def python.scripts.cmake_depends.run (   args)
Inspect cmake build dependencies

Definition at line 318 of file cmake_depends.py.

318 def run(args):
319  """Inspect cmake build dependencies"""
320 
321  # Find packages.dot:
322  if not args.cmakedot:
323  try:
324  args.cmakedot = os.path.join(os.environ['AtlasArea'],'InstallArea',
325  os.environ['BINARY_TAG'],'packages.dot')
326  except KeyError:
327  main.parser.error("Cannot find 'packages.dot'. Setup a release or use --cmakedot.")
328 
329  # Find packages.py.dot:
330  pydeps = None
331  if args.py:
332  if args.target:
333  main.parser.error("Python dependencies not possible in target mode.")
334 
335  args.pydot = args.pydot or args.cmakedot.replace('.dot','.py.dot')
336  try:
337  pydeps = pygraphviz.AGraph(args.pydot)
338  except Exception:
339  main.parser.error(f"Cannot read '{args.pydot}'. Setup a release or use --pydot.")
340 
341  # Read packages.txt if needed:
342  package_paths = {}
343  if args.long:
344  try:
345  package_paths = read_package_list(os.path.join(os.environ['AtlasArea'],'InstallArea',
346  os.environ['BINARY_TAG'],'packages.txt'))
347  except Exception:
348  main.parser.error("Cannot read 'packages.txt'. Setup a release or run without -l/--long.")
349 
350  # Read dependencies:
351  deps = AthGraph(args.cmakedot)
352 
353  # Create combined graph for all given targets:
354  if not args.batch:
355  subgraphs = [create_dep_graph(target, deps, pydeps, args) for target in args.names]
356  if len(subgraphs)>1:
357  graph = pygraphviz.AGraph(name='AthGraph', directed=True, strict=False)
358  for g in subgraphs:
359  graph.add_subgraph(name=g.get_name())
360  copy_graph(g, graph.subgraphs()[-1])
361  else:
362  graph = subgraphs[0]
363 
364  print_dep_graph(graph, args, package_paths)
365 
366  # Batch mode: create separte graph for each target:
367  else:
368  import multiprocessing
369  global doit # required for use in multiprocessing
370  def doit(target):
371  graph = create_dep_graph(target, deps, pydeps, args)
372  print_dep_graph(graph, args, package_paths)
373 
374  pool = multiprocessing.Pool(args.batch)
375  pool.map(doit, args.names)
376 
377 

◆ subgraph()

def python.scripts.cmake_depends.subgraph (   graph,
  sources,
  reverse = False,
  maxdepth = None,
  nodegetter = lambda n : n.attr.get('label') 
)
Extract subgraph created by traversing from one or more sources.
Parameters are the same as in `traverse`. Return list of edge tuples.

Definition at line 85 of file cmake_depends.py.

85 def subgraph(graph, sources, reverse=False, maxdepth=None, nodegetter=lambda n : n.attr.get('label')):
86  """Extract subgraph created by traversing from one or more sources.
87  Parameters are the same as in `traverse`. Return list of edge tuples.
88  """
89  edges = set()
90  for root in sources:
91  for a,b in traverse(graph, root, reverse=reverse, maxdepth=maxdepth, nodegetter=nodegetter):
92  if a and b and a!=b:
93  if reverse: edges.add((b,a))
94  else: edges.add((a,b))
95 
96  return edges
97 
98 

◆ traverse()

def python.scripts.cmake_depends.traverse (   graph,
  root,
  reverse = False,
  maxdepth = None,
  nodegetter = lambda n:n 
)
Depth-limited BFS edge traversal of graph starting at root.

@param graph       graph
@param root        start node for traversal
@param reverse     traverse graph in reverse
@param maxdepth    maximum traversal depth (1 = only direct neighbors)
@param nodegetter  functor returning node names
@return            edge tuple (parent,node)

Inspired by https://github.com/networkx/networkx/tree/master/networkx/algorithms/traversal

Definition at line 51 of file cmake_depends.py.

51 def traverse(graph, root, reverse=False, maxdepth=None, nodegetter=lambda n:n):
52  """Depth-limited BFS edge traversal of graph starting at root.
53 
54  @param graph graph
55  @param root start node for traversal
56  @param reverse traverse graph in reverse
57  @param maxdepth maximum traversal depth (1 = only direct neighbors)
58  @param nodegetter functor returning node names
59  @return edge tuple (parent,node)
60 
61  Inspired by https://github.com/networkx/networkx/tree/master/networkx/algorithms/traversal
62  """
63  visited_nodes = set()
64  visited_edges = set()
65  queue = deque([(root,root,0)])
66  neighbors = graph.iterpred if reverse else graph.itersucc
67  while queue:
68  parent,node,level = queue.popleft()
69 
70  if node not in visited_nodes:
71  visited_nodes.add(node)
72 
73  # Add edges to neighbors into queue:
74  if maxdepth is None or level < maxdepth:
75  queue.extend((node,n,level+1) for n in neighbors(node))
76  # For the last level only edges to already visited nodes:
77  elif level==maxdepth:
78  queue.extend((node,n,level+1) for n in neighbors(node) if n in visited_nodes)
79 
80  if (parent,node) not in visited_edges:
81  visited_edges.add((parent,node))
82  yield nodegetter(parent), nodegetter(node)
83 
84 

Variable Documentation

◆ py_style

python.scripts.cmake_depends.py_style

Definition at line 21 of file cmake_depends.py.

python.scripts.cmake_depends.create_dep_graph
def create_dep_graph(target, deps, pydeps, args)
Definition: cmake_depends.py:171
python.scripts.cmake_depends.copy_graph
def copy_graph(source, dest)
Definition: cmake_depends.py:109
python.scripts.cmake_depends.traverse
def traverse(graph, root, reverse=False, maxdepth=None, nodegetter=lambda n:n)
Definition: cmake_depends.py:51
run
int run(int argc, char *argv[])
Definition: ttree2hdf5.cxx:28
covarianceTool.filter
filter
Definition: covarianceTool.py:514
python.scripts.cmake_depends.print_dep_graph
def print_dep_graph(graph, args, package_paths={})
Definition: cmake_depends.py:245
fillPileUpNoiseLumi.next
next
Definition: fillPileUpNoiseLumi.py:52
python.scripts.cmake_depends.read_package_list
def read_package_list(package_file)
Definition: cmake_depends.py:23
python.scripts.cmake_depends.subgraph
def subgraph(graph, sources, reverse=False, maxdepth=None, nodegetter=lambda n :n.attr.get('label'))
Definition: cmake_depends.py:85
python.scripts.cmake_depends.externals_name
def externals_name(lib)
Definition: cmake_depends.py:32
run
Definition: run.py:1
DerivationFramework::TriggerMatchingUtils::sorted
std::vector< typename T::value_type > sorted(T begin, T end)
Helper function to create a sorted vector from an unsorted one.
CxxUtils::set
constexpr std::enable_if_t< is_bitmask_v< E >, E & > set(E &lhs, E rhs)
Convenience function to set bits in a class enum bitmask.
Definition: bitmask.h:232
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
Cut::all
@ all
Definition: SUSYToolsAlg.cxx:67
Trk::open
@ open
Definition: BinningType.h:40
python.scripts.cmake_depends.lrstrip
def lrstrip(s, prefix, postfix)
Definition: cmake_depends.py:44
python.scripts.cmake_depends.add_legend
def add_legend(graph)
Definition: cmake_depends.py:99
python.scripts.cmake_depends.main
def main(args)
Definition: cmake_depends.py:378