ATLAS Offline Software
periods.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 
3 # Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
4 
5 from __future__ import with_statement, print_function
6 
7 from os import listdir, makedirs
8 from os.path import basename, exists, join as pjoin
9 
10 from .sugar import define_iov_type, IOVSet
11 
12 import logging
13 log = logging.getLogger("DQUtils.periods")
14 
15 DATA_PERIODS_PATH = "/afs/cern.ch/atlas/www/GROUPS/DATAPREPARATION/DataPeriods/"
16 
17 @define_iov_type
18 def PERIODIOV_VAL(project_tag, period):
19  pass
20 
22  """
23  Warning: Being deprecated
24  """
25  for period_file in listdir(DATA_PERIODS_PATH):
26  if not period_file.endswith(".runs.list"):
27  continue
28 
29 def get_period_iovset_from_file(project_tag, period):
30  runs_file = "%s.%s.runs.list" % (project_tag, period)
31  with open(pjoin(DATA_PERIODS_PATH, runs_file)) as fd:
32  period_data = fd.read()
33 
34  try:
35  period_data = map(int, period_data.strip().split("\n"))
36  except ValueError:
37  raise RuntimeError("Invalid period data")
38 
39  return IOVSet.from_runs(period_data)
40 
41 def get_period_iovset_from_ami(ptag, period):
42  #from pyAMI.pyAMI import AMI
43  #import pyAMI
44 
45  # This import needed to ensure correct python environment since pyAMI 4.0.3
46  # Yuriy Ilchenko: commented out (not sure needed since ATHENA 17.X)
47  #import setup_pyAMI
48  import pyAMI
49  from pyAMI.client import Client
50  amiclient = Client('atlas')
51  amiclient.config.read()
52 
53  args = ['GetRunsForDataPeriod',
54  '-projectName=%s' % ptag,
55  '-period=%s' % period]
56  try:
57  runs = amiclient.execute(args, format='dict_object').get_rows()
58  except pyAMI.exceptions.AMI_Error:
59  raise
60 
61  #return IOVSet.from_runs(int(x['runNumber']) for x in runs.values())
62  return sorted(int(x['runNumber']) for x in runs.values())
63 
65  """
66  Returns a dictionary {project_tag: {period: [runs]}}
67 
68  retrieved from the coma database.
69  """
70 
71  from sqlalchemy import (select, Table, Column,
72  String, Integer, ForeignKey)
73 
74  from re import compile
75 
76  LETTER_PART = compile(r"^(\D+)")
77 
78  from DQUtils.oracle import make_oracle_connection
79 
80  engine, metadata = make_oracle_connection("oracle://atlas_dd/atlasdd")
81 
82  schema = "ATLAS_TAGS_METADATA"
83 
84  coma_runs = Table("coma_runs", metadata,
85  Column("run_index", Integer),
86  Column("run_number", Integer),
87  schema=schema
88  )
89 
90  coma_period_definitions = Table("coma_period_defs", metadata,
91  Column("p_index", Integer),
92  Column("p_project_period", String),
93  schema=schema
94  )
95  project_period = coma_period_definitions.c.p_project_period
96 
97  coma_period_p1_to_runs = Table("coma_period_p1_to_runs", metadata,
98  Column("p2r_index", Integer),
99  Column("p_index", Integer, ForeignKey(coma_period_definitions.c.p_index)),
100  Column("run_index", Integer, ForeignKey(coma_runs.c.run_index)),
101  schema=schema
102  )
103  joined = coma_period_p1_to_runs.join(coma_runs).join(coma_period_definitions)
104 
105  smt = (select([project_period, coma_runs.c.run_number], from_obj=joined)
106  .order_by(coma_runs.c.run_number))
107 
108  projectperiod_runs = smt.execute().fetchall()
109 
110  mapping = {}
111  for projectperiod, run in projectperiod_runs:
112  project, period = projectperiod.split(".")
113  period = period.replace("period", "")
114 
115  project_dict = mapping.setdefault(project, {})
116  period_runs = project_dict.setdefault(period, [])
117  if not period_runs or period_runs[-1] != run:
118  period_runs.append(run)
119 
120  period_letter = LETTER_PART.search(period)
121  if period_letter:
122  (period_letter,) = period_letter.groups()
123  period_runs = project_dict.setdefault(period_letter, [])
124  if not period_runs or period_runs[-1] != run:
125  period_runs.append(run)
126 
127  return mapping
128 
129 def split_grl(project_tag, periods, grl_file, out_directory):
130  grl = IOVSet.from_grl(grl_file)
131 
132  period_info = fetch_project_period_runs()
133  if project_tag not in period_info:
134  raise RuntimeError("Unknown period {0}".format(project_tag))
135  ptag_periods = period_info[project_tag]
136 
137  if periods is None:
138  # If periods are unspecified, use all
139  periods = sorted(ptag_periods.keys())
140 
141  name_left, _, extension = basename(grl_file).rpartition(".")
142  assert name_left, "No '.' in filename?"
143 
144  if not exists(out_directory):
145  makedirs(out_directory)
146 
147  for period in periods:
148  iovs = grl & IOVSet.from_runs(ptag_periods[period])
149 
150  output_name = "{0}_period{1}.{2}".format(name_left, period, extension)
151  log.info("{0:4s} : {1:3} runs, {2:5} lumiblocks".format(
152  period, len(iovs.runs), iovs.lb_counts))
153 
154  output_path = pjoin(out_directory, output_name)
155  iovs.to_grl(output_path)
vtune_athena.format
format
Definition: vtune_athena.py:14
CaloCellPos2Ntuple.int
int
Definition: CaloCellPos2Ntuple.py:24
python.dummyaccess.listdir
def listdir(dirname)
Definition: dummyaccess.py:6
FakeBkgTools::Client
Client
Definition: FakeBkgInternals.h:141
python.oracle.make_oracle_connection
def make_oracle_connection(connection_string)
Definition: oracle.py:59
python.periods.PERIODIOV_VAL
def PERIODIOV_VAL(project_tag, period)
Definition: periods.py:18
python.periods.fetch_project_period_runs
def fetch_project_period_runs()
Definition: periods.py:64
python.periods.get_periods_from_file
def get_periods_from_file()
Definition: periods.py:21
DerivationFramework::TriggerMatchingUtils::sorted
std::vector< typename T::value_type > sorted(T begin, T end)
Helper function to create a sorted vector from an unsorted one.
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
Trk::open
@ open
Definition: BinningType.h:40
python.periods.split_grl
def split_grl(project_tag, periods, grl_file, out_directory)
Definition: periods.py:129
python.periods.get_period_iovset_from_ami
def get_period_iovset_from_ami(ptag, period)
Definition: periods.py:41
python.dummyaccess.exists
def exists(filename)
Definition: dummyaccess.py:9
python.periods.get_period_iovset_from_file
def get_period_iovset_from_file(project_tag, period)
Definition: periods.py:29
DerivationFramework::ClustersInCone::select
void select(const xAOD::IParticle *particle, const float coneSize, const xAOD::CaloClusterContainer *clusters, std::vector< bool > &mask)
Definition: ClustersInCone.cxx:14
Trk::split
@ split
Definition: LayerMaterialProperties.h:38
beamspotman.basename
basename
Definition: beamspotman.py:640