ATLAS Offline Software
quick_retrieve.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
3 */
4 
5 #define likely(x) __builtin_expect((x),1)
6 #define unlikely(x) __builtin_expect((x),0)
7 
8 #include <Python.h>
9 
10 #include <CoolKernel/ChannelSelection.h>
11 #include <CoolKernel/ITime.h>
12 #include <CoolKernel/IField.h>
13 #include <CoolKernel/FieldSelection.h>
14 #include <CoolKernel/CompositeSelection.h>
15 #include <CoolKernel/IObject.h>
16 #include <CoolKernel/IObjectIterator.h>
17 #include <CoolKernel/IFolder.h>
18 #include <CoolKernel/IDatabase.h>
19 #include <CoolKernel/IDatabaseSvc.h>
20 
21 #include <CoolApplication/DatabaseSvcFactory.h>
22 
23 #include <boost/typeof/typeof.hpp>
24 #include <boost/bind/bind.hpp>
25 #include <boost/function.hpp>
26 
27 #include <vector>
28 #include <string>
29 #include <iostream>
30 #include <stdexcept>
31 
34 
35 using std::cout;
36 using std::endl;
37 using std::string;
38 using std::vector;
39 
40 using boost::bind;
41 using boost::placeholders::_1;
42 
43 using cool::DatabaseSvcFactory;
44 using cool::IDatabasePtr;
45 using cool::IField;
46 using cool::IObject;
47 using cool::IFolder;
48 using cool::IFolderPtr;
49 using cool::ChannelSelection;
50 using cool::IObjectIteratorPtr;
51 using cool::IObjectVectorPtr;
52 using cool::ValidityKey;
53 using cool::ITime;
54 
55 cool::IRecordSelection* make_fieldselection(
56  const std::string& name,
57  const cool::StorageType::TypeId typeId,
58  cool::FieldSelection::Relation relation,
59  PyObject* refValue)
60 {
61  #define MAKE_FS(type, converter) \
62  if (typeId == cool::StorageType::type) \
63  return new cool::FieldSelection(name, typeId, relation, \
64  static_cast<cool::type>(converter(refValue)));
65 
66  MAKE_FS(Bool, PyLong_AsLong)
67  MAKE_FS(Float, PyFloat_AsDouble)
68  MAKE_FS(Double, PyFloat_AsDouble)
69 
70  MAKE_FS(UChar, PyLong_AsUnsignedLong)
71  MAKE_FS(Int16, PyLong_AsLong)
72  MAKE_FS(UInt16, PyLong_AsUnsignedLong)
73  MAKE_FS(Int32, PyLong_AsLong)
74  MAKE_FS(UInt32, PyLong_AsUnsignedLong)
75  MAKE_FS(Int64, PyLong_AsLongLong)
76  MAKE_FS(UInt63, PyLong_AsUnsignedLongLong)
77 
78  MAKE_FS(String255, _PyUnicode_AsString)
79  MAKE_FS(String4k, _PyUnicode_AsString)
80  MAKE_FS(String64k, _PyUnicode_AsString)
81  MAKE_FS(String16M, _PyUnicode_AsString)
82 
83  //MAKE_FS(Blob16M, PyString_AsString)
84  //MAKE_FS(Blob64k, PyString_AsString)
85  throw (std::runtime_error("Unsupported cool type encountered in python conversion"));
86 }
87 
88 vector<const cool::IRecordSelection*> make_selection_vector()
89 {
90  return vector<const cool::IRecordSelection*>();
91 
92 }
93 
94 // A function taking an IObject and returning a PyObject*
95 typedef boost::function<PyObject* (const IObject&)> payload_fetcher_t;
96 
97 // A function to signal that a conversion object could not be found
98 PyObject *no_conversion_available(const IObject&) {return NULL;}
99 
100 // Helpers to create python strings/byte objects from Blob and std::string
102 {
103  const char* data = reinterpret_cast<const char*>(blob.startingAddress());
104  return PyBytes_FromStringAndSize(data, blob.size());
105 }
106 
108 {
109  return PyUnicode_FromStringAndSize(str.c_str(), str.size());
110 }
111 
113 {
114  return PyUnicode_DecodeUTF8(str.c_str(), str.size(), "backslashreplace");
115 }
116 
117 // payload_fetcher takes an IObject `o` to be retrieved from, a `name` to
118 // retrieve and a converter_function which can take a COOL type and create a
119 // python object for it.
120 template<typename T, typename FUNC_TYPE>
121 inline PyObject* payload_fetcher(const IObject& o, const string& name,
122  FUNC_TYPE* converter_function)
123 {
124  // We have to call this, because if one calls payloadValue on a null record,
125  // it throws an exception.
126  const IField& field = o.payload()[name];
127  if (field.isNull())
128  Py_RETURN_NONE;
129  //cppcheck-suppress CastIntegerToAddressAtReturn
130  return converter_function(field.data<T>());
131 }
132 
133 // create_payload_fetcher is called for each field being retrieved from COOL,
134 // once for a given browseObjects() on the first object.
136  const string& type_name,
137  bool string_to_unicode = false)
138 {
139  // A macro which takes a cool `type` and a python api function which returns
140  // a PyObject*, `converter`.
141  #define MAKE_FETCHER(type, converter) \
142  if (type_name == #type) \
143  return bind(payload_fetcher<cool::type, BOOST_TYPEOF(converter)>, \
144  _1, name, &converter);
145 
146  // See the python c-api reference for python conversion functions
147  // Python/C API Reference Manual >> Concrete Objects Layer
148  // http://docs.python.org/c-api/concrete.html
149 
150  MAKE_FETCHER(Bool, PyBool_FromLong)
151  MAKE_FETCHER(Float, PyFloat_FromDouble)
152  MAKE_FETCHER(Double, PyFloat_FromDouble)
153  MAKE_FETCHER(UChar, PyLong_FromLong)
154  MAKE_FETCHER(Int16, PyLong_FromLong)
155  MAKE_FETCHER(UInt16, PyLong_FromUnsignedLong)
156  MAKE_FETCHER(Int32, PyLong_FromLong)
157  MAKE_FETCHER(UInt32, PyLong_FromUnsignedLong)
158  MAKE_FETCHER(Int64, PyLong_FromLong)
159  MAKE_FETCHER(UInt63, PyLong_FromUnsignedLongLong)
160  MAKE_FETCHER(UInt64, PyLong_FromUnsignedLongLong)
161  if (string_to_unicode)
162  {
167  } else
168  {
173  }
176 
177  PyErr_Format(PyExc_RuntimeError,
178  "Type '%s' is not in type conversion table. "
179  "Please add it to quick_retrieve.cxx. "
180  "Can't convert field '%s'.",
181  type_name.c_str(),
182  name);
184 }
185 
186 // Scope guard IObjectIterator so that resource is freed even on exception
188 {
189  const IObjectIteratorPtr& m_iterator;
190 public:
191  IObjectIterator_Guard(const IObjectIteratorPtr& iterator)
192  : m_iterator(iterator) {}
194 };
195 
196 inline PyObject* make_iov_key(PyObject *iovkey_wrapper,
197  unsigned long long value)
198 {
199  static const char * const argtypes = const_cast<char *>("K");
200  if (iovkey_wrapper && iovkey_wrapper != Py_None)
201  return PyObject_CallFunction(iovkey_wrapper, argtypes, value);
202  return PyLong_FromUnsignedLongLong(value);
203 }
204 
205 inline PyObject *apply_function(PyObject *function, PyObject *object)
206 {
207  // Convert object according to function, taking care of references
208  // If function is null, return unmodified object
209  if (!function || function == Py_None) return object;
210  PyObject *old_object = object;
211  PyObject *new_object = PyObject_CallObject(function, object);
212  Py_DECREF(old_object);
213  return new_object;
214 }
215 
216 // Quickly retrieve IObjects from an IObjectIterator
217 // to_fetch is a list of strings specifying which variables to look at
218 // This function closes the iterator when it completes.
219 // The return result is a list of tuples
220 // [(since, until, channel, ...) ] where ... are variables specified by to_fetch
221 PyObject* quick_retrieve(const IObjectIteratorPtr& objects,
222  PyObject *object_converter,
223  PyObject *to_fetch = NULL,
224  const long max_records = -1,
225  const bool with_channel = true,
226  const bool loud = false,
227  PyObject *iovkey_wrapper = NULL,
228  PyObject *channel_name_mapping = NULL,
229  const bool with_time = false,
230  const bool as_unicode = false)
231 {
232  IObjectIterator_Guard closeiterator_guard(objects);
233 
234  PyObject* result = PyList_New(0);
235 
236  const long long records = objects->size();
237  const Py_ssize_t count = to_fetch ? PySequence_Size(to_fetch) : 0;
238  const unsigned int fetch_start = 2 + with_channel + with_time;
239  bool first = true;
240  int j = 0;
241  PyObject *py_datetime_class = NULL;
242 
243  vector<payload_fetcher_t> payload_fetchers;
244 
245  while (objects->goToNext())
246  {
247  const IObject& object = objects->currentRef();
248 
249  if (unlikely(first))
250  {
251  // On the first iteration, compute the types in to_fetch, and bind
252  // them into the payload_fetchers.
253  first = false;
254  for (Py_ssize_t i = 0; i < count; i++)
255  {
256  PyObject *py_name = PySequence_GetItem(to_fetch, i);
257  const char *name = _PyUnicode_AsString(py_name);
258  const string type = (object.payload()
259  .specification()[name]
260  .storageType()
261  .name());
262 
264  as_unicode);
266  return NULL;
267  payload_fetchers.push_back(pf);
268  Py_DECREF(py_name);
269  }
270 
271  if (with_time)
272  {
273  PyObject *py_datetime_module = PyImport_ImportModule("datetime");
274  if (!py_datetime_module)
275  throw (std::runtime_error("Could not import python datetime_module"));
276 
277  py_datetime_class = PyObject_GetAttrString(py_datetime_module,
278  "datetime");
279 
280  Py_DECREF(py_datetime_module);
281  }
282  }
283 
284  PyObject *one = PyTuple_New(fetch_start + count);
285 
286  // Fetch (since, until, channel[optional])
287 
288  PyTuple_SET_ITEM(one, 0, make_iov_key(iovkey_wrapper, object.since()));
289  PyTuple_SET_ITEM(one, 1, make_iov_key(iovkey_wrapper, object.until()));
290 
291  if (with_time)
292  {
293  const ITime& t = object.insertionTime();
294  static const char * const argtypes = const_cast<char *>("iiiiiil");
295 
296  PyObject *py_record_time =
297  PyObject_CallFunction(py_datetime_class, argtypes,
298  t.year(), t.month(), t.day(),
299  t.hour(), t.minute(), t.second(), t.nanosecond() / 1000);
300 
301  PyTuple_SetItem(one, 2, py_record_time);
302  }
303 
304  if (with_channel)
305  {
306  if (channel_name_mapping)
307  {
308  // Use a channel name if it is available, otherwise fall back
309  // to standard channelid.
310  PyObject *channelId = PyLong_FromUnsignedLong(object.channelId());
311  PyObject *channelName = PyDict_GetItem(channel_name_mapping,
312  channelId);
313  if (channelName)
314  {
315  Py_INCREF(channelName);
316  Py_DECREF(channelId);
317  PyTuple_SET_ITEM(one, 2+with_time, channelName);
318  } else
319  PyTuple_SET_ITEM(one, 2+with_time, channelId);
320 
321  } else
322  PyTuple_SET_ITEM(one, 2+with_time,
323  PyLong_FromLong(object.channelId()));
324  }
325 
326  // Fetch the objects specified in to_fetch
327  for (Py_ssize_t i = 0; i < count; i++)
328  PyTuple_SET_ITEM(one, fetch_start + i, payload_fetchers[i](object));
329 
330  one = apply_function(object_converter, one);
331  if (!one)
332  {
333  // apply_function returned an error
334  // We won't return the list to python so we need to tell python it
335  // can be deleted
336  Py_DECREF(result);
337  return NULL;
338  }
339 
340  PyList_Append(result, one);
341  Py_DECREF(one);
342 
343  // Print status if loud is active
344  if (unlikely(loud && ((++j) % 1000 == 0)))
345  {
346  cout << "Progress: " << j << " / " << records << "\r";
347  cout.flush();
348  }
349 
350  if (unlikely(max_records > 0 && j >= max_records))
351  break;
352  }
353 
354  if (loud)
355  cout << "Done fetching " << j << " records.\x1B[K" << endl;
356 
357  if (py_datetime_class)
358  {
359  Py_DECREF(py_datetime_class);
360  }
361 
362  return result;
363 }
364 
xAOD::iterator
JetConstituentVector::iterator iterator
Definition: JetConstituentVector.cxx:68
create_payload_fetcher
payload_fetcher_t create_payload_fetcher(const char *name, const string &type_name, bool string_to_unicode=false)
Definition: quick_retrieve.cxx:135
data
char data[hepevt_bytes_allocation_ATLAS]
Definition: HepEvt.cxx:11
get_generator_info.result
result
Definition: get_generator_info.py:21
LArConditions2Ntuple.objects
objects
Definition: LArConditions2Ntuple.py:64
unlikely
#define unlikely(x)
Definition: quick_retrieve.cxx:6
make_fieldselection
cool::IRecordSelection * make_fieldselection(const std::string &name, const cool::StorageType::TypeId typeId, cool::FieldSelection::Relation relation, PyObject *refValue)
Definition: quick_retrieve.cxx:55
Trk::one
@ one
Definition: TrkDetDescr/TrkSurfaces/TrkSurfaces/RealQuadraticEquation.h:22
python.subdetectors.tile.Blob
Blob
Definition: tile.py:17
qr_PyString_FromStdString
PyObject * qr_PyString_FromStdString(const string &str)
Definition: quick_retrieve.cxx:107
athena.value
value
Definition: athena.py:124
MAKE_FETCHER
#define MAKE_FETCHER(type, converter)
payload_fetcher
PyObject * payload_fetcher(const IObject &o, const string &name, FUNC_TYPE *converter_function)
Definition: quick_retrieve.cxx:121
ReadOfcFromCool.field
field
Definition: ReadOfcFromCool.py:48
read_hist_ntuple.t
t
Definition: read_hist_ntuple.py:5
python.CaloAddPedShiftConfig.type
type
Definition: CaloAddPedShiftConfig.py:42
MAKE_FS
#define MAKE_FS(type, converter)
XMLtoHeader.count
count
Definition: XMLtoHeader.py:84
quick_retrieve
PyObject * quick_retrieve(const IObjectIteratorPtr &objects, PyObject *object_converter, PyObject *to_fetch=NULL, const long max_records=-1, const bool with_channel=true, const bool loud=false, PyObject *iovkey_wrapper=NULL, PyObject *channel_name_mapping=NULL, const bool with_time=false, const bool as_unicode=false)
Definition: quick_retrieve.cxx:221
python.AthDsoLogger.argtypes
argtypes
Definition: AthDsoLogger.py:43
dq_defect_copy_defect_database.since
def since
Definition: dq_defect_copy_defect_database.py:54
dq_defect_copy_defect_database.until
def until
Definition: dq_defect_copy_defect_database.py:55
IObjectIterator_Guard::m_iterator
const IObjectIteratorPtr & m_iterator
Definition: quick_retrieve.cxx:189
qr_PyString_FromBlob
PyObject * qr_PyString_FromBlob(const coral::Blob &blob)
Definition: quick_retrieve.cxx:101
lumiFormat.i
int i
Definition: lumiFormat.py:85
qr_PyUnicode_FromStdString
PyObject * qr_PyUnicode_FromStdString(const string &str)
Definition: quick_retrieve.cxx:112
apply_function
PyObject * apply_function(PyObject *function, PyObject *object)
Definition: quick_retrieve.cxx:205
CaloCondBlobAlgs_fillNoiseFromASCII.channelId
channelId
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:121
payload_fetcher_t
boost::function< PyObject *(const IObject &)> payload_fetcher_t
Definition: quick_retrieve.cxx:95
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
no_conversion_available
PyObject * no_conversion_available(const IObject &)
Definition: quick_retrieve.cxx:98
make_iov_key
PyObject * make_iov_key(PyObject *iovkey_wrapper, unsigned long long value)
Definition: quick_retrieve.cxx:196
IObjectIterator_Guard
Definition: quick_retrieve.cxx:188
python.output.AtlRunQueryRoot.pf
pf
Definition: AtlRunQueryRoot.py:988
DeMoScan.first
bool first
Definition: DeMoScan.py:534
IObjectIterator_Guard::IObjectIterator_Guard
IObjectIterator_Guard(const IObjectIteratorPtr &iterator)
Definition: quick_retrieve.cxx:191
ATLAS_NO_CHECK_FILE_THREAD_SAFETY
ATLAS_NO_CHECK_FILE_THREAD_SAFETY
Definition: quick_retrieve.cxx:33
pickleTool.object
object
Definition: pickleTool.py:29
str
Definition: BTagTrackIpAccessor.cxx:11
checker_macros.h
Define macros for attributes used to control the static checker.
make_selection_vector
vector< const cool::IRecordSelection * > make_selection_vector()
Definition: quick_retrieve.cxx:88
PyObject
_object PyObject
Definition: IPyComponent.h:26
Crest::TypeId
TypeId
Definition: CrestContainer.h:22
TSU::T
unsigned long long T
Definition: L1TopoDataTypes.h:35
CaloCondBlobAlgs_fillNoiseFromASCII.blob
blob
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:95
IObjectIterator_Guard::~IObjectIterator_Guard
~IObjectIterator_Guard()
Definition: quick_retrieve.cxx:193