ATLAS Offline Software
quick_retrieve.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
3 */
4 
5 #define likely(x) __builtin_expect((x),1)
6 #define unlikely(x) __builtin_expect((x),0)
7 
8 #include <Python.h>
9 
10 #include <CoolKernel/ChannelSelection.h>
11 #include <CoolKernel/ITime.h>
12 #include <CoolKernel/IField.h>
13 #include <CoolKernel/FieldSelection.h>
14 #include <CoolKernel/CompositeSelection.h>
15 #include <CoolKernel/IObject.h>
16 #include <CoolKernel/IObjectIterator.h>
17 #include <CoolKernel/IFolder.h>
18 #include <CoolKernel/IDatabase.h>
19 #include <CoolKernel/IDatabaseSvc.h>
20 
21 #include <CoolApplication/DatabaseSvcFactory.h>
22 
23 #include <functional>
24 
25 #include <vector>
26 #include <string>
27 #include <iostream>
28 #include <stdexcept>
29 
32 
33 using std::cout;
34 using std::endl;
35 using std::string;
36 using std::vector;
37 
38 using std::bind;
39 using std::placeholders::_1;
40 
41 using cool::DatabaseSvcFactory;
42 using cool::IDatabasePtr;
43 using cool::IField;
44 using cool::IObject;
45 using cool::IFolder;
46 using cool::IFolderPtr;
47 using cool::ChannelSelection;
48 using cool::IObjectIteratorPtr;
49 using cool::IObjectVectorPtr;
50 using cool::ValidityKey;
51 using cool::ITime;
52 
53 cool::IRecordSelection* make_fieldselection(
54  const std::string& name,
55  const cool::StorageType::TypeId typeId,
56  cool::FieldSelection::Relation relation,
57  PyObject* refValue)
58 {
59  #define MAKE_FS(type, converter) \
60  if (typeId == cool::StorageType::type) \
61  return new cool::FieldSelection(name, typeId, relation, \
62  static_cast<cool::type>(converter(refValue)));
63 
64  MAKE_FS(Bool, PyLong_AsLong)
65  MAKE_FS(Float, PyFloat_AsDouble)
66  MAKE_FS(Double, PyFloat_AsDouble)
67 
68  MAKE_FS(UChar, PyLong_AsUnsignedLong)
69  MAKE_FS(Int16, PyLong_AsLong)
70  MAKE_FS(UInt16, PyLong_AsUnsignedLong)
71  MAKE_FS(Int32, PyLong_AsLong)
72  MAKE_FS(UInt32, PyLong_AsUnsignedLong)
73  MAKE_FS(Int64, PyLong_AsLongLong)
74  MAKE_FS(UInt63, PyLong_AsUnsignedLongLong)
75 
76  MAKE_FS(String255, _PyUnicode_AsString)
77  MAKE_FS(String4k, _PyUnicode_AsString)
78  MAKE_FS(String64k, _PyUnicode_AsString)
79  MAKE_FS(String16M, _PyUnicode_AsString)
80 
81  //MAKE_FS(Blob16M, PyString_AsString)
82  //MAKE_FS(Blob64k, PyString_AsString)
83  throw (std::runtime_error("Unsupported cool type encountered in python conversion"));
84 }
85 
86 vector<const cool::IRecordSelection*> make_selection_vector()
87 {
88  return vector<const cool::IRecordSelection*>();
89 
90 }
91 
92 // A function taking an IObject and returning a PyObject*
93 typedef std::function<PyObject* (const IObject&)> payload_fetcher_t;
94 
95 // A function to signal that a conversion object could not be found
96 PyObject *no_conversion_available(const IObject&) {return NULL;}
97 
98 // Helpers to create python strings/byte objects from Blob and std::string
100 {
101  const char* data = reinterpret_cast<const char*>(blob.startingAddress());
102  return PyBytes_FromStringAndSize(data, blob.size());
103 }
104 
106 {
107  return PyUnicode_FromStringAndSize(str.c_str(), str.size());
108 }
109 
111 {
112  return PyUnicode_DecodeUTF8(str.c_str(), str.size(), "backslashreplace");
113 }
114 
115 // payload_fetcher takes an IObject `o` to be retrieved from, a `name` to
116 // retrieve and a converter_function which can take a COOL type and create a
117 // python object for it.
118 template<typename T, typename FUNC_TYPE>
119 inline PyObject* payload_fetcher(const IObject& o, const string& name,
120  FUNC_TYPE* converter_function)
121 {
122  // We have to call this, because if one calls payloadValue on a null record,
123  // it throws an exception.
124  const IField& field = o.payload()[name];
125  if (field.isNull())
126  Py_RETURN_NONE;
127  //cppcheck-suppress CastIntegerToAddressAtReturn
128  return converter_function(field.data<T>());
129 }
130 
131 // create_payload_fetcher is called for each field being retrieved from COOL,
132 // once for a given browseObjects() on the first object.
134  const string& type_name,
135  bool string_to_unicode = false)
136 {
137  // A macro which takes a cool `type` and a python api function which returns
138  // a PyObject*, `converter`.
139  #define MAKE_FETCHER(type, converter) \
140  if (type_name == #type) \
141  return bind(payload_fetcher<cool::type, decltype(converter)>, \
142  _1, name, &converter);
143 
144  // See the python c-api reference for python conversion functions
145  // Python/C API Reference Manual >> Concrete Objects Layer
146  // http://docs.python.org/c-api/concrete.html
147 
148  MAKE_FETCHER(Bool, PyBool_FromLong)
149  MAKE_FETCHER(Float, PyFloat_FromDouble)
150  MAKE_FETCHER(Double, PyFloat_FromDouble)
151  MAKE_FETCHER(UChar, PyLong_FromLong)
152  MAKE_FETCHER(Int16, PyLong_FromLong)
153  MAKE_FETCHER(UInt16, PyLong_FromUnsignedLong)
154  MAKE_FETCHER(Int32, PyLong_FromLong)
155  MAKE_FETCHER(UInt32, PyLong_FromUnsignedLong)
156  MAKE_FETCHER(Int64, PyLong_FromLong)
157  MAKE_FETCHER(UInt63, PyLong_FromUnsignedLongLong)
158  MAKE_FETCHER(UInt64, PyLong_FromUnsignedLongLong)
159  if (string_to_unicode)
160  {
165  } else
166  {
171  }
174 
175  PyErr_Format(PyExc_RuntimeError,
176  "Type '%s' is not in type conversion table. "
177  "Please add it to quick_retrieve.cxx. "
178  "Can't convert field '%s'.",
179  type_name.c_str(),
180  name);
182 }
183 
184 // Scope guard IObjectIterator so that resource is freed even on exception
186 {
187  const IObjectIteratorPtr& m_iterator;
188 public:
189  IObjectIterator_Guard(const IObjectIteratorPtr& iterator)
190  : m_iterator(iterator) {}
192 };
193 
194 inline PyObject* make_iov_key(PyObject *iovkey_wrapper,
195  unsigned long long value)
196 {
197  static const char * const argtypes = const_cast<char *>("K");
198  if (iovkey_wrapper && iovkey_wrapper != Py_None)
199  return PyObject_CallFunction(iovkey_wrapper, argtypes, value);
200  return PyLong_FromUnsignedLongLong(value);
201 }
202 
203 inline PyObject *apply_function(PyObject *function, PyObject *object)
204 {
205  // Convert object according to function, taking care of references
206  // If function is null, return unmodified object
207  if (!function || function == Py_None) return object;
208  PyObject *old_object = object;
209  PyObject *new_object = PyObject_CallObject(function, object);
210  Py_DECREF(old_object);
211  return new_object;
212 }
213 
214 // Quickly retrieve IObjects from an IObjectIterator
215 // to_fetch is a list of strings specifying which variables to look at
216 // This function closes the iterator when it completes.
217 // The return result is a list of tuples
218 // [(since, until, channel, ...) ] where ... are variables specified by to_fetch
219 PyObject* quick_retrieve(const IObjectIteratorPtr& objects,
220  PyObject *object_converter,
221  PyObject *to_fetch = NULL,
222  const long max_records = -1,
223  const bool with_channel = true,
224  const bool loud = false,
225  PyObject *iovkey_wrapper = NULL,
226  PyObject *channel_name_mapping = NULL,
227  const bool with_time = false,
228  const bool as_unicode = false)
229 {
230  IObjectIterator_Guard closeiterator_guard(objects);
231 
232  PyObject* result = PyList_New(0);
233 
234  const long long records = objects->size();
235  const Py_ssize_t count = to_fetch ? PySequence_Size(to_fetch) : 0;
236  const unsigned int fetch_start = 2 + with_channel + with_time;
237  bool first = true;
238  int j = 0;
239  PyObject *py_datetime_class = NULL;
240 
241  vector<payload_fetcher_t> payload_fetchers;
242 
243  while (objects->goToNext())
244  {
245  const IObject& object = objects->currentRef();
246 
247  if (unlikely(first))
248  {
249  // On the first iteration, compute the types in to_fetch, and bind
250  // them into the payload_fetchers.
251  first = false;
252  for (Py_ssize_t i = 0; i < count; i++)
253  {
254  PyObject *py_name = PySequence_GetItem(to_fetch, i);
255  const char *name = _PyUnicode_AsString(py_name);
256  const string type = (object.payload()
257  .specification()[name]
258  .storageType()
259  .name());
260 
262  as_unicode);
263  auto pff = pf.target<PyObject* (*)(const IObject&)>();
264  if ( pff && *pff == &no_conversion_available)
265  return NULL;
266  payload_fetchers.push_back(pf);
267  Py_DECREF(py_name);
268  }
269 
270  if (with_time)
271  {
272  PyObject *py_datetime_module = PyImport_ImportModule("datetime");
273  if (!py_datetime_module)
274  throw (std::runtime_error("Could not import python datetime_module"));
275 
276  py_datetime_class = PyObject_GetAttrString(py_datetime_module,
277  "datetime");
278 
279  Py_DECREF(py_datetime_module);
280  }
281  }
282 
283  PyObject *one = PyTuple_New(fetch_start + count);
284 
285  // Fetch (since, until, channel[optional])
286 
287  PyTuple_SET_ITEM(one, 0, make_iov_key(iovkey_wrapper, object.since()));
288  PyTuple_SET_ITEM(one, 1, make_iov_key(iovkey_wrapper, object.until()));
289 
290  if (with_time)
291  {
292  const ITime& t = object.insertionTime();
293  static const char * const argtypes = const_cast<char *>("iiiiiil");
294 
295  PyObject *py_record_time =
296  PyObject_CallFunction(py_datetime_class, argtypes,
297  t.year(), t.month(), t.day(),
298  t.hour(), t.minute(), t.second(), t.nanosecond() / 1000);
299 
300  PyTuple_SetItem(one, 2, py_record_time);
301  }
302 
303  if (with_channel)
304  {
305  if (channel_name_mapping)
306  {
307  // Use a channel name if it is available, otherwise fall back
308  // to standard channelid.
309  PyObject *channelId = PyLong_FromUnsignedLong(object.channelId());
310  PyObject *channelName = PyDict_GetItem(channel_name_mapping,
311  channelId);
312  if (channelName)
313  {
314  Py_INCREF(channelName);
315  Py_DECREF(channelId);
316  PyTuple_SET_ITEM(one, 2+with_time, channelName);
317  } else
318  PyTuple_SET_ITEM(one, 2+with_time, channelId);
319 
320  } else
321  PyTuple_SET_ITEM(one, 2+with_time,
322  PyLong_FromLong(object.channelId()));
323  }
324 
325  // Fetch the objects specified in to_fetch
326  for (Py_ssize_t i = 0; i < count; i++)
327  PyTuple_SET_ITEM(one, fetch_start + i, payload_fetchers[i](object));
328 
329  one = apply_function(object_converter, one);
330  if (!one)
331  {
332  // apply_function returned an error
333  // We won't return the list to python so we need to tell python it
334  // can be deleted
335  Py_DECREF(result);
336  return NULL;
337  }
338 
339  PyList_Append(result, one);
340  Py_DECREF(one);
341 
342  // Print status if loud is active
343  if (unlikely(loud && ((++j) % 1000 == 0)))
344  {
345  cout << "Progress: " << j << " / " << records << "\r";
346  cout.flush();
347  }
348 
349  if (unlikely(max_records > 0 && j >= max_records))
350  break;
351  }
352 
353  if (loud)
354  cout << "Done fetching " << j << " records.\x1B[K" << endl;
355 
356  if (py_datetime_class)
357  {
358  Py_DECREF(py_datetime_class);
359  }
360 
361  return result;
362 }
363 
xAOD::iterator
JetConstituentVector::iterator iterator
Definition: JetConstituentVector.cxx:68
create_payload_fetcher
payload_fetcher_t create_payload_fetcher(const char *name, const string &type_name, bool string_to_unicode=false)
Definition: quick_retrieve.cxx:133
data
char data[hepevt_bytes_allocation_ATLAS]
Definition: HepEvt.cxx:11
get_generator_info.result
result
Definition: get_generator_info.py:21
LArConditions2Ntuple.objects
objects
Definition: LArConditions2Ntuple.py:64
unlikely
#define unlikely(x)
Definition: quick_retrieve.cxx:6
make_fieldselection
cool::IRecordSelection * make_fieldselection(const std::string &name, const cool::StorageType::TypeId typeId, cool::FieldSelection::Relation relation, PyObject *refValue)
Definition: quick_retrieve.cxx:53
Trk::one
@ one
Definition: TrkDetDescr/TrkSurfaces/TrkSurfaces/RealQuadraticEquation.h:22
python.subdetectors.tile.Blob
Blob
Definition: tile.py:17
qr_PyString_FromStdString
PyObject * qr_PyString_FromStdString(const string &str)
Definition: quick_retrieve.cxx:105
athena.value
value
Definition: athena.py:124
MAKE_FETCHER
#define MAKE_FETCHER(type, converter)
payload_fetcher
PyObject * payload_fetcher(const IObject &o, const string &name, FUNC_TYPE *converter_function)
Definition: quick_retrieve.cxx:119
defaults::Float
constexpr float Float
Definition: MSVtxValidationAlg.h:45
ReadOfcFromCool.field
field
Definition: ReadOfcFromCool.py:48
read_hist_ntuple.t
t
Definition: read_hist_ntuple.py:5
python.CaloAddPedShiftConfig.type
type
Definition: CaloAddPedShiftConfig.py:42
MAKE_FS
#define MAKE_FS(type, converter)
XMLtoHeader.count
count
Definition: XMLtoHeader.py:84
quick_retrieve
PyObject * quick_retrieve(const IObjectIteratorPtr &objects, PyObject *object_converter, PyObject *to_fetch=NULL, const long max_records=-1, const bool with_channel=true, const bool loud=false, PyObject *iovkey_wrapper=NULL, PyObject *channel_name_mapping=NULL, const bool with_time=false, const bool as_unicode=false)
Definition: quick_retrieve.cxx:219
python.AthDsoLogger.argtypes
argtypes
Definition: AthDsoLogger.py:43
dq_defect_copy_defect_database.since
def since
Definition: dq_defect_copy_defect_database.py:54
dq_defect_copy_defect_database.until
def until
Definition: dq_defect_copy_defect_database.py:55
IObjectIterator_Guard::m_iterator
const IObjectIteratorPtr & m_iterator
Definition: quick_retrieve.cxx:187
qr_PyString_FromBlob
PyObject * qr_PyString_FromBlob(const coral::Blob &blob)
Definition: quick_retrieve.cxx:99
lumiFormat.i
int i
Definition: lumiFormat.py:85
qr_PyUnicode_FromStdString
PyObject * qr_PyUnicode_FromStdString(const string &str)
Definition: quick_retrieve.cxx:110
apply_function
PyObject * apply_function(PyObject *function, PyObject *object)
Definition: quick_retrieve.cxx:203
CaloCondBlobAlgs_fillNoiseFromASCII.channelId
channelId
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:121
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
no_conversion_available
PyObject * no_conversion_available(const IObject &)
Definition: quick_retrieve.cxx:96
make_iov_key
PyObject * make_iov_key(PyObject *iovkey_wrapper, unsigned long long value)
Definition: quick_retrieve.cxx:194
IObjectIterator_Guard
Definition: quick_retrieve.cxx:186
payload_fetcher_t
std::function< PyObject *(const IObject &)> payload_fetcher_t
Definition: quick_retrieve.cxx:93
python.output.AtlRunQueryRoot.pf
pf
Definition: AtlRunQueryRoot.py:988
DeMoScan.first
bool first
Definition: DeMoScan.py:534
IObjectIterator_Guard::IObjectIterator_Guard
IObjectIterator_Guard(const IObjectIteratorPtr &iterator)
Definition: quick_retrieve.cxx:189
ATLAS_NO_CHECK_FILE_THREAD_SAFETY
ATLAS_NO_CHECK_FILE_THREAD_SAFETY
Definition: quick_retrieve.cxx:31
pickleTool.object
object
Definition: pickleTool.py:29
str
Definition: BTagTrackIpAccessor.cxx:11
checker_macros.h
Define macros for attributes used to control the static checker.
make_selection_vector
vector< const cool::IRecordSelection * > make_selection_vector()
Definition: quick_retrieve.cxx:86
PyObject
_object PyObject
Definition: IPyComponent.h:26
Crest::TypeId
TypeId
Definition: CrestContainer.h:22
TSU::T
unsigned long long T
Definition: L1TopoDataTypes.h:35
CaloCondBlobAlgs_fillNoiseFromASCII.blob
blob
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:95
IObjectIterator_Guard::~IObjectIterator_Guard
~IObjectIterator_Guard()
Definition: quick_retrieve.cxx:191