ATLAS Offline Software
Classes | Macros | Typedefs | Functions | Variables
quick_retrieve.cxx File Reference
#include <Python.h>
#include <CoolKernel/ChannelSelection.h>
#include <CoolKernel/ITime.h>
#include <CoolKernel/IField.h>
#include <CoolKernel/FieldSelection.h>
#include <CoolKernel/CompositeSelection.h>
#include <CoolKernel/IObject.h>
#include <CoolKernel/IObjectIterator.h>
#include <CoolKernel/IFolder.h>
#include <CoolKernel/IDatabase.h>
#include <CoolKernel/IDatabaseSvc.h>
#include <CoolApplication/DatabaseSvcFactory.h>
#include <functional>
#include <vector>
#include <string>
#include <iostream>
#include <stdexcept>
#include "CxxUtils/checker_macros.h"

Go to the source code of this file.

Classes

class  IObjectIterator_Guard
 

Macros

#define likely(x)   __builtin_expect((x),1)
 
#define unlikely(x)   __builtin_expect((x),0)
 
#define MAKE_FS(type, converter)
 
#define MAKE_FETCHER(type, converter)
 

Typedefs

typedef std::function< PyObject *(const IObject &)> payload_fetcher_t
 

Functions

cool::IRecordSelection * make_fieldselection (const std::string &name, const cool::StorageType::TypeId typeId, cool::FieldSelection::Relation relation, PyObject *refValue)
 
vector< const cool::IRecordSelection * > make_selection_vector ()
 
PyObjectno_conversion_available (const IObject &)
 
PyObjectqr_PyString_FromBlob (const coral::Blob &blob)
 
PyObjectqr_PyString_FromStdString (const string &str)
 
PyObjectqr_PyUnicode_FromStdString (const string &str)
 
template<typename T , typename FUNC_TYPE >
PyObjectpayload_fetcher (const IObject &o, const string &name, FUNC_TYPE *converter_function)
 
payload_fetcher_t create_payload_fetcher (const char *name, const string &type_name, bool string_to_unicode=false)
 
PyObjectmake_iov_key (PyObject *iovkey_wrapper, unsigned long long value)
 
PyObjectapply_function (PyObject *function, PyObject *object)
 
PyObjectquick_retrieve (const IObjectIteratorPtr &objects, PyObject *object_converter, PyObject *to_fetch=NULL, const long max_records=-1, const bool with_channel=true, const bool loud=false, PyObject *iovkey_wrapper=NULL, PyObject *channel_name_mapping=NULL, const bool with_time=false, const bool as_unicode=false)
 

Variables

 ATLAS_NO_CHECK_FILE_THREAD_SAFETY
 

Macro Definition Documentation

◆ likely

#define likely (   x)    __builtin_expect((x),1)

Definition at line 6 of file quick_retrieve.cxx.

◆ MAKE_FETCHER

#define MAKE_FETCHER (   type,
  converter 
)
Value:
if (type_name == #type) \
return bind(payload_fetcher<cool::type, decltype(converter)>, \
_1, name, &converter);

◆ MAKE_FS

#define MAKE_FS (   type,
  converter 
)
Value:
if (typeId == cool::StorageType::type) \
return new cool::FieldSelection(name, typeId, relation, \
static_cast<cool::type>(converter(refValue)));

◆ unlikely

#define unlikely (   x)    __builtin_expect((x),0)

Definition at line 7 of file quick_retrieve.cxx.

Typedef Documentation

◆ payload_fetcher_t

typedef std::function<PyObject* (const IObject&)> payload_fetcher_t

Definition at line 93 of file quick_retrieve.cxx.

Function Documentation

◆ apply_function()

PyObject* apply_function ( PyObject function,
PyObject object 
)
inline

Definition at line 203 of file quick_retrieve.cxx.

204 {
205  // Convert object according to function, taking care of references
206  // If function is null, return unmodified object
207  if (!function || function == Py_None) return object;
208  PyObject *old_object = object;
209  PyObject *new_object = PyObject_CallObject(function, object);
210  Py_DECREF(old_object);
211  return new_object;
212 }

◆ create_payload_fetcher()

payload_fetcher_t create_payload_fetcher ( const char *  name,
const string &  type_name,
bool  string_to_unicode = false 
)

Definition at line 133 of file quick_retrieve.cxx.

136 {
137  // A macro which takes a cool `type` and a python api function which returns
138  // a PyObject*, `converter`.
139  #define MAKE_FETCHER(type, converter) \
140  if (type_name == #type) \
141  return bind(payload_fetcher<cool::type, decltype(converter)>, \
142  _1, name, &converter);
143 
144  // See the python c-api reference for python conversion functions
145  // Python/C API Reference Manual >> Concrete Objects Layer
146  // http://docs.python.org/c-api/concrete.html
147 
148  MAKE_FETCHER(Bool, PyBool_FromLong)
149  MAKE_FETCHER(Float, PyFloat_FromDouble)
150  MAKE_FETCHER(Double, PyFloat_FromDouble)
151  MAKE_FETCHER(UChar, PyLong_FromLong)
152  MAKE_FETCHER(Int16, PyLong_FromLong)
153  MAKE_FETCHER(UInt16, PyLong_FromUnsignedLong)
154  MAKE_FETCHER(Int32, PyLong_FromLong)
155  MAKE_FETCHER(UInt32, PyLong_FromUnsignedLong)
156  MAKE_FETCHER(Int64, PyLong_FromLong)
157  MAKE_FETCHER(UInt63, PyLong_FromUnsignedLongLong)
158  MAKE_FETCHER(UInt64, PyLong_FromUnsignedLongLong)
159  if (string_to_unicode)
160  {
165  } else
166  {
171  }
174 
175  PyErr_Format(PyExc_RuntimeError,
176  "Type '%s' is not in type conversion table. "
177  "Please add it to quick_retrieve.cxx. "
178  "Can't convert field '%s'.",
179  type_name.c_str(),
180  name);
182 }

◆ make_fieldselection()

cool::IRecordSelection* make_fieldselection ( const std::string &  name,
const cool::StorageType::TypeId  typeId,
cool::FieldSelection::Relation  relation,
PyObject refValue 
)

Definition at line 53 of file quick_retrieve.cxx.

58 {
59  #define MAKE_FS(type, converter) \
60  if (typeId == cool::StorageType::type) \
61  return new cool::FieldSelection(name, typeId, relation, \
62  static_cast<cool::type>(converter(refValue)));
63 
64  MAKE_FS(Bool, PyLong_AsLong)
65  MAKE_FS(Float, PyFloat_AsDouble)
66  MAKE_FS(Double, PyFloat_AsDouble)
67 
68  MAKE_FS(UChar, PyLong_AsUnsignedLong)
69  MAKE_FS(Int16, PyLong_AsLong)
70  MAKE_FS(UInt16, PyLong_AsUnsignedLong)
71  MAKE_FS(Int32, PyLong_AsLong)
72  MAKE_FS(UInt32, PyLong_AsUnsignedLong)
73  MAKE_FS(Int64, PyLong_AsLongLong)
74  MAKE_FS(UInt63, PyLong_AsUnsignedLongLong)
75 
76  MAKE_FS(String255, _PyUnicode_AsString)
77  MAKE_FS(String4k, _PyUnicode_AsString)
78  MAKE_FS(String64k, _PyUnicode_AsString)
79  MAKE_FS(String16M, _PyUnicode_AsString)
80 
81  //MAKE_FS(Blob16M, PyString_AsString)
82  //MAKE_FS(Blob64k, PyString_AsString)
83  throw (std::runtime_error("Unsupported cool type encountered in python conversion"));
84 }

◆ make_iov_key()

PyObject* make_iov_key ( PyObject iovkey_wrapper,
unsigned long long  value 
)
inline

Definition at line 194 of file quick_retrieve.cxx.

196 {
197  static const char * const argtypes = const_cast<char *>("K");
198  if (iovkey_wrapper && iovkey_wrapper != Py_None)
199  return PyObject_CallFunction(iovkey_wrapper, argtypes, value);
200  return PyLong_FromUnsignedLongLong(value);
201 }

◆ make_selection_vector()

vector<const cool::IRecordSelection*> make_selection_vector ( )

Definition at line 86 of file quick_retrieve.cxx.

87 {
88  return vector<const cool::IRecordSelection*>();
89 
90 }

◆ no_conversion_available()

PyObject* no_conversion_available ( const IObject &  )

Definition at line 96 of file quick_retrieve.cxx.

96 {return NULL;}

◆ payload_fetcher()

template<typename T , typename FUNC_TYPE >
PyObject* payload_fetcher ( const IObject &  o,
const string &  name,
FUNC_TYPE *  converter_function 
)
inline

Definition at line 119 of file quick_retrieve.cxx.

121 {
122  // We have to call this, because if one calls payloadValue on a null record,
123  // it throws an exception.
124  const IField& field = o.payload()[name];
125  if (field.isNull())
126  Py_RETURN_NONE;
127  //cppcheck-suppress CastIntegerToAddressAtReturn
128  return converter_function(field.data<T>());
129 }

◆ qr_PyString_FromBlob()

PyObject* qr_PyString_FromBlob ( const coral::Blob &  blob)

Definition at line 99 of file quick_retrieve.cxx.

100 {
101  const char* data = reinterpret_cast<const char*>(blob.startingAddress());
102  return PyBytes_FromStringAndSize(data, blob.size());
103 }

◆ qr_PyString_FromStdString()

PyObject* qr_PyString_FromStdString ( const string &  str)

Definition at line 105 of file quick_retrieve.cxx.

106 {
107  return PyUnicode_FromStringAndSize(str.c_str(), str.size());
108 }

◆ qr_PyUnicode_FromStdString()

PyObject* qr_PyUnicode_FromStdString ( const string &  str)

Definition at line 110 of file quick_retrieve.cxx.

111 {
112  return PyUnicode_DecodeUTF8(str.c_str(), str.size(), "backslashreplace");
113 }

◆ quick_retrieve()

PyObject* quick_retrieve ( const IObjectIteratorPtr &  objects,
PyObject object_converter,
PyObject to_fetch = NULL,
const long  max_records = -1,
const bool  with_channel = true,
const bool  loud = false,
PyObject iovkey_wrapper = NULL,
PyObject channel_name_mapping = NULL,
const bool  with_time = false,
const bool  as_unicode = false 
)

Definition at line 219 of file quick_retrieve.cxx.

229 {
230  IObjectIterator_Guard closeiterator_guard(objects);
231 
232  PyObject* result = PyList_New(0);
233 
234  const long long records = objects->size();
235  const Py_ssize_t count = to_fetch ? PySequence_Size(to_fetch) : 0;
236  const unsigned int fetch_start = 2 + with_channel + with_time;
237  bool first = true;
238  int j = 0;
239  PyObject *py_datetime_class = NULL;
240 
241  vector<payload_fetcher_t> payload_fetchers;
242 
243  while (objects->goToNext())
244  {
245  const IObject& object = objects->currentRef();
246 
247  if (unlikely(first))
248  {
249  // On the first iteration, compute the types in to_fetch, and bind
250  // them into the payload_fetchers.
251  first = false;
252  for (Py_ssize_t i = 0; i < count; i++)
253  {
254  PyObject *py_name = PySequence_GetItem(to_fetch, i);
255  const char *name = _PyUnicode_AsString(py_name);
256  const string type = (object.payload()
257  .specification()[name]
258  .storageType()
259  .name());
260 
262  as_unicode);
263  auto pff = pf.target<PyObject* (*)(const IObject&)>();
264  if ( pff && *pff == &no_conversion_available)
265  return NULL;
266  payload_fetchers.push_back(pf);
267  Py_DECREF(py_name);
268  }
269 
270  if (with_time)
271  {
272  PyObject *py_datetime_module = PyImport_ImportModule("datetime");
273  if (!py_datetime_module)
274  throw (std::runtime_error("Could not import python datetime_module"));
275 
276  py_datetime_class = PyObject_GetAttrString(py_datetime_module,
277  "datetime");
278 
279  Py_DECREF(py_datetime_module);
280  }
281  }
282 
283  PyObject *one = PyTuple_New(fetch_start + count);
284 
285  // Fetch (since, until, channel[optional])
286 
287  PyTuple_SET_ITEM(one, 0, make_iov_key(iovkey_wrapper, object.since()));
288  PyTuple_SET_ITEM(one, 1, make_iov_key(iovkey_wrapper, object.until()));
289 
290  if (with_time)
291  {
292  const ITime& t = object.insertionTime();
293  static const char * const argtypes = const_cast<char *>("iiiiiil");
294 
295  PyObject *py_record_time =
296  PyObject_CallFunction(py_datetime_class, argtypes,
297  t.year(), t.month(), t.day(),
298  t.hour(), t.minute(), t.second(), t.nanosecond() / 1000);
299 
300  PyTuple_SetItem(one, 2, py_record_time);
301  }
302 
303  if (with_channel)
304  {
305  if (channel_name_mapping)
306  {
307  // Use a channel name if it is available, otherwise fall back
308  // to standard channelid.
309  PyObject *channelId = PyLong_FromUnsignedLong(object.channelId());
310  PyObject *channelName = PyDict_GetItem(channel_name_mapping,
311  channelId);
312  if (channelName)
313  {
314  Py_INCREF(channelName);
315  Py_DECREF(channelId);
316  PyTuple_SET_ITEM(one, 2+with_time, channelName);
317  } else
318  PyTuple_SET_ITEM(one, 2+with_time, channelId);
319 
320  } else
321  PyTuple_SET_ITEM(one, 2+with_time,
322  PyLong_FromLong(object.channelId()));
323  }
324 
325  // Fetch the objects specified in to_fetch
326  for (Py_ssize_t i = 0; i < count; i++)
327  PyTuple_SET_ITEM(one, fetch_start + i, payload_fetchers[i](object));
328 
329  one = apply_function(object_converter, one);
330  if (!one)
331  {
332  // apply_function returned an error
333  // We won't return the list to python so we need to tell python it
334  // can be deleted
335  Py_DECREF(result);
336  return NULL;
337  }
338 
339  PyList_Append(result, one);
340  Py_DECREF(one);
341 
342  // Print status if loud is active
343  if (unlikely(loud && ((++j) % 1000 == 0)))
344  {
345  cout << "Progress: " << j << " / " << records << "\r";
346  cout.flush();
347  }
348 
349  if (unlikely(max_records > 0 && j >= max_records))
350  break;
351  }
352 
353  if (loud)
354  cout << "Done fetching " << j << " records.\x1B[K" << endl;
355 
356  if (py_datetime_class)
357  {
358  Py_DECREF(py_datetime_class);
359  }
360 
361  return result;
362 }

Variable Documentation

◆ ATLAS_NO_CHECK_FILE_THREAD_SAFETY

ATLAS_NO_CHECK_FILE_THREAD_SAFETY

Definition at line 31 of file quick_retrieve.cxx.

create_payload_fetcher
payload_fetcher_t create_payload_fetcher(const char *name, const string &type_name, bool string_to_unicode=false)
Definition: quick_retrieve.cxx:133
data
char data[hepevt_bytes_allocation_ATLAS]
Definition: HepEvt.cxx:11
get_generator_info.result
result
Definition: get_generator_info.py:21
LArConditions2Ntuple.objects
objects
Definition: LArConditions2Ntuple.py:64
unlikely
#define unlikely(x)
Definition: quick_retrieve.cxx:6
Trk::one
@ one
Definition: TrkDetDescr/TrkSurfaces/TrkSurfaces/RealQuadraticEquation.h:22
qr_PyString_FromStdString
PyObject * qr_PyString_FromStdString(const string &str)
Definition: quick_retrieve.cxx:105
athena.value
value
Definition: athena.py:124
MAKE_FETCHER
#define MAKE_FETCHER(type, converter)
payload_fetcher
PyObject * payload_fetcher(const IObject &o, const string &name, FUNC_TYPE *converter_function)
Definition: quick_retrieve.cxx:119
defaults::Float
constexpr float Float
Definition: MSVtxValidationAlg.h:45
ReadOfcFromCool.field
field
Definition: ReadOfcFromCool.py:48
read_hist_ntuple.t
t
Definition: read_hist_ntuple.py:5
python.CaloAddPedShiftConfig.type
type
Definition: CaloAddPedShiftConfig.py:42
MAKE_FS
#define MAKE_FS(type, converter)
XMLtoHeader.count
count
Definition: XMLtoHeader.py:84
python.AthDsoLogger.argtypes
argtypes
Definition: AthDsoLogger.py:43
dq_defect_copy_defect_database.since
def since
Definition: dq_defect_copy_defect_database.py:54
dq_defect_copy_defect_database.until
def until
Definition: dq_defect_copy_defect_database.py:55
qr_PyString_FromBlob
PyObject * qr_PyString_FromBlob(const coral::Blob &blob)
Definition: quick_retrieve.cxx:99
lumiFormat.i
int i
Definition: lumiFormat.py:85
qr_PyUnicode_FromStdString
PyObject * qr_PyUnicode_FromStdString(const string &str)
Definition: quick_retrieve.cxx:110
apply_function
PyObject * apply_function(PyObject *function, PyObject *object)
Definition: quick_retrieve.cxx:203
CaloCondBlobAlgs_fillNoiseFromASCII.channelId
channelId
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:121
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
no_conversion_available
PyObject * no_conversion_available(const IObject &)
Definition: quick_retrieve.cxx:96
make_iov_key
PyObject * make_iov_key(PyObject *iovkey_wrapper, unsigned long long value)
Definition: quick_retrieve.cxx:194
IObjectIterator_Guard
Definition: quick_retrieve.cxx:186
payload_fetcher_t
std::function< PyObject *(const IObject &)> payload_fetcher_t
Definition: quick_retrieve.cxx:93
python.output.AtlRunQueryRoot.pf
pf
Definition: AtlRunQueryRoot.py:988
DeMoScan.first
bool first
Definition: DeMoScan.py:534
pickleTool.object
object
Definition: pickleTool.py:29
str
Definition: BTagTrackIpAccessor.cxx:11
PyObject
_object PyObject
Definition: IPyComponent.h:26
TSU::T
unsigned long long T
Definition: L1TopoDataTypes.h:35
CaloCondBlobAlgs_fillNoiseFromASCII.blob
blob
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:95