quick_retrieve.cxx File Reference

#include <Python.h>
#include <CoolKernel/ChannelSelection.h>
#include <CoolKernel/ITime.h>
#include <CoolKernel/IField.h>
#include <CoolKernel/FieldSelection.h>
#include <CoolKernel/CompositeSelection.h>
#include <CoolKernel/IObject.h>
#include <CoolKernel/IObjectIterator.h>
#include <CoolKernel/IFolder.h>
#include <CoolKernel/IDatabase.h>
#include <CoolKernel/IDatabaseSvc.h>
#include <CoolApplication/DatabaseSvcFactory.h>
#include <functional>
#include <vector>
#include <string>
#include <iostream>
#include <stdexcept>
#include "CxxUtils/checker_macros.h"

Go to the source code of this file.

Classes
class	IObjectIterator_Guard

Macros
#define	likely(x)
#define	unlikely(x)
#define	MAKE_FS(type, converter)
#define	MAKE_FETCHER(type, converter)

Typedefs
typedef std::function< PyObject *(const IObject &)>	payload_fetcher_t

Functions
cool::IRecordSelection *	make_fieldselection (const std::string &name, const cool::StorageType::TypeId typeId, cool::FieldSelection::Relation relation, PyObject *refValue)
vector< const cool::IRecordSelection * >	make_selection_vector ()
PyObject *	no_conversion_available (const IObject &)
PyObject *	qr_PyString_FromBlob (const coral::Blob &blob)
PyObject *	qr_PyString_FromStdString (const string &str)
PyObject *	qr_PyUnicode_FromStdString (const string &str)
template<typename T, typename FUNC_TYPE>
PyObject *	payload_fetcher (const IObject &o, const string &name, FUNC_TYPE *converter_function)
payload_fetcher_t	create_payload_fetcher (const char *name, const string &type_name, bool string_to_unicode=false)
PyObject *	make_iov_key (PyObject *iovkey_wrapper, unsigned long long value)
PyObject *	apply_function (PyObject *function, PyObject *object)
PyObject *	quick_retrieve (const IObjectIteratorPtr &objects, PyObject *object_converter, PyObject *to_fetch=NULL, const long max_records=-1, const bool with_channel=true, const bool loud=false, PyObject *iovkey_wrapper=NULL, PyObject *channel_name_mapping=NULL, const bool with_time=false, const bool as_unicode=false)

Variables
	ATLAS_NO_CHECK_FILE_THREAD_SAFETY

Macro Definition Documentation

likely

#define likely

(

x

)

Value:

__builtin_expect((x),1)

Definition at line 5 of file quick_retrieve.cxx.

MAKE_FETCHER

#define MAKE_FETCHER	(		type,
			converter )

Value:

        if (type_name == #type)                                                \
            return bind(payload_fetcher<cool::type, decltype(converter)>,  \
                        _1, name, &converter);

MAKE_FS

#define MAKE_FS	(		type,
			converter )

Value:

        if (typeId == cool::StorageType::type) \
            return new cool::FieldSelection(name, typeId, relation, \
                static_cast<cool::type>(converter(refValue)));

unlikely

#define unlikely

(

x

)

Value:

__builtin_expect((x),0)

Definition at line 6 of file quick_retrieve.cxx.

Typedef Documentation

payload_fetcher_t

typedef std::function<PyObject* (const IObject&)> payload_fetcher_t

Definition at line 93 of file quick_retrieve.cxx.

Function Documentation

apply_function()

PyObject * apply_function ( PyObject * function, PyObject * object )

inline

Definition at line 203 of file quick_retrieve.cxx.

{
    // Convert object according to function, taking care of references
    // If function is null, return unmodified object
    if (!function || function == Py_None) return object;
    PyObject *old_object = object;
    PyObject *new_object = PyObject_CallObject(function, object);
    Py_DECREF(old_object);
    return new_object;
}

create_payload_fetcher()

payload_fetcher_t create_payload_fetcher	(	const char *	name,
		const string &	type_name,
		bool	string_to_unicode = false )

Definition at line 133 of file quick_retrieve.cxx.

{
    // A macro which takes a cool `type` and a python api function which returns
    // a PyObject*, `converter`.
    #define MAKE_FETCHER(type, converter)                                      \
        if (type_name == #type)                                                \
            return bind(payload_fetcher<cool::type, decltype(converter)>,  \
                        _1, name, &converter);
    
    // See the python c-api reference for python conversion functions
    // Python/C API Reference Manual >> Concrete Objects Layer
    // http://docs.python.org/c-api/concrete.html
    
    MAKE_FETCHER(Bool,      PyBool_FromLong)
    MAKE_FETCHER(Float,     PyFloat_FromDouble)
    MAKE_FETCHER(Double,    PyFloat_FromDouble)
    MAKE_FETCHER(UChar,     PyLong_FromLong)
    MAKE_FETCHER(Int16,     PyLong_FromLong)
    MAKE_FETCHER(UInt16,    PyLong_FromUnsignedLong)
    MAKE_FETCHER(Int32,     PyLong_FromLong)
    MAKE_FETCHER(UInt32,    PyLong_FromUnsignedLong)
    MAKE_FETCHER(Int64,     PyLong_FromLong)
    MAKE_FETCHER(UInt63,    PyLong_FromUnsignedLongLong)
    MAKE_FETCHER(UInt64,    PyLong_FromUnsignedLongLong)
    if (string_to_unicode)
    {
        MAKE_FETCHER(String255, qr_PyUnicode_FromStdString)
        MAKE_FETCHER(String4k,  qr_PyUnicode_FromStdString)
        MAKE_FETCHER(String64k, qr_PyUnicode_FromStdString)
        MAKE_FETCHER(String16M, qr_PyUnicode_FromStdString)
    } else
    {
        MAKE_FETCHER(String255, qr_PyString_FromStdString)
        MAKE_FETCHER(String4k,  qr_PyString_FromStdString)
        MAKE_FETCHER(String64k, qr_PyString_FromStdString)
        MAKE_FETCHER(String16M, qr_PyString_FromStdString)
    }
    MAKE_FETCHER(Blob16M,   qr_PyString_FromBlob)
    MAKE_FETCHER(Blob64k,   qr_PyString_FromBlob)
 
    PyErr_Format(PyExc_RuntimeError, 
                 "Type '%s' is not in type conversion table. "
                 "Please add it to quick_retrieve.cxx. "
                 "Can't convert field '%s'.",
                 type_name.c_str(),
                 name);
    return no_conversion_available;
}

make_fieldselection()

cool::IRecordSelection * make_fieldselection	(	const std::string &	name,
		const cool::StorageType::TypeId	typeId,
		cool::FieldSelection::Relation	relation,
		PyObject *	refValue )

Definition at line 53 of file quick_retrieve.cxx.

{    
    #define MAKE_FS(type, converter) \
        if (typeId == cool::StorageType::type) \
            return new cool::FieldSelection(name, typeId, relation, \
                static_cast<cool::type>(converter(refValue)));
    
    MAKE_FS(Bool,      PyLong_AsLong)
    MAKE_FS(Float,     PyFloat_AsDouble)
    MAKE_FS(Double,    PyFloat_AsDouble)
    
    MAKE_FS(UChar,     PyLong_AsUnsignedLong)
    MAKE_FS(Int16,     PyLong_AsLong)
    MAKE_FS(UInt16,    PyLong_AsUnsignedLong)
    MAKE_FS(Int32,     PyLong_AsLong)
    MAKE_FS(UInt32,    PyLong_AsUnsignedLong)
    MAKE_FS(Int64,     PyLong_AsLongLong)
    MAKE_FS(UInt63,    PyLong_AsUnsignedLongLong)
    
    MAKE_FS(String255, _PyUnicode_AsString)
    MAKE_FS(String4k,  _PyUnicode_AsString)
    MAKE_FS(String64k, _PyUnicode_AsString)
    MAKE_FS(String16M, _PyUnicode_AsString)
 
    //MAKE_FS(Blob16M,   PyString_AsString)
    //MAKE_FS(Blob64k,   PyString_AsString)
    throw (std::runtime_error("Unsupported cool type encountered in python conversion"));
}

make_iov_key()

PyObject * make_iov_key ( PyObject * iovkey_wrapper, unsigned long long value )

inline

Definition at line 194 of file quick_retrieve.cxx.

{
    static const char * const argtypes = const_cast<char *>("K");
    if (iovkey_wrapper && iovkey_wrapper != Py_None)
        return PyObject_CallFunction(iovkey_wrapper, argtypes, value);
    return PyLong_FromUnsignedLongLong(value);
}

make_selection_vector()

vector< const cool::IRecordSelection * > make_selection_vector

(

)

Definition at line 86 of file quick_retrieve.cxx.

{
    return vector<const cool::IRecordSelection*>();
 
}

no_conversion_available()

PyObject * no_conversion_available

(

const IObject &

)

Definition at line 96 of file quick_retrieve.cxx.

{return NULL;}

payload_fetcher()

template<typename T, typename FUNC_TYPE>

PyObject * payload_fetcher ( const IObject & o, const string & name, FUNC_TYPE * converter_function )

inline

Definition at line 119 of file quick_retrieve.cxx.

{
    // We have to call this, because if one calls payloadValue on a null record,
    // it throws an exception.
    const IField& field = o.payload()[name];
    if (field.isNull())
        Py_RETURN_NONE;
    //cppcheck-suppress CastIntegerToAddressAtReturn
    return converter_function(field.data<T>());
}

qr_PyString_FromBlob()

PyObject * qr_PyString_FromBlob

(

const coral::Blob &

blob

)

Definition at line 99 of file quick_retrieve.cxx.

{
    const char* data = reinterpret_cast<const char*>(blob.startingAddress());
    return PyBytes_FromStringAndSize(data, blob.size());
}

qr_PyString_FromStdString()

PyObject * qr_PyString_FromStdString

(

const string &

str

)

Definition at line 105 of file quick_retrieve.cxx.

{
    return PyUnicode_FromStringAndSize(str.c_str(), str.size());
}

qr_PyUnicode_FromStdString()

PyObject * qr_PyUnicode_FromStdString

(

const string &

str

)

Definition at line 110 of file quick_retrieve.cxx.

{
    return PyUnicode_DecodeUTF8(str.c_str(), str.size(), "backslashreplace");
}

quick_retrieve()

PyObject * quick_retrieve	(	const IObjectIteratorPtr &	objects,
		PyObject *	object_converter,
		PyObject *	to_fetch = NULL,
		const long	max_records = -1,
		const bool	with_channel = true,
		const bool	loud = false,
		PyObject *	iovkey_wrapper = NULL,
		PyObject *	channel_name_mapping = NULL,
		const bool	with_time = false,
		const bool	as_unicode = false )

Definition at line 219 of file quick_retrieve.cxx.

{
    IObjectIterator_Guard closeiterator_guard(objects);
    
    PyObject* result = PyList_New(0);
    
    const long long records = objects->size();
    const Py_ssize_t count = to_fetch ? PySequence_Size(to_fetch) : 0;
    const unsigned int fetch_start = 2 + with_channel + with_time;
    bool first = true;
    int j = 0;
    PyObject *py_datetime_class = NULL;
    
    vector<payload_fetcher_t> payload_fetchers;
    
    while (objects->goToNext())
    {
        const IObject& object = objects->currentRef();
        
        if (unlikely(first))
        {
            // On the first iteration, compute the types in to_fetch, and bind
            // them into the payload_fetchers.
            first = false;
            for (Py_ssize_t i = 0; i < count; i++)
            {
                PyObject *py_name = PySequence_GetItem(to_fetch, i);
                const char *name = _PyUnicode_AsString(py_name);
                const string type = (object.payload()
                                           .specification()[name]
                                           .storageType()
                                           .name());
                
                payload_fetcher_t pf = create_payload_fetcher(name, type, 
                                                              as_unicode);
                auto pff = pf.target<PyObject* (*)(const IObject&)>();
                if ( pff && *pff == &no_conversion_available)
                    return NULL;
                payload_fetchers.push_back(std::move(pf));
                Py_DECREF(py_name);
            }
            
            if (with_time)
            {
                PyObject *py_datetime_module = PyImport_ImportModule("datetime");
                if (!py_datetime_module)
                    throw (std::runtime_error("Could not import python datetime_module"));
                
                py_datetime_class = PyObject_GetAttrString(py_datetime_module,
                                                           "datetime"); 
                
                Py_DECREF(py_datetime_module);
            }
        }
        
        PyObject *one = PyTuple_New(fetch_start + count);
        
        // Fetch (since, until, channel[optional])
        
        PyTuple_SET_ITEM(one, 0, make_iov_key(iovkey_wrapper, object.since()));
        PyTuple_SET_ITEM(one, 1, make_iov_key(iovkey_wrapper, object.until()));
 
        if (with_time)
        {
            const ITime& t = object.insertionTime();
            static const char * const argtypes = const_cast<char *>("iiiiiil");
            
            PyObject *py_record_time = 
                PyObject_CallFunction(py_datetime_class, argtypes, 
                    t.year(), t.month(), t.day(), 
                    t.hour(), t.minute(), t.second(), t.nanosecond() / 1000);
                    
            PyTuple_SetItem(one, 2, py_record_time);
        }
        
        if (with_channel)
        {
            if (channel_name_mapping)
            {
                // Use a channel name if it is available, otherwise fall back
                // to standard channelid.
                PyObject *channelId = PyLong_FromUnsignedLong(object.channelId());
                PyObject *channelName = PyDict_GetItem(channel_name_mapping,    
                                                       channelId);
                if (channelName)
                {
                    Py_INCREF(channelName);
                    Py_DECREF(channelId);
                    PyTuple_SET_ITEM(one, 2+with_time, channelName);
                } else
                    PyTuple_SET_ITEM(one, 2+with_time, channelId);
                
            } else
                PyTuple_SET_ITEM(one, 2+with_time, 
                                 PyLong_FromLong(object.channelId()));
        }
        
        // Fetch the objects specified in to_fetch
        for (Py_ssize_t i = 0; i < count; i++)
            PyTuple_SET_ITEM(one, fetch_start + i, payload_fetchers[i](object));
        
        one = apply_function(object_converter, one);
        if (!one)
        {
            // apply_function returned an error
            // We won't return the list to python so we need to tell python it
            // can be deleted
            Py_DECREF(result);
            return NULL;
        }
        
        PyList_Append(result, one);
        Py_DECREF(one);
        
        // Print status if loud is active
        if (unlikely(loud && ((++j) % 1000 == 0)))
        {
            cout << "Progress: " << j << " / " << records << "\r";
            cout.flush();
        }
        
        if (unlikely(max_records > 0 && j >= max_records))
            break;
    }
    
    if (loud)
        cout << "Done fetching " << j << " records.\x1B[K" << endl;
    
    if (py_datetime_class)
    {
        Py_DECREF(py_datetime_class);
    }
    
    return result;
}

Variable Documentation

ATLAS_NO_CHECK_FILE_THREAD_SAFETY

ATLAS_NO_CHECK_FILE_THREAD_SAFETY

Definition at line 31 of file quick_retrieve.cxx.