python.db Namespace Reference

Classes
class	Databases
class	DefectsDB

Functions
def	make_safe_fields (fields)
def	get_query_range (since, until, runs)
def	fetch_iovs (folder_name, since=None, until=None, channels=None, tag="", what="all", max_records=-1, with_channel=True, loud=False, database=None, convert_time=False, named_channels=False, selection=None, runs=None, with_time=False, unicode_strings=False)
def	write_iovs (folder_name, iovs, record, multiversion=True, tag="", create=False, storage_buffer=False)

Variables
	log
string	DEFAULT_DBNAME = "CONDBR2"

Function Documentation

fetch_iovs()

def python.db.fetch_iovs	(		folder_name,
			since = None,
			until = None,
			channels = None,
			tag = "",
			what = "all",
			max_records = -1,
			with_channel = True,
			loud = False,
			database = None,
			convert_time = False,
			named_channels = False,
			selection = None,
			runs = None,
			with_time = False,
			unicode_strings = False
	)

Helper to fetch objects in a pythonic manner
`folder_name` may be an abbreviated name (DQMFONL) or a fully-qualified name
    (e.g. /GLOBAL/DETSTATUS/DQMFONL)
`since`, `until` can be (run, lumi) tuples, or standard iov keys
`channels` can be a cool ChannelSelection object or a list of ids/names
`tag` COOL folder tag
`what` is a list of strings specifying which records should be fetched
    if it is the string "all" (not a list), then all records are fetched,
    and naming is turned on.
`max_records` specifies the maximum number of records to fetch. -1 means all
`with_channel` specifies whether the channel number should be in the result
    list of each tuple
`loud` specifies whether quick_retrieve (C++ function) should print its
    status every 1000 objects
`database` can be used to specify an abbreviated database, or a connection 
    string
`convert_time` performs a conversion of `since` and `until` from runlumi
    to nanoseconds since the epoch.
`named_channels` causes the iovs returned to contain strings in the channel 
    identifier
`selection` [NOT IMPLEMENTED YET] create a cool selection object
`runs` if it is an integer, it is a run number. If it is a tuple, it is a
    run range.
`with_time` retrieve insertiontime for iovs
`unicode_strings` return unicode string objects, assuming database content 
                  is UTF-8

Definition at line 67 of file DQUtils/python/db.py.

def fetch_iovs(folder_name, since=None, until=None, channels=None, tag="",
               what="all", max_records=-1, with_channel=True, loud=False,
               database=None, convert_time=False, named_channels=False,
               selection=None, runs=None, with_time=False, unicode_strings=False):
    """
    Helper to fetch objects in a pythonic manner
    `folder_name` may be an abbreviated name (DQMFONL) or a fully-qualified name
        (e.g. /GLOBAL/DETSTATUS/DQMFONL)
    `since`, `until` can be (run, lumi) tuples, or standard iov keys
    `channels` can be a cool ChannelSelection object or a list of ids/names
    `tag` COOL folder tag
    `what` is a list of strings specifying which records should be fetched
        if it is the string "all" (not a list), then all records are fetched,
        and naming is turned on.
    `max_records` specifies the maximum number of records to fetch. -1 means all
    `with_channel` specifies whether the channel number should be in the result
        list of each tuple
    `loud` specifies whether quick_retrieve (C++ function) should print its
        status every 1000 objects
    `database` can be used to specify an abbreviated database, or a connection 
        string
    `convert_time` performs a conversion of `since` and `until` from runlumi
        to nanoseconds since the epoch.
    `named_channels` causes the iovs returned to contain strings in the channel 
        identifier
    `selection` [NOT IMPLEMENTED YET] create a cool selection object
    `runs` if it is an integer, it is a run number. If it is a tuple, it is a
        run range.
    `with_time` retrieve insertiontime for iovs
    `unicode_strings` return unicode string objects, assuming database content 
                      is UTF-8
    """
    from .quick_retrieve import quick_retrieve, browse_coracool, get_coracool_payload_spec
    
    if channels == []: return IOVSet()
            
    since, until = get_query_range(since, until, runs)
 
    channel_mapping = None
    if isinstance(folder_name, str):
        folder = Databases.get_folder(folder_name, database) 
    else:
        try:
            folder = folder_name
            folder_name = folder.fullPath()
        except Exception:
            log.error("Exception when interpreting folder: {0}".format(folder_name))
            raise
 
    log.info("Querying %s", folder_name)
    log.debug("Query range: [%s, %s]", since, until)
    
    short_folder = folder.fullPath().split("/")[-1]
    
    time_based_folder = "<timeStamp>time</timeStamp>" in folder.description()
    coracool_folder = "<coracool>" in folder.description()
    iov_key_type = TimestampType if time_based_folder else RunLumiType
    
    if time_based_folder and (convert_time or runs):
        # Perform a conversion of the run IoV to a time-based one.
        # Note: probably inadvisable to do this for long ranges since
        #       it has to retrieve all of the luminosity blocks that took place
        #       in the query range.
        
        until = min(until, RunLumi(100000000, 0))
        runrange = fetch_iovs("LBLB", since, until)
        if runrange:
            # If the runrange is empty for some reason, fall back.
            since, until = runrange.first.StartTime, runrange.last.EndTime
            return fetch_iovs(folder_name, since, until, channels, tag, what, 
                              max_records, with_channel, loud, 
                              database, convert_time=False, 
                              named_channels=named_channels, selection=selection, 
                              with_time=with_time, 
                              unicode_strings=unicode_strings)
        else:
            return IOVSet()
    
    detstatus_names = "DQMFOFL", "DCSOFL", "DQMFONL", "SHIFTOFL", "SHIFTONL", "LBSUMM"
    if any(short_folder.endswith(x) for x in detstatus_names):
        channel_mapping = None # get channel mapping from channel_mapping.py
    else:
        _, _, channelmap = get_channel_ids_names(folder)
        cm_reversed = dict((value, key) for key, value in six.iteritems(channelmap))
        channelmap.update(cm_reversed)
        channel_mapping = channelmap
    
    channels = make_channelselection(channels, channel_mapping)    
        
    field_name = "%s_VAL" % short_folder
    
    if not coracool_folder:
        if what == "all":
            what = folder.folderSpecification().payloadSpecification().keys()
        
        channelmap = None
        if named_channels:
            _, _, channelmap = get_channel_ids_names(folder)
                    
        folder.setPrefetchAll(False)
        
        if selection:
            sel = make_browse_objects_selection(folder, selection)
            iterator = folder.browseObjects(since, until, channels, tag, sel)
        else:
            iterator = folder.browseObjects(since, until, channels, tag)
        
        fields = []
        if with_time:
            fields.append("insertion_time")
        if with_channel:
            fields.append("channel")
        fields.extend(what)
        
        record = make_iov_type(field_name, make_safe_fields(fields))
        
        result = quick_retrieve(iterator, record, what, max_records, with_channel, 
                                loud, iov_key_type, channelmap, with_time,
                                unicode_strings)
        
    else:
        args = folder_name, database
        database, folder_path = Databases.resolve_folder_string(*args)
    
        # Coracool
        assert database, "Coracool folder - you must specify a database"
        
        db = Databases.get_instance(database)
        spec =  get_coracool_payload_spec(db, folder_path)
        if what == "all":
            what = spec.keys()
        
        assert isinstance(what, list), ("Coracool folder - you must specify "
            "`what` to fetch (it cannot be inferred, as with non-coracool.)")
        
        record = make_iov_type(field_name, ["channel", "elements"])
        
        element = namedtuple("element", " ".join(make_safe_fields(what)))
    
        result = browse_coracool(db, folder_path, since, until, channels, "", 
                                 what, record, element, iov_key_type)
                
    result = IOVSet(result, iov_type=record, origin=short_folder)
        
    return result
 

get_query_range()

def python.db.get_query_range	(		since,
			until,
			runs
	)

Take `since`, `until` and `runs` and turn them into parameters.

If nothing is specified, an infinite range is used.
If `runs` is an integer, just that run is used
If `runs` is a two-tuple, then it is used as a (from_run, to_run)

Definition at line 29 of file DQUtils/python/db.py.

def get_query_range(since, until, runs):
    """
    Take `since`, `until` and `runs` and turn them into parameters.
    
    If nothing is specified, an infinite range is used.
    If `runs` is an integer, just that run is used
    If `runs` is a two-tuple, then it is used as a (from_run, to_run)
    """
    if runs and (since is None and until is None):
        from builtins import int
        if isinstance(runs, tuple):
            since, until = (runs[0], 0), (runs[1], 0)
            
        elif isinstance(runs, int):
            since, until = (runs, 0), (runs+1, 0)
        else:
            raise RuntimeError("Invalid type for `runs`, should be int or tuple")
            
    elif runs:
        raise RuntimeError("Specify (since and/or until), OR runs, not both")
        
    else:
        if since is None: since = 0
        if until is None: until = 2**63-1
    
    if isinstance(since, tuple): since = RunLumi(*since)
    if isinstance(until, tuple): until = RunLumi(*until)
 
    if isinstance(since, str): since = TimestampType.from_string(since)
    if isinstance(until, str): until = TimestampType.from_string(until)
 
    if isinstance(since, datetime): since = TimestampType.from_date(since)
    if isinstance(until, datetime): until = TimestampType.from_date(until)
 
    assert since <= until, "Bad query range (since > until?)"
    
    return since, until
    

make_safe_fields()

def python.db.make_safe_fields

(

fields

)

Definition at line 26 of file DQUtils/python/db.py.

def make_safe_fields(fields):
    return [(field + "_") if iskeyword(field) else field for field in fields]
 

write_iovs()

def python.db.write_iovs	(		folder_name,
			iovs,
			record,
			multiversion = True,
			tag = "",
			create = False,
			storage_buffer = False
	)

Definition at line 213 of file DQUtils/python/db.py.

def write_iovs(folder_name, iovs, record, multiversion=True, tag="",
               create=False, storage_buffer=False):    
    args = folder_name, multiversion, record, create
    db, folder, payload = Databases.fetch_for_writing(*args)
    
    if storage_buffer:
        folder.setupStorageBuffer()
        
    total_iovs = len(iovs)
    for i, iov in enumerate(iovs):
        for field_name, field_value in zip(iov._fields[3:], iov[3:]):
            payload[field_name] = field_value
        
        folder.storeObject(iov.since, iov.until, payload, iov.channel, tag)
        if not i % 1000:
            log.debug("Wrote %5i / %5i records", i, total_iovs)
 
    if storage_buffer:
        log.debug("Flushing records to database...")
        folder.flushStorageBuffer()
        log.debug("... done")
 

Variable Documentation

DEFAULT_DBNAME

string python.db.DEFAULT_DBNAME = "CONDBR2"

Definition at line 24 of file DQUtils/python/db.py.

log

python.db.log

Definition at line 17 of file DQDefects/python/db.py.