python.DiskUtils.FileSet Class Reference

Collaboration diagram for python.DiskUtils.FileSet:

Public Member Functions
def	__init__ (self, iterator, backend)
def	from_single_file (cls, path, backend=None)
def	from_directory (cls, path, backend=None)
def	from_file_containing_list (cls, path, backend=None)
def	from_glob (cls, pattern, backend=None)
def	from_ds_info (cls, run, project, stream, base, backend=None)
def	from_input (cls, input_string, backend=None)
def	__iter__ (self)
def	strict_mode (self, setting=True)
def	matching (self, pattern)
def	excluding (self, pattern)
def	use_files_from (self, path)
def	only_existing (self, setting=True)
def	only_latest (self, setting=True)
def	only_single_dataset (self, setting=True)
def	with_lumi_blocks (self, map_file=None)

Public Attributes
	backend
	broken
	lb_map

Private Member Functions
def	_with_lumi_blocks_from_map (self, map_file)
def	_with_lumi_blocks_from_ara (self)

Private Attributes
	_iter
	_existing
	_white_pattern
	_black_pattern
	_strict
	_explicit
	_dedup
	_single_dataset

Detailed Description

Represents a list of input files.
This class abstracts over the different ways files can be specified, and
the different storage backends/protocols on which they reside. It is an
iterator, and provides some methods for filtering the file set. E.g.:

    fs = FileSet.from_input('/eos/atlas/path/to/dataset/')
    for f in fs.matching(r'.*AOD.*').only_existing():
        print(f)

Definition at line 281 of file DiskUtils.py.

Constructor & Destructor Documentation

__init__()

def python.DiskUtils.FileSet.__init__	(		self,
			iterator,
			backend
	)

Definition at line 292 of file DiskUtils.py.

    def __init__(self, iterator, backend):
        self.backend = backend
        self._iter = iterator
        self._existing = False
        self._white_pattern = None
        self._black_pattern = None
        self._strict = True
        self._explicit = None
        self._dedup = False
        self._single_dataset = False
        self.broken = []
        self.lb_map = {}
 

Member Function Documentation

__iter__()

def python.DiskUtils.FileSet.__iter__

(

self

)

Definition at line 349 of file DiskUtils.py.

    def __iter__(self):
        it = self._iter
        if self._white_pattern:
            it = filter(lambda x: self._white_pattern.search(x), it)
        if self._black_pattern:
            it = filter(lambda x: not self._black_pattern.search(x), it)
        if self._existing: # see: only_existing
            if self._strict:
                def generator(i, b):
                    for f in i:
                        if b.exists(f):
                            yield f
                        else:
                            raise AccessError('File not found: ' + f)
                it = generator(it, self.backend)
            else:
                it = filter(lambda x: self.backend.exists(x), it)
        if self._explicit is not None: # see: use_files_from
            def generator(i, strict):
                for f in i:
                    name = os.path.basename(f)
                    if self._explicit.pop(name, False):
                        yield f
                if strict and self._explicit:
                    for f in self._explicit: print('Missing:', f)
                    raise FilterError('Not all explicit files were found.')
            it = generator(it, self._strict)
        if self._dedup: # see: only_latest
            def fn(m, f):
                name, ext = os.path.splitext(f)
                if name in m:
                    m[name] = str(max(int(m[name]), int(ext[1:])))
                else:
                    m[name] = ext[1:]
                return m
            def generator(em):
                for name, ext in em.items():
                    yield '.'.join([name, ext])
            it = generator(functools.reduce(fn, self, {}))
        if self._single_dataset: # see: only_single_dataset
            def generator(i):
                dataset = None
                for f in i:
                    ds = '.'.join(f.split('.')[0:3])
                    if dataset is None:
                        dataset = ds
                    if ds == dataset:
                        yield f
                    else:
                        raise FilterError(
                                "Files found from more than one dataset: '{}' != '{}'"
                                .format(ds, dataset))
            it = generator(it)
        it = map(lambda x: self.backend.wrap(x), it)
        return it
 

_with_lumi_blocks_from_ara()

def python.DiskUtils.FileSet._with_lumi_blocks_from_ara (   self )

private

Definition at line 483 of file DiskUtils.py.

    def _with_lumi_blocks_from_ara(self):
        def generator(s):
            for f in s:
                try:
                    lbs = get_lumi_blocks(f)
                except AccessError:
                    if s._strict:
                        raise
                    else:
                        s.broken.append(f)
                        continue
                yield f, set(lbs)
        return generator(self)

_with_lumi_blocks_from_map()

def python.DiskUtils.FileSet._with_lumi_blocks_from_map (   self,   map_file  )

private

Definition at line 463 of file DiskUtils.py.

    def _with_lumi_blocks_from_map(self, map_file):
        with open(map_file) as mf:
            for line in mf:
                print(line)
                fname = line.split(' ')[0]
                print(line.split(' ')[0])
                print(line.split(' ')[1])
                lbs = set(int(l) for l in line.split(' ')[1].split(','))
                self.lb_map[fname] = lbs
        def generator(s):
            for f in s:
                try:
                    yield f, s.lb_map[os.path.basename(f)]
                except KeyError:
                    if s._strict:
                        raise
                    else:
                        s.broken.append(f)
        return generator(self)
 

excluding()

def python.DiskUtils.FileSet.excluding	(		self,
			pattern
	)

Skip filenames matching the provided regular expression. 

Definition at line 423 of file DiskUtils.py.

    def excluding(self, pattern):
        ''' Skip filenames matching the provided regular expression. '''
        self._black_pattern = re.compile(pattern) if pattern else None
        return self
 

from_directory()

def python.DiskUtils.FileSet.from_directory	(		cls,
			path,
			backend = None
	)

Definition at line 310 of file DiskUtils.py.

    def from_directory(cls, path, backend=None):
        be = backend or Local()
        return cls(be.children(path), be)
 

from_ds_info()

def python.DiskUtils.FileSet.from_ds_info	(		cls,
			run,
			project,
			stream,
			base,
			backend = None
	)

Definition at line 327 of file DiskUtils.py.

    def from_ds_info(cls, run, project, stream, base, backend=None):
        path = os.path.join(base, project, stream,
                '{:0{digits}d}'.format(int(run), digits=8))
        return cls.from_directory(path, backend=backend)
 

from_file_containing_list()

def python.DiskUtils.FileSet.from_file_containing_list	(		cls,
			path,
			backend = None
	)

Definition at line 315 of file DiskUtils.py.

    def from_file_containing_list(cls, path, backend=None):
        with open(path) as lf:
            listtoiter = [l.strip() for l in lf.readlines()]
            iterator = iter(listtoiter)
            return cls(iterator, backend or Local())
 

from_glob()

def python.DiskUtils.FileSet.from_glob	(		cls,
			pattern,
			backend = None
	)

Definition at line 322 of file DiskUtils.py.

    def from_glob(cls, pattern, backend=None):
        be = backend or Local()
        return cls(be.glob(pattern), be)
 

from_input()

def python.DiskUtils.FileSet.from_input	(		cls,
			input_string,
			backend = None
	)

Guess what kind of input file specification was provided. 

Definition at line 333 of file DiskUtils.py.

    def from_input(cls, input_string, backend=None):
        ''' Guess what kind of input file specification was provided. '''
        be = backend or Local()
        if be.is_directory(input_string):
            return cls.from_directory(input_string, be)
        elif Local().is_file(input_string) and not (
                input_string.endswith('.root') or
                input_string[-7:-2] == '.root'):
            return cls.from_file_containing_list(input_string, be)
        elif be.is_file(input_string):
            return cls.from_single_file(input_string, be)
        elif '*' in input_string or '?' in input_string or '[' in input_string:
            return cls.from_glob(input_string, be)
        else:
            raise AccessError('Unable to resolve input: ' + repr(input_string))
 

from_single_file()

def python.DiskUtils.FileSet.from_single_file	(		cls,
			path,
			backend = None
	)

Definition at line 306 of file DiskUtils.py.

    def from_single_file(cls, path, backend=None):
        return cls(iter([path]), backend or Local())
 

matching()

def python.DiskUtils.FileSet.matching	(		self,
			pattern
	)

Only accept filenames matching the provided regular expression. 

Definition at line 418 of file DiskUtils.py.

    def matching(self, pattern):
        ''' Only accept filenames matching the provided regular expression. '''
        self._white_pattern = re.compile(pattern) if pattern else None
        return self
 

only_existing()

def python.DiskUtils.FileSet.only_existing	(		self,
			setting = True
	)

Only use existing files. 

Definition at line 437 of file DiskUtils.py.

    def only_existing(self, setting=True):
        ''' Only use existing files. '''
        self._existing = setting
        return self
 

only_latest()

def python.DiskUtils.FileSet.only_latest	(		self,
			setting = True
	)

Keep only the latest retry from sets like `*.1`, `*.2`. 

Definition at line 442 of file DiskUtils.py.

    def only_latest(self, setting=True):
        ''' Keep only the latest retry from sets like `*.1`, `*.2`. '''
        self._dedup = setting
        return self
 

only_single_dataset()

def python.DiskUtils.FileSet.only_single_dataset	(		self,
			setting = True
	)

Require all files to be from the same dataset. 

Definition at line 447 of file DiskUtils.py.

    def only_single_dataset(self, setting=True):
        ''' Require all files to be from the same dataset. '''
        self._single_dataset = setting
        return self
 

strict_mode()

def python.DiskUtils.FileSet.strict_mode	(		self,
			setting = True
	)

When strict, errors are raised in the following cases (which
otherwise cause the corresponding files to be silently skipped):

  * When LB info is requested but cannot be found for a file (because
    it was not in the map file, or we couldn't open the ROOT file).
  * When `only_existing` is set and a file is missing.
  * When a file list is provided and not all of the files it mentions
    were encountered by the end of iteration.

Definition at line 405 of file DiskUtils.py.

    def strict_mode(self, setting=True):
        """ When strict, errors are raised in the following cases (which
        otherwise cause the corresponding files to be silently skipped):
 
          * When LB info is requested but cannot be found for a file (because
            it was not in the map file, or we couldn't open the ROOT file).
          * When `only_existing` is set and a file is missing.
          * When a file list is provided and not all of the files it mentions
            were encountered by the end of iteration.
        """
        self._strict = setting
        return self
 

use_files_from()

def python.DiskUtils.FileSet.use_files_from	(		self,
			path
	)

Use specific filenames from within the provided dataset. 

Definition at line 428 of file DiskUtils.py.

    def use_files_from(self, path):
        ''' Use specific filenames from within the provided dataset. '''
        if path:
            with open(path) as lf:
                self._explicit = [l.strip() for l in lf.readlines()]
        else:
            self._explicit = None
        return self
 

with_lumi_blocks()

def python.DiskUtils.FileSet.with_lumi_blocks	(		self,
			map_file = None
	)

Lookup the luminosity blocks contained in each file.
If a map file is provided it will be queried for the LB mapping,
otherwise each file will be opened and accessed using AthenaROOTAccess
which can be a little slow.

Definition at line 452 of file DiskUtils.py.

    def with_lumi_blocks(self, map_file=None):
        """ Lookup the luminosity blocks contained in each file.
        If a map file is provided it will be queried for the LB mapping,
        otherwise each file will be opened and accessed using AthenaROOTAccess
        which can be a little slow.
        """
        if map_file:
            return self._with_lumi_blocks_from_map(map_file)
        else:
            return self._with_lumi_blocks_from_ara()
 

Member Data Documentation

_black_pattern

python.DiskUtils.FileSet._black_pattern

private

Definition at line 297 of file DiskUtils.py.

_dedup

python.DiskUtils.FileSet._dedup

private

Definition at line 300 of file DiskUtils.py.

_existing

python.DiskUtils.FileSet._existing

private

Definition at line 295 of file DiskUtils.py.

_explicit

python.DiskUtils.FileSet._explicit

private

Definition at line 299 of file DiskUtils.py.

_iter

python.DiskUtils.FileSet._iter

private

Definition at line 294 of file DiskUtils.py.

_single_dataset

python.DiskUtils.FileSet._single_dataset

private

Definition at line 301 of file DiskUtils.py.

_strict

python.DiskUtils.FileSet._strict

private

Definition at line 298 of file DiskUtils.py.

_white_pattern

python.DiskUtils.FileSet._white_pattern

private

Definition at line 296 of file DiskUtils.py.

backend

python.DiskUtils.FileSet.backend

Definition at line 293 of file DiskUtils.py.

broken

python.DiskUtils.FileSet.broken

Definition at line 302 of file DiskUtils.py.

lb_map

python.DiskUtils.FileSet.lb_map

Definition at line 303 of file DiskUtils.py.

---

The documentation for this class was generated from the following file:

• DiskUtils.py