Collaboration diagram for python.DiskUtils.FileSet:

Public Member Functions
	__init__ (self, iterator, backend)
	from_single_file (cls, path, backend=None)
	from_directory (cls, path, backend=None)
	from_file_containing_list (cls, path, backend=None)
	from_glob (cls, pattern, backend=None)
	from_ds_info (cls, run, project, stream, base, backend=None)
	from_input (cls, input_string, backend=None)
	__iter__ (self)
	strict_mode (self, setting=True)
	matching (self, pattern)
	excluding (self, pattern)
	use_files_from (self, path)
	only_existing (self, setting=True)
	only_latest (self, setting=True)
	only_single_dataset (self, setting=True)
	with_lumi_blocks (self, map_file=None)

Public Attributes
	backend = backend
list	broken = []
dict	lb_map = {}

Protected Member Functions
	_with_lumi_blocks_from_map (self, map_file)
	_with_lumi_blocks_from_ara (self)

Protected Attributes
	_iter = iterator
bool	_existing = False
	_white_pattern = None
	_black_pattern = None
bool	_strict = True
print(, f)	_explicit = None
bool	_dedup = False
bool	_single_dataset = False

Detailed Description

 Represents a list of input files.
This class abstracts over the different ways files can be specified, and
the different storage backends/protocols on which they reside. It is an
iterator, and provides some methods for filtering the file set. E.g.:

    fs = FileSet.from_input('/eos/atlas/path/to/dataset/')
    for f in fs.matching(r'.*AOD.*').only_existing():
        print(f)

Definition at line 252 of file DiskUtils.py.

Constructor & Destructor Documentation

◆ init()

python.DiskUtils.FileSet.__init__	(	self,
		iterator,
		backend )

Definition at line 263 of file DiskUtils.py.

    def __init__(self, iterator, backend):
        self.backend = backend
        self._iter = iterator
        self._existing = False
        self._white_pattern = None
        self._black_pattern = None
        self._strict = True
        self._explicit = None
        self._dedup = False
        self._single_dataset = False
        self.broken = []
        self.lb_map = {}
 

Member Function Documentation

◆ iter()

python.DiskUtils.FileSet.__iter__ ( self )

Definition at line 320 of file DiskUtils.py.

    def __iter__(self):
        it = self._iter
        if self._white_pattern:
            it = filter(lambda x: self._white_pattern.search(x), it)
        if self._black_pattern:
            it = filter(lambda x: not self._black_pattern.search(x), it)
        if self._existing: # see: only_existing
            if self._strict:
                def generator(i, b):
                    for f in i:
                        if b.exists(f):
                            yield f
                        else:
                            raise AccessError('File not found: ' + f)
                it = generator(it, self.backend)
            else:
                it = filter(lambda x: self.backend.exists(x), it)
        if self._explicit is not None: # see: use_files_from
            def generator(i, strict):
                for f in i:
                    name = os.path.basename(f)
                    if self._explicit.pop(name, False):
                        yield f
                if strict and self._explicit:
                    for f in self._explicit: print('Missing:', f)
                    raise FilterError('Not all explicit files were found.')
            it = generator(it, self._strict)
        if self._dedup: # see: only_latest
            def fn(m, f):
                name, ext = os.path.splitext(f)
                if name in m:
                    m[name] = str(max(int(m[name]), int(ext[1:])))
                else:
                    m[name] = ext[1:]
                return m
            def generator(em):
                for name, ext in em.items():
                    yield '.'.join([name, ext])
            it = generator(functools.reduce(fn, self, {}))
        if self._single_dataset: # see: only_single_dataset
            def generator(i):
                dataset = None
                for f in i:
                    ds = '.'.join(f.split('.')[0:3])
                    if dataset is None:
                        dataset = ds
                    if ds == dataset:
                        yield f
                    else:
                        raise FilterError(
                                "Files found from more than one dataset: '{}' != '{}'"
                                .format(ds, dataset))
            it = generator(it)
        it = map(lambda x: self.backend.wrap(x), it)
        return it
 

◆ _with_lumi_blocks_from_ara()

python.DiskUtils.FileSet._with_lumi_blocks_from_ara ( self )

protected

Definition at line 454 of file DiskUtils.py.

    def _with_lumi_blocks_from_ara(self):
        def generator(s):
            for f in s:
                try:
                    lbs = get_lumi_blocks(f)
                except AccessError:
                    if s._strict:
                        raise
                    else:
                        s.broken.append(f)
                        continue
                yield f, set(lbs)
        return generator(self)

◆ _with_lumi_blocks_from_map()

python.DiskUtils.FileSet._with_lumi_blocks_from_map	(		self,
			map_file )

protected

Definition at line 434 of file DiskUtils.py.

    def _with_lumi_blocks_from_map(self, map_file):
        with open(map_file) as mf:
            for line in mf:
                print(line)
                fname = line.split(' ')[0]
                print(line.split(' ')[0])
                print(line.split(' ')[1])
                lbs = set(int(l) for l in line.split(' ')[1].split(','))
                self.lb_map[fname] = lbs
        def generator(s):
            for f in s:
                try:
                    yield f, s.lb_map[os.path.basename(f)]
                except KeyError:
                    if s._strict:
                        raise
                    else:
                        s.broken.append(f)
        return generator(self)
 

◆ excluding()

python.DiskUtils.FileSet.excluding	(		self,
			pattern )

Skip filenames matching the provided regular expression.

Definition at line 394 of file DiskUtils.py.

    def excluding(self, pattern):
        ''' Skip filenames matching the provided regular expression. '''
        self._black_pattern = re.compile(pattern) if pattern else None
        return self
 

◆ from_directory()

python.DiskUtils.FileSet.from_directory	(	cls,
		path,
		backend = None )

Definition at line 281 of file DiskUtils.py.

    def from_directory(cls, path, backend=None):
        be = backend or Local()
        return cls(be.children(path), be)
 

◆ from_ds_info()

python.DiskUtils.FileSet.from_ds_info	(	cls,
		run,
		project,
		stream,
		base,
		backend = None )

Definition at line 298 of file DiskUtils.py.

    def from_ds_info(cls, run, project, stream, base, backend=None):
        path = os.path.join(base, project, stream,
                '{:0{digits}d}'.format(int(run), digits=8))
        return cls.from_directory(path, backend=backend)
 

◆ from_file_containing_list()

python.DiskUtils.FileSet.from_file_containing_list	(	cls,
		path,
		backend = None )

Definition at line 286 of file DiskUtils.py.

    def from_file_containing_list(cls, path, backend=None):
        with open(path) as lf:
            listtoiter = [l.strip() for l in lf.readlines()]
            iterator = iter(listtoiter)
            return cls(iterator, backend or Local())
 

◆ from_glob()

python.DiskUtils.FileSet.from_glob	(	cls,
		pattern,
		backend = None )

Definition at line 293 of file DiskUtils.py.

    def from_glob(cls, pattern, backend=None):
        be = backend or Local()
        return cls(be.glob(pattern), be)
 

◆ from_input()

python.DiskUtils.FileSet.from_input	(	cls,
		input_string,
		backend = None )

Guess what kind of input file specification was provided.

Definition at line 304 of file DiskUtils.py.

    def from_input(cls, input_string, backend=None):
        ''' Guess what kind of input file specification was provided. '''
        be = backend or Local()
        if be.is_directory(input_string):
            return cls.from_directory(input_string, be)
        elif Local().is_file(input_string) and not (
                input_string.endswith('.root') or
                input_string[-7:-2] == '.root'):
            return cls.from_file_containing_list(input_string, be)
        elif be.is_file(input_string):
            return cls.from_single_file(input_string, be)
        elif '*' in input_string or '?' in input_string or '[' in input_string:
            return cls.from_glob(input_string, be)
        else:
            raise AccessError('Unable to resolve input: ' + repr(input_string))
 

◆ from_single_file()

python.DiskUtils.FileSet.from_single_file	(	cls,
		path,
		backend = None )

Definition at line 277 of file DiskUtils.py.

    def from_single_file(cls, path, backend=None):
        return cls(iter([path]), backend or Local())
 

◆ matching()

python.DiskUtils.FileSet.matching	(		self,
			pattern )

Only accept filenames matching the provided regular expression.

Definition at line 389 of file DiskUtils.py.

    def matching(self, pattern):
        ''' Only accept filenames matching the provided regular expression. '''
        self._white_pattern = re.compile(pattern) if pattern else None
        return self
 

◆ only_existing()

python.DiskUtils.FileSet.only_existing	(		self,
			setting = True )

Only use existing files.

Definition at line 408 of file DiskUtils.py.

    def only_existing(self, setting=True):
        ''' Only use existing files. '''
        self._existing = setting
        return self
 

◆ only_latest()

python.DiskUtils.FileSet.only_latest	(		self,
			setting = True )

Keep only the latest retry from sets like `*.1`, `*.2`.

Definition at line 413 of file DiskUtils.py.

    def only_latest(self, setting=True):
        ''' Keep only the latest retry from sets like `*.1`, `*.2`. '''
        self._dedup = setting
        return self
 

◆ only_single_dataset()

python.DiskUtils.FileSet.only_single_dataset	(		self,
			setting = True )

Require all files to be from the same dataset.

Definition at line 418 of file DiskUtils.py.

    def only_single_dataset(self, setting=True):
        ''' Require all files to be from the same dataset. '''
        self._single_dataset = setting
        return self
 

◆ strict_mode()

python.DiskUtils.FileSet.strict_mode	(		self,
			setting = True )

 When strict, errors are raised in the following cases (which
otherwise cause the corresponding files to be silently skipped):

  * When LB info is requested but cannot be found for a file (because
    it was not in the map file, or we couldn't open the ROOT file).
  * When `only_existing` is set and a file is missing.
  * When a file list is provided and not all of the files it mentions
    were encountered by the end of iteration.

Definition at line 376 of file DiskUtils.py.

    def strict_mode(self, setting=True):
        """ When strict, errors are raised in the following cases (which
        otherwise cause the corresponding files to be silently skipped):
 
          * When LB info is requested but cannot be found for a file (because
            it was not in the map file, or we couldn't open the ROOT file).
          * When `only_existing` is set and a file is missing.
          * When a file list is provided and not all of the files it mentions
            were encountered by the end of iteration.
        """
        self._strict = setting
        return self
 

◆ use_files_from()

python.DiskUtils.FileSet.use_files_from	(		self,
			path )

Use specific filenames from within the provided dataset.

Definition at line 399 of file DiskUtils.py.

    def use_files_from(self, path):
        ''' Use specific filenames from within the provided dataset. '''
        if path:
            with open(path) as lf:
                self._explicit = [l.strip() for l in lf.readlines()]
        else:
            self._explicit = None
        return self
 

◆ with_lumi_blocks()

python.DiskUtils.FileSet.with_lumi_blocks	(		self,
			map_file = None )

 Lookup the luminosity blocks contained in each file.
If a map file is provided it will be queried for the LB mapping,
otherwise each file will be opened and accessed using AthenaROOTAccess
which can be a little slow.

Definition at line 423 of file DiskUtils.py.

    def with_lumi_blocks(self, map_file=None):
        """ Lookup the luminosity blocks contained in each file.
        If a map file is provided it will be queried for the LB mapping,
        otherwise each file will be opened and accessed using AthenaROOTAccess
        which can be a little slow.
        """
        if map_file:
            return self._with_lumi_blocks_from_map(map_file)
        else:
            return self._with_lumi_blocks_from_ara()
 

Member Data Documentation

◆ _black_pattern

python.DiskUtils.FileSet._black_pattern = None

protected

Definition at line 268 of file DiskUtils.py.

◆ _dedup

bool python.DiskUtils.FileSet._dedup = False

protected

Definition at line 271 of file DiskUtils.py.

◆ _existing

bool python.DiskUtils.FileSet._existing = False

protected

Definition at line 266 of file DiskUtils.py.

◆ _explicit

print(, f) python.DiskUtils.FileSet._explicit = None

protected

Definition at line 270 of file DiskUtils.py.

◆ _iter

python.DiskUtils.FileSet._iter = iterator

protected

Definition at line 265 of file DiskUtils.py.

◆ _single_dataset

bool python.DiskUtils.FileSet._single_dataset = False

protected

Definition at line 272 of file DiskUtils.py.

◆ _strict

bool python.DiskUtils.FileSet._strict = True

protected

Definition at line 269 of file DiskUtils.py.

◆ _white_pattern

python.DiskUtils.FileSet._white_pattern = None

protected

Definition at line 267 of file DiskUtils.py.

◆ backend

python.DiskUtils.FileSet.backend = backend

Definition at line 264 of file DiskUtils.py.

◆ broken

list python.DiskUtils.FileSet.broken = []

Definition at line 273 of file DiskUtils.py.

◆ lb_map

dict python.DiskUtils.FileSet.lb_map = {}

Definition at line 274 of file DiskUtils.py.

The documentation for this class was generated from the following file:

DiskUtils.py

Public Member Functions

Public Attributes

Protected Member Functions

Protected Attributes

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ __iter__()

◆ _with_lumi_blocks_from_ara()

◆ _with_lumi_blocks_from_map()

◆ excluding()

◆ from_directory()

◆ from_ds_info()

◆ from_file_containing_list()

◆ from_glob()

◆ from_input()

◆ from_single_file()

◆ matching()

◆ only_existing()

◆ only_latest()

◆ only_single_dataset()

◆ strict_mode()

◆ use_files_from()

◆ with_lumi_blocks()

Member Data Documentation

◆ _black_pattern

◆ _dedup

◆ _existing

◆ _explicit

◆ _iter

◆ _single_dataset

◆ _strict

◆ _white_pattern

◆ backend

◆ broken

◆ lb_map

◆ init()

◆ iter()