SH Namespace Reference

This module provides a lot of global definitions, forward declarations and includes that are used by all modules. More...

Classes
class	DiskList
	an interface for listing directory contents, locally or on a file server More...
class	DiskListEOS
	a DiskList implementation for EOS More...
class	DiskListLocal
	a DiskList implementation for local directories More...
class	DiskListSRM
	a DiskList implementation for the SRM protocol More...
class	DiskListXRD
	a DiskList implementation for the XRD protocol More...
class	DiskOutput
	a class/interface representing an output location for files More...
class	DiskOutputLocal
	an implementation of DiskOutput for local disks More...
class	DiskOutputXRD
	an implementation of DiskOutput using the XRD protocol More...
class	DiskWriter
	an interface that manages writing a single output file More...
class	DiskWriterLocal
	an implementation of DiskWriter for local files More...
class	DiskWriterXRD
	an implementation of DiskWriter for the XRD protocol More...
class	Meta
	A base class for classes that implement arbitrary meta-data. More...
class	MetaData
	A class implementing a templatized version of the meta-data. More...
struct	MetaDataQuery
	an object containing the result of an AMI meta-data query More...
struct	MetaDataSample
	all the meta-data fields that we may try to read from AMI. More...
struct	MetaFields
	This class contains the known meta-data names. More...
struct	MetaNames
	the names of common meta-data More...
class	MetaObject
	A class that manages meta-data to be associated with an object. More...
class	MetaVector
	This class defines a templatized version of the meta-data in vector form. More...
struct	RucioDownloadResult
	the result from rucio_download More...
struct	RucioListDidsEntry
	one entry from the rucio-list-dids command More...
struct	RucioListFileReplicasEntry
	one entry from the rucio-list-file-replicas command More...
class	Sample
	a base class that manages a set of files belonging to a particular data set and the associated meta-data. More...
class	SampleComposite
	This module defines an implementation of Sample that contains composite samples. More...
class	SampleGrid
	This class implements a Sample located on the grid. More...
class	SampleHandler
	A class that manages a list of Sample objects. More...
class	SampleHist
	A sample that represents a single histogram file. More...
class	SampleLocal
	A Sample based on a simple file list. More...
class	SampleMeta
	A Sample that consists only of Meta-Information. More...
class	SamplePtr
	A smart pointer class that holds a single Sample object. More...
struct	ScanDir
	the class used for scanning local directories and file servers for samples More...
class	TagList
	A class that manages a list of tags for identifying samples. More...

Functions
void	fetchMetaData (MetaDataQuery &query)
	effects: fetch information on all the samples/datasets specified guarantee: basic
void	fetchMetaData (SampleHandler &sh, bool override)
	effects: fetch information on all the samples in the sample handler.
const std::string &	downloadStageEnvVar ()
	the name of the environment variable containing the directory for staging files from the grid
bool	checkVomsProxy ()
	return whether we have a valid VOMS proxy available
void	ensureVomsProxy ()
	ensure that we have a valid VOMS proxy available
std::vector< std::string >	faxListFilesGlob (const std::string &name, const std::string &filter)
	list the FAX URLs for all the files in the dataset or dataset container matching the given filter (as glob expression)
std::vector< std::string >	faxListFilesRegex (const std::string &name, const std::string &filter)
	list the FAX URLs for all the files in the dataset or dataset container matching the given filter (as regular expression)
std::vector< std::string >	rucioDirectAccessGlob (const std::string &name, const std::string &filter, const std::string &selectOptions)
	list the rucio URLs for all the files in the dataset or dataset container matching the given filter (as glob expression)
std::vector< std::string >	rucioDirectAccessRegex (const std::string &name, const std::string &filter, const std::string &selectOptions)
	list the rucio URLs for all the files in the dataset or dataset container matching the given filter (as regular expression)
std::vector< RucioListDidsEntry >	rucioListDids (const std::string &dataset)
	run rucio-list-dids for the given dataset
std::vector< RucioListFileReplicasEntry >	rucioListFileReplicas (const std::string &dataset)
	run rucio-list-file-replicas for the given dataset
std::map< std::string, std::unique_ptr< MetaObject > >	rucioGetMetadata (const std::set< std::string > &datasets)
	run rucio-get-metadata for the given list of datasets
RucioDownloadResult	rucioDownload (const std::string &location, const std::string &dataset)
	run rucio-download
std::vector< RucioDownloadResult >	rucioDownloadList (const std::string &location, const std::vector< std::string > &datasets)
	run rucio-download with multiple datasets
std::vector< std::string >	rucioCacheDatasetGlob (const std::string &location, const std::string &dataset, const std::string &fileGlob)
	download the dataset, and return a list matching the pattern
std::string	dbg (const Meta &, unsigned)
std::string	dbg (const TagList &obj, unsigned)
void	scanDir (SampleHandler &sh, DiskList &list, const std::string &pattern="*.root*", const std::string &samplePattern="*", const std::string &samplePostfix="")
	effects: scan the given directory and add all subdirectories as samples that contain root files.
void	scanDir (SampleHandler &sh, const std::string &dir)
void	scanDir (SampleHandler &sh, const std::string &dir, const std::string &prefix)
void	scanFiles (SampleHandler &sh, DiskList &list, const std::string &pattern="*.root*")
	effects: scan the given directory tree and make a separate sample for each file (using the file name before any "." as the sample name).
Sample *	makeFromTChain (const std::string &name, const TChain &chain)
	effects: create a sample with the given name from the given TChain object guarantee: strong failures: out of memory II
void	scanSingleDir (SampleHandler &sh, const std::string &name, DiskList &list, const std::string &pattern="*.root*")
	effects: scan the given directory tree and turn it into a single sample of the given name guarantee: strong failures: out of memory III
void	scanDQ2 (SampleHandler &sh, const std::string &pattern)
	effects: make a list from DQ2 using the given pattern guarantee: basic, may add partially failures: out of memory III failures: dq2-ls errors
void	scanRucio (SampleHandler &sh, const std::string &pattern, bool alwaysQuery=false)
	make a list of grid datasets using the given pattern
void	addGrid (SampleHandler &sh, const std::string &ds)
	effects: add a grid dataset for dataset ds guarantee: strong failures: out of memory II requires: ds.find ("*") == std::string::npos
void	addGridCombined (SampleHandler &sh, const std::string &dsName, const std::vector< std::string > &dsList)
	effects: add a combined grid dataset with name dsName for dataset list dsList guarantee: strong failures: out of memory II requires: for each dataset ds: ds.find ("*") == std::string::npos
void	addGridCombinedFromFile (SampleHandler &sh, const std::string &dsName, const std::string &dsFile)
	effects: add a combined grid dataset with name dsName for dataset list file dsFile guarantee: strong failures: out of memory II requires: for each dataset ds: ds.find ("*") == std::string::npos
void	makeGridDirect (SampleHandler &sh, const std::string &disk, const std::string &from, const std::string &to, bool allow_partial)
	effects: update all grid samples in the sample handler that are located on the given disk to be opened directly.
void	scanForTrees (SampleHandler &sh, Sample &sample, const std::string &pattern=".*")
	effects: scan for trees in the given sample (or sample handler), and create a separate sample for each tree.
void	scanForTrees (SampleHandler &sh, const std::string &pattern)
void	readFileList (SampleHandler &sh, const std::string &name, const std::string &file)
	effects: read a file list from a text file guarantee: strong failures: out of memory III failures: i/o errors
void	printDuplicateEvents (const Sample &sample)
	effects: check the given sample for duplicate events and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors
void	printDuplicateEventsSplit (const SampleHandler &sh)
	effects: check each sample for duplicate events and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors
void	printDuplicateEventsJoint (const SampleHandler &sh)
	effects: check for duplicate events between all the samples and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors
void	mergeSamples (SampleHandler &sh, const std::string &sampleName, const std::string &pattern)
	effects: remove all samples matching the name pattern, and join them into a single sample named sampleName guarantee: strong failures: out of memory II failures: i/o errors
void	readSusyMeta (const SampleHandler &sh, const std::string &inputFile)
	effects: read the susy-meta-data file and add its information to the samples from the given sample handler guarantee: basic failures: i/o errors warning: there are no checks for duplicate and inconsistent sample definitions
void	readSusyMetaDir (const SampleHandler &sh, const std::string &inputDir)
	effects: read an entire directory of susy-meta-data files and add their information to the samples from the given sample handle.
std::unique_ptr< TFile >	openFile (const std::string &name, const MetaObject &options)
	open a file with the given options
std::unique_ptr< SampleLocal >	mergeFiles (const Sample &sample, const std::string &location, bool overwrite)
	merge all the files in the sample into a single file in the given location
SampleHandler	mergeFiles (const SampleHandler &sh, const std::string &location, bool overwrite)
	merge all the files in the sample handles into a single file per sample in the given location
void	scanNEvents (SampleHandler &sh)
	effects: scan each sample in the sample handler and store the number of entries per file in the meta-data guarantee: basic, may only scan some failures: out of memory failures: read errors failures: invalid sample type
void	scanNEvents (Sample &sample)
	effects: scan the given sample and store the number of entries per file in the meta-data guarantee: strong failures: out of memory failures: read errors failures: invalid sample type
SampleHandler	splitSample (Sample &sample, Long64_t nevt)
	effects: split the given sample into a set of samples, with each sample containing either exactly one file or at most nevt events side effects: if scanNEvents hasn't been run on this sample, run it.
std::string	dbg (const Meta &obj, unsigned verbosity=0)
	the debugging info of this object
std::string	dbg (const MetaObject &obj, unsigned verbosity=0)
	the debugging info of this object
void	swap (MetaObject &a, MetaObject &b)
	standard swap
std::string	dbg (const Sample &obj, unsigned verbosity=0)
	the debugging info of this object
std::string	dbg (const SampleHandler &obj, unsigned verbosity=0)
	the debugging info of this object
void	swap (SampleHandler &a, SampleHandler &b)
	standard swap
std::string	dbg (const TagList &obj, unsigned verbosity=0)
	the debugging info of this object

Detailed Description

This module provides a lot of global definitions, forward declarations and includes that are used by all modules.

This module defines utility functions used for splitting samples.

This module defines utility functions used for joining samples.

Author

Alexander Madsen

Nils Krumnack

Nils Krumnack

As such it doesn't fall into the user vs. expert classification. This module is considered to be in the pre-alpha stage.

The interface provided in this module is intended for the general user. The module is considered to be in the pre-alpha stage.

Function Documentation

addGrid()

void SH::addGrid	(	SampleHandler &	sh,
		const std::string &	ds )

effects: add a grid dataset for dataset ds guarantee: strong failures: out of memory II requires: ds.find ("*") == std::string::npos

Definition at line 175 of file ToolsDiscovery.cxx.

  {
    RCU_ASSERT_SOFT (ds.find ("*") == std::string::npos);
 
    std::string name;
    if (ds[ds.size()-1] == '/')
      name = ds.substr (0, ds.size()-1);
    else
      name = ds;
 
    auto sample = std::make_unique<SampleGrid> (name);
    sample->meta()->setString (MetaFields::gridName, ds);
    sample->meta()->setString (MetaFields::gridFilter, MetaFields::gridFilter_default);
    sh.add (sample.release());
  }

addGridCombined()

void SH::addGridCombined	(	SampleHandler &	sh,
		const std::string &	dsName,
		const std::vector< std::string > &	dsList )

effects: add a combined grid dataset with name dsName for dataset list dsList guarantee: strong failures: out of memory II requires: for each dataset ds: ds.find ("*") == std::string::npos

Definition at line 193 of file ToolsDiscovery.cxx.

  {
    std::string name;
    for (const std::string &ds : dsList)
    {
      RCU_ASSERT_SOFT (ds.find ("*") == std::string::npos);
 
      if (!name.empty())
        name.append(",");
 
      if (ds.at(ds.size() - 1) == '/')
        name.append(ds.substr (0, ds.size() - 1));
      else
        name.append(ds);
    }
 
    auto sample = std::make_unique<SampleGrid> (dsName);
    sample->meta()->setString (MetaFields::gridName, name);
    sample->meta()->setString (MetaFields::gridFilter, MetaFields::gridFilter_default);
    sh.add (sample.release());
  }

addGridCombinedFromFile()

void SH::addGridCombinedFromFile	(	SampleHandler &	sh,
		const std::string &	dsName,
		const std::string &	dsFile )

effects: add a combined grid dataset with name dsName for dataset list file dsFile guarantee: strong failures: out of memory II requires: for each dataset ds: ds.find ("*") == std::string::npos

Definition at line 215 of file ToolsDiscovery.cxx.

  {
    std::ifstream file (dsFile.c_str());
 
    std::string name;
    std::string ds;
    const std::set<char> whitespaces{'\t',' ','\n','\r'};
    while (std::getline (file, ds))
    {
      while ((!ds.empty()) && whitespaces.count(ds.back())) ds.pop_back();
      if (ds.empty() || ds.at(0) == '#')
        continue;
 
      RCU_ASSERT_SOFT (ds.find ("*") == std::string::npos);
 
      if (!name.empty())
        name.append(",");
 
      if (ds.at(ds.size() - 1) == '/')
        name.append(ds.substr (0, ds.size() - 1));
      else
        name.append(ds);
    }
    if (!file.eof())
      RCU_THROW_MSG ("failed to read file: " + dsFile);
 
    auto sample = std::make_unique<SampleGrid> (dsName);
    sample->meta()->setString (MetaFields::gridName, name);
    sample->meta()->setString (MetaFields::gridFilter, MetaFields::gridFilter_default);
    sh.add (sample.release());
  }

checkVomsProxy()

bool SH::checkVomsProxy

(

)

return whether we have a valid VOMS proxy available

Guarantee

basic

Failures

grid utility failures

Definition at line 206 of file GridTools.cxx.

  {
    return proxyData().checkVomsProxy();
  }

dbg() [1/7]

std::string SH::dbg ( const Meta & , unsigned  )

related

Definition at line 28 of file Meta.cxx.

  {
    return "meta-object";
  }

dbg() [2/7]

std::string SH::dbg ( const TagList & obj, unsigned  )

related

Definition at line 30 of file TagList.cxx.

  {
    std::string result;
 
    for (TagList::iterator tag = obj.begin(), end = obj.end();
     tag != end; ++ tag)
    {
      if (!result.empty())
    result += ",";
      result += *tag;
    };
    return "(" + result + ")";
  }

downloadStageEnvVar()

const std::string & SH::downloadStageEnvVar

(

)

the name of the environment variable containing the directory for staging files from the grid

Definition at line 198 of file GridTools.cxx.

  {
    static const std::string result = "SAMPLEHANDLER_RUCIO_DOWNLOAD";
    return result;
  }

ensureVomsProxy()

void SH::ensureVomsProxy

(

)

ensure that we have a valid VOMS proxy available

First this checks whether we have a valid PROXY, and if not it sets up a new certificate.

Guarantee

basic

Failures

grid utility failures
failure to set up new VOMS proxy

Definition at line 213 of file GridTools.cxx.

  {
    proxyData().ensureVomsProxy();
  }

faxListFilesGlob()

std::vector< std::string > SH::faxListFilesGlob	(	const std::string &	name,
		const std::string &	filter )

list the FAX URLs for all the files in the dataset or dataset container matching the given filter (as glob expression)

Guarantee

strong

Failures

grid utility failures

Precondition

!name.empty()

name.find('*') == std::string::npos

!filter.empty()

Definition at line 221 of file GridTools.cxx.

  {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpragmas"
#pragma GCC diagnostic ignored "-Wunknown-pragmas"
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
    return faxListFilesRegex (name, RCU::glob_to_regexp (filter));
#pragma GCC diagnostic pop
  }

faxListFilesRegex()

std::vector< std::string > SH::faxListFilesRegex	(	const std::string &	name,
		const std::string &	filter )

list the FAX URLs for all the files in the dataset or dataset container matching the given filter (as regular expression)

Guarantee

strong

Failures

grid utility failures

Precondition

!name.empty()

name.find('*') == std::string::npos

!filter.empty()

Definition at line 234 of file GridTools.cxx.

  {
    RCU_REQUIRE_SOFT (!name.empty());
    RCU_REQUIRE_SOFT (name.find('*') == std::string::npos);
    RCU_REQUIRE_SOFT (!filter.empty());
 
    ensureVomsProxy ();
 
    static const std::string separator = "------- SampleHandler Split -------";
    std::vector<std::string> result;
 
    ANA_MSG_INFO ("querying FAX for dataset " << name);
    std::string output = sh::exec_read ("source $ATLAS_LOCAL_ROOT_BASE/user/atlasLocalSetup.sh -q && lsetup --force fax && echo " + separator + " && fax-get-gLFNs " + sh::quote (name));
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      RCU_THROW_MSG ("couldn't find separator in: " + output);
 
    std::istringstream str (output.substr (split + separator.size() + 1));
    std::regex pattern (filter);
    std::string line;
    while (std::getline (str, line))
    {
      if (!line.empty())
      {
    if (!line.starts_with ("root:"))
      RCU_THROW_MSG ("couldn't parse line: " + line);
 
    std::string::size_type split1 = line.rfind (":");
    std::string::size_type split2 = line.rfind ("/");
    if (split1 < split2)
      split1 = split2;
    if (split1 != std::string::npos)
    {
      if (RCU::match_expr (pattern, line.substr (split1+1)))
        result.push_back (line);
    } else
      RCU_THROW_MSG ("couldn't parse line: " + line);
      }
    }
    if (result.size() == 0)
      RCU_WARN_MSG ("dataset " + name + " did not contain any files.  this is likely not right");
    return result;
  }

fetchMetaData() [1/2]

void SH::fetchMetaData

(

MetaDataQuery &

query

)

effects: fetch information on all the samples/datasets specified guarantee: basic

Definition at line 31 of file fetch.cxx.

  {
    static std::once_flag loaded;
    auto do_load = []() {
      TString path = "$ROOTCOREBIN/python/SampleHandler/SampleHandler_QueryAMI.py";
      gSystem->ExpandPathName (path);
      TPython::LoadMacro (path.Data());
    };
    std::call_once (loaded, do_load);
 
    std::ostringstream command;
#if ROOT_VERSION_CODE >= ROOT_VERSION(6,33,01)
    command << "_anyresult = ";
#endif
    command << "SampleHandler_QueryAmi([";
    for (std::size_t iter = 0, end = query.samples.size(); iter != end; ++ iter)
    {
      if (iter != 0)
    command << ", ";
      command << "'" << query.samples[iter].name << "'";
    }
    command << "])";
#if ROOT_VERSION_CODE >= ROOT_VERSION(6,33,01)
    std::any result;
    TPython::Exec (command.str().c_str(), &result);
    query = std::any_cast<MetaDataQuery>(result);
#else
    MetaDataQuery* myquery = static_cast<MetaDataQuery*>
      ((void*) TPython::Eval (command.str().c_str()));
    query = *myquery;
#endif
  }

fetchMetaData() [2/2]

void SH::fetchMetaData	(	SampleHandler &	sh,
		bool	override )

effects: fetch information on all the samples in the sample handler.

if override is specified it will override the existing meta-data fields, otherwise they will only be used if the given fields do not yet exist guarantee: basic

Definition at line 65 of file fetch.cxx.

  {
    std::vector<SH::Sample*> samples;
    // typedef std::vector<SH::Sample*> SamplesIter;
    MetaDataQuery query;
    for (SH::SampleHandler::iterator sample = sh.begin(),
       end = sh.end(); sample != end; ++ sample)
    {
      std::string name = (*sample)->meta()->castString (SH::MetaFields::gridName,(*sample)->name());
      query.samples.push_back (MetaDataSample (name));
      samples.push_back (&**sample);
    }
    fetchMetaData (query);
 
    if (!query.messages.empty())
      RCU_PRINT_MSG (query.messages);
    for (std::size_t iter = 0, end = query.samples.size(); iter != end; ++ iter)
    {
      if (query.samples[iter].unknown)
      {
    RCU_WARN_MSG ("failed to find sample " + query.samples[iter].name);
      } else
      {
    RCU_ASSERT (iter != samples.size());
    SH::Sample *sample = samples[iter];
 
    if (!override)
    {
      query.samples[iter].isData = sample->meta()->castDouble (SH::MetaFields::isData, query.samples[iter].isData);
      query.samples[iter].luminosity = sample->meta()->castDouble (SH::MetaFields::lumi, query.samples[iter].luminosity);
      query.samples[iter].crossSection = sample->meta()->castDouble (SH::MetaFields::crossSection, query.samples[iter].crossSection);
      query.samples[iter].nevents = sample->meta()->castDouble (SH::MetaFields::numEvents, query.samples[iter].nevents);
      query.samples[iter].kfactor = sample->meta()->castDouble (SH::MetaFields::kfactor, query.samples[iter].kfactor);
      query.samples[iter].filterEfficiency = sample->meta()->castDouble (SH::MetaFields::filterEfficiency, query.samples[iter].filterEfficiency);
    }
    if (query.samples[iter].isData != -1)
      sample->meta()->setDouble (SH::MetaFields::isData, query.samples[iter].isData);
    if (query.samples[iter].luminosity != -1)
      sample->meta()->setDouble (SH::MetaFields::lumi, query.samples[iter].luminosity);
    if (query.samples[iter].crossSection != -1)
      sample->meta()->setDouble (SH::MetaFields::crossSection, query.samples[iter].crossSection);
    if (query.samples[iter].nevents != -1)
      sample->meta()->setDouble (SH::MetaFields::numEvents, query.samples[iter].nevents);
    if (query.samples[iter].kfactor != -1)
      sample->meta()->setDouble (SH::MetaFields::kfactor, query.samples[iter].kfactor);
    if (query.samples[iter].filterEfficiency != -1)
      sample->meta()->setDouble (SH::MetaFields::filterEfficiency, query.samples[iter].filterEfficiency);
      }
    }
  }

makeFromTChain()

Sample * SH::makeFromTChain	(	const std::string &	name,
		const TChain &	chain )

effects: create a sample with the given name from the given TChain object guarantee: strong failures: out of memory II

Definition at line 98 of file ToolsDiscovery.cxx.

  {
    std::unique_ptr<SampleLocal> result (new SampleLocal (name));
    result->meta()->setString (MetaFields::treeName, chain.GetName());
 
    TIter chainIter (chain.GetListOfFiles());
    TChainElement *chainElement = 0;
    while ((chainElement = dynamic_cast<TChainElement*>(chainIter.Next())) != 0)
      result->add (chainElement->GetTitle());
    return result.release();
  }

makeGridDirect()

void SH::makeGridDirect	(	SampleHandler &	sh,
		const std::string &	disk,
		const std::string &	from,
		const std::string &	to,
		bool	allow_partial )

effects: update all grid samples in the sample handler that are located on the given disk to be opened directly.

for that it replaces the from part of the file with the to part. if allow_partial is set, it allows for partial datasets (meaning you may lose some files that are not there). guarantee: basic, may convert only some samples failures: out of memory III failures: dq2-ls errors failures: from not part of file name

Definition at line 249 of file ToolsDiscovery.cxx.

  {
    SampleHandler mysh;
 
    for (SampleHandler::iterator sample = sh.begin(),
       end = sh.end(); sample != end; ++ sample)
    {
      SampleGrid *grid = dynamic_cast<SampleGrid*>(*sample);
 
      if (grid == 0)
      {
    mysh.add (*sample);
      } else
      {
    const std::string ds = grid->meta()->castString (MetaFields::gridName);
    if (ds.empty())
      RCU_THROW_MSG ("no dataset configured for grid dataset " + ds);
 
    std::regex pattern (RCU::glob_to_regexp (grid->meta()->castString (MetaFields::gridFilter, MetaFields::gridFilter_default)));
 
        std::set<std::string> knownFiles;
        std::map<std::string,std::string> usedFiles;
        for (auto& entry : rucioListFileReplicas (ds))
        {
          if (RCU::match_expr (pattern, entry.name))
          {
            knownFiles.insert (entry.name);
            if (entry.disk == disk)
            {
              std::string url = entry.replica;
              const auto split = url.find (from);
              if (split != std::string::npos)
                url.replace(split, from.size(), to);
              usedFiles[entry.name] = url;
            }
          }
        }
 
        if (usedFiles.empty())
        {
          if (allow_partial)
        RCU_WARN_MSG ("dataset " + ds + " not at " + disk + ", skipped");
        } else if (knownFiles.size() != usedFiles.size())
        {
          if (allow_partial)
          {
            RCU_WARN_MSG ("only incomplete version of dataset " + ds + " at " + disk);
          } else
          {
            usedFiles.clear ();
          }
        }
 
        if (usedFiles.size() == 0)
        {
      sh.add (*sample);
        } else
    {
      std::unique_ptr<SampleLocal> mysample
        (new SampleLocal (grid->name()));
      *mysample->meta() = *grid->meta();
 
      for (const auto& file : usedFiles)
      {
            mysample->add (file.second);
      }
      mysh.add (mysample.release());
    }
      }
    }
    swap (sh, mysh);
  }

mergeFiles() [1/2]

std::unique_ptr< SampleLocal > SH::mergeFiles	(	const Sample &	sample,
		const std::string &	location,
		bool	overwrite )

merge all the files in the sample into a single file in the given location

This is mostly meant to allow merging histogram files (particularly in EventLoop), but in some cases it can also be used to merge n-tuple fiels together.

Returns

a sample containing the merged file

Guarantee

basic

Failures

i/o errors
out of memory III

Definition at line 73 of file ToolsOther.cxx.

  {
    std::unique_ptr<SampleLocal> result (new SampleLocal (sample.name()));
    result->add (location);
    *result->meta() = *sample.meta();
    if (overwrite || gSystem->AccessPathName (location.c_str()) != 0)
    {
      std::vector<std::string> files = sample.makeFileList();
      RCU::hadd (location, files);
    }
    return result;
  }

mergeFiles() [2/2]

SampleHandler SH::mergeFiles	(	const SampleHandler &	sh,
		const std::string &	location,
		bool	overwrite )

merge all the files in the sample handles into a single file per sample in the given location

This is mostly meant to allow merging histogram files (particularly in EventLoop), but in some cases it can also be used to merge n-tuple files together.

Returns

a sample handler containing the merged files

Guarantee

basic

Failures

i/o errors
out of memory III

Definition at line 90 of file ToolsOther.cxx.

  {
    SampleHandler result;
    for (auto& sample : sh)
    {
      std::string sublocation = location + sample->name() + ".root";
      result.add (mergeFiles (*sample, sublocation, overwrite));
    }
    return result;
  }

mergeSamples()

void SH::mergeSamples	(	SampleHandler &	sh,
		const std::string &	sampleName,
		const std::string &	pattern )

effects: remove all samples matching the name pattern, and join them into a single sample named sampleName guarantee: strong failures: out of memory II failures: i/o errors

Definition at line 31 of file ToolsJoin.cxx.

  {
    SampleHandler mysh;
    std::unique_ptr<SampleLocal> mysample (new SampleLocal (sampleName));
 
    std::regex mypattern (pattern.c_str());
    for (SampleHandler::iterator sample = sh.begin(),
       end = sh.end(); sample != end; ++ sample)
    {
      if (RCU::match_expr (mypattern, (*sample)->name()))
      {
    for (unsigned file = 0, end = (*sample)->numFiles();
         file != end; ++ file)
    {
      mysample->add ((*sample)->fileName (file));
    }
      } else
      {
    mysh.add (*sample);
      }
    }
    mysh.add (mysample.release());
    swap (mysh, sh);
  }

openFile()

std::unique_ptr< TFile > SH::openFile	(	const std::string &	name,
		const MetaObject &	options )

open a file with the given options

for now this is to allow opening files with retries, but in the future there may be other options as well.

Guarantee

strong

Failures

i/o errors
file not found

Postcondition

result != nullptr

Definition at line 35 of file ToolsOther.cxx.

  {
    const unsigned retries
      = options.castInteger (MetaNames::openRetries(),
                 MetaNames::openRetries_default());
    const double wait
      = options.castDouble (MetaNames::openRetriesWait(),
                MetaNames::openRetriesWait_default());
    if (wait < 0)
      RCU_THROW_MSG ("negative values not allowed for: " + MetaNames::openRetriesWait());
 
    std::unique_ptr<TFile> result;
 
    static std::atomic<unsigned> seed = 0;
    std::default_random_engine re (++seed);
    std::uniform_real_distribution wait_dist (wait/2, wait);
 
    for (unsigned tries = 0; tries <= retries; ++ tries)
    {
      if (tries > 0)
      {
    unsigned mywait = 1000 * wait_dist (re);
    std::cout << "open failed, waiting " << (mywait/1000.) << " seconds: "
          << name << std::endl;
    std::this_thread::sleep_for (std::chrono::milliseconds (mywait));
      }
      result.reset (TFile::Open (name.c_str(), "READ"));
      if (result != nullptr)
    return result;
    }
    RCU_THROW_MSG ("failed to open file: " + name);
    //cppcheck-suppress rethrowNoCurrentException
    throw; //compiler dummy
  }

printDuplicateEvents()

void SH::printDuplicateEvents

(

const Sample &

sample

)

effects: check the given sample for duplicate events and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors

Definition at line 142 of file ToolsDuplicates.cxx.

  {
    RunEventList list;
    std::unique_ptr<TChain> chain (sample.makeTChain ());
    printDuplicateEvents (*chain, list);
  }

printDuplicateEventsJoint()

void SH::printDuplicateEventsJoint

(

const SampleHandler &

sh

)

effects: check for duplicate events between all the samples and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors

Definition at line 162 of file ToolsDuplicates.cxx.

  {
    RunEventList list;
    for (SampleHandler::iterator sample = sh.begin(),
       end = sh.end(); sample != end; ++ sample)
    {
      std::unique_ptr<TChain> chain ((*sample)->makeTChain ());
      printDuplicateEvents (*chain, list);
    }
  }

printDuplicateEventsSplit()

void SH::printDuplicateEventsSplit

(

const SampleHandler &

sh

)

effects: check each sample for duplicate events and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors

Definition at line 151 of file ToolsDuplicates.cxx.

  {
    for (SampleHandler::iterator sample = sh.begin(),
       end = sh.end(); sample != end; ++ sample)
    {
      printDuplicateEvents (**sample);
    }
  }

readFileList()

void SH::readFileList	(	SampleHandler &	sh,
		const std::string &	name,
		const std::string &	file )

effects: read a file list from a text file guarantee: strong failures: out of memory III failures: i/o errors

Definition at line 371 of file ToolsDiscovery.cxx.

  {
    std::ifstream myfile (file.c_str());
 
    auto sample = std::make_unique<SampleLocal> (name);
    std::string line;
    const std::set<char> whitespaces{'\t',' ','\n','\r'};
    while (std::getline (myfile, line))
    {
      while ((!line.empty()) && whitespaces.count(line.back())) line.pop_back();
      if (!line.empty() && line.at(0) != '#')
      {
    sample->add (line);
      }
    }
    if (!myfile.eof())
      RCU_THROW_MSG ("failed to read file: " + file);
    sh.add (sample.release());
  }

readSusyMeta()

void SH::readSusyMeta	(	const SampleHandler &	sh,
		const std::string &	inputFile )

effects: read the susy-meta-data file and add its information to the samples from the given sample handler guarantee: basic failures: i/o errors warning: there are no checks for duplicate and inconsistent sample definitions

Definition at line 30 of file ToolsMeta.cxx.

  {
    TString myfile = inputFile;
    gSystem->ExpandPathName (myfile);
    std::ifstream file (myfile.Data());
    std::string line;
 
    if (!file)
      RCU_THROW_MSG ("failed to read file: " + inputFile);
 
    while (std::getline (file, line))
    {
      if (line[0] >= '0' && line[0] <= '9')
      {
    std::istringstream str (line);
    std::string DSID, name;
    float crossSection = 0, kFactor = 0, filterEfficiency = 0,
      xsUncertainty = 0;
 
    if (!(str >> DSID >> name >> crossSection >> kFactor >> filterEfficiency
          >> xsUncertainty))
      RCU_THROW_MSG ("failed to parse line: " + line);
 
    std::string mydsid = "." + DSID + ".";
    for (SampleHandler::iterator sample = sh.begin(),
           end = sh.end(); sample != end; ++ sample)
    {
      if ((*sample)->name().find (mydsid) != std::string::npos)
      {
            double oldCrossSection
              = (*sample)->meta()->castDouble (MetaFields::crossSection);
        (*sample)->meta()->setDouble (MetaFields::crossSection, oldCrossSection + crossSection);
        (*sample)->meta()->setDouble (MetaFields::kfactor, kFactor);
        (*sample)->meta()->setDouble (MetaFields::filterEfficiency, filterEfficiency);
        (*sample)->meta()->setDouble (MetaFields::crossSectionRelUncertainty, xsUncertainty);
      }
    }
      }
    }
  }

readSusyMetaDir()

void SH::readSusyMetaDir	(	const SampleHandler &	sh,
		const std::string &	inputDir )

effects: read an entire directory of susy-meta-data files and add their information to the samples from the given sample handle.

for that it assumes that all files ending in ".txt" are susy-meta-data files. guarantee: basic failures: i/o errors warning: there are no checks for duplicate and inconsistent sample definitions

Definition at line 71 of file ToolsMeta.cxx.

  {
    TString mydir = inputDir;
    gSystem->ExpandPathName (mydir);
    void *dirp = 0;
 
    try
    {
      dirp = gSystem->OpenDirectory (mydir.Data());
      const char *file = 0;
      while ((file = gSystem->GetDirEntry (dirp)))
      {
    std::string myfile = inputDir + "/" + file;
    if (myfile.size() > 4 && myfile.substr (myfile.size()-4) == ".txt")
      readSusyMeta (sh, myfile);
      }
      gSystem->FreeDirectory (dirp);
    } catch (...)
    {
      gSystem->FreeDirectory (dirp);
      throw;
    }
  }

rucioCacheDatasetGlob()

std::vector< std::string > SH::rucioCacheDatasetGlob	(	const std::string &	location,
		const std::string &	dataset,
		const std::string &	fileGlob )

download the dataset, and return a list matching the pattern

Guarantee

basic

Failures

grid utility failures i/o errors

Definition at line 535 of file GridTools.cxx.

  {
    std::vector<std::string> result;
 
    std::string path = location;
    if (path.back() != '/')
      path += "/";
    if (dataset.find (':') != std::string::npos)
      path += dataset.substr (dataset.find (':')+1);
    else
      path += dataset;
    const std::string finished {
      path + "-finished"};
 
    // check if the finished file does not exist
    // note that AccessPathName has the weirdest calling convention
    if (gSystem->AccessPathName (finished.c_str()) != 0)
    {
      RucioDownloadResult status = rucioDownload (location, dataset);
      if (status.downloadedFiles + status.alreadyLocal < status.totalFiles)
        throw std::runtime_error ("failed to download all files of " + dataset);
      //  this just creates an empty file
      std::ofstream (finished.c_str());
    }
 
    std::string output = sh::exec_read ("find " + sh::quote (path) + " -type f -name " + sh::quote (fileGlob));
    std::istringstream str (output);
    std::string line;
    while (std::getline (str, line))
    {
      if (!line.empty())
        result.push_back (line);
    }
    return result;
  }

rucioDirectAccessGlob()

std::vector< std::string > SH::rucioDirectAccessGlob	(	const std::string &	name,
		const std::string &	filter,
		const std::string &	selectOptions )

list the rucio URLs for all the files in the dataset or dataset container matching the given filter (as glob expression)

Guarantee

strong

Failures

grid utility failures

Precondition

!name.empty()

name.find('*') == std::string::npos

!filter.empty()

Definition at line 281 of file GridTools.cxx.

  {
    return rucioDirectAccessRegex (name, RCU::glob_to_regexp (filter),
                                   selectOptions);
  }

rucioDirectAccessRegex()

std::vector< std::string > SH::rucioDirectAccessRegex	(	const std::string &	name,
		const std::string &	filter,
		const std::string &	selectOptions )

list the rucio URLs for all the files in the dataset or dataset container matching the given filter (as regular expression)

Guarantee

strong

Failures

grid utility failures

Precondition

!name.empty()

name.find('*') == std::string::npos

!filter.empty()

Definition at line 291 of file GridTools.cxx.

  {
    RCU_REQUIRE_SOFT (!name.empty());
    RCU_REQUIRE_SOFT (name.find('*') == std::string::npos);
    RCU_REQUIRE_SOFT (!filter.empty());
 
    ensureVomsProxy ();
 
    static const std::string separator = "------- SampleHandler Split -------";
 
    ANA_MSG_INFO ("querying rucio for dataset " << name);
    std::string output = sh::exec_read (rucioSetupCommand() + " && echo " + separator + " && rucio list-file-replicas --pfns --protocols root " + selectOptions + " " + sh::quote (name));
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      RCU_THROW_MSG ("couldn't find separator in: " + output);
    std::istringstream str (output.substr (split + separator.size() + 1));
 
    // this is used to avoid getting two copies of the same file.  we
    // first fill them in a map by filename, then copy them into a
    // vector
    std::map<std::string,std::string> resultMap;
 
    std::regex urlPattern ("^root://.*");
    std::regex pattern (filter);
    std::string line;
    while (std::getline (str, line))
    {
      if (line.empty())
      {
        // no-op
      } else if (!RCU::match_expr (urlPattern, line))
      {
        ANA_MSG_INFO ("couldn't handle line: " << line);
      } else
      {
    std::string::size_type split = line.rfind ("/");
    if (split != std::string::npos)
    {
          std::string filename = line.substr (split+1);
      if (RCU::match_expr (pattern, filename))
        resultMap[filename] = line;
    } else
      RCU_THROW_MSG ("couldn't parse line: " + line);
      }
    }
 
    std::vector<std::string> result;
    for (const auto& file : resultMap)
      result.push_back (file.second);
    if (result.size() == 0)
      ANA_MSG_WARNING ("dataset " + name + " did not contain any files.  this is likely not right");
    return result;
  }

rucioDownload()

RucioDownloadResult SH::rucioDownload	(	const std::string &	location,
		const std::string &	dataset )

run rucio-download

Guarantee

basic

Failures

grid utility failures

Definition at line 496 of file GridTools.cxx.

  {
    ensureVomsProxy ();
    
    const std::string separator = "------- SampleHandler Split -------";
    std::string command = rucioSetupCommand() + " && echo " + separator + " && cd " + sh::quote (location) + " && rucio download " + sh::quote (dataset) + " 2>&1";
 
    ANA_MSG_INFO ("starting rucio download " + dataset + " into " + location);
    std::string output = sh::exec_read (command);
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      RCU_THROW_MSG ("couldn't find separator in: " + output);
    output = output.substr (split + separator.size() + 1);
 
    RucioDownloadResult result;
    result.did = readLine (output, "DID ");
    result.totalFiles = readLineUnsigned (output, "Total files (DID): ");
    result.downloadedFiles = readLineUnsigned (output, "Downloaded files: ");
    result.alreadyLocal = readLineUnsigned (output, "Files already found locally: ");
    result.notDownloaded = readLineUnsigned (output, "Files that cannot be downloaded: ");
    return result;
  }

rucioDownloadList()

std::vector< RucioDownloadResult > SH::rucioDownloadList	(	const std::string &	location,
		const std::vector< std::string > &	datasets )

run rucio-download with multiple datasets

Guarantee

basic

Failures

grid utility failures

Definition at line 523 of file GridTools.cxx.

  {
    std::vector<RucioDownloadResult> result;
    for (auto& dataset : datasets)
      result.push_back (rucioDownload (location, dataset));
    return result;
  }

rucioGetMetadata()

std::map< std::string, std::unique_ptr< MetaObject > > SH::rucioGetMetadata

(

const std::set< std::string > &

datasets

)

run rucio-get-metadata for the given list of datasets

Guarantee

strong

Failures

grid utility failures

Precondition

!datasets.empty()

!dataset.empty() (for each dataset)

Definition at line 426 of file GridTools.cxx.

  {
    RCU_REQUIRE_SOFT (!datasets.empty());
 
    ensureVomsProxy ();
 
    static const std::string separator = "------- SampleHandler Split -------";
    std::map<std::string,std::unique_ptr<MetaObject> > result;
 
    std::string command = rucioSetupCommand() + " && echo " + separator + " && rucio get-metadata";
    for (auto& dataset : datasets)
    {
      RCU_REQUIRE_SOFT (!dataset.empty());
      command += " " + sh::quote (dataset);
    }
 
    ANA_MSG_INFO ("querying rucio for meta-data");
    std::string output = sh::exec_read (command);
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      RCU_THROW_MSG ("couldn't find separator in: " + output);
 
    std::istringstream str (output.substr (split + separator.size() + 1));
    std::regex pattern ("^([^:]+): *(.+)$");
    std::string line;
    std::unique_ptr<MetaObject> meta (new MetaObject);
 
    auto addMeta = [&] ()
    {
      std::string name = meta->castString ("scope") + ":" + meta->castString ("name");
      if (result.find (name) != result.end())
    RCU_THROW_MSG ("read " + name + " twice");
      result[name] = std::move (meta);
    };
 
    while (std::getline (str, line))
    {
      std::smatch what;
      if (line == "------")
      {
        addMeta ();
        meta.reset (new MetaObject);
      } else  if (std::regex_match (line, what, pattern))
      {
    if (meta->get (what[1]))
          throw std::runtime_error (std::string("duplicate entry: ") + what[1].str());
    meta->setString (what[1], what[2]);
      } else if (!line.empty())
      {
    ANA_MSG_WARNING ("couldn't parse line: " << line);
      }
    }
    addMeta ();
 
    for (auto& subresult : result)
    {
      if (datasets.find (subresult.first) == datasets.end())
    RCU_THROW_MSG ("received result for dataset not requested: " + subresult.first);
    }
    for (auto& dataset : datasets)
    {
      if (result.find (dataset) == result.end())
    RCU_THROW_MSG ("received no result for dataset: " + dataset);
    }
 
    return result;
  }

rucioListDids()

std::vector< RucioListDidsEntry > SH::rucioListDids

(

const std::string &

dataset

)

run rucio-list-dids for the given dataset

Guarantee

strong

Failures

grid utility failures

Precondition

!dataset.empty()

Definition at line 348 of file GridTools.cxx.

  {
    RCU_REQUIRE_SOFT (!dataset.empty());
 
    ensureVomsProxy ();
 
    static const std::string separator = "------- SampleHandler Split -------";
    std::vector<RucioListDidsEntry> result;
 
    ANA_MSG_INFO ("querying rucio for dataset " << dataset);
    std::string output = sh::exec_read (rucioSetupCommand() + " && echo " + separator + " && rucio list-dids " + sh::quote (dataset));
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      RCU_THROW_MSG ("couldn't find separator in: " + output);
 
    std::istringstream str (output.substr (split + separator.size() + 1));
    std::regex pattern ("^\\| ([a-zA-Z0-9_.-]+):([a-zA-Z0-9_.-]+) +\\| ([a-zA-Z0-9_.-]+) +\\| *$");
    std::string line;
    while (std::getline (str, line))
    {
      std::smatch what;
      if (std::regex_match (line, what, pattern))
      {
    RucioListDidsEntry entry;
    entry.scope = what[1];
    entry.name = what[2];
    entry.type = what[3];
    result.push_back (entry);
      }
    }
    return result;
  }

rucioListFileReplicas()

std::vector< RucioListFileReplicasEntry > SH::rucioListFileReplicas

(

const std::string &

dataset

)

run rucio-list-file-replicas for the given dataset

Guarantee

strong

Failures

grid utility failures

Precondition

!dataset.empty()

Definition at line 384 of file GridTools.cxx.

  {
    RCU_REQUIRE_SOFT (!dataset.empty());
 
    ensureVomsProxy ();
 
    static const std::string separator = "------- SampleHandler Split -------";
    std::vector<RucioListFileReplicasEntry> result;
 
    std::string command = rucioSetupCommand() + " && echo " + separator + " && rucio list-file-replicas --protocols root " + sh::quote (dataset);
 
    ANA_MSG_INFO ("querying rucio for dataset " << dataset);
    std::string output = sh::exec_read ( command );
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      RCU_THROW_MSG ("couldn't find separator in: " + output);
 
    std::istringstream str (output.substr (split + separator.size() + 1));
    std::regex pattern ("^\\| +([^ ]+) +\\| +([^ ]+) +\\| +([^ ]+ [^ ]+) +\\| +([^ ]+) +\\| +([^: ]+): ([^ ]+) +\\| *$");
    std::string line;
    while (std::getline (str, line))
    {
      std::smatch what;
      if (std::regex_match (line, what, pattern) &&
          what[1] != "SCOPE")
      {
    RucioListFileReplicasEntry entry;
    entry.scope    = what[1];
    entry.name     = what[2];
    entry.filesize = what[3];
    entry.adler32  = what[4];
    entry.disk     = what[5];
    entry.replica  = what[6];
    result.push_back (entry);
      }
    }
    return result;
  }

scanDir() [1/3]

void SH::scanDir	(	SampleHandler &	sh,
		const std::string &	dir )

Definition at line 62 of file ToolsDiscovery.cxx.

  {
    ScanDir()
      .sampleDepth (0)
      .minDepth (1)
      .maxDepth (1)
      .scan (sh, dir);
  }

scanDir() [2/3]

void SH::scanDir	(	SampleHandler &	sh,
		const std::string &	dir,
		const std::string &	prefix )

Definition at line 73 of file ToolsDiscovery.cxx.

  {
    DiskListLocal list (dir, prefix);
    ScanDir()
      .sampleDepth (0)
      .minDepth (1)
      .maxDepth (1)
      .scan (sh, list);
  }

scanDir() [3/3]

void SH::scanDir	(	SampleHandler &	sh,
		DiskList &	list,
		const std::string &	pattern = "*.root*",
		const std::string &	samplePattern = "*",
		const std::string &	samplePostfix = "" )

effects: scan the given directory and add all subdirectories as samples that contain root files.

if prefix is provided, use that instead of dir when constructing the file names. guarantee: basic, only some samples might be added failures: low level errors IV failures: directory not found failures: duplicate samples rationale: the prefix option is for the Tier 3 prototype, where you scan the local directory, but then access the files through xrootd

Definition at line 45 of file ToolsDiscovery.cxx.

  {
    ScanDir()
      .sampleDepth (0)
      .minDepth (1)
      .maxDepth (1)
      .filePattern (pattern)
      .samplePattern (samplePattern)
      .samplePostfix (samplePostfix)
      .scan (sh, list);
  }

scanDQ2()

void SH::scanDQ2	(	SampleHandler &	sh,
		const std::string &	pattern )

effects: make a list from DQ2 using the given pattern guarantee: basic, may add partially failures: out of memory III failures: dq2-ls errors

Definition at line 124 of file ToolsDiscovery.cxx.

  {
    if (pattern.find ("*") == std::string::npos)
    {
      addGrid (sh, pattern);
    } else
    {
      std::set<std::string> types = {"DATASET", "DIDType.DATASET"};
      if (pattern.back() == '/')
    types = {"CONTAINER", "DIDType.CONTAINER"};
 
      auto subresult = rucioListDids (pattern);
      for (auto& ds : subresult)
      {
    if (types.find (ds.type) != types.end())
      addGrid (sh, ds.scope + ":" + ds.name);
      }
    }
  }

scanFiles()

void SH::scanFiles	(	SampleHandler &	sh,
		DiskList &	list,
		const std::string &	pattern = "*.root*" )

effects: scan the given directory tree and make a separate sample for each file (using the file name before any "." as the sample name).

guarantee: basic, only some samples might be added failures: out of memory III failures: i/o errors failures: duplicate samples

Definition at line 86 of file ToolsDiscovery.cxx.

  {
    ScanDir()
      .sampleDepth (-1)
      .filePattern (pattern)
      .samplePostfix (".*")
      .scan (sh, list);
  }

scanForTrees() [1/2]

void SH::scanForTrees	(	SampleHandler &	sh,
		const std::string &	pattern )

Definition at line 357 of file ToolsDiscovery.cxx.

  {
    SH::SampleHandler sh_new;
 
    for (SampleHandler::iterator sample = sh.begin(),
       end = sh.end(); sample != end; ++ sample)
    {
      scanForTrees (sh_new, **sample, pattern);
    }
    swap (sh, sh_new);
  }

scanForTrees() [2/2]

void SH::scanForTrees	(	SampleHandler &	sh,
		Sample &	sample,
		const std::string &	pattern = ".*" )

effects: scan for trees in the given sample (or sample handler), and create a separate sample for each tree.

if pattern is specified it is applied to the allowed tree names. guarantee: strong failures: out of memory III failures: i/o errors

Definition at line 326 of file ToolsDiscovery.cxx.

  {
    SamplePtr mysample = sample.makeLocal();
    if (mysample->numFiles() == 0)
    {
      sh.add (&sample);
      return;
    }
    std::unique_ptr<TFile> file (TFile::Open (mysample->fileName(0).c_str()));
    if (!file.get())
      RCU_THROW_MSG ("could not open file: " + mysample->fileName(0));
    TObject *object = 0;
    std::regex mypattern (pattern);
    for (TIter iter (file->GetListOfKeys()); (object = iter.Next()); )
    {
      if (RCU::match_expr (mypattern, object->GetName()) &&
      dynamic_cast<TTree*>(file->Get(object->GetName())))
      {
    std::string newName = sample.name() + "_" + object->GetName();
        std::unique_ptr<Sample> newSample
      (dynamic_cast<Sample*>(sample.Clone (newName.c_str())));
        newSample->name (newName);
        newSample->meta()->setString (MetaFields::treeName, object->GetName());
    sh.add (newSample.release());
      }
    }
  }

scanNEvents() [1/2]

void SH::scanNEvents

(

Sample &

sample

)

effects: scan the given sample and store the number of entries per file in the meta-data guarantee: strong failures: out of memory failures: read errors failures: invalid sample type

Definition at line 47 of file ToolsSplit.cxx.

  {
    SampleLocal *const mysample = dynamic_cast<SampleLocal*>(&sample);
    if (!mysample)
      RCU_THROW_MSG ("sample not of type SampleLocal");
 
    const std::string tree_name = sample.meta()->castString (MetaFields::treeName, MetaFields::treeName_default);
    if (tree_name.empty())
      RCU_THROW_MSG ("sample doesn't contain a tree name");
 
    Long64_t tot_entries = 0;
    std::vector<Long64_t> entries;
    const auto fileList = sample.makeFileList ();
    for (const std::string& file_name : fileList)
    {
      std::unique_ptr<TFile> file (TFile::Open (file_name.c_str(), "READ"));
      if (!file.get())
    RCU_THROW_MSG ("failed to open file " + file_name);
      Long64_t treeEntries = 0;
      TTree *const tree = dynamic_cast<TTree*>(file->Get (tree_name.c_str()));
      if (tree != 0)
    treeEntries = tree->GetEntries ();
      entries.push_back (treeEntries);
      tot_entries += treeEntries;
    }
    RCU_ASSERT (entries.size() == fileList.size());
    sample.meta()->addReplace (new MetaVector<Long64_t>(MetaFields::numEventsPerFile, entries));
    sample.meta()->setDouble (MetaFields::numEvents, tot_entries);
  }

scanNEvents() [2/2]

void SH::scanNEvents

(

SampleHandler &

sh

)

effects: scan each sample in the sample handler and store the number of entries per file in the meta-data guarantee: basic, may only scan some failures: out of memory failures: read errors failures: invalid sample type

Definition at line 38 of file ToolsSplit.cxx.

  {
    for (SampleHandler::iterator sample = sh.begin(),
       end = sh.end(); sample != end; ++ sample)
      scanNEvents (**sample);
  }

scanRucio()

void SH::scanRucio	(	SampleHandler &	sh,
		const std::string &	pattern,
		bool	alwaysQuery = false )

make a list of grid datasets using the given pattern

This will first look for dataset containers, and if none are found, it will instead look for datasets. If those are also not found it will produce an error.

For patterns that don't contain a wild-card (i.e. that can only match one dataset/container) it instead adds the dataset directly without querying rucio. This is a performance optimization based on the assumption that you probably checked the dataset names beforehand, and that if you made a mistake it will pop up at a later stage anyways. If you want to query rucio even in those cases set alwaysQuery to true.

Guarantee

basic, may add some datasets

Failures

no pattern match
rucio failures
out of memory III

Definition at line 146 of file ToolsDiscovery.cxx.

  {
    if (pattern.find ("*") == std::string::npos && !alwaysQuery)
    {
      addGrid (sh, pattern);
    } else
    {
      auto subresult = rucioListDids (pattern);
      bool added = false;
      for (std::string type : {"CONTAINER", "DIDType.CONTAINER", "DATASET", "DIDType.DATASET"})
      {
    for (auto& ds : subresult)
    {
      if (ds.type == type)
      {
        addGrid (sh, ds.scope + ":" + ds.name);
        added = true;
      }
    }
    if (added)
      return;
      }
      RCU_THROW_MSG ("failed to find any datasets matching pattern: " + pattern);
    }
  }

scanSingleDir()

void SH::scanSingleDir	(	SampleHandler &	sh,
		const std::string &	name,
		DiskList &	list,
		const std::string &	pattern )

effects: scan the given directory tree and turn it into a single sample of the given name guarantee: strong failures: out of memory III

Definition at line 112 of file ToolsDiscovery.cxx.

  {
    ScanDir()
      .sampleDepth (0)
      .filePattern (pattern)
      .sampleRename ("*", name)
      .scan (sh, list);
  }

splitSample()

SampleHandler SH::splitSample	(	Sample &	sample,
		Long64_t	nevt )

effects: split the given sample into a set of samples, with each sample containing either exactly one file or at most nevt events side effects: if scanNEvents hasn't been run on this sample, run it.

guarantee: strong failures: out of memory failures: scanning errors

Definition at line 79 of file ToolsSplit.cxx.

  {
    if (!dynamic_cast<SampleLocal*>(&sample))
      RCU_THROW_MSG ("sample not of type SampleLocal");
 
    TObject *meta = sample.meta()->get (MetaFields::numEventsPerFile);
    if (!meta)
    {
      RCU_WARN_MSG ("sample " + sample.name() + " lacks nc_nevtfile, running scanNEvents, please save sample");
      scanNEvents (sample);
      meta = sample.meta()->get (MetaFields::numEventsPerFile);
    }
    RCU_ASSERT (meta != 0);
    MetaVector<Long64_t> *const nentries
      = dynamic_cast<MetaVector<Long64_t> *>(meta);
    if (nentries == 0)
      RCU_THROW_MSG ("nc_nevtfile is of the wrong type");
    if (nentries->value.size() != sample.numFiles())
      RCU_THROW_MSG ("nc_nevtfile has the wrong number of entries");
 
    SampleHandler result;
    std::unique_ptr<SampleLocal> res;
    Long64_t num = 0;
    const std::string meta_tree = sample.meta()->castString (MetaFields::treeName, MetaFields::treeName_default);
    const double meta_xs = sample.meta()->castDouble (MetaFields::crossSection, 0);
    for (unsigned file = 0, end = nentries->value.size(); file != end; ++ file)
    {
      if (num > 0 && num + nentries->value[file] > nevt)
      {
    result.add (res.release());
    num = 0;
      }
      if (res.get() == 0)
      {
    std::ostringstream name;
    name << sample.name() << "_" << result.size();
    res.reset (new SampleLocal (name.str()));
    res->tags (sample.tags());
    res->meta()->fetch (*sample.meta());
    if (!meta_tree.empty())
      res->meta()->setString (MetaFields::treeName, meta_tree);
    if (meta_xs != 0)
      res->meta()->setDouble (MetaFields::crossSection, meta_xs);
      }
      res->add (sample.fileName (file));
      num += nentries->value[file];
    }
    if (num > 0)
      result.add (res.release());
    return result;
  }