SH Namespace Reference

This module provides a lot of global definitions, forward declarations and includes that are used by all modules. More...

Classes
class	DiskList
	an interface for listing directory contents, locally or on a file server More...
class	DiskListEOS
	a DiskList implementation for EOS More...
class	DiskListLocal
	a DiskList implementation for local directories More...
class	DiskListSRM
	a DiskList implementation for the SRM protocol More...
class	DiskListXRD
	a DiskList implementation for the XRD protocol More...
class	DiskOutput
	a class/interface representing an output location for files More...
class	DiskOutputLocal
	an implementation of DiskOutput for local disks More...
class	DiskOutputXRD
	an implementation of DiskOutput using the XRD protocol More...
class	DiskWriter
	an interface that manages writing a single output file More...
class	DiskWriterLocal
	an implementation of DiskWriter for local files More...
class	DiskWriterXRD
	an implementation of DiskWriter for the XRD protocol More...
class	Meta
	A base class for classes that implement arbitrary meta-data. More...
class	MetaData
	A class implementing a templatized version of the meta-data. More...
struct	MetaDataQuery
	an object containing the result of an AMI meta-data query More...
struct	MetaDataSample
	all the meta-data fields that we may try to read from AMI. More...
struct	MetaFields
	This class contains the known meta-data names. More...
struct	MetaNames
	the names of common meta-data More...
class	MetaObject
	A class that manages meta-data to be associated with an object. More...
class	MetaVector
	This class defines a templatized version of the meta-data in vector form. More...
struct	RucioDownloadResult
	the result from rucio_download More...
struct	RucioListDidsEntry
	one entry from the rucio-list-dids command More...
struct	RucioListFileReplicasEntry
	one entry from the rucio-list-file-replicas command More...
class	Sample
	a base class that manages a set of files belonging to a particular data set and the associated meta-data. More...
class	SampleComposite
	This module defines an implementation of Sample that contains composite samples. More...
class	SampleGrid
	This class implements a Sample located on the grid. More...
class	SampleHandler
	A class that manages a list of Sample objects. More...
class	SampleHist
	A sample that represents a single histogram file. More...
class	SampleLocal
	A Sample based on a simple file list. More...
class	SampleMeta
	A Sample that consists only of Meta-Information. More...
struct	ScanDir
	the class used for scanning local directories and file servers for samples More...
class	TagList
	A class that manages a list of tags for identifying samples. More...

Functions
void	fetchMetaData (MetaDataQuery &query)
	effects: fetch information on all the samples/datasets specified guarantee: basic
void	fetchMetaData (SampleHandler &sh, bool override)
	effects: fetch information on all the samples in the sample handler.
const std::string &	downloadStageEnvVar ()
	the name of the environment variable containing the directory for staging files from the grid
bool	checkVomsProxy ()
	return whether we have a valid VOMS proxy available
void	ensureVomsProxy ()
	ensure that we have a valid VOMS proxy available
std::vector< std::string >	faxListFilesGlob (const std::string &name, const std::string &filter)
	list the FAX URLs for all the files in the dataset or dataset container matching the given filter (as glob expression)
std::vector< std::string >	faxListFilesRegex (const std::string &name, const std::string &filter)
	list the FAX URLs for all the files in the dataset or dataset container matching the given filter (as regular expression)
std::vector< std::string >	rucioDirectAccessGlob (const std::string &name, const std::string &filter, const std::string &selectOptions)
	list the rucio URLs for all the files in the dataset or dataset container matching the given filter (as glob expression)
std::vector< std::string >	rucioDirectAccessRegex (const std::string &name, const std::string &filter, const std::string &selectOptions)
	list the rucio URLs for all the files in the dataset or dataset container matching the given filter (as regular expression)
std::vector< RucioListDidsEntry >	rucioListDids (const std::string &dataset)
	run rucio-list-dids for the given dataset
std::vector< RucioListFileReplicasEntry >	rucioListFileReplicas (const std::string &dataset)
	run rucio-list-file-replicas for the given dataset
std::map< std::string, std::unique_ptr< MetaObject > >	rucioGetMetadata (const std::set< std::string > &datasets)
	run rucio-get-metadata for the given list of datasets
RucioDownloadResult	rucioDownload (const std::string &location, const std::string &dataset)
	run rucio-download
std::vector< RucioDownloadResult >	rucioDownloadList (const std::string &location, const std::vector< std::string > &datasets)
	run rucio-download with multiple datasets
std::vector< std::string >	rucioCacheDatasetGlob (const std::string &location, const std::string &dataset, const std::string &fileGlob)
	download the dataset, and return a list matching the pattern
std::string	dbg (const Meta &, unsigned)
std::string	dbg (const TagList &obj, unsigned)
void	scanDir (SampleHandler &sh, DiskList &list, const std::string &pattern="*.root*", const std::string &samplePattern="*", const std::string &samplePostfix="")
	effects: scan the given directory and add all subdirectories as samples that contain root files.
void	scanDir (SampleHandler &sh, const std::string &dir)
void	scanDir (SampleHandler &sh, const std::string &dir, const std::string &prefix)
void	scanFiles (SampleHandler &sh, DiskList &list, const std::string &pattern="*.root*")
	effects: scan the given directory tree and make a separate sample for each file (using the file name before any "." as the sample name).
std::unique_ptr< Sample >	makeFromTChainUnique (const std::string &name, const TChain &chain)
	effects: create a sample with the given name from the given TChain object guarantee: strong failures: out of memory II
Sample *	makeFromTChain (const std::string &name, const TChain &chain)
	effects: create a sample with the given name from the given TChain object guarantee: strong failures: out of memory II
void	scanSingleDir (SampleHandler &sh, const std::string &name, DiskList &list, const std::string &pattern="*.root*")
	effects: scan the given directory tree and turn it into a single sample of the given name guarantee: strong failures: out of memory III
void	scanDQ2 (SampleHandler &sh, const std::string &pattern)
	effects: make a list from DQ2 using the given pattern guarantee: basic, may add partially failures: out of memory III failures: dq2-ls errors
void	scanRucio (SampleHandler &sh, const std::string &pattern, bool alwaysQuery=false)
	make a list of grid datasets using the given pattern
void	addGrid (SampleHandler &sh, const std::string &ds)
	effects: add a grid dataset for dataset ds guarantee: strong failures: out of memory II requires: ds.find ("*") == std::string::npos
void	addGridCombined (SampleHandler &sh, const std::string &dsName, const std::vector< std::string > &dsList)
	effects: add a combined grid dataset with name dsName for dataset list dsList guarantee: strong failures: out of memory II requires: for each dataset ds: ds.find ("*") == std::string::npos
void	addGridCombinedFromFile (SampleHandler &sh, const std::string &dsName, const std::string &dsFile)
	effects: add a combined grid dataset with name dsName for dataset list file dsFile guarantee: strong failures: out of memory II requires: for each dataset ds: ds.find ("*") == std::string::npos
void	makeGridDirect (SampleHandler &sh, const std::string &disk, const std::string &from, const std::string &to, bool allow_partial)
	effects: update all grid samples in the sample handler that are located on the given disk to be opened directly.
void	scanForTrees (SampleHandler &sh, const std::shared_ptr< Sample > &sample, const std::string &pattern=".*")
	effects: scan for trees in the given sample (or sample handler), and create a separate sample for each tree.
void	scanForTrees (SampleHandler &sh, const std::string &pattern)
void	readFileList (SampleHandler &sh, const std::string &name, const std::string &file)
	effects: read a file list from a text file guarantee: strong failures: out of memory III failures: i/o errors
void	printDuplicateEvents (const Sample &sample)
	effects: check the given sample for duplicate events and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors
void	printDuplicateEventsSplit (const SampleHandler &sh)
	effects: check each sample for duplicate events and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors
void	printDuplicateEventsJoint (const SampleHandler &sh)
	effects: check for duplicate events between all the samples and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors
void	mergeSamples (SampleHandler &sh, const std::string &sampleName, const std::string &pattern)
	effects: remove all samples matching the name pattern, and join them into a single sample named sampleName guarantee: strong failures: out of memory II failures: i/o errors
void	readSusyMeta (const SampleHandler &sh, const std::string &inputFile)
	effects: read the susy-meta-data file and add its information to the samples from the given sample handler guarantee: basic failures: i/o errors warning: there are no checks for duplicate and inconsistent sample definitions
void	readSusyMetaDir (const SampleHandler &sh, const std::string &inputDir)
	effects: read an entire directory of susy-meta-data files and add their information to the samples from the given sample handle.
std::unique_ptr< TFile >	openFile (const std::string &name, const MetaObject &options)
	open a file with the given options
std::unique_ptr< SampleLocal >	mergeFiles (const Sample &sample, const std::string &location, bool overwrite)
	merge all the files in the sample into a single file in the given location
SampleHandler	mergeFiles (const SampleHandler &sh, const std::string &location, bool overwrite)
	merge all the files in the sample handles into a single file per sample in the given location
void	scanNEvents (SampleHandler &sh)
	effects: scan each sample in the sample handler and store the number of entries per file in the meta-data guarantee: basic, may only scan some failures: out of memory failures: read errors failures: invalid sample type
void	scanNEvents (Sample &sample)
	effects: scan the given sample and store the number of entries per file in the meta-data guarantee: strong failures: out of memory failures: read errors failures: invalid sample type
SampleHandler	splitSample (Sample &sample, Long64_t nevt)
	effects: split the given sample into a set of samples, with each sample containing either exactly one file or at most nevt events side effects: if scanNEvents hasn't been run on this sample, run it.
std::string	dbg (const Meta &obj, unsigned verbosity=0)
	the debugging info of this object
std::string	dbg (const MetaObject &obj, unsigned verbosity=0)
	the debugging info of this object
void	swap (MetaObject &a, MetaObject &b)
	standard swap
std::string	dbg (const Sample &obj, unsigned verbosity=0)
	the debugging info of this object
std::string	dbg (const SampleHandler &obj, unsigned verbosity=0)
	the debugging info of this object
void	swap (SampleHandler &a, SampleHandler &b)
	standard swap
std::string	dbg (const TagList &obj, unsigned verbosity=0)
	the debugging info of this object

Detailed Description

This module provides a lot of global definitions, forward declarations and includes that are used by all modules.

This module defines utility functions used for splitting samples.

This module defines utility functions used for joining samples.

Author

Nils Krumnack

Function Documentation

addGrid()

void SH::addGrid	(	SampleHandler &	sh,
		const std::string &	ds )

effects: add a grid dataset for dataset ds guarantee: strong failures: out of memory II requires: ds.find ("*") == std::string::npos

Definition at line 175 of file ToolsDiscovery.cxx.

  {
    RCU_ASSERT_SOFT (ds.find ("*") == std::string::npos);
 
    std::string name;
    if (!ds.empty() && ds.back() == '/')
      name = ds.substr (0, ds.size()-1);
    else
      name = ds;
 
    auto sample = std::make_unique<SampleGrid> (name);
    sample->meta()->setString (MetaFields::gridName, ds);
    sample->meta()->setString (MetaFields::gridFilter, MetaFields::gridFilter_default);
    sh.add (std::move (sample));
  }

addGridCombined()

void SH::addGridCombined	(	SampleHandler &	sh,
		const std::string &	dsName,
		const std::vector< std::string > &	dsList )

effects: add a combined grid dataset with name dsName for dataset list dsList guarantee: strong failures: out of memory II requires: for each dataset ds: ds.find ("*") == std::string::npos

Definition at line 193 of file ToolsDiscovery.cxx.

  {
    std::string name;
    for (const std::string &ds : dsList)
    {
      RCU_ASSERT_SOFT (ds.find ("*") == std::string::npos);
 
      if (!name.empty())
        name.append(",");
 
      if (ds.at(ds.size() - 1) == '/')
        name.append(ds.substr (0, ds.size() - 1));
      else
        name.append(ds);
    }
 
    auto sample = std::make_unique<SampleGrid> (dsName);
    sample->meta()->setString (MetaFields::gridName, name);
    sample->meta()->setString (MetaFields::gridFilter, MetaFields::gridFilter_default);
    sh.add (std::move (sample));
  }

addGridCombinedFromFile()

void SH::addGridCombinedFromFile	(	SampleHandler &	sh,
		const std::string &	dsName,
		const std::string &	dsFile )

effects: add a combined grid dataset with name dsName for dataset list file dsFile guarantee: strong failures: out of memory II requires: for each dataset ds: ds.find ("*") == std::string::npos

Definition at line 215 of file ToolsDiscovery.cxx.

  {
    std::ifstream file (dsFile.c_str());
 
    std::string name;
    std::string ds;
    const std::set<char> whitespaces{'\t',' ','\n','\r'};
    while (std::getline (file, ds))
    {
      while ((!ds.empty()) && whitespaces.count(ds.back())) ds.pop_back();
      if (ds.empty() || ds.at(0) == '#')
        continue;
 
      RCU_ASSERT_SOFT (ds.find ("*") == std::string::npos);
 
      if (!name.empty())
        name.append(",");
 
      if (ds.at(ds.size() - 1) == '/')
        name.append(ds.substr (0, ds.size() - 1));
      else
        name.append(ds);
    }
    if (!file.eof())
      throw std::runtime_error ("failed to read file: " + dsFile);
 
    auto sample = std::make_unique<SampleGrid> (dsName);
    sample->meta()->setString (MetaFields::gridName, name);
    sample->meta()->setString (MetaFields::gridFilter, MetaFields::gridFilter_default);
    sh.add (std::move (sample));
  }

checkVomsProxy()

bool SH::checkVomsProxy

(

)

return whether we have a valid VOMS proxy available

Guarantee

basic

Failures

grid utility failures

Definition at line 221 of file GridTools.cxx.

  {
    return proxyData().checkVomsProxy();
  }

dbg() [1/7]

std::string SH::dbg ( const Meta & , unsigned  )

related

Definition at line 21 of file Meta.cxx.

  {
    return "meta-object";
  }

dbg() [2/7]

std::string SH::dbg ( const TagList & obj, unsigned  )

related

Definition at line 23 of file TagList.cxx.

  {
    std::string result;
 
    for (const auto& tag : obj)
    {
      if (!result.empty())
    result += ",";
      result += tag;
    }
    return "(" + result + ")";
  }

downloadStageEnvVar()

const std::string & SH::downloadStageEnvVar

(

)

the name of the environment variable containing the directory for staging files from the grid

Definition at line 213 of file GridTools.cxx.

  {
    static const std::string result = "SAMPLEHANDLER_RUCIO_DOWNLOAD";
    return result;
  }

ensureVomsProxy()

void SH::ensureVomsProxy

(

)

ensure that we have a valid VOMS proxy available

First this checks whether we have a valid PROXY, and if not it sets up a new certificate.

Guarantee

basic

Failures

grid utility failures
failure to set up new VOMS proxy

Definition at line 228 of file GridTools.cxx.

  {
    proxyData().ensureVomsProxy();
  }

faxListFilesGlob()

std::vector< std::string > SH::faxListFilesGlob	(	const std::string &	name,
		const std::string &	filter )

list the FAX URLs for all the files in the dataset or dataset container matching the given filter (as glob expression)

Guarantee

strong

Failures

grid utility failures

Precondition

!name.empty()

name.find('*') == std::string::npos

!filter.empty()

Definition at line 236 of file GridTools.cxx.

  {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpragmas"
#pragma GCC diagnostic ignored "-Wunknown-pragmas"
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
    return faxListFilesRegex (name, RCU::glob_to_regexp (filter));
#pragma GCC diagnostic pop
  }

faxListFilesRegex()

std::vector< std::string > SH::faxListFilesRegex	(	const std::string &	name,
		const std::string &	filter )

list the FAX URLs for all the files in the dataset or dataset container matching the given filter (as regular expression)

Guarantee

strong

Failures

grid utility failures

Precondition

!name.empty()

name.find('*') == std::string::npos

!filter.empty()

Definition at line 249 of file GridTools.cxx.

  {
    RCU_REQUIRE_SOFT (!name.empty());
    RCU_REQUIRE_SOFT (name.find('*') == std::string::npos);
    RCU_REQUIRE_SOFT (!filter.empty());
 
    ensureVomsProxy ();
 
    static const std::string separator = "------- SampleHandler Split -------";
    std::vector<std::string> result;
 
    ANA_MSG_INFO ("querying FAX for dataset " << name);
    std::string output = sh::exec_read ("source $ATLAS_LOCAL_ROOT_BASE/user/atlasLocalSetup.sh -q && lsetup --force fax && echo " + separator + " && fax-get-gLFNs " + sh::quote (name));
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      throw std::runtime_error ("couldn't find separator in: " + output);
 
    std::istringstream str (output.substr (split + separator.size() + 1));
    std::regex pattern (filter);
    std::string line;
    while (std::getline (str, line))
    {
      if (!line.empty())
      {
    if (!line.starts_with ("root:"))
      throw std::runtime_error ("faxListFilesRegex: couldn't parse line: " + line);
 
    std::string::size_type split1 = line.rfind (":");
    std::string::size_type split2 = line.rfind ("/");
    if (split1 < split2)
      split1 = split2;
    if (split1 != std::string::npos)
    {
      if (RCU::match_expr (pattern, line.substr (split1+1)))
        result.push_back (line);
    } else
      throw std::runtime_error ("faxListFilesRegex: couldn't parse line: " + line);
      }
    }
    if (result.size() == 0)
      ANA_MSG_WARNING ("dataset " << name << " did not contain any files.  this is likely not right");
    return result;
  }

fetchMetaData() [1/2]

void SH::fetchMetaData

(

MetaDataQuery &

query

)

effects: fetch information on all the samples/datasets specified guarantee: basic

Definition at line 51 of file fetch.cxx.

  {
    static std::once_flag loaded;
    auto do_load = []() {
      // rationale: the helper module is installed via
      //   atlas_install_python_modules and is importable through
      //   PYTHONPATH, so import it by name rather than loading it from
      //   the dead RootCore $ROOTCOREBIN path.
      if (!TPython::Exec ("import SampleHandler.SampleHandler_QueryAMI"))
        throw std::runtime_error ("failed to import python module SampleHandler.SampleHandler_QueryAMI");
    };
    std::call_once (loaded, do_load);
 
    std::ostringstream command;
#if ROOT_VERSION_CODE >= ROOT_VERSION(6,33,01)
    command << "_anyresult = ";
#endif
    command << "SampleHandler.SampleHandler_QueryAMI.SampleHandler_QueryAmi([";
    for (std::size_t iter = 0, end = query.samples.size(); iter != end; ++ iter)
    {
      if (iter != 0)
    command << ", ";
      command << pyQuote (query.samples[iter].name);
    }
    command << "])";
#if ROOT_VERSION_CODE >= ROOT_VERSION(6,33,01)
    std::any result;
    if (!TPython::Exec (command.str().c_str(), &result))
      throw std::runtime_error ("failed to execute AMI metadata query");
    query = std::any_cast<MetaDataQuery>(result);
#else
    MetaDataQuery* myquery = static_cast<MetaDataQuery*>
      ((void*) TPython::Eval (command.str().c_str()));
    if (myquery == nullptr)
      throw std::runtime_error ("failed to execute AMI metadata query");
    query = *myquery;
#endif
  }

fetchMetaData() [2/2]

void SH::fetchMetaData	(	SampleHandler &	sh,
		bool	override )

effects: fetch information on all the samples in the sample handler.

if override is specified it will override the existing meta-data fields, otherwise they will only be used if the given fields do not yet exist guarantee: basic

Definition at line 91 of file fetch.cxx.

  {
    using namespace msgFetch;
 
    std::vector<SH::Sample*> samples;
    // typedef std::vector<SH::Sample*> SamplesIter;
    MetaDataQuery query;
    for (auto *sample : sh)
    {
      std::string name = sample->meta()->castString (SH::MetaFields::gridName, sample->name());
      query.samples.push_back (MetaDataSample (name));
      samples.push_back (sample);
    }
    fetchMetaData (query);
 
    if (!query.messages.empty())
      ANA_MSG_INFO (query.messages);
    for (std::size_t iter = 0, end = query.samples.size(); iter != end; ++ iter)
    {
      if (query.samples[iter].unknown)
      {
        ANA_MSG_WARNING ("failed to find sample " << query.samples[iter].name);
      } else
      {
    if (iter >= samples.size())
      throw std::runtime_error ("AMI query returned more samples than were requested");
    SH::Sample *sample = samples[iter];
 
    if (!override)
    {
      query.samples[iter].isData = sample->meta()->castDouble (SH::MetaFields::isData, query.samples[iter].isData);
      query.samples[iter].luminosity = sample->meta()->castDouble (SH::MetaFields::lumi, query.samples[iter].luminosity);
      query.samples[iter].crossSection = sample->meta()->castDouble (SH::MetaFields::crossSection, query.samples[iter].crossSection);
      query.samples[iter].nevents = sample->meta()->castDouble (SH::MetaFields::numEvents, query.samples[iter].nevents);
      query.samples[iter].kfactor = sample->meta()->castDouble (SH::MetaFields::kfactor, query.samples[iter].kfactor);
      query.samples[iter].filterEfficiency = sample->meta()->castDouble (SH::MetaFields::filterEfficiency, query.samples[iter].filterEfficiency);
    }
    if (query.samples[iter].isData != -1)
      sample->meta()->setDouble (SH::MetaFields::isData, query.samples[iter].isData);
    if (query.samples[iter].luminosity != -1)
      sample->meta()->setDouble (SH::MetaFields::lumi, query.samples[iter].luminosity);
    if (query.samples[iter].crossSection != -1)
      sample->meta()->setDouble (SH::MetaFields::crossSection, query.samples[iter].crossSection);
    if (query.samples[iter].nevents != -1)
      sample->meta()->setDouble (SH::MetaFields::numEvents, query.samples[iter].nevents);
    if (query.samples[iter].kfactor != -1)
      sample->meta()->setDouble (SH::MetaFields::kfactor, query.samples[iter].kfactor);
    if (query.samples[iter].filterEfficiency != -1)
      sample->meta()->setDouble (SH::MetaFields::filterEfficiency, query.samples[iter].filterEfficiency);
      }
    }
  }

makeFromTChain()

Sample * SH::makeFromTChain	(	const std::string &	name,
		const TChain &	chain )

effects: create a sample with the given name from the given TChain object guarantee: strong failures: out of memory II

Definition at line 105 of file ToolsDiscovery.cxx.

  {
    return makeFromTChainUnique (name, chain).release();
  }

makeFromTChainUnique()

std::unique_ptr< Sample > SH::makeFromTChainUnique	(	const std::string &	name,
		const TChain &	chain )

effects: create a sample with the given name from the given TChain object guarantee: strong failures: out of memory II

Definition at line 91 of file ToolsDiscovery.cxx.

  {
    auto result = std::make_unique<SampleLocal> (name);
    result->meta()->setString (MetaFields::treeName, chain.GetName());
 
    TIter chainIter (chain.GetListOfFiles());
    TChainElement *chainElement = nullptr;
    while ((chainElement = dynamic_cast<TChainElement*>(chainIter.Next())) != nullptr)
      result->add (chainElement->GetTitle());
    return result;
  }

makeGridDirect()

void SH::makeGridDirect	(	SampleHandler &	sh,
		const std::string &	disk,
		const std::string &	from,
		const std::string &	to,
		bool	allow_partial )

effects: update all grid samples in the sample handler that are located on the given disk to be opened directly.

for that it replaces the from part of the file with the to part. if allow_partial is set, it allows for partial datasets (meaning you may lose some files that are not there). guarantee: basic, may convert only some samples failures: out of memory III failures: dq2-ls errors failures: from not part of file name

Definition at line 249 of file ToolsDiscovery.cxx.

  {
    using namespace msgDiscovery;
 
    SampleHandler mysh;
 
    for (auto sample : sh.samples())
    {
      SampleGrid *grid = dynamic_cast<SampleGrid*>(sample.get());
 
      if (grid == nullptr)
      {
        mysh.add (sample);
      } else
      {
        const std::string ds = grid->meta()->castString (MetaFields::gridName);
        if (ds.empty())
          throw std::runtime_error ("no dataset configured for grid sample " + grid->name());
 
        std::regex pattern (RCU::glob_to_regexp (grid->meta()->castString (MetaFields::gridFilter, MetaFields::gridFilter_default)));
 
        std::set<std::string> knownFiles;
        std::map<std::string,std::string> usedFiles;
        for (auto& entry : rucioListFileReplicas (ds))
        {
          if (RCU::match_expr (pattern, entry.name))
          {
            knownFiles.insert (entry.name);
            if (entry.disk == disk)
            {
              std::string url = entry.replica;
              const auto split = url.find (from);
              if (split == std::string::npos)
                throw std::runtime_error ("prefix \"" + from + "\" not part of replica URL: " + url);
              url.replace(split, from.size(), to);
              usedFiles[entry.name] = url;
            }
          }
        }
 
        if (usedFiles.empty())
        {
          if (allow_partial)
            ANA_MSG_WARNING ("dataset " << ds << " not at " << disk << ", skipped");
        } else if (knownFiles.size() != usedFiles.size())
        {
          if (allow_partial)
          {
            ANA_MSG_WARNING ("only incomplete version of dataset " << ds << " at " << disk);
          } else
          {
            usedFiles.clear ();
          }
        }
 
        if (usedFiles.size() == 0)
        {
          mysh.add (sample);
        } else
        {
          auto mysample = std::make_unique<SampleLocal> (grid->name());
          *mysample->meta() = *grid->meta();
 
          for (const auto& file : usedFiles)
          {
            mysample->add (file.second);
          }
          mysh.add (std::move (mysample));
        }
      }
    }
    swap (sh, mysh);
  }

mergeFiles() [1/2]

std::unique_ptr< SampleLocal > SH::mergeFiles	(	const Sample &	sample,
		const std::string &	location,
		bool	overwrite )

merge all the files in the sample into a single file in the given location

This is mostly meant to allow merging histogram files (particularly in EventLoop), but in some cases it can also be used to merge n-tuple fiels together.

Returns

a sample containing the merged file

Guarantee

basic

Failures

i/o errors
out of memory III

Definition at line 71 of file ToolsOther.cxx.

  {
    auto result = std::make_unique<SampleLocal> (sample.name());
    result->add (location);
    *result->meta() = *sample.meta();
    if (overwrite || gSystem->AccessPathName (location.c_str()) != 0)
    {
      std::vector<std::string> files = sample.makeFileList();
      RCU::hadd (location, files);
    }
    return result;
  }

mergeFiles() [2/2]

SampleHandler SH::mergeFiles	(	const SampleHandler &	sh,
		const std::string &	location,
		bool	overwrite )

merge all the files in the sample handles into a single file per sample in the given location

This is mostly meant to allow merging histogram files (particularly in EventLoop), but in some cases it can also be used to merge n-tuple files together.

Returns

a sample handler containing the merged files

Guarantee

basic

Failures

i/o errors
out of memory III

Definition at line 88 of file ToolsOther.cxx.

  {
    SampleHandler result;
    for (auto sample : sh)
    {
      std::string sublocation = location + sample->name() + ".root";
      result.add (mergeFiles (*sample, sublocation, overwrite));
    }
    return result;
  }

mergeSamples()

void SH::mergeSamples	(	SampleHandler &	sh,
		const std::string &	sampleName,
		const std::string &	pattern )

effects: remove all samples matching the name pattern, and join them into a single sample named sampleName guarantee: strong failures: out of memory II failures: i/o errors

Definition at line 25 of file ToolsJoin.cxx.

  {
    SampleHandler mysh;
    auto mysample = std::make_shared<SampleLocal> (sampleName);
    bool matched = false;
 
    std::regex mypattern (pattern);
    for (auto& sample : sh.samples())
    {
      if (RCU::match_expr (mypattern, sample->name()))
      {
        if (!matched)
        {
          // rationale: inherit the metadata (tree name, cross section,
          //   ...) from the first matched sample instead of falling
          //   back to the defaults.
          *mysample->meta() = *sample->meta();
          matched = true;
        }
        for (unsigned file = 0, end = sample->numFiles();
            file != end; ++ file)
        {
          mysample->add (sample->fileName (file));
        }
      } else
      {
        mysh.add (sample);
      }
    }
    // rationale: only add the merged sample if at least one sample
    //   matched, so we do not create a spurious empty sample.
    if (matched)
      mysh.add (mysample);
    swap (mysh, sh);
  }

openFile()

std::unique_ptr< TFile > SH::openFile	(	const std::string &	name,
		const MetaObject &	options )

open a file with the given options

for now this is to allow opening files with retries, but in the future there may be other options as well.

Guarantee

strong

Failures

i/o errors
file not found

Postcondition

result != nullptr

Definition at line 35 of file ToolsOther.cxx.

  {
    const unsigned retries
      = options.castInteger (MetaNames::openRetries(),
                 MetaNames::openRetries_default());
    const double wait
      = options.castDouble (MetaNames::openRetriesWait(),
                MetaNames::openRetriesWait_default());
    if (wait < 0)
      throw std::runtime_error ("negative values not allowed for: " + MetaNames::openRetriesWait());
 
    std::unique_ptr<TFile> result;
 
    static std::atomic<unsigned> seed = 0;
    std::default_random_engine re (++seed);
    std::uniform_real_distribution wait_dist (wait/2, wait);
 
    for (unsigned tries = 0; tries <= retries; ++ tries)
    {
      if (tries > 0)
      {
    unsigned mywait = 1000 * wait_dist (re);
    std::cout << "open failed, waiting " << (mywait/1000.) << " seconds: "
          << name << std::endl;
    std::this_thread::sleep_for (std::chrono::milliseconds (mywait));
      }
      result.reset (TFile::Open (name.c_str(), "READ"));
      if (result != nullptr)
    return result;
    }
    throw std::runtime_error ("failed to open file: " + name);
  }

printDuplicateEvents()

void SH::printDuplicateEvents

(

const Sample &

sample

)

effects: check the given sample for duplicate events and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors

Definition at line 172 of file ToolsDuplicates.cxx.

  {
    RunEventList list;
    std::unique_ptr<TChain> chain (sample.makeTChain ());
    printDuplicateEvents (*chain, list);
  }

printDuplicateEventsJoint()

void SH::printDuplicateEventsJoint

(

const SampleHandler &

sh

)

effects: check for duplicate events between all the samples and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors

Definition at line 191 of file ToolsDuplicates.cxx.

  {
    RunEventList list;
    for (auto *sample : sh)
    {
      std::unique_ptr<TChain> chain (sample->makeTChain ());
      printDuplicateEvents (*chain, list);
    }
  }

printDuplicateEventsSplit()

void SH::printDuplicateEventsSplit

(

const SampleHandler &

sh

)

effects: check each sample for duplicate events and then print them out guarantee: basic, may print partially failures: out of memory III failures: i/o errors

Definition at line 181 of file ToolsDuplicates.cxx.

  {
    for (auto *sample : sh)
    {
      printDuplicateEvents (*sample);
    }
  }

readFileList()

void SH::readFileList	(	SampleHandler &	sh,
		const std::string &	name,
		const std::string &	file )

effects: read a file list from a text file guarantee: strong failures: out of memory III failures: i/o errors

Definition at line 377 of file ToolsDiscovery.cxx.

  {
    std::ifstream myfile (file.c_str());
 
    auto sample = std::make_unique<SampleLocal> (name);
    std::string line;
    const std::set<char> whitespaces{'\t',' ','\n','\r'};
    while (std::getline (myfile, line))
    {
      while ((!line.empty()) && whitespaces.count(line.back())) line.pop_back();
      if (!line.empty() && line.at(0) != '#')
      {
    sample->add (line);
      }
    }
    if (!myfile.eof())
      throw std::runtime_error ("failed to read file: " + file);
    sh.add (std::move (sample));
  }

readSusyMeta()

void SH::readSusyMeta	(	const SampleHandler &	sh,
		const std::string &	inputFile )

effects: read the susy-meta-data file and add its information to the samples from the given sample handler guarantee: basic failures: i/o errors warning: there are no checks for duplicate and inconsistent sample definitions

Definition at line 30 of file ToolsMeta.cxx.

  {
    TString myfile = inputFile;
    gSystem->ExpandPathName (myfile);
    std::ifstream file (myfile.Data());
    std::string line;
 
    if (!file)
      throw std::runtime_error ("failed to read file: " + inputFile);
 
    while (std::getline (file, line))
    {
      if (line[0] >= '0' && line[0] <= '9')
      {
    std::istringstream str (line);
    std::string DSID, name;
    float crossSection = 0, kFactor = 0, filterEfficiency = 0,
      xsUncertainty = 0;
 
    if (!(str >> DSID >> name >> crossSection >> kFactor >> filterEfficiency
          >> xsUncertainty))
      throw std::runtime_error ("readSusyMeta: failed to parse line: " + line);
 
    std::string mydsid = "." + DSID + ".";
    for (auto *sample : sh)
    {
      if (sample->name().find (mydsid) != std::string::npos)
      {
            double oldCrossSection
              = sample->meta()->castDouble (MetaFields::crossSection);
        sample->meta()->setDouble (MetaFields::crossSection, oldCrossSection + crossSection);
        sample->meta()->setDouble (MetaFields::kfactor, kFactor);
        sample->meta()->setDouble (MetaFields::filterEfficiency, filterEfficiency);
        sample->meta()->setDouble (MetaFields::crossSectionRelUncertainty, xsUncertainty);
      }
    }
      }
    }
  }

readSusyMetaDir()

void SH::readSusyMetaDir	(	const SampleHandler &	sh,
		const std::string &	inputDir )

effects: read an entire directory of susy-meta-data files and add their information to the samples from the given sample handle.

for that it assumes that all files ending in ".txt" are susy-meta-data files. guarantee: basic failures: i/o errors warning: there are no checks for duplicate and inconsistent sample definitions

Definition at line 70 of file ToolsMeta.cxx.

  {
    TString mydir = inputDir;
    gSystem->ExpandPathName (mydir);
    void *dirp = 0;
 
    try
    {
      dirp = gSystem->OpenDirectory (mydir.Data());
      // rationale: OpenDirectory returns null for a missing/mistyped
      //   directory, and GetDirEntry/FreeDirectory tolerate a null
      //   handle, so this would otherwise silently read nothing; throw
      //   for consistency with readSusyMeta on a missing file.
      if (dirp == nullptr)
        throw std::runtime_error ("could not open directory: " + inputDir);
      const char *file = 0;
      while ((file = gSystem->GetDirEntry (dirp)))
      {
    std::string myfile = inputDir + "/" + file;
    if (myfile.size() > 4 && myfile.substr (myfile.size()-4) == ".txt")
      readSusyMeta (sh, myfile);
      }
      gSystem->FreeDirectory (dirp);
    } catch (...)
    {
      gSystem->FreeDirectory (dirp);
      throw;
    }
  }

rucioCacheDatasetGlob()

std::vector< std::string > SH::rucioCacheDatasetGlob	(	const std::string &	location,
		const std::string &	dataset,
		const std::string &	fileGlob )

download the dataset, and return a list matching the pattern

Guarantee

basic

Failures

grid utility failures i/o errors

Definition at line 550 of file GridTools.cxx.

  {
    std::vector<std::string> result;
 
    std::string path = location;
    if (path.empty() || path.back() != '/')
      path += "/";
    if (dataset.find (':') != std::string::npos)
      path += dataset.substr (dataset.find (':')+1);
    else
      path += dataset;
    const std::string finished {
      path + "-finished"};
 
    // check if the finished file does not exist
    // note that AccessPathName has the weirdest calling convention
    //
    // rationale: this check-then-download is not safe against two jobs
    //   caching the same dataset into the same directory concurrently
    //   (they can both see the marker missing and download at the same
    //   time); guarding that properly would need an exclusive lock on
    //   the directory.  we do at least check that the marker file was
    //   created, so an unwritable directory fails loudly instead of
    //   silently re-downloading on every call.
    if (gSystem->AccessPathName (finished.c_str()) != 0)
    {
      RucioDownloadResult status = rucioDownload (location, dataset);
      if (status.downloadedFiles + status.alreadyLocal < status.totalFiles)
        throw std::runtime_error ("failed to download all files of " + dataset);
      //  this just creates an empty file
      std::ofstream finishedFile (finished.c_str());
      if (!finishedFile)
        throw std::runtime_error ("failed to create marker file: " + finished);
    }
 
    std::string output = sh::exec_read ("find " + sh::quote (path) + " -type f -name " + sh::quote (fileGlob));
    std::istringstream str (output);
    std::string line;
    while (std::getline (str, line))
    {
      if (!line.empty())
        result.push_back (line);
    }
    return result;
  }

rucioDirectAccessGlob()

std::vector< std::string > SH::rucioDirectAccessGlob	(	const std::string &	name,
		const std::string &	filter,
		const std::string &	selectOptions )

list the rucio URLs for all the files in the dataset or dataset container matching the given filter (as glob expression)

Guarantee

strong

Failures

grid utility failures

Precondition

!name.empty()

name.find('*') == std::string::npos

!filter.empty()

Definition at line 296 of file GridTools.cxx.

  {
    return rucioDirectAccessRegex (name, RCU::glob_to_regexp (filter),
                                   selectOptions);
  }

rucioDirectAccessRegex()

std::vector< std::string > SH::rucioDirectAccessRegex	(	const std::string &	name,
		const std::string &	filter,
		const std::string &	selectOptions )

list the rucio URLs for all the files in the dataset or dataset container matching the given filter (as regular expression)

Guarantee

strong

Failures

grid utility failures

Precondition

!name.empty()

name.find('*') == std::string::npos

!filter.empty()

Definition at line 306 of file GridTools.cxx.

  {
    RCU_REQUIRE_SOFT (!name.empty());
    RCU_REQUIRE_SOFT (name.find('*') == std::string::npos);
    RCU_REQUIRE_SOFT (!filter.empty());
 
    ensureVomsProxy ();
 
    static const std::string separator = "------- SampleHandler Split -------";
 
    ANA_MSG_INFO ("querying rucio for dataset " << name);
    std::string output = sh::exec_read (rucioSetupCommand() + " && echo " + separator + " && rucio list-file-replicas --pfns --protocols root " + selectOptions + " " + sh::quote (name));
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      throw std::runtime_error ("couldn't find separator in: " + output);
    std::istringstream str (output.substr (split + separator.size() + 1));
 
    // this is used to avoid getting two copies of the same file.  we
    // first fill them in a map by filename, then copy them into a
    // vector
    std::map<std::string,std::string> resultMap;
 
    std::regex urlPattern ("^root://.*");
    std::regex pattern (filter);
    std::string line;
    while (std::getline (str, line))
    {
      if (line.empty())
      {
        // no-op
      } else if (!RCU::match_expr (urlPattern, line))
      {
        ANA_MSG_INFO ("couldn't handle line: " << line);
      } else
      {
    std::string::size_type split = line.rfind ("/");
    if (split != std::string::npos)
    {
          std::string filename = line.substr (split+1);
      if (RCU::match_expr (pattern, filename))
        resultMap[filename] = line;
    } else
      throw std::runtime_error ("rucioDirectAccessRegex: couldn't parse line: " + line);
      }
    }
 
    std::vector<std::string> result;
    for (const auto& file : resultMap)
      result.push_back (file.second);
    if (result.size() == 0)
      ANA_MSG_WARNING ("dataset " + name + " did not contain any files.  this is likely not right");
    return result;
  }

rucioDownload()

RucioDownloadResult SH::rucioDownload	(	const std::string &	location,
		const std::string &	dataset )

run rucio-download

Guarantee

basic

Failures

grid utility failures

Definition at line 511 of file GridTools.cxx.

  {
    ensureVomsProxy ();
    
    const std::string separator = "------- SampleHandler Split -------";
    std::string command = rucioSetupCommand() + " && echo " + separator + " && cd " + sh::quote (location) + " && rucio download " + sh::quote (dataset) + " 2>&1";
 
    ANA_MSG_INFO ("starting rucio download " + dataset + " into " + location);
    std::string output = sh::exec_read (command);
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      throw std::runtime_error ("couldn't find separator in: " + output);
    output = output.substr (split + separator.size() + 1);
 
    RucioDownloadResult result;
    result.did = readLine (output, "DID ");
    result.totalFiles = readLineUnsigned (output, "Total files (DID): ");
    result.downloadedFiles = readLineUnsigned (output, "Downloaded files: ");
    result.alreadyLocal = readLineUnsigned (output, "Files already found locally: ");
    result.notDownloaded = readLineUnsigned (output, "Files that cannot be downloaded: ");
    return result;
  }

rucioDownloadList()

std::vector< RucioDownloadResult > SH::rucioDownloadList	(	const std::string &	location,
		const std::vector< std::string > &	datasets )

run rucio-download with multiple datasets

Guarantee

basic

Failures

grid utility failures

Definition at line 538 of file GridTools.cxx.

  {
    std::vector<RucioDownloadResult> result;
    for (auto& dataset : datasets)
      result.push_back (rucioDownload (location, dataset));
    return result;
  }

rucioGetMetadata()

std::map< std::string, std::unique_ptr< MetaObject > > SH::rucioGetMetadata

(

const std::set< std::string > &

datasets

)

run rucio-get-metadata for the given list of datasets

Guarantee

strong

Failures

grid utility failures

Precondition

!datasets.empty()

!dataset.empty() (for each dataset)

Definition at line 441 of file GridTools.cxx.

  {
    RCU_REQUIRE_SOFT (!datasets.empty());
 
    ensureVomsProxy ();
 
    static const std::string separator = "------- SampleHandler Split -------";
    std::map<std::string,std::unique_ptr<MetaObject> > result;
 
    std::string command = rucioSetupCommand() + " && echo " + separator + " && rucio get-metadata";
    for (auto& dataset : datasets)
    {
      RCU_REQUIRE_SOFT (!dataset.empty());
      command += " " + sh::quote (dataset);
    }
 
    ANA_MSG_INFO ("querying rucio for meta-data");
    std::string output = sh::exec_read (command);
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      throw std::runtime_error ("couldn't find separator in: " + output);
 
    std::istringstream str (output.substr (split + separator.size() + 1));
    std::regex pattern ("^([^:]+): *(.+)$");
    std::string line;
    auto meta = std::make_unique<MetaObject>();
 
    auto addMeta = [&] ()
    {
      std::string name = meta->castString ("scope") + ":" + meta->castString ("name");
      if (result.find (name) != result.end())
        throw std::runtime_error ("rucioGetMetadata: read " + name + " twice");
      result[name] = std::move (meta);
    };
 
    while (std::getline (str, line))
    {
      std::smatch what;
      if (line == "------")
      {
        addMeta ();
        meta = std::make_unique<MetaObject>();
      } else  if (std::regex_match (line, what, pattern))
      {
    if (meta->get (what[1]))
          throw std::runtime_error (std::string("duplicate entry: ") + what[1].str());
    meta->setString (what[1], what[2]);
      } else if (!line.empty())
      {
    ANA_MSG_WARNING ("couldn't parse line: " << line);
      }
    }
    addMeta ();
 
    for (auto& subresult : result)
    {
      if (datasets.find (subresult.first) == datasets.end())
        throw std::runtime_error ("received result for dataset not requested: " + subresult.first);
    }
    for (auto& dataset : datasets)
    {
      if (result.find (dataset) == result.end())
        throw std::runtime_error ("received no result for dataset: " + dataset);
    }
 
    return result;
  }

rucioListDids()

std::vector< RucioListDidsEntry > SH::rucioListDids

(

const std::string &

dataset

)

run rucio-list-dids for the given dataset

Guarantee

strong

Failures

grid utility failures

Precondition

!dataset.empty()

Definition at line 363 of file GridTools.cxx.

  {
    RCU_REQUIRE_SOFT (!dataset.empty());
 
    ensureVomsProxy ();
 
    static const std::string separator = "------- SampleHandler Split -------";
    std::vector<RucioListDidsEntry> result;
 
    ANA_MSG_INFO ("querying rucio for dataset " << dataset);
    std::string output = sh::exec_read (rucioSetupCommand() + " && echo " + separator + " && rucio list-dids " + sh::quote (dataset));
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      throw std::runtime_error ("couldn't find separator in: " + output);
 
    std::istringstream str (output.substr (split + separator.size() + 1));
    std::regex pattern ("^\\| ([a-zA-Z0-9_.-]+):([a-zA-Z0-9_.-]+) +\\| ([a-zA-Z0-9_.-]+) +\\| *$");
    std::string line;
    while (std::getline (str, line))
    {
      std::smatch what;
      if (std::regex_match (line, what, pattern))
      {
    RucioListDidsEntry entry;
    entry.scope = what[1];
    entry.name = what[2];
    entry.type = what[3];
    result.push_back (entry);
      }
    }
    return result;
  }

rucioListFileReplicas()

std::vector< RucioListFileReplicasEntry > SH::rucioListFileReplicas

(

const std::string &

dataset

)

run rucio-list-file-replicas for the given dataset

Guarantee

strong

Failures

grid utility failures

Precondition

!dataset.empty()

Definition at line 399 of file GridTools.cxx.

  {
    RCU_REQUIRE_SOFT (!dataset.empty());
 
    ensureVomsProxy ();
 
    static const std::string separator = "------- SampleHandler Split -------";
    std::vector<RucioListFileReplicasEntry> result;
 
    std::string command = rucioSetupCommand() + " && echo " + separator + " && rucio list-file-replicas --protocols root " + sh::quote (dataset);
 
    ANA_MSG_INFO ("querying rucio for dataset " << dataset);
    std::string output = sh::exec_read ( command );
    auto split = output.rfind (separator + "\n");
    if (split == std::string::npos)
      throw std::runtime_error ("couldn't find separator in: " + output);
 
    std::istringstream str (output.substr (split + separator.size() + 1));
    std::regex pattern ("^\\| +([^ ]+) +\\| +([^ ]+) +\\| +([^ ]+ [^ ]+) +\\| +([^ ]+) +\\| +([^: ]+): ([^ ]+) +\\| *$");
    std::string line;
    while (std::getline (str, line))
    {
      std::smatch what;
      if (std::regex_match (line, what, pattern) &&
          what[1] != "SCOPE")
      {
    RucioListFileReplicasEntry entry;
    entry.scope    = what[1];
    entry.name     = what[2];
    entry.filesize = what[3];
    entry.adler32  = what[4];
    entry.disk     = what[5];
    entry.replica  = what[6];
    result.push_back (entry);
      }
    }
    return result;
  }

scanDir() [1/3]

void SH::scanDir	(	SampleHandler &	sh,
		const std::string &	dir )

Definition at line 55 of file ToolsDiscovery.cxx.

  {
    ScanDir()
      .sampleDepth (0)
      .minDepth (1)
      .maxDepth (1)
      .scan (sh, dir);
  }

scanDir() [2/3]

void SH::scanDir	(	SampleHandler &	sh,
		const std::string &	dir,
		const std::string &	prefix )

Definition at line 66 of file ToolsDiscovery.cxx.

  {
    DiskListLocal list (dir, prefix);
    ScanDir()
      .sampleDepth (0)
      .minDepth (1)
      .maxDepth (1)
      .scan (sh, list);
  }

scanDir() [3/3]

void SH::scanDir	(	SampleHandler &	sh,
		DiskList &	list,
		const std::string &	pattern = "*.root*",
		const std::string &	samplePattern = "*",
		const std::string &	samplePostfix = "" )

effects: scan the given directory and add all subdirectories as samples that contain root files.

if prefix is provided, use that instead of dir when constructing the file names. guarantee: basic, only some samples might be added failures: low level errors IV failures: directory not found failures: duplicate samples rationale: the prefix option is for the Tier 3 prototype, where you scan the local directory, but then access the files through xrootd

Definition at line 38 of file ToolsDiscovery.cxx.

  {
    ScanDir()
      .sampleDepth (0)
      .minDepth (1)
      .maxDepth (1)
      .filePattern (pattern)
      .samplePattern (samplePattern)
      .samplePostfix (samplePostfix)
      .scan (sh, list);
  }

scanDQ2()

void SH::scanDQ2	(	SampleHandler &	sh,
		const std::string &	pattern )

effects: make a list from DQ2 using the given pattern guarantee: basic, may add partially failures: out of memory III failures: dq2-ls errors

Definition at line 124 of file ToolsDiscovery.cxx.

  {
    if (pattern.find ("*") == std::string::npos)
    {
      addGrid (sh, pattern);
    } else
    {
      std::set<std::string> types = {"DATASET", "DIDType.DATASET"};
      if (!pattern.empty() && pattern.back() == '/')
    types = {"CONTAINER", "DIDType.CONTAINER"};
 
      auto subresult = rucioListDids (pattern);
      for (auto& ds : subresult)
      {
    if (types.find (ds.type) != types.end())
      addGrid (sh, ds.scope + ":" + ds.name);
      }
    }
  }

scanFiles()

void SH::scanFiles	(	SampleHandler &	sh,
		DiskList &	list,
		const std::string &	pattern = "*.root*" )

effects: scan the given directory tree and make a separate sample for each file (using the file name before any "." as the sample name).

guarantee: basic, only some samples might be added failures: out of memory III failures: i/o errors failures: duplicate samples

Definition at line 79 of file ToolsDiscovery.cxx.

  {
    ScanDir()
      .sampleDepth (-1)
      .filePattern (pattern)
      .samplePostfix (".*")
      .scan (sh, list);
  }

scanForTrees() [1/2]

void SH::scanForTrees	(	SampleHandler &	sh,
		const std::shared_ptr< Sample > &	sample,
		const std::string &	pattern = ".*" )

effects: scan for trees in the given sample (or sample handler), and create a separate sample for each tree.

if pattern is specified it is applied to the allowed tree names. guarantee: strong failures: out of memory III failures: i/o errors

Definition at line 327 of file ToolsDiscovery.cxx.

  {
    auto mysample = sample->makeLocal();
    if (mysample->numFiles() == 0)
    {
      sh.add (sample);
      return;
    }
    std::unique_ptr<TFile> file (TFile::Open (mysample->fileName(0).c_str()));
    if (!file.get())
      throw std::runtime_error ("could not open file: " + mysample->fileName(0));
    TObject *object = nullptr;
    std::regex mypattern (pattern);
    // rationale: GetListOfKeys() contains one key per tree cycle
    //   (e.g. "physics;1", "physics;2"), all with the same name.  we
    //   process each name only once, so that repeated cycles do not
    //   make sh.add throw on a duplicate sample name.
    std::set<std::string> seenTrees;
    for (TIter iter (file->GetListOfKeys()); (object = iter.Next()); )
    {
      if (RCU::match_expr (mypattern, object->GetName()) &&
      seenTrees.insert (object->GetName()).second &&
      dynamic_cast<TTree*>(file->Get(object->GetName())))
      {
    std::string newName = sample->name() + "_" + object->GetName();
        std::unique_ptr<Sample> newSample
      (dynamic_cast<Sample*>(sample->Clone (newName.c_str())));
        newSample->name (newName);
        newSample->meta()->setString (MetaFields::treeName, object->GetName());
    sh.add (std::move (newSample));
      }
    }
  }

scanForTrees() [2/2]

void SH::scanForTrees	(	SampleHandler &	sh,
		const std::string &	pattern )

Definition at line 364 of file ToolsDiscovery.cxx.

  {
    SH::SampleHandler sh_new;
 
    for (auto sample : sh.samples())
    {
      scanForTrees (sh_new, sample, pattern);
    }
    swap (sh, sh_new);
  }

scanNEvents() [1/2]

void SH::scanNEvents

(

Sample &

sample

)

effects: scan the given sample and store the number of entries per file in the meta-data guarantee: strong failures: out of memory failures: read errors failures: invalid sample type

Definition at line 40 of file ToolsSplit.cxx.

  {
    SampleLocal *const mysample = dynamic_cast<SampleLocal*>(&sample);
    if (!mysample)
      throw std::runtime_error ("sample not of type SampleLocal");
 
    const std::string tree_name = sample.meta()->castString (MetaFields::treeName, MetaFields::treeName_default);
    if (tree_name.empty())
      throw std::runtime_error ("scanNEvents: sample doesn't contain a tree name");
 
    Long64_t tot_entries = 0;
    std::vector<Long64_t> entries;
    const auto fileList = sample.makeFileList ();
    for (const std::string& file_name : fileList)
    {
      std::unique_ptr<TFile> file (TFile::Open (file_name.c_str(), "READ"));
      if (!file.get())
        throw std::runtime_error ("failed to open file " + file_name);
      Long64_t treeEntries = 0;
      TTree *const tree = dynamic_cast<TTree*>(file->Get (tree_name.c_str()));
      if (tree != 0)
    treeEntries = tree->GetEntries ();
      entries.push_back (treeEntries);
      tot_entries += treeEntries;
    }
    RCU_ASSERT (entries.size() == fileList.size());
    sample.meta()->addReplace (new MetaVector<Long64_t>(MetaFields::numEventsPerFile, entries));
    sample.meta()->setDouble (MetaFields::numEvents, tot_entries);
  }

scanNEvents() [2/2]

void SH::scanNEvents

(

SampleHandler &

sh

)

effects: scan each sample in the sample handler and store the number of entries per file in the meta-data guarantee: basic, may only scan some failures: out of memory failures: read errors failures: invalid sample type

Definition at line 32 of file ToolsSplit.cxx.

  {
    for (auto *sample : sh)
      scanNEvents (*sample);
  }

scanRucio()

void SH::scanRucio	(	SampleHandler &	sh,
		const std::string &	pattern,
		bool	alwaysQuery = false )

make a list of grid datasets using the given pattern

This will first look for dataset containers, and if none are found, it will instead look for datasets. If those are also not found it will produce an error.

For patterns that don't contain a wild-card (i.e. that can only match one dataset/container) it instead adds the dataset directly without querying rucio. This is a performance optimization based on the assumption that you probably checked the dataset names beforehand, and that if you made a mistake it will pop up at a later stage anyways. If you want to query rucio even in those cases set alwaysQuery to true.

Guarantee

basic, may add some datasets

Failures

no pattern match
rucio failures
out of memory III

Definition at line 146 of file ToolsDiscovery.cxx.

  {
    if (pattern.find ("*") == std::string::npos && !alwaysQuery)
    {
      addGrid (sh, pattern);
    } else
    {
      auto subresult = rucioListDids (pattern);
      bool added = false;
      for (std::string type : {"CONTAINER", "DIDType.CONTAINER", "DATASET", "DIDType.DATASET"})
      {
    for (auto& ds : subresult)
    {
      if (ds.type == type)
      {
        addGrid (sh, ds.scope + ":" + ds.name);
        added = true;
      }
    }
    if (added)
      return;
      }
      throw std::runtime_error ("failed to find any datasets matching pattern: " + pattern);
    }
  }

scanSingleDir()

void SH::scanSingleDir	(	SampleHandler &	sh,
		const std::string &	name,
		DiskList &	list,
		const std::string &	pattern )

effects: scan the given directory tree and turn it into a single sample of the given name guarantee: strong failures: out of memory III

Definition at line 112 of file ToolsDiscovery.cxx.

  {
    ScanDir()
      .sampleDepth (0)
      .filePattern (pattern)
      .sampleRename ("*", name)
      .scan (sh, list);
  }

splitSample()

SampleHandler SH::splitSample	(	Sample &	sample,
		Long64_t	nevt )

effects: split the given sample into a set of samples, with each sample containing either exactly one file or at most nevt events side effects: if scanNEvents hasn't been run on this sample, run it.

guarantee: strong failures: out of memory failures: scanning errors

Definition at line 72 of file ToolsSplit.cxx.

  {
    using namespace msgSplit;
 
    if (!dynamic_cast<SampleLocal*>(&sample))
      throw std::runtime_error ("sample not of type SampleLocal");
 
    TObject *meta = sample.meta()->get (MetaFields::numEventsPerFile);
    if (!meta)
    {
      ANA_MSG_WARNING ("sample " << sample.name() << " lacks nc_nevtfile, running scanNEvents, please save sample");
      scanNEvents (sample);
      meta = sample.meta()->get (MetaFields::numEventsPerFile);
    }
    RCU_ASSERT (meta != 0);
    MetaVector<Long64_t> *const nentries
      = dynamic_cast<MetaVector<Long64_t> *>(meta);
    if (nentries == 0)
      throw std::runtime_error ("nc_nevtfile is of the wrong type");
    if (nentries->value.size() != sample.numFiles())
      throw std::runtime_error ("nc_nevtfile has the wrong number of entries");
 
    SampleHandler result;
    std::unique_ptr<SampleLocal> res;
    Long64_t num = 0;
    const std::string meta_tree = sample.meta()->castString (MetaFields::treeName, MetaFields::treeName_default);
    const double meta_xs = sample.meta()->castDouble (MetaFields::crossSection, 0);
    for (unsigned file = 0, end = nentries->value.size(); file != end; ++ file)
    {
      if (num > 0 && num + nentries->value[file] > nevt)
      {
    result.add (std::move (res));
    res.reset();
    num = 0;
      }
      if (res.get() == 0)
      {
    std::ostringstream name;
    name << sample.name() << "_" << result.size();
    res = std::make_unique<SampleLocal> (name.str());
    res->tags (sample.tags());
    res->meta()->fetch (*sample.meta());
    if (!meta_tree.empty())
      res->meta()->setString (MetaFields::treeName, meta_tree);
    if (meta_xs != 0)
      res->meta()->setDouble (MetaFields::crossSection, meta_xs);
      }
      res->add (sample.fileName (file));
      num += nentries->value[file];
    }
    if (num > 0)
      result.add (std::move (res));
    return result;
  }