CaloGPUHybridClusterProcessor.h

Go to the documentation of this file.

//
// Copyright (C) 2002-2023 CERN for the benefit of the ATLAS collaboration
//
// Dear emacs, this is -*- c++ -*-
//
 
#ifndef CALORECGPU_CALOGPUHYBRIDCLUSTERPROCESSOR_H
#define CALORECGPU_CALOGPUHYBRIDCLUSTERPROCESSOR_H
 
 
#include "AthenaBaseComps/AthReentrantAlgorithm.h"
#include "GaudiKernel/ToolHandle.h"
#include "AthenaMonitoringKernel/GenericMonitoringTool.h"
 
#include "CxxUtils/checker_macros.h"
 
#include "CaloUtils/CaloClusterCollectionProcessor.h"
#include "CaloRecGPU/CaloClusterGPUTransformers.h"
#include "CaloRecGPU/CaloClusterGPUProcessor.h"
#include "CaloRecGPU/CaloGPUTimed.h"
#include "CaloRecGPU/CaloGPUCUDAInitialization.h"
#include "CaloRecGPU/DataHolders.h"
 
#include "xAODCaloEvent/CaloClusterContainer.h"
 
#include "StoreGate/ReadDecorHandle.h"
#include "StoreGate/WriteDecorHandleKey.h"
 
#include "CaloConditions/CaloNoise.h"
 
#include <string>
#include <mutex>
#include <atomic>
#include <utility>
 
class CaloGPUHybridClusterProcessor : public AthReentrantAlgorithm, public CaloGPUTimed, public CaloGPUCUDAInitialization
{
 public:
 
  CaloGPUHybridClusterProcessor(const std::string & name, ISvcLocator * pSvcLocator);
  virtual ~CaloGPUHybridClusterProcessor() override = default;
 
  virtual StatusCode initialize() override
  {
    return CaloGPUCUDAInitialization::initialize();
  }
 
  virtual StatusCode initialize_non_CUDA() override;
 
  virtual StatusCode initialize_CUDA() override;
 
  virtual StatusCode execute(const EventContext & ctx) const override;
  virtual StatusCode finalize() override;
 
 private:
 
 
  ToolHandle<ICaloClusterGPUConstantTransformer> m_transformConstantData{this, "ConstantDataToGPUTool", "", "Tool for transforming the constant data and sending it to the GPU"};
 
  ToolHandleArray<CaloClusterCollectionProcessor> m_preGPUoperations{this, "BeforeGPUTools", {}, "Tools to be applied to the clusters on the CPU before processing them on the GPU"};
 
 
  ToolHandle<ICaloClusterGPUInputTransformer> m_transformForGPU{this, "EventDataToGPUTool", "", "Tool for transforming the event data and sending it to the GPU"};
 
  ToolHandleArray<CaloClusterGPUProcessor> m_GPUoperations{this, "GPUTools", {}, "Tools to be applied to the clusters on the GPU"};
 
  ToolHandle<ICaloClusterGPUOutputTransformer> m_transformBackToCPU{this, "GPUToEventDataTool", {}, "Tool for getting the data from the GPU back to the CPU Athena data structures"};
 
  ToolHandleArray<CaloClusterCollectionProcessor> m_postGPUoperations{this, "AfterGPUTools", {}, "Tools to be applied to the clusters on the CPU after returning from the GPU"};
 
 
  Gaudi::Property<bool> m_doPlots{this, "DoPlots", false, "Do plots based on the plotter tool optionally provided."};
 
  ToolHandle<ICaloClusterGPUPlotter> m_plotterTool{this, "PlotterTool", "", "An optional plotter, for testing and/or debugging purposes"};
 
 
  Gaudi::Property<bool> m_doMonitoring{this, "DoMonitoring", false, "Do monitoring."};
  Gaudi::Property<bool> m_monitorCells{this, "MonitorCells", false, "Whether to monitor cells too."};
  Gaudi::Property<float> m_monitoring1thr { this, "Thr1", 2, "First Threshold to pass for cell monitoring" };
  Gaudi::Property<float> m_monitoring2thr { this, "Thr2", 4, "Second Threshold to pass for cell monitoring" };
 
 
  // adding noise handle for monitoring purposes
  SG::ReadCondHandleKey<CaloNoise> m_noiseCDOKey{this, "CaloNoiseKey", "totalNoise", "SG Key of CaloNoise data object"};
 
  ToolHandle<GenericMonitoringTool> m_moniTool { this, "MonitoringTool", "", "Monitoring tool" };
 
  SG::ReadDecorHandleKey<xAOD::EventInfo> m_avgMuKey { this, "averageInteractionsPerCrossingKey", "EventInfo.averageInteractionsPerCrossing", "Decoration for Average Interaction Per Crossing" };
 
  Gaudi::Property<bool> m_writeTriggerSpecificInfo{this, "WriteTriggerSpecificInfo", false, "Write some trigger-specific decorations and use the trigger auxiliary container."};
 
  SG::WriteDecorHandleKey<xAOD::CaloClusterContainer> m_mDecor_ncells {this, "Decor_ncells", "nCells", "Decorator containing the number of cells associated to a cluster"};
 
  Gaudi::Property<size_t> m_numPreAllocatedGPUData{this, "NumPreAllocatedDataHolders", 0, "Number of event data holders to pre-allocate on GPU memory"};
 
  SG::ReadHandleKey<CaloCellContainer> m_cellsKey {this, "CellsName", "", "Name(s) of Cell Containers"};
 
  SG::WriteHandleKey<xAOD::CaloClusterContainer> m_clusterOutput {this, "ClustersOutputName", "", "The name of the key in StoreGate for the output CaloClusterContainer"};
 
  SG::WriteHandleKey<CaloClusterCellLinkContainer> m_clusterCellLinkOutput{this, "ClusterCellLinksOutputName", "", "The name of the key in StoreGate for the output CaloClusterCellLinkContainer"};
 
  Gaudi::Property<bool> m_deferConstantDataToFirstEvent {this, "DeferConstantDataPreparationToFirstEvent", true, "Convert and send event data on first event instead of during initialize (needed for exporting geometry and noise properly)?"};
 
 
  Gaudi::Property<bool> m_skipConversions {this, "SkipConversions", false, "If true, skip converting CPU to GPU data (useful if only instanting CPU tools)"};
 
  mutable CaloRecGPU::Helpers::separate_thread_holder<CaloRecGPU::EventDataHolder> m_eventDataThreadedHolder ATLAS_THREAD_SAFE;
 
  struct simple_GPU_pointer_holder
  {
   private:
    void * m_ptr;
 
   public:
 
    void allocate(const size_t size)
    {
      if (m_ptr == nullptr && size > 0)
        {
          m_ptr = CaloRecGPU::CUDA_Helpers::allocate(size);
        }
    }
 
    simple_GPU_pointer_holder():
      m_ptr(nullptr)
    {
    }
    simple_GPU_pointer_holder(const simple_GPU_pointer_holder &) = delete;
    simple_GPU_pointer_holder(simple_GPU_pointer_holder && other)
    {
      m_ptr = other.m_ptr;
      other.m_ptr = nullptr;
    }
    simple_GPU_pointer_holder & operator= (const simple_GPU_pointer_holder &) = delete;
 
    simple_GPU_pointer_holder & operator= (simple_GPU_pointer_holder && other)
    {
      std::swap(m_ptr, other.m_ptr);
      return (*this);
    }
 
    ~simple_GPU_pointer_holder()
    {
      if (m_ptr != nullptr)
        //This check might still be needed to ensure the code behaves on non-CUDA enabled platforms
        //where some destructors might still be called with nullptr.
        {
          CaloRecGPU::CUDA_Helpers::deallocate(m_ptr);
        }
    }
 
    void * operator* ()
    {
      return m_ptr;
    }
 
    void * get_pointer()
    {
      return m_ptr;
    }
 
  };
 
  mutable CaloRecGPU::Helpers::separate_thread_holder<simple_GPU_pointer_holder> m_temporariesThreadedHolder ATLAS_THREAD_SAFE;
 
  size_t m_temporariesSize;
 
  mutable CaloRecGPU::ConstantDataHolder m_constantData ATLAS_THREAD_SAFE;
 
  mutable std::atomic<bool> m_constantDataSent;
 
 
  mutable std::mutex m_mutex;
 
};
 
#endif //CALORECGPU_CALOGPUHYBRIDCLUSTERPROCESSOR_H