ATLAS Offline Software
DataPreparationPipeline.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
3 */
4 
13 
14 #include "AthContainers/debug.h"
16 
18 {
19  // Check if we are using the pass through tool
20  // If yes, check if we are running in software mode. sw mode doesn't require an accelerator
21  if (m_usePassThrough)
22  {
23  ATH_CHECK(m_passThroughTool.retrieve());
24  if (m_passThroughTool->runSW())
25  {
26  ATH_MSG_INFO("Running software pass through");
27  }
28  else
29  {
30  ATH_MSG_INFO("Running hardware pass through");
34  }
35  }
36  // We are doing the Data Preparation Pipeline on FPGA, so we need to setup hardware
37  else
38  {
39  ATH_MSG_INFO("Running the DataPrep pipeline on the FPGA accelerator");
43 
44  // Kernel objects shouldn't change so they can be initialized here
46  m_spacepointKernel = cl::Kernel(m_program, m_spacepointKernelName.value().data());
47 
50 
51  if (m_useTV)
52  {
57  ATH_CHECK(m_testVectorTool.retrieve());
58  }
59  }
60 
61  ATH_CHECK(m_xAODContainerMaker.retrieve());
62 
63  return StatusCode::SUCCESS;
64 }
65 
66 StatusCode DataPreparationPipeline::execute(const EventContext &ctx) const
67 {
68  if (m_usePassThrough)
69  {
72  std::unique_ptr<EFTrackingDataFormats::Metadata> metadata =
73  std::make_unique<EFTrackingDataFormats::Metadata>();
74 
75  ATH_CHECK(m_passThroughTool->runPassThrough(scAux, pxAux, metadata.get(), ctx));
76  ATH_CHECK(m_xAODContainerMaker->makeStripClusterContainer(scAux, metadata.get(), ctx));
77  ATH_CHECK(m_xAODContainerMaker->makePixelClusterContainer(pxAux, metadata.get(), ctx));
78 
79  return StatusCode::SUCCESS;
80  }
81 
82  // HW kernels to be added here in the future
83  ATH_MSG_INFO("Running the FPGA Data Preparation Pipeline");
84 
85  // Order of execution
86  // 1. Pixel/strip clustering
87  // 2. Pixel/strip L2G
88  // 3. Pixel/strip Spacepoint
89 
90  // Are we using TV or RDO?
91  if (m_useTV)
92  {
93  // Make queue of commands
94  cl::CommandQueue acc_queue(m_context, m_accelerator);
95 
96  ATH_MSG_INFO("Using test vectors");
97  EFTrackingFPGAIntegration::TVHolder pixelTV("PixelClustering");
100 
101  std::vector<uint64_t> fpgaOutput(pixelTV.inputTV.size() * 2, 0);
102 
103  ATH_MSG_INFO("Running the pixel clustering kernel");
104  acc_queue.enqueueWriteBuffer(m_pxlClusteringInBuff, CL_TRUE, 0, sizeof(uint64_t) * pixelTV.inputTV.size(), pixelTV.inputTV.data(), NULL, NULL);
105  acc_queue.enqueueTask(m_pixelClusteringKernel);
106  acc_queue.finish();
107 
108  // The current implementation of the kernel Read/Write the same buffer
109  // So the line below reads the inbuff insatead of outbuff
110  acc_queue.enqueueReadBuffer(m_pxlClusteringInBuff, CL_TRUE, 0, sizeof(uint64_t) * fpgaOutput.size(), fpgaOutput.data(), NULL, NULL);
111 
112  // compare the output with the reference
113  ATH_CHECK(m_testVectorTool->compare(pixelTV, fpgaOutput));
114 
115  // do the same for spacepoint
116  EFTrackingFPGAIntegration::TVHolder spacepointTV("Spacepoint");
117  ATH_CHECK(m_testVectorTool->prepareTV(m_spacepointTVPath, spacepointTV.inputTV));
118  ATH_CHECK(m_testVectorTool->prepareTV(m_spacepointRefTVPath, spacepointTV.refTV));
119 
120  // reset the output vector
121  fpgaOutput = std::vector<uint64_t>(spacepointTV.inputTV.size() * 2, 0);
122 
123  ATH_MSG_INFO("Running the spacepoint kernel");
124  acc_queue.enqueueWriteBuffer(m_spacepointInBuff, CL_TRUE, 0, sizeof(uint64_t) * spacepointTV.inputTV.size(), spacepointTV.inputTV.data(), NULL, NULL);
125  acc_queue.enqueueTask(m_spacepointKernel);
126  acc_queue.finish();
127 
128  acc_queue.enqueueReadBuffer(m_spacepointInBuff, CL_TRUE, 0, sizeof(uint64_t) * fpgaOutput.size(), fpgaOutput.data(), NULL, NULL);
129 
130  // compare the output with the reference
131  ATH_CHECK(m_testVectorTool->compare(spacepointTV, fpgaOutput));
132  }
133  else
134  {
135  // Use FPGADataFormatTool to prepare input
136  ATH_MSG_INFO("Using RDOs for FPGA is not ready yet");
137  }
138 
139  return StatusCode::SUCCESS;
140 }
141 
143 {
144  // Allocate buffers on accelerator
145  cl_int err = 0;
146 
147  // Set the size of the buffer to be 100000, should be further discussed!
148  unsigned int size = 100000;
149  m_pxlClusteringInBuff = cl::Buffer(m_context, CL_MEM_READ_ONLY, sizeof(uint64_t) * size, NULL, &err);
150  if (err != CL_SUCCESS)
151  {
152  ATH_MSG_ERROR("Failed to allocate intput buffer for pixel clustering");
153  return StatusCode::FAILURE;
154  }
155 
156  m_pxlClusteringOutBuff = cl::Buffer(m_context, CL_MEM_READ_WRITE, sizeof(uint64_t) * size, NULL, &err);
157  if (err != CL_SUCCESS)
158  {
159  ATH_MSG_ERROR("Failed to allocate output buffer for pixel clustering");
160  return StatusCode::FAILURE;
161  }
162 
163  m_spacepointInBuff = cl::Buffer(m_context, CL_MEM_READ_ONLY, sizeof(uint64_t) * size, NULL, &err);
164  if (err != CL_SUCCESS)
165  {
166  ATH_MSG_ERROR("Failed to allocate input buffer for spacepoint");
167  return StatusCode::FAILURE;
168  }
169 
170  m_spacepointOutBuff = cl::Buffer(m_context, CL_MEM_READ_WRITE, sizeof(uint64_t) * size, NULL, &err);
171  if (err != CL_SUCCESS)
172  {
173  ATH_MSG_ERROR("Failed to allocate output buffer for spacepoint");
174  return StatusCode::FAILURE;
175  }
176 
177  return StatusCode::SUCCESS;
178 }
179 
181 {
182  // Prepare kernel
183  m_pixelClusteringKernel.setArg<uint>(0, 0);
184  m_pixelClusteringKernel.setArg<cl::Buffer>(1, m_pxlClusteringInBuff);
185  m_pixelClusteringKernel.setArg<cl::Buffer>(2, m_pxlClusteringOutBuff);
186 
187  m_spacepointKernel.setArg<cl::Buffer>(0, m_spacepointInBuff);
188  m_spacepointKernel.setArg<cl::Buffer>(1, m_spacepointOutBuff);
189  // Hardcoded the size for now, shouldn't need the size eventually
190  m_spacepointKernel.setArg<uint>(2, 33);
191 
192  return StatusCode::SUCCESS;
193 }
DataPreparationPipeline::m_pixelClusterRefTVPath
Gaudi::Property< std::string > m_pixelClusterRefTVPath
Pixel cluster reference test vector.
Definition: DataPreparationPipeline.h:74
EFTrackingFPGAIntegration::TVHolder::refTV
std::vector< uint64_t > refTV
Definition: TestVectorTool.h:32
EFTrackingDataFormats::StripClusterAuxInput
The StripClusterAuxInput struct is used to simplify the creaction of the xAOD::StripClusterContainer.
Definition: EFTrackingDataFormats.h:186
IntegrationBase::m_accelerator
cl::Device m_accelerator
Device object for the accelerator card.
Definition: IntegrationBase.h:66
IntegrationBase::initialize
virtual StatusCode initialize() override
Detect the OpenCL devices and prepare OpenCL context.
Definition: IntegrationBase.cxx:14
DataPreparationPipeline::m_pixelClusteringKernelName
Gaudi::Property< std::string > m_pixelClusteringKernelName
Pixle clustering kernel name.
Definition: DataPreparationPipeline.h:64
ATH_MSG_INFO
#define ATH_MSG_INFO(x)
Definition: AthMsgStreamMacros.h:31
EFTrackingDataFormats::PixelClusterAuxInput
The PixelClusterAuxInput struct is used to simplify the creaction of the xAOD::PixelClusterContainer.
Definition: EFTrackingDataFormats.h:200
DataPreparationPipeline::m_spacepointOutBuff
cl::Buffer m_spacepointOutBuff
Buffer for spacepoint output.
Definition: DataPreparationPipeline.h:55
IntegrationBase::m_context
cl::Context m_context
Context object for the application.
Definition: IntegrationBase.h:67
DataPreparationPipeline::execute
StatusCode execute(const EventContext &ctx) const override final
Should be overriden by derived classes to perform meaningful work.
Definition: DataPreparationPipeline.cxx:66
DataPreparationPipeline::m_pixelClusterTVPath
Gaudi::Property< std::string > m_pixelClusterTVPath
Pixel cluster test vector.
Definition: DataPreparationPipeline.h:72
DataPreparationPipeline::m_pxlClusteringOutBuff
cl::Buffer m_pxlClusteringOutBuff
Buffer for pixel clustering output.
Definition: DataPreparationPipeline.h:52
DataPreparationPipeline::m_spacepointTVPath
Gaudi::Property< std::string > m_spacepointTVPath
Spacepoint test vector.
Definition: DataPreparationPipeline.h:77
DataPreparationPipeline::m_spacepointRefTVPath
Gaudi::Property< std::string > m_spacepointRefTVPath
Spacepoint reference test vector.
Definition: DataPreparationPipeline.h:79
python.checkMetadata.metadata
metadata
Definition: checkMetadata.py:175
python.setupRTTAlg.size
int size
Definition: setupRTTAlg.py:39
uint
unsigned int uint
Definition: LArOFPhaseFill.cxx:20
DataPreparationPipeline.h
Class for the data preparation pipeline.
ATH_MSG_ERROR
#define ATH_MSG_ERROR(x)
Definition: AthMsgStreamMacros.h:33
DataPreparationPipeline::m_passThroughTool
ToolHandle< PassThroughTool > m_passThroughTool
Tool handle for PassThroughTool.
Definition: DataPreparationPipeline.h:93
dqt_zlumi_pandas.err
err
Definition: dqt_zlumi_pandas.py:182
EL::StatusCode
::StatusCode StatusCode
StatusCode definition for legacy code.
Definition: PhysicsAnalysis/D3PDTools/EventLoop/EventLoop/StatusCode.h:22
DataPreparationPipeline::m_useTV
Gaudi::Property< bool > m_useTV
Use test vector.
Definition: DataPreparationPipeline.h:86
DataPreparationPipeline::m_spacepointKernel
cl::Kernel m_spacepointKernel
Kernel for spacepoint.
Definition: DataPreparationPipeline.h:49
xAOD::uint64_t
uint64_t
Definition: EventInfo_v1.cxx:123
ATH_CHECK
#define ATH_CHECK
Definition: AthCheckMacros.h:40
EFTrackingFPGAIntegration::TVHolder
Definition: TestVectorTool.h:27
DataPreparationPipeline::m_passThroughKernelName
Gaudi::Property< std::string > m_passThroughKernelName
Pass through kernel name.
Definition: DataPreparationPipeline.h:61
DataPreparationPipeline::m_xclbin
Gaudi::Property< std::string > m_xclbin
Path and name of the xclbin file.
Definition: DataPreparationPipeline.h:58
IntegrationBase::loadProgram
StatusCode loadProgram(const std::string &xclbin)
Find the xclbin file and load it into the OpenCL program object.
Definition: IntegrationBase.cxx:82
IntegrationBase::precheck
StatusCode precheck(std::vector< Gaudi::Property< std::string >> inputs) const
Check if the the desired Gaudi properties are set.
Definition: IntegrationBase.cxx:121
DataPreparationPipeline::m_spacepointKernelName
Gaudi::Property< std::string > m_spacepointKernelName
Spacepoint kernel name.
Definition: DataPreparationPipeline.h:67
DataPreparationPipeline::initialize
StatusCode initialize() override final
Detect the OpenCL devices and prepare OpenCL context.
Definition: DataPreparationPipeline.cxx:17
DataPreparationPipeline::setupBuffers
StatusCode setupBuffers()
Definition: DataPreparationPipeline.cxx:142
DataPreparationPipeline::m_usePassThrough
Gaudi::Property< bool > m_usePassThrough
Use the pass through tool instead of data prep pipeline.
Definition: DataPreparationPipeline.h:83
DataPreparationPipeline::m_testVectorTool
ToolHandle< TestVectorTool > m_testVectorTool
Tool handle for TestVectorTool.
Definition: DataPreparationPipeline.h:97
DataPreparationPipeline::m_spacepointInBuff
cl::Buffer m_spacepointInBuff
Buffer for spacepoint input.
Definition: DataPreparationPipeline.h:54
StripClusterAuxContainer.h
DataPreparationPipeline::setupKernelArgs
StatusCode setupKernelArgs()
Definition: DataPreparationPipeline.cxx:180
DataPreparationPipeline::m_pixelClusteringKernel
cl::Kernel m_pixelClusteringKernel
Kernel for pixel clustering.
Definition: DataPreparationPipeline.h:48
IntegrationBase::m_program
cl::Program m_program
Program object containing the kernel.
Definition: IntegrationBase.h:68
DataPreparationPipeline::m_xAODContainerMaker
ToolHandle< xAODContainerMaker > m_xAODContainerMaker
Tool handle for xAODContainerMaker.
Definition: DataPreparationPipeline.h:89
debug.h
Helper functions intended to be called from the debugger.
EFTrackingFPGAIntegration::TVHolder::inputTV
std::vector< uint64_t > inputTV
Definition: TestVectorTool.h:31
DataPreparationPipeline::m_pxlClusteringInBuff
cl::Buffer m_pxlClusteringInBuff
Buffer for pixel clustering input.
Definition: DataPreparationPipeline.h:51