ATLAS Offline Software
DataPreparationPipeline.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
3 */
4 
13 
14 #include "AthContainers/debug.h"
16 
18 {
19  // Check if we are using the pass through tool
20  // If yes, check if we are running in software mode. sw mode doesn't require an accelerator
21  if (m_usePassThrough)
22  {
23  ATH_CHECK(m_passThroughTool.retrieve());
24  if (m_passThroughTool->runSW())
25  {
26  ATH_MSG_INFO("Running software pass through");
27  }
28  else
29  {
30  ATH_MSG_INFO("Running hardware pass through");
34  }
35  }
36  // We are doing the Data Preparation Pipeline on FPGA, so we need to setup hardware
37  else
38  {
39  ATH_MSG_INFO("Running the DataPrep pipeline on the FPGA accelerator");
43 
44  // Kernel objects shouldn't change so they can be initialized here
46  m_spacepointKernel = cl::Kernel(m_program, m_spacepointKernelName.value().data());
47 
50 
51  if (m_useTV)
52  {
57  ATH_CHECK(m_testVectorTool.retrieve());
58  }
59  }
60 
61  ATH_CHECK(m_xAODClusterMaker.retrieve());
62  ATH_CHECK(m_xAODSpacePointMaker.retrieve());
63 
64  return StatusCode::SUCCESS;
65 }
66 
67 StatusCode DataPreparationPipeline::execute(const EventContext &ctx) const
68 {
69  if (m_usePassThrough)
70  {
73 
74  // Create separate structures for pixel and strip space points
77 
78  std::unique_ptr<EFTrackingTransient::Metadata> metadata =
79  std::make_unique<EFTrackingTransient::Metadata>();
80 
81  // Pass both aux structures to the PassThroughTool
82  ATH_CHECK(m_passThroughTool->runPassThrough(scAux, pxAux, stripSpAux, pixelSpAux, metadata.get(), ctx));
83  ATH_CHECK(m_xAODClusterMaker->makeStripClusterContainer(scAux, metadata.get(), ctx));
84  ATH_CHECK(m_xAODClusterMaker->makePixelClusterContainer(pxAux, metadata.get(), ctx));
85 
86  ATH_CHECK(m_xAODSpacePointMaker->makeStripSpacePointContainer(stripSpAux, metadata.get(), ctx));
87  ATH_CHECK(m_xAODSpacePointMaker->makePixelSpacePointContainer(pixelSpAux, metadata.get(), ctx));
88 
89  return StatusCode::SUCCESS;
90  }
91 
92  // HW kernels to be added here in the future
93  ATH_MSG_INFO("Running the FPGA Data Preparation Pipeline");
94 
95  // Order of execution
96  // 1. Pixel/strip clustering
97  // 2. Pixel/strip L2G
98  // 3. Pixel/strip Spacepoint
99 
100  // Are we using TV or RDO?
101  if (m_useTV)
102  {
103  // Make queue of commands
104  cl::CommandQueue acc_queue(m_context, m_accelerator);
105 
106  ATH_MSG_INFO("Using test vectors");
107  EFTrackingFPGAIntegration::TVHolder pixelTV("PixelClustering");
110 
111  std::vector<uint64_t> fpgaOutput(pixelTV.inputTV.size() * 2, 0);
112 
113  ATH_MSG_INFO("Running the pixel clustering kernel");
114  acc_queue.enqueueWriteBuffer(m_pxlClusteringInBuff, CL_TRUE, 0, sizeof(uint64_t) * pixelTV.inputTV.size(), pixelTV.inputTV.data(), NULL, NULL);
115  acc_queue.enqueueTask(m_pixelClusteringKernel);
116  acc_queue.finish();
117 
118  // The current implementation of the kernel Read/Write the same buffer
119  // So the line below reads the inbuff insatead of outbuff
120  acc_queue.enqueueReadBuffer(m_pxlClusteringInBuff, CL_TRUE, 0, sizeof(uint64_t) * fpgaOutput.size(), fpgaOutput.data(), NULL, NULL);
121 
122  // compare the output with the reference
123  ATH_CHECK(m_testVectorTool->compare(pixelTV, fpgaOutput));
124 
125  // do the same for spacepoint
126  EFTrackingFPGAIntegration::TVHolder spacepointTV("Spacepoint");
127  ATH_CHECK(m_testVectorTool->prepareTV(m_spacepointTVPath, spacepointTV.inputTV));
128  ATH_CHECK(m_testVectorTool->prepareTV(m_spacepointRefTVPath, spacepointTV.refTV));
129 
130  // reset the output vector
131  fpgaOutput = std::vector<uint64_t>(spacepointTV.inputTV.size() * 2, 0);
132 
133  ATH_MSG_INFO("Running the spacepoint kernel");
134  acc_queue.enqueueWriteBuffer(m_spacepointInBuff, CL_TRUE, 0, sizeof(uint64_t) * spacepointTV.inputTV.size(), spacepointTV.inputTV.data(), NULL, NULL);
135  acc_queue.enqueueTask(m_spacepointKernel);
136  acc_queue.finish();
137 
138  acc_queue.enqueueReadBuffer(m_spacepointInBuff, CL_TRUE, 0, sizeof(uint64_t) * fpgaOutput.size(), fpgaOutput.data(), NULL, NULL);
139 
140  // compare the output with the reference
141  ATH_CHECK(m_testVectorTool->compare(spacepointTV, fpgaOutput));
142  }
143  else
144  {
145  // Use FPGADataFormatTool to prepare input
146  ATH_MSG_INFO("Using RDOs for FPGA is not ready yet");
147  }
148 
149  return StatusCode::SUCCESS;
150 }
151 
153 {
154  // Allocate buffers on accelerator
155  cl_int err = 0;
156 
157  // Set the size of the buffer to be 100000, should be further discussed!
158  unsigned int size = 100000;
159  m_pxlClusteringInBuff = cl::Buffer(m_context, CL_MEM_READ_ONLY, sizeof(uint64_t) * size, NULL, &err);
160  if (err != CL_SUCCESS)
161  {
162  ATH_MSG_ERROR("Failed to allocate intput buffer for pixel clustering");
163  return StatusCode::FAILURE;
164  }
165 
166  m_pxlClusteringOutBuff = cl::Buffer(m_context, CL_MEM_READ_WRITE, sizeof(uint64_t) * size, NULL, &err);
167  if (err != CL_SUCCESS)
168  {
169  ATH_MSG_ERROR("Failed to allocate output buffer for pixel clustering");
170  return StatusCode::FAILURE;
171  }
172 
173  m_spacepointInBuff = cl::Buffer(m_context, CL_MEM_READ_ONLY, sizeof(uint64_t) * size, NULL, &err);
174  if (err != CL_SUCCESS)
175  {
176  ATH_MSG_ERROR("Failed to allocate input buffer for spacepoint");
177  return StatusCode::FAILURE;
178  }
179 
180  m_spacepointOutBuff = cl::Buffer(m_context, CL_MEM_READ_WRITE, sizeof(uint64_t) * size, NULL, &err);
181  if (err != CL_SUCCESS)
182  {
183  ATH_MSG_ERROR("Failed to allocate output buffer for spacepoint");
184  return StatusCode::FAILURE;
185  }
186 
187  return StatusCode::SUCCESS;
188 }
189 
191 {
192  // Prepare kernel
193  m_pixelClusteringKernel.setArg<uint>(0, 0);
194  m_pixelClusteringKernel.setArg<cl::Buffer>(1, m_pxlClusteringInBuff);
195  m_pixelClusteringKernel.setArg<cl::Buffer>(2, m_pxlClusteringOutBuff);
196 
197  m_spacepointKernel.setArg<cl::Buffer>(0, m_spacepointInBuff);
198  m_spacepointKernel.setArg<cl::Buffer>(1, m_spacepointOutBuff);
199  // Hardcoded the size for now, shouldn't need the size eventually
200  m_spacepointKernel.setArg<uint>(2, 33);
201 
202  return StatusCode::SUCCESS;
203 }
DataPreparationPipeline::m_pixelClusterRefTVPath
Gaudi::Property< std::string > m_pixelClusterRefTVPath
Pixel cluster reference test vector.
Definition: DataPreparationPipeline.h:74
EFTrackingFPGAIntegration::TVHolder::refTV
std::vector< uint64_t > refTV
Definition: TestVectorTool.h:34
IntegrationBase::m_accelerator
cl::Device m_accelerator
Device object for the accelerator card.
Definition: IntegrationBase.h:66
DataPreparationPipeline::m_xAODSpacePointMaker
ToolHandle< xAODSpacePointMaker > m_xAODSpacePointMaker
Tool handle for xAODSpacePointMaker.
Definition: DataPreparationPipeline.h:93
IntegrationBase::initialize
virtual StatusCode initialize() override
Detect the OpenCL devices and prepare OpenCL context.
Definition: IntegrationBase.cxx:16
DataPreparationPipeline::m_pixelClusteringKernelName
Gaudi::Property< std::string > m_pixelClusteringKernelName
Pixle clustering kernel name.
Definition: DataPreparationPipeline.h:64
EFTrackingTransient::PixelClusterAuxInput
The PixelClusterAuxInput struct is used to simplify the creaction of the xAOD::PixelClusterContainer.
Definition: EFTrackingTransient.h:226
ATH_MSG_INFO
#define ATH_MSG_INFO(x)
Definition: AthMsgStreamMacros.h:31
EFTrackingTransient::SpacePointAuxInput
The SpacePointAuxInput struct is used to simplify the creaction of the xAOD::SpacePointContainer.
Definition: EFTrackingTransient.h:254
DataPreparationPipeline::m_spacepointOutBuff
cl::Buffer m_spacepointOutBuff
Buffer for spacepoint output.
Definition: DataPreparationPipeline.h:55
IntegrationBase::m_context
cl::Context m_context
Context object for the application.
Definition: IntegrationBase.h:67
DataPreparationPipeline::execute
StatusCode execute(const EventContext &ctx) const override final
Should be overriden by derived classes to perform meaningful work.
Definition: DataPreparationPipeline.cxx:67
DataPreparationPipeline::m_pixelClusterTVPath
Gaudi::Property< std::string > m_pixelClusterTVPath
Pixel cluster test vector.
Definition: DataPreparationPipeline.h:72
DataPreparationPipeline::m_pxlClusteringOutBuff
cl::Buffer m_pxlClusteringOutBuff
Buffer for pixel clustering output.
Definition: DataPreparationPipeline.h:52
DataPreparationPipeline::m_spacepointTVPath
Gaudi::Property< std::string > m_spacepointTVPath
Spacepoint test vector.
Definition: DataPreparationPipeline.h:77
DataPreparationPipeline::m_spacepointRefTVPath
Gaudi::Property< std::string > m_spacepointRefTVPath
Spacepoint reference test vector.
Definition: DataPreparationPipeline.h:79
python.checkMetadata.metadata
metadata
Definition: checkMetadata.py:175
python.setupRTTAlg.size
int size
Definition: setupRTTAlg.py:39
uint
unsigned int uint
Definition: LArOFPhaseFill.cxx:20
DataPreparationPipeline.h
ATH_MSG_ERROR
#define ATH_MSG_ERROR(x)
Definition: AthMsgStreamMacros.h:33
DataPreparationPipeline::m_passThroughTool
ToolHandle< PassThroughTool > m_passThroughTool
Tool handle for PassThroughTool.
Definition: DataPreparationPipeline.h:97
dqt_zlumi_pandas.err
err
Definition: dqt_zlumi_pandas.py:183
EL::StatusCode
::StatusCode StatusCode
StatusCode definition for legacy code.
Definition: PhysicsAnalysis/D3PDTools/EventLoop/EventLoop/StatusCode.h:22
DataPreparationPipeline::m_useTV
Gaudi::Property< bool > m_useTV
Use test vector.
Definition: DataPreparationPipeline.h:86
DataPreparationPipeline::m_spacepointKernel
cl::Kernel m_spacepointKernel
Kernel for spacepoint.
Definition: DataPreparationPipeline.h:49
xAOD::uint64_t
uint64_t
Definition: EventInfo_v1.cxx:123
ATH_CHECK
#define ATH_CHECK
Definition: AthCheckMacros.h:40
EFTrackingFPGAIntegration::TVHolder
Definition: TestVectorTool.h:29
IntegrationBase::precheck
StatusCode precheck(const std::vector< Gaudi::Property< std::string >> &inputs) const
Check if the the desired Gaudi properties are set.
Definition: IntegrationBase.cxx:154
DataPreparationPipeline::m_passThroughKernelName
Gaudi::Property< std::string > m_passThroughKernelName
Pass through kernel name.
Definition: DataPreparationPipeline.h:61
DataPreparationPipeline::m_xclbin
Gaudi::Property< std::string > m_xclbin
Path and name of the xclbin file.
Definition: DataPreparationPipeline.h:58
IntegrationBase::loadProgram
StatusCode loadProgram(const std::string &xclbin)
Find the xclbin file and load it into the OpenCL program object.
Definition: IntegrationBase.cxx:115
EFTrackingTransient::StripClusterAuxInput
The StripClusterAuxInput struct is used to simplify the creaction of the xAOD::StripClusterContainer.
Definition: EFTrackingTransient.h:211
DataPreparationPipeline::m_spacepointKernelName
Gaudi::Property< std::string > m_spacepointKernelName
Spacepoint kernel name.
Definition: DataPreparationPipeline.h:67
DataPreparationPipeline::initialize
StatusCode initialize() override final
Detect the OpenCL devices and prepare OpenCL context.
Definition: DataPreparationPipeline.cxx:17
DataPreparationPipeline::setupBuffers
StatusCode setupBuffers()
Definition: DataPreparationPipeline.cxx:152
DataPreparationPipeline::m_usePassThrough
Gaudi::Property< bool > m_usePassThrough
Use the pass through tool instead of data prep pipeline.
Definition: DataPreparationPipeline.h:83
DataPreparationPipeline::m_testVectorTool
ToolHandle< TestVectorTool > m_testVectorTool
Tool handle for TestVectorTool.
Definition: DataPreparationPipeline.h:101
DataPreparationPipeline::m_spacepointInBuff
cl::Buffer m_spacepointInBuff
Buffer for spacepoint input.
Definition: DataPreparationPipeline.h:54
StripClusterAuxContainer.h
DataPreparationPipeline::m_xAODClusterMaker
ToolHandle< xAODClusterMaker > m_xAODClusterMaker
Tool handle for xAODClusterMaker.
Definition: DataPreparationPipeline.h:89
DataPreparationPipeline::setupKernelArgs
StatusCode setupKernelArgs()
Definition: DataPreparationPipeline.cxx:190
DataPreparationPipeline::m_pixelClusteringKernel
cl::Kernel m_pixelClusteringKernel
Kernel for pixel clustering.
Definition: DataPreparationPipeline.h:48
IntegrationBase::m_program
cl::Program m_program
Program object containing the kernel.
Definition: IntegrationBase.h:68
debug.h
Helper functions intended to be called from the debugger.
EFTrackingFPGAIntegration::TVHolder::inputTV
std::vector< uint64_t > inputTV
Definition: TestVectorTool.h:33
DataPreparationPipeline::m_pxlClusteringInBuff
cl::Buffer m_pxlClusteringInBuff
Buffer for pixel clustering input.
Definition: DataPreparationPipeline.h:51