ATLAS Offline Software
F600IntegrationAlg.h
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
3 */
4 
5 
6  #ifndef EFTRACKING_FPGA_INTEGRATION_F600INTEGRATIONALG_H
7  #define EFTRACKING_FPGA_INTEGRATION_F600INTEGRATIONALG_H
8 
9  // EFTracking include
17 
18  // Athena include
19  #include "GaudiKernel/ServiceHandle.h"
20  #include "GaudiKernel/IChronoSvc.h"
21 
22 // XRT includes
23 #include "xrt/xrt_bo.h"
24 #include "xrt/xrt_device.h"
25 #include "xrt/xrt_kernel.h"
26 #include <experimental/xrt_xclbin.h>
27 #include <experimental/xrt_ip.h>
28 #include <experimental/xrt_queue.h>
29 
30 // XRT -> CL includes
31 #include "CL/cl2xrt.hpp"
32 
33 #include <filesystem>
34 #include <fstream>
35 
36 #define IP_START 0x1 // USER_CTRL[0]
37 #define IP_DONE 0x2 // USER_CTRL[1]
38 #define IP_IDLE 0x4 // USER_CTRL[2]
39 #define EVENT_COUNT_RST 0x80000000 // USER_CTRL[31]
40 // Addresses
41 #define USER_CTRL_OFFSET 0x10
42 #define EVENT_RX_COUNT_OFFSET 0x14
43 #define EVENT_TX_COUNT_OFFSET 0x1C
44 
46  {
48  {
49  public:
50  using IntegrationBase::IntegrationBase;
51  virtual StatusCode initialize() override final;
52  virtual StatusCode execute(const EventContext &ctx) const override final;
53  virtual StatusCode finalize() override final;
54 
55 
56  private:
58  "ChronoStatSvc", name()};
59 
60  ToolHandle<xAODClusterMaker> m_xaodClusterMaker{
61  this,
62  "xAODClusterMaker",
63  "xAODClusterMaker",
64  "Tool for creating xAOD cluster containers"};
65 
66  ToolHandle<TestVectorTool> m_testVectorTool{
67  this, "TestVectorTool", "TestVectorTool", "Tool for preparing test vectors"};
68 
69  ToolHandle<FPGADataFormatTool> m_FPGADataFormatTool{
70  this, "FPGADataFormatTool", "FPGADataFormatTool", "Tool for formatting FPGA data"};
71 
72  Gaudi::Property<std::string> m_xclbin{
73  this, "xclbin", "", "xclbin path and name"};
74 
75  SG::ReadHandleKey<FPGATrackSimTrackCollection> m_FPGATrackKey{this, "FPGATrackSimTrack1stKey","FPGATracks_1st_reg34","FPGATrackSim Tracks 1st stage key"};
76 
77  // Tool for output conversion
78  ToolHandle<OutputConversionTool> m_outputConversionTool{this, "OutputConversionTool", "OutputConversionTool", "tool for output conversion"};
79 
80  // XRT Kernels and IPs Name Properties
81  // Data Prep
82  Gaudi::Property<std::string> m_pixelClusteringKernelName{
83  this, "PixelClusteringKernelName", "", "Name of the pixel clustering kernel (pxl cls)"};
84  Gaudi::Property<std::string> m_processHitsKernelName{
85  this, "ProcessHitsKernelName", "", "Name of the strip clustering kernel (strp cls)"};
86  Gaudi::Property<std::string> m_pixelL2gKernelName{
87  this, "PixelL2gKernelName", "", "Name of the pixel L2G transformation kernel"};
88  Gaudi::Property<std::string> m_stripL2gKernelName{
89  this, "StripL2gKernelName", "", "Name of the strip L2G transformation kernel"};
90  Gaudi::Property<std::string> m_pixelEdmPrepKernelName{
91  this, "PixelEdmPrepKernelName", "", "Name of the pixel EDM prep kernel"};
92  Gaudi::Property<std::string> m_stripEdmPrepKernelName{
93  this, "StripEdmPrepKernelName", "", "Name of the strip EDM prep kernel"};
94 
95  // Slicing Engine
96  Gaudi::Property<std::string> m_pixelFirstStageSlicingIPName{
97  this, "PixelFirstStageSlicingIPName", "", "Name of the slicing engine IP"};
98  Gaudi::Property<std::string> m_pixelFirstStageInputKernelName{
99  this, "PixelFirstStageInputKernelName", "", "Name of the input stage kernel"};
100  Gaudi::Property<std::string> m_pixelFirstStageOutputKernelName{
101  this, "PixelFirstStageOutputKernelName", "", "Name of the output stage kernel"};
102 
103  // Inside Out
104  Gaudi::Property<std::string> m_memReadKernelName{
105  this, "MemReadKernelName", "", "Name of the memory read kernel"};
106  Gaudi::Property<std::string> m_memWriteKernelName{
107  this, "MemWriteKernelName", "", "Name of the memory write kernel"};
108 
109  // Space Points
110  Gaudi::Property<std::string> m_spacepointKernelName{
111  this, "SpacepointKernelName", "", "Name of the spacepoint kernel"};
112 
113  // NN Pathfinder
114  Gaudi::Property<std::string> m_loaderKernelName{
115  this, "LoaderKernelName", "", "Name of the loader kernel"};
116  Gaudi::Property<std::string> m_unloaderKernelName{
117  this, "UnloaderKernelName", "", "Name of the unloader kernel"};
118 
119  // NN Classifier
120  Gaudi::Property<std::string> m_nnOverlapDecoratorKernelName{
121  this, "NnOverlapDecoratorKernelName", "", "Name of the NN overlap decorator kernel"};
122 
123  // Duplicate Remover
124  Gaudi::Property<std::string> m_runnerKernelName{
125  this, "RunnerKernelName", "", "Name of the duplicate remover runner kernel"};
126 
127  // NOTE: Temporary and for injecting TV data
128  Gaudi::Property<std::string> m_pixelClusterInputPath{
129  this, "PixelClusterInputPath", "", "Filepath for the pixel clustering input data"};
130  Gaudi::Property<std::string> m_pixelStageOneSlicingInputPath{
131  this, "PixelStageOneSlicingInputPath", "", "Filepath for the slicing engine (pixel first stage) input data"};
132  Gaudi::Property<std::string> m_insideOutInputPath{
133  this, "InsideOutInputPath", "", "Filepath for the inside out input data"};
134 
135  // NOTE WHA TI NEED: max size for IO for all the kernels. working SE other stages
136 
137  mutable std::atomic<cl_ulong> m_kernelTime{0};
138  mutable std::atomic<cl_ulong> m_sum_kernelTime{0};
139  mutable std::atomic<ulonglong> m_num_Events{0};
140 
141  // For IP access through XRT
142  xrt::device m_xrt_accelerator;
143 
144  mutable std::mutex m_fpgaHandleMtx ATLAS_THREAD_SAFE;
145 
146  mutable std::vector<cl::Event> m_pixelFirstStageSlicingInputEndEvents ATLAS_THREAD_SAFE;
147  mutable std::vector<cl::Event> m_pixelFirstStageSlicingOutputEndEvents ATLAS_THREAD_SAFE;
148  mutable std::vector<cl::Event> m_insideOutInputEndEvents ATLAS_THREAD_SAFE;
149  mutable std::vector<cl::Event> m_insideOutOutputEndEvents ATLAS_THREAD_SAFE;
150 
151 
152  // Pixel CLS
153  mutable std::vector<cl::Kernel> m_pixelClusterL2GKernels ATLAS_THREAD_SAFE;
154  // Slicing Engine
155  mutable std::vector<cl::Kernel> m_pixelFirstStageSlicingInputKernels ATLAS_THREAD_SAFE;
156  mutable std::vector<xrt::ip> m_pixelFirstStageSlicingIPs ATLAS_THREAD_SAFE;
157  mutable std::vector<cl::Kernel> m_pixelFirstStageSlicingOutputKernels ATLAS_THREAD_SAFE;
158  // Inside Out
159  mutable std::vector<cl::Kernel> m_insideOutInputKernels ATLAS_THREAD_SAFE;
160  mutable std::vector<cl::Kernel> m_insideOutOutputKernels ATLAS_THREAD_SAFE;
161 
162 
163 
164  // Buffers for Clustering & L2G
165  mutable std::vector<cl::Buffer> m_pixelClusterL2GInputBufferList ATLAS_THREAD_SAFE;
166  mutable std::vector<cl::Buffer> m_pixelClusterL2GOutputBufferList ATLAS_THREAD_SAFE;
167  mutable std::vector<cl::Buffer> m_pixelClusterL2GEDMOutputBufferList ATLAS_THREAD_SAFE;
168  // Buffers for slicing engine
169  mutable std::vector<cl::Buffer> m_pixelFirstStageSlicingInputBufferList ATLAS_THREAD_SAFE;
170  mutable std::vector<cl::Buffer> m_pixelFirstStageSlicingOutputBufferList ATLAS_THREAD_SAFE;
171  // Buffers for inside out
172  mutable std::vector<cl::Buffer> m_insideOutInputBufferList ATLAS_THREAD_SAFE;
173  mutable std::vector<cl::Buffer> m_insideOutOutputBufferList ATLAS_THREAD_SAFE;
174 
175  // Command queue
176  cl::CommandQueue m_queue;
177 
178  std::string get_cu_name(const std::string& kernel_name, int cu);
179 
180  // NOTE: These are temporary and are used to inject TVs in where kernels would otherwise go.
181  // Once all kernels are working these will be removed.
182  int readTVLength(std::string filepath) const;
183  void readTV(const std::string filepath, int n_words, uint64_t *data) const;
184 
185  void outputHexData(size_t dataLen, uint64_t *data, const std::string dataDescriptor) const;
186  };
187  }
188 
189  #endif // EFTRACKING_FPGA_INTEGRATION_F600INTEGRATIONALG_H
190 
EFTrackingFPGAIntegration::F600IntegrationAlg::readTV
void readTV(const std::string filepath, int n_words, uint64_t *data) const
Definition: F600IntegrationAlg.cxx:89
EFTrackingFPGAIntegration::F600IntegrationAlg::outputHexData
void outputHexData(size_t dataLen, uint64_t *data, const std::string dataDescriptor) const
Definition: F600IntegrationAlg.cxx:18
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Kernel > m_pixelFirstStageSlicingOutputKernels ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:157
EFTrackingFPGAIntegration::F600IntegrationAlg::m_pixelFirstStageOutputKernelName
Gaudi::Property< std::string > m_pixelFirstStageOutputKernelName
Definition: F600IntegrationAlg.h:100
IntegrationBase
The base class for the EFTracking FPGA integration development.
Definition: IntegrationBase.h:38
data
char data[hepevt_bytes_allocation_ATLAS]
Definition: HepEvt.cxx:11
EFTrackingFPGAIntegration::F600IntegrationAlg::m_stripL2gKernelName
Gaudi::Property< std::string > m_stripL2gKernelName
Definition: F600IntegrationAlg.h:88
EFTrackingFPGAIntegration::F600IntegrationAlg::finalize
virtual StatusCode finalize() override final
Definition: F600IntegrationAlg.cxx:247
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Buffer > m_pixelClusterL2GEDMOutputBufferList ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:167
EFTrackingFPGAIntegration::F600IntegrationAlg::m_pixelL2gKernelName
Gaudi::Property< std::string > m_pixelL2gKernelName
Definition: F600IntegrationAlg.h:86
EFTrackingFPGAIntegration::F600IntegrationAlg::m_queue
cl::CommandQueue m_queue
Definition: F600IntegrationAlg.h:176
BeamSpot::mutex
std::mutex mutex
Definition: InDetBeamSpotVertex.cxx:18
EFTrackingFPGAIntegration::F600IntegrationAlg::m_stripEdmPrepKernelName
Gaudi::Property< std::string > m_stripEdmPrepKernelName
Definition: F600IntegrationAlg.h:92
EFTrackingFPGAIntegration::F600IntegrationAlg::m_loaderKernelName
Gaudi::Property< std::string > m_loaderKernelName
Definition: F600IntegrationAlg.h:114
EFTrackingFPGAIntegration::F600IntegrationAlg::m_insideOutInputPath
Gaudi::Property< std::string > m_insideOutInputPath
Definition: F600IntegrationAlg.h:132
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Buffer > m_pixelFirstStageSlicingOutputBufferList ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:170
EFTrackingFPGAIntegration::F600IntegrationAlg::m_kernelTime
std::atomic< cl_ulong > m_kernelTime
Time for kernel execution.
Definition: F600IntegrationAlg.h:137
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::mutex m_fpgaHandleMtx ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:144
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Event > m_insideOutOutputEndEvents ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:149
IntegrationBase.h
SG::ReadHandleKey< FPGATrackSimTrackCollection >
EFTrackingFPGAIntegration::F600IntegrationAlg::m_pixelFirstStageSlicingIPName
Gaudi::Property< std::string > m_pixelFirstStageSlicingIPName
Definition: F600IntegrationAlg.h:96
EFTrackingFPGAIntegration::F600IntegrationAlg::m_pixelClusterInputPath
Gaudi::Property< std::string > m_pixelClusterInputPath
Definition: F600IntegrationAlg.h:128
EFTrackingFPGAIntegration::F600IntegrationAlg::m_unloaderKernelName
Gaudi::Property< std::string > m_unloaderKernelName
Definition: F600IntegrationAlg.h:116
EFTrackingFPGAIntegration::F600IntegrationAlg::m_FPGADataFormatTool
ToolHandle< FPGADataFormatTool > m_FPGADataFormatTool
Tool for formatting FPGA data.
Definition: F600IntegrationAlg.h:69
EFTrackingFPGAIntegration::F600IntegrationAlg::m_sum_kernelTime
std::atomic< cl_ulong > m_sum_kernelTime
Sum for the average time of the kernel execution.
Definition: F600IntegrationAlg.h:138
Muon::nsw::STGTPPad::n_words
constexpr std::size_t n_words
Definition: NSWSTGTPDecodeBitmaps.h:62
EFTrackingFPGAIntegration::F600IntegrationAlg::initialize
virtual StatusCode initialize() override final
Detect the OpenCL devices and prepare OpenCL context.
Definition: F600IntegrationAlg.cxx:26
DataPreparationPipeline.h
FPGATrackSimTrackCollection.h
EFTrackingFPGAIntegration::F600IntegrationAlg::m_FPGATrackKey
SG::ReadHandleKey< FPGATrackSimTrackCollection > m_FPGATrackKey
Definition: F600IntegrationAlg.h:75
EFTrackingFPGAIntegration::F600IntegrationAlg::execute
virtual StatusCode execute(const EventContext &ctx) const override final
Should be overriden by derived classes to perform meaningful work.
Definition: F600IntegrationAlg.cxx:109
EFTrackingFPGAIntegration::F600IntegrationAlg::m_pixelClusteringKernelName
Gaudi::Property< std::string > m_pixelClusteringKernelName
Definition: F600IntegrationAlg.h:82
EL::StatusCode
::StatusCode StatusCode
StatusCode definition for legacy code.
Definition: PhysicsAnalysis/D3PDTools/EventLoop/EventLoop/StatusCode.h:22
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Event > m_pixelFirstStageSlicingOutputEndEvents ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:147
EFTrackingFPGAIntegration::F600IntegrationAlg::get_cu_name
std::string get_cu_name(const std::string &kernel_name, int cu)
Definition: F600IntegrationAlg.cxx:12
EFTrackingFPGAIntegration::F600IntegrationAlg::m_xaodClusterMaker
ToolHandle< xAODClusterMaker > m_xaodClusterMaker
Tool for creating xAOD containers.
Definition: F600IntegrationAlg.h:60
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Buffer > m_insideOutInputBufferList ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:172
EFTrackingFPGAIntegration::F600IntegrationAlg
Definition: F600IntegrationAlg.h:48
EFTrackingFPGAIntegration::F600IntegrationAlg::m_memReadKernelName
Gaudi::Property< std::string > m_memReadKernelName
Definition: F600IntegrationAlg.h:104
EFTrackingFPGAIntegration::F600IntegrationAlg::m_testVectorTool
ToolHandle< TestVectorTool > m_testVectorTool
Tool for preparing test vectors.
Definition: F600IntegrationAlg.h:66
EFTrackingFPGAIntegration::F600IntegrationAlg::m_xclbin
Gaudi::Property< std::string > m_xclbin
Path and name of the xclbin file.
Definition: F600IntegrationAlg.h:72
xAOD::uint64_t
uint64_t
Definition: EventInfo_v1.cxx:123
OutputConversionTool.h
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Buffer > m_insideOutOutputBufferList ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:173
EFTrackingFPGAIntegration
The class for enconding RDO to FPGA format.
Definition: BenchmarkAlg.h:28
EFTrackingFPGAIntegration::F600IntegrationAlg::m_pixelEdmPrepKernelName
Gaudi::Property< std::string > m_pixelEdmPrepKernelName
Definition: F600IntegrationAlg.h:90
EFTrackingFPGAIntegration::F600IntegrationAlg::m_processHitsKernelName
Gaudi::Property< std::string > m_processHitsKernelName
Definition: F600IntegrationAlg.h:84
EFTrackingFPGAIntegration::F600IntegrationAlg::m_memWriteKernelName
Gaudi::Property< std::string > m_memWriteKernelName
Definition: F600IntegrationAlg.h:106
EFTrackingFPGAIntegration::F600IntegrationAlg::m_num_Events
std::atomic< ulonglong > m_num_Events
Number of events for the average time of the kernel execution.
Definition: F600IntegrationAlg.h:139
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
EFTrackingFPGAIntegration::F600IntegrationAlg::m_runnerKernelName
Gaudi::Property< std::string > m_runnerKernelName
Definition: F600IntegrationAlg.h:124
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Buffer > m_pixelClusterL2GInputBufferList ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:165
EFTrackingFPGAIntegration::F600IntegrationAlg::m_outputConversionTool
ToolHandle< OutputConversionTool > m_outputConversionTool
Definition: F600IntegrationAlg.h:78
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Kernel > m_pixelFirstStageSlicingInputKernels ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:155
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Buffer > m_pixelFirstStageSlicingInputBufferList ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:169
xAODClusterMaker.h
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< xrt::ip > m_pixelFirstStageSlicingIPs ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:156
FPGADataFormatTool.h
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Buffer > m_pixelClusterL2GOutputBufferList ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:166
EFTrackingFPGAIntegration::F600IntegrationAlg::m_xrt_accelerator
xrt::device m_xrt_accelerator
Definition: F600IntegrationAlg.h:142
TestVectorTool.h
EFTrackingFPGAIntegration::F600IntegrationAlg::readTVLength
int readTVLength(std::string filepath) const
Definition: F600IntegrationAlg.cxx:79
EFTrackingFPGAIntegration::F600IntegrationAlg::m_chronoSvc
ServiceHandle< IChronoSvc > m_chronoSvc
Service for timing the algorithm.
Definition: F600IntegrationAlg.h:57
EFTrackingFPGAIntegration::F600IntegrationAlg::m_pixelStageOneSlicingInputPath
Gaudi::Property< std::string > m_pixelStageOneSlicingInputPath
Definition: F600IntegrationAlg.h:130
EFTrackingFPGAIntegration::F600IntegrationAlg::m_spacepointKernelName
Gaudi::Property< std::string > m_spacepointKernelName
Definition: F600IntegrationAlg.h:110
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Kernel > m_insideOutOutputKernels ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:160
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Kernel > m_insideOutInputKernels ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:159
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Event > m_insideOutInputEndEvents ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:148
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Kernel > m_pixelClusterL2GKernels ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:153
EFTrackingFPGAIntegration::F600IntegrationAlg::m_nnOverlapDecoratorKernelName
Gaudi::Property< std::string > m_nnOverlapDecoratorKernelName
Definition: F600IntegrationAlg.h:120
EFTrackingFPGAIntegration::F600IntegrationAlg::m_pixelFirstStageInputKernelName
Gaudi::Property< std::string > m_pixelFirstStageInputKernelName
Definition: F600IntegrationAlg.h:98
ServiceHandle< IChronoSvc >
EFTrackingFPGAIntegration::F600IntegrationAlg::ATLAS_THREAD_SAFE
std::vector< cl::Event > m_pixelFirstStageSlicingInputEndEvents ATLAS_THREAD_SAFE
Definition: F600IntegrationAlg.h:146