Service used for programming XRT compatible accelerators.
More...
#include <DeviceMgmtSvc.h>
|
struct | AthClContext |
| Struct to hold information about a context, as well as the devices, the program and XCLBIN file associated with the context. More...
|
|
struct | SystemInfo |
|
struct | XclbinInfo |
| Struct to hold information about an XCLBIN file, as well as the kernels it contains. More...
|
|
Service used for programming XRT compatible accelerators.
This service aims to be a central point for managing XRT compatible accelerators. It will enumerate the available devices, program them with the specified XCLBIN files and provide a way for client algorithms to request lists of devices by kernel name, either using native XRT API or OpenCL API.
- Author
- Quentin Berthet quent.nosp@m.in.b.nosp@m.erthe.nosp@m.t@ce.nosp@m.rn.ch
Definition at line 31 of file DeviceMgmtSvc.h.
◆ finalize()
StatusCode AthXRT::DeviceMgmtSvc::finalize |
( |
| ) |
|
|
overridevirtual |
Finalise the service.
Definition at line 485 of file DeviceMgmtSvc.cxx.
491 return StatusCode::SUCCESS;
◆ get_device_bdf()
std::string AthXRT::DeviceMgmtSvc::get_device_bdf |
( |
const cl::Device & |
device | ) |
const |
|
private |
Get the BDF (bus:device:function) string of a cl::device.
- Parameters
-
device | The device to get the BDF from. |
- Returns
- The BDF of the device as a string, or "error" in case of failure.
Definition at line 194 of file DeviceMgmtSvc.cxx.
196 cl_int
err = CL_SUCCESS;
197 std::string device_bdf;
198 err = device.getInfo(CL_DEVICE_PCIE_BDF, &device_bdf);
199 if (
err != CL_SUCCESS) {
201 return std::string(
"error");
◆ get_device_name()
std::string AthXRT::DeviceMgmtSvc::get_device_name |
( |
const cl::Device & |
device | ) |
const |
|
private |
Get the name of a cl::device.
- Parameters
-
device | The device to get the name from. |
- Returns
- The name of the device as a string, or "error" in case of failure.
Definition at line 179 of file DeviceMgmtSvc.cxx.
181 cl_int
err = CL_SUCCESS;
182 std::string device_name;
183 err = device.getInfo(CL_DEVICE_NAME, &device_name);
184 if (
err != CL_SUCCESS) {
186 return std::string(
"error");
◆ get_opencl_handles_by_kernel_name()
Get a list of OpenCL handles providing the specified kernel.
- Parameters
-
name | The name of the kernel to search for. |
- Returns
- A vector of OpenCL handles structs providing the specified kernel.
Definition at line 524 of file DeviceMgmtSvc.cxx.
527 std::vector<IDeviceMgmtSvc::OpenCLHandle> handles;
532 if (
std::find(ath_cl_context.xclbin_info.kernel_names.begin(),
533 ath_cl_context.xclbin_info.kernel_names.end(),
534 name) != ath_cl_context.xclbin_info.kernel_names.end()) {
535 IDeviceMgmtSvc::OpenCLHandle handle = {ath_cl_context.context,
536 ath_cl_context.program};
537 handles.push_back(handle);
◆ get_xrt_devices_by_kernel_name()
const std::vector< std::shared_ptr< xrt::device > > AthXRT::DeviceMgmtSvc::get_xrt_devices_by_kernel_name |
( |
const std::string & |
name | ) |
const |
|
overridevirtual |
Get a list of XRT devices providing the specified kernel.
- Parameters
-
name | The name of the kernel to search for. |
- Returns
- A vector of XRT device handles providing the specified kernel.
Definition at line 495 of file DeviceMgmtSvc.cxx.
497 std::vector<std::shared_ptr<xrt::device>> devices;
502 if (
std::find(ath_cl_context.xclbin_info.kernel_names.begin(),
503 ath_cl_context.xclbin_info.kernel_names.end(),
504 name) != ath_cl_context.xclbin_info.kernel_names.end()) {
505 for (
const cl::Device &device : ath_cl_context.devices) {
506 devices.push_back(std::make_shared<xrt::device>(
507 xrt::opencl::get_xrt_device(device())));
510 else if (
std::find(ath_cl_context.xclbin_info.cu_names.begin(),
511 ath_cl_context.xclbin_info.cu_names.end(),
512 name) != ath_cl_context.xclbin_info.cu_names.end()) {
513 for (
const cl::Device &device : ath_cl_context.devices) {
514 devices.push_back(std::make_shared<xrt::device>(
515 xrt::opencl::get_xrt_device(device())));
◆ initialize()
StatusCode AthXRT::DeviceMgmtSvc::initialize |
( |
| ) |
|
|
overridevirtual |
Initialise the service.
Initialize the service.
This function will inspect the available devices and XCLBIN files, pair them and create contexts based on the information gathered. It will then program the devices with the XCLBIN files and create contexts.
- Returns
- StatusCode::SUCCESS if the initialization was successful, or StatusCode::FAILURE otherwise.
Definition at line 461 of file DeviceMgmtSvc.cxx.
482 return StatusCode::SUCCESS;
◆ inspect_devices()
StatusCode AthXRT::DeviceMgmtSvc::inspect_devices |
( |
SystemInfo & |
si | ) |
|
|
private |
Inspect the available devices and fill the SystemInfo structure.
Find all available XRT compatible accelerator devices and group them by type.
- Parameters
-
Definition at line 19 of file DeviceMgmtSvc.cxx.
21 std::vector<cl::Platform> platforms;
25 for (
const cl::Platform &
platform : platforms) {
29 std::string platform_name;
31 if (platform_name !=
"Xilinx") {
37 std::vector<cl::Device> devices;
40 ATH_MSG_DEBUG(
"Found XRT/OpenCL platform '" << platform_name <<
"' with "
43 si.device_count += devices.size();
48 for (
const cl::Device &device : devices) {
53 for (std::vector<cl::Device> &
list : si.device_types) {
54 std::string list_device_name;
55 ATH_CHECK(
list[0].getInfo(CL_DEVICE_NAME, &list_device_name) ==
57 if (device_name == list_device_name) {
59 list.push_back(device);
64 std::vector<cl::Device> new_list = {device};
65 si.device_types.push_back(new_list);
71 if (si.device_count < 1) {
76 return StatusCode::FAILURE;
79 << si.device_count <<
" AMD FPGA device(s) ("
80 << si.device_types.size() <<
" device type(s))");
83 return StatusCode::SUCCESS;
◆ inspect_xclbins()
StatusCode AthXRT::DeviceMgmtSvc::inspect_xclbins |
( |
SystemInfo & |
si | ) |
|
|
private |
Inspect the provided XCLBIN files and fill the SystemInfo structure.
Gather information using XRT native API about the XCLBIN files and the kernel(s) they contain and perform some basic sanity checks.
- Parameters
-
Definition at line 91 of file DeviceMgmtSvc.cxx.
96 return StatusCode::FAILURE;
103 "More XCLBIN file(s) specified than "
104 "devices type available ("
105 << si.device_count <<
"): ");
109 return StatusCode::FAILURE;
116 ATH_MSG_ERROR(
"XCLBIN file does not exist: " << xclbin_path);
117 return StatusCode::FAILURE;
123 DeviceMgmtSvc::XclbinInfo xclbin_info;
125 xrt::xclbin xrt_xclbin(xclbin_path);
126 xclbin_info.path = xclbin_path;
127 xclbin_info.xsa_name = xrt_xclbin.get_xsa_name();
128 xclbin_info.fpga_device_name = xrt_xclbin.get_fpga_device_name();
129 xclbin_info.uuid = xrt_xclbin.get_uuid().to_string();
131 for (
const xrt::xclbin::kernel &kernel : xrt_xclbin.get_kernels()) {
132 const std::string& kernelName = kernel.get_name();
138 if (!kernel.get_cus().empty()) {
139 xclbin_info.kernel_names.push_back(kernelName);
143 const std::string& computeUnitName = computeUnit.get_name();
144 const std::string computeUnitIsolatedName =
145 computeUnitName.substr(kernelName.size() + 1);
147 const std::string computeUnitUsableName = kernelName +
":{" + computeUnitIsolatedName +
"}";
149 xclbin_info.cu_names.push_back(std::move(computeUnitUsableName));
154 ATH_MSG_ERROR(
"Could not create xrt::xclbin from " << xclbin_path);
155 return StatusCode::FAILURE;
163 std::set<std::string> uuids;
164 std::set<std::string> fpga_device_names;
166 uuids.insert(
info.uuid);
167 fpga_device_names.insert(
info.fpga_device_name);
169 si.different_xclbin_count = uuids.size();
170 si.different_xclbin_fpga_device_name = fpga_device_names.size();
172 return StatusCode::SUCCESS;
◆ is_xclbin_compatible_with_device()
Helper function to check if an XCLBIN file is compatible with a device.
Check if an XCLBIN is compatible with a device.
This is done by comparing the device name and the XSA name used in the XCLBIN file up to the second occurrence of the underscore character. This check is not a guarantee that the XCLBIN will work on the device, but mismatching XSA names are a strong indicator of incompatibility.
- Parameters
-
xclbin_info | The XCLBIN to check compatibility with. |
device | The device to check compatibility with. |
- Returns
- True if the XCLBIN is compatible with the device, false otherwise.
Definition at line 241 of file DeviceMgmtSvc.cxx.
245 const std::string device_prefix =
247 const std::string xsa_prefix =
248 getPrefixUpToNthOccurrence(xclbin_info.xsa_name,
'_', 2);
249 if (device_prefix == xsa_prefix) {
◆ pair_devices_and_xclbins()
StatusCode AthXRT::DeviceMgmtSvc::pair_devices_and_xclbins |
( |
const SystemInfo & |
si | ) |
|
|
private |
Pair devices and XCLBINs and create contexts.
This function will pair devices and XCLBINsd epending on the number of devices and XCLBINs, will attempt to program all provided XCLBINs according to the following rules:
- If we have only one FPGA type and one XCLBIN: Program all devices with the same XCLBIN and create one context.
- If we have only one FPGA type and multiple identical XCLBIN: Program the same number of devices that we have XCLBIN files, but put them in only one context as the XCLBIN will be identical for all programmed devices.
- If we have multiple type and multiple different XCLBIN, but all targeting the same device: Program all devices with a different XCLBIN and create one context per device/XCLBIN.
- If we have multiple type and multiple different XCLBIN, and the XCLBIN files target multiple device types: Program each device with a matching XCLBIN and create one context per device/XCLBIN. Some devices might be left un-programmed if no matching XCLBIN is found.
- Parameters
-
si | The SystemInfo structure containing information about the devices and XCLBINs available on the system. |
- Returns
- StatusCode::SUCCESS if the pairing was successful, or StatusCode::FAILURE otherwise.
Definition at line 276 of file DeviceMgmtSvc.cxx.
279 if (si.device_types.size() == 1) {
286 ATH_MSG_DEBUG(
"Case 1: One or multiple identical device(s), one xclbin");
287 DeviceMgmtSvc::AthClContext ath_cl_context;
288 for (
const cl::Device &device : si.device_types[0]) {
289 ath_cl_context.devices.push_back(device);
296 if (si.different_xclbin_fpga_device_name > 1) {
301 "Specified XCLBINs target multiple device types, but only one "
302 "device type is present");
303 return StatusCode::FAILURE;
306 if (si.different_xclbin_count == 1) {
312 "Case 2: Multiple identical devices, multiple identical xclbins");
313 DeviceMgmtSvc::AthClContext ath_cl_context;
315 ath_cl_context.devices.push_back(si.device_types[0][
i]);
327 "Case 3: Multiple identical devices, multiple different XCLBIN "
328 "files, but targeting the same device type");
330 DeviceMgmtSvc::AthClContext ath_cl_context;
332 ath_cl_context.devices.push_back(si.device_types[0][
i]);
343 ATH_MSG_DEBUG(
"Case 4: Multiple different devices, multiple xclbins");
345 for (
const std::vector<cl::Device> &device_type : si.device_types) {
346 for (
const cl::Device &device : device_type) {
347 DeviceMgmtSvc::AthClContext ath_cl_context;
348 ath_cl_context.devices.push_back(device);
353 for (
iter = unaffected_xclbin_infos.begin();
354 iter != unaffected_xclbin_infos.end();) {
356 ath_cl_context.xclbin_info = *
iter;
357 iter = unaffected_xclbin_infos.erase(
iter);
378 for (
const XclbinInfo &xclbin_info : unaffected_xclbin_infos) {
382 "No compatible device found for XCLBIN: " << xclbin_info.path);
386 return StatusCode::SUCCESS;
◆ program_devices()
StatusCode AthXRT::DeviceMgmtSvc::program_devices |
( |
| ) |
|
|
private |
Program the devices with the XCLBIN files and create contexts.
This function will program the devices with the XCLBIN files and create contexts based on the information in m_ath_cl_contexts. If an incompatible XCLBIN is found for a device, the initialization will fail.
- Returns
- StatusCode::SUCCESS if the programming was successful, or StatusCode::FAILURE otherwise.
Definition at line 396 of file DeviceMgmtSvc.cxx.
398 cl_int
err = CL_SUCCESS;
403 ath_cl_context.context = std::make_shared<cl::Context>(
404 ath_cl_context.devices,
nullptr,
nullptr,
nullptr, &
err);
405 if (
err != CL_SUCCESS) {
407 return StatusCode::FAILURE;
413 file.open(ath_cl_context.xclbin_info.path.c_str(), std::ios::binary);
415 ATH_MSG_ERROR(
"Could not open " << ath_cl_context.xclbin_info.path
417 return StatusCode::FAILURE;
419 std::vector<char> xclbin_buffer((std::istreambuf_iterator<char>(
file)),
420 std::istreambuf_iterator<char>());
425 cl::Program::Binaries binary;
426 for (std::size_t
i = 0;
i < ath_cl_context.devices.size(); ++
i) {
427 binary.push_back({xclbin_buffer.data(), xclbin_buffer.size()});
432 ath_cl_context.program = std::make_shared<cl::Program>(
433 *ath_cl_context.context, ath_cl_context.devices, binary,
nullptr, &
err);
434 if (
err != CL_SUCCESS) {
436 return StatusCode::FAILURE;
440 std::string bdfs =
"";
441 for (
const cl::Device &device : ath_cl_context.devices) {
445 ATH_MSG_INFO(
"Loaded " << ath_cl_context.xclbin_info.path <<
" on "
446 << ath_cl_context.devices.size() <<
" "
448 <<
" device(s): " << bdfs);
451 return StatusCode::SUCCESS;
◆ m_ath_cl_contexts
std::vector<AthClContext> AthXRT::DeviceMgmtSvc::m_ath_cl_contexts |
|
private |
List of contexts configured for the service.
Definition at line 138 of file DeviceMgmtSvc.h.
◆ m_xclbin_infos
std::vector<XclbinInfo> AthXRT::DeviceMgmtSvc::m_xclbin_infos |
|
private |
List of XCLBIN files info configured for the service.
Definition at line 126 of file DeviceMgmtSvc.h.
◆ m_xclbin_path_list
Gaudi::Property<std::vector<std::string> > AthXRT::DeviceMgmtSvc::m_xclbin_path_list |
|
private |
Initial value:{
this,
"XclbinPathsList",
{},
"The list of XCLBIN files to program on FPGAs"}
The list of xclbin files to use.
This is a list of paths to XCLBIN files to load on accelerator devices.
Definition at line 72 of file DeviceMgmtSvc.h.
The documentation for this class was generated from the following files: