560{
561
562
563
564
565 uint64_t exec_start_ns = 0;
566 SET_TIMESTAMP(exec_start_ns);
567
568
569
570
571
572
573
574
576 RETURN_IF_ERROR(TRITONBACKEND_ModelInstanceState(
577 instance,
reinterpret_cast<void**
>(&instance_state)));
579
580
581
582
583 cudaError_t err = cudaSetDevice(instance_state->DeviceId());
584 if (err != cudaSuccess)
585 {
586 return TRITONSERVER_ErrorNew(
587 TRITONSERVER_ERROR_INTERNAL,
588 ("Failed to set CUDA device: " + std::string(cudaGetErrorString(err))).c_str());
589 }
590
591
592
593
594
595
596
597
598
599
600
601
602 std::vector<TRITONBACKEND_Response*> responses;
603 responses.reserve(request_count);
604 for (uint32_t
r = 0;
r < request_count; ++
r) {
605 TRITONBACKEND_Request* request = requests[
r];
607 RETURN_IF_ERROR(TRITONBACKEND_ResponseNew(&
response, request));
609 }
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658 bool all_inputs_present = true;
659 for (uint32_t
r = 0;
r < request_count; ++
r) {
660 for (const std::string& name :
662 TRITONBACKEND_Input* input = nullptr;
663 TRITONSERVER_Error* input_err =
664 TRITONBACKEND_RequestInput(requests[
r], name.c_str(), &input);
665 if (input_err != nullptr) {
666 all_inputs_present = false;
667 RESPOND_AND_SET_NULL_IF_ERROR(&responses[
r], input_err);
668 break;
669 }
670 }
671 }
672
673 BackendInputCollector collector(
674 requests, request_count, &responses, model_state->TritonMemoryManager(),
675 false , nullptr );
676
677
678
679
680
681 std::vector<std::pair<TRITONSERVER_MemoryType, int64_t>> allowed_input_types =
682 {{TRITONSERVER_MEMORY_CPU_PINNED, 0}, {TRITONSERVER_MEMORY_CPU, 0}};
683
684 const char *input_cells_buffer = nullptr;
685 size_t input_cells_buffer_byte_size = 0;
686 TRITONSERVER_MemoryType input_cells_buffer_memory_type;
687 int64_t input_cells_buffer_memory_type_id;
688
689 const char *input_iov_buffer = nullptr;
690 size_t input_iov_buffer_byte_size = 0;
691 TRITONSERVER_MemoryType input_iov_buffer_memory_type;
692 int64_t input_iov_buffer_memory_type_id;
693
694 if (all_inputs_present)
695 {
696 RESPOND_ALL_AND_SET_NULL_IF_ERROR(
697 responses, request_count,
698 collector.ProcessTensor(
700 0 , allowed_input_types, &input_cells_buffer,
701 &input_cells_buffer_byte_size, &input_cells_buffer_memory_type,
702 &input_cells_buffer_memory_type_id));
703 RESPOND_ALL_AND_SET_NULL_IF_ERROR(
704 responses, request_count,
705 collector.ProcessTensor(
707 0 , allowed_input_types, &input_iov_buffer,
708 &input_iov_buffer_byte_size, &input_iov_buffer_memory_type,
709 &input_iov_buffer_memory_type_id));
710
711
712
713
714
715
716 const bool need_cuda_input_sync = collector.Finalize();
717 if (need_cuda_input_sync)
718 {
719 LOG_MESSAGE(
720 TRITONSERVER_LOG_ERROR,
721 "'Traccc' backend: unexpected CUDA sync required by collector");
722 }
723 }
724
725 bool print_stats = true;
726
727
728
729
730
732 SET_TIMESTAMP(compute_start_ns);
733
735 bool run_succeeded = false;
736 try
737 {
738 if (!all_inputs_present) {
739 throw std::runtime_error(
740 "request is missing the '" +
743 }
744
745 std::array<int64_t, iovSize>
iov;
746 if (input_iov_buffer_byte_size != sizeof(iov)) {
747 throw std::runtime_error(
"'" +
iovInputName +
"' has the wrong size");
748 }
749 std::memcpy(
iov.data(), input_iov_buffer,
sizeof(iov));
751 const EventIDBase eventID(
752 static_cast<number_type>(iov[0]), EventIDBase::UNDEFEVT,
756 reinterpret_cast<const uint8_t*
>(input_cells_buffer),
757 input_cells_buffer_byte_size, eventID,
758 static_cast<number_type>(iov[4]), print_stats, traccc_result);
759 run_succeeded = true;
760 }
761 catch (const std::exception& e)
762 {
763 RESPOND_ALL_AND_SET_NULL_IF_ERROR(
764 responses, request_count,
765 TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL,
e.what()));
766 }
767
769 SET_TIMESTAMP(compute_end_ns);
770
771 bool supports_first_dim_batching;
772 RESPOND_ALL_AND_SET_NULL_IF_ERROR(
773 responses, request_count,
774 model_state->SupportsFirstDimBatching(&supports_first_dim_batching));
775
776
777
778
779
780
781
782
783
784 BackendOutputResponder responder(
785 requests, request_count, &responses, model_state->TritonMemoryManager(),
786 supports_first_dim_batching, false ,
787 nullptr );
788
789 auto output_proc_start = std::chrono::high_resolution_clock::now();
790
791
792 std::vector<std::vector<std::uint8_t>> output_bytes;
794
795
796
797 if (run_succeeded)
798 {
799
800
801
802 const auto sendCollection = [&](
const std::string&
name,
803 const auto& collection) {
804 std::vector<std::uint8_t>& bytes = output_bytes.emplace_back();
806 std::vector<int64_t> shape = {
807 static_cast<int64_t>(bytes.size())};
808 responder.ProcessTensor(
809 name.c_str(), TRITONSERVER_TYPE_UINT8, shape,
810 reinterpret_cast<const char*>(bytes.data()),
811 TRITONSERVER_MEMORY_CPU, 0);
812 };
813 sendCollection("MEASUREMENTS", traccc_result.measurements);
814 sendCollection("CLUSTERS", traccc_result.clusters);
815 sendCollection("TRACKS", traccc_result.tracks.tracks);
816 sendCollection("TRACK_STATES", traccc_result.tracks.states);
817 }
818
819 if (print_stats)
820 {
821 auto output_proc_end = std::chrono::high_resolution_clock::now();
822 std::cout << "[TIMING] Output processing: "
823 << std::chrono::duration_cast<std::chrono::milliseconds>(
824 output_proc_end - output_proc_start).count()
825 << " ms" << std::endl;
826 }
827
828
829
830
831
832
833
834 const bool need_cuda_output_sync = responder.Finalize();
835 if (need_cuda_output_sync) {
836 LOG_MESSAGE(
837 TRITONSERVER_LOG_ERROR,
838 "'traccc' backend: unexpected CUDA sync required by responder");
839 }
840
841
842
845 LOG_IF_ERROR(
846 TRITONBACKEND_ResponseSend(
847 response, TRITONSERVER_RESPONSE_COMPLETE_FINAL,
nullptr),
848 "failed to send response");
849 }
850 }
851
853 SET_TIMESTAMP(exec_end_ns);
854
855#ifdef TRITON_ENABLE_STATS
856
857
858
859
860 size_t total_batch_size = 0;
861 if (!supports_first_dim_batching) {
862 total_batch_size = request_count;
863 } else {
864 for (uint32_t
r = 0;
r < request_count; ++
r)
865 {
866 auto& request = requests[
r];
867 TRITONBACKEND_Input*
input =
nullptr;
868 LOG_IF_ERROR(
869 TRITONBACKEND_RequestInputByIndex(request, 0 , &input),
870 "failed getting request input");
871 if (input != nullptr)
872 {
873 const int64_t* shape = nullptr;
874 LOG_IF_ERROR(
875 TRITONBACKEND_InputProperties(
876 input, nullptr, nullptr, &shape, nullptr, nullptr, nullptr),
877 "failed getting input properties");
878 if (shape != nullptr)
879 {
880 total_batch_size += shape[0];
881 }
882 }
883 }
884 }
885#else
886 (void)exec_start_ns;
887 (void)exec_end_ns;
888 (void)compute_start_ns;
889 (void)compute_end_ns;
890#endif
891
892
893for (uint32_t
r = 0;
r < request_count; ++
r)
894{
895 auto& request = requests[
r];
896
897 #ifdef TRITON_ENABLE_STATS
898 LOG_IF_ERROR(
899 TRITONBACKEND_ModelInstanceReportStatistics(
900 instance_state->TritonModelInstance(), request,
901 (responses[
r] !=
nullptr) , exec_start_ns,
902 compute_start_ns, compute_end_ns, exec_end_ns),
903 "failed reporting request statistics");
904 #endif
905
906 LOG_IF_ERROR(
907 TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL),
908 "failed releasing request");
909}
910
911#ifdef TRITON_ENABLE_STATS
912
913 LOG_IF_ERROR(
914 TRITONBACKEND_ModelInstanceReportBatchStatistics(
915 instance_state->TritonModelInstance(), total_batch_size,
916 exec_start_ns, compute_start_ns, compute_end_ns, exec_end_ns),
917 "failed reporting batch request statistics");
918#endif
919
920return nullptr;
921}
std::unique_ptr< TracccTritonRunner > traccc_triton_runner_
ModelState * StateForModel()
const std::string & InputCellsTensorName() const
void serialize(const vecmem::edm::host< vecmem::edm::schema< VARTYPES... >, INTERFACE > &container, std::vector< std::uint8_t > &out)
Append container to out in the format described above.
const std::array< std::string, 4 > outputNames
const std::string iovInputName
The host-resident reconstruction result of one request: the track container, the measurements its sta...