From e66bc301a955cdac3355cce27433f59ea916ef98 Mon Sep 17 00:00:00 2001 From: ConnorC432 Date: Fri, 13 Mar 2026 15:49:37 +0000 Subject: [PATCH 1/7] detect system onnxruntime with pkg-config --- SerialPrograms/CMakeLists.txt | 137 +++++++++++++++++++--------------- 1 file changed, 75 insertions(+), 62 deletions(-) diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index ec7415075..14aedfe16 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -379,74 +379,87 @@ else() # macOS and Linux # extract it, and then link it up # Set the ONNXRUNTIME version and arch, allows for quick updates - set(ONNXRUNTIME_VERSION "1.23.0") - set(ONNXRUNTIME_ARCH "x64") - set(ONNXRUNTIME_DIR_NAME "onnxruntime-linux-${ONNXRUNTIME_ARCH}-${ONNXRUNTIME_VERSION}") - set(ONNXRUNTIME_TGZ_NAME "${ONNXRUNTIME_DIR_NAME}.tgz") - # build up the URL based on the version and name - set(ONNXRUNTIME_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/${ONNXRUNTIME_TGZ_NAME}") - - # Cache variable is editable by users *and* is properly stored - set(ONNX_ROOT_PATH "" CACHE PATH "Path to the root of a pre-installed ONNX Runtime (e.g., /path/to/onnxruntime-linux-x64-1.23.0)") - - # Download the onnxruntime if it doesn't exist or if the path is bad - if(NOT ONNX_ROOT_PATH OR NOT EXISTS "${ONNX_ROOT_PATH}/include/onnxruntime_cxx_api.h") - set(EXTRACTED_ONNX_PATH "${CMAKE_BINARY_DIR}/${ONNXRUNTIME_DIR_NAME}") - - if(NOT IS_DIRECTORY ${EXTRACTED_ONNX_PATH}) - message(STATUS "ONNX_ROOT_PATH not found. Downloading ONNX Runtime v${ONNXRUNTIME_VERSION}...") - - # logic for downloading... - set(DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/${ONNXRUNTIME_TGZ_NAME}") - file(DOWNLOAD - ${ONNXRUNTIME_URL} - ${DOWNLOAD_LOCATION} - SHOW_PROGRESS - STATUS download_status - ) - list(GET download_status 0 error_code) - if(NOT error_code EQUAL 0) - list(GET download_status 1 error_message) - message(FATAL_ERROR "Failed to download ONNX Runtime: ${error_message}") - endif() + pkg_check_modules(ONNXRUNTIME QUIET libonnxruntime) - # logic for extracting the tarball to our working directory - message(STATUS "Extracting ONNX Runtime...") - execute_process( - COMMAND ${CMAKE_COMMAND} -E tar xzf "${DOWNLOAD_LOCATION}" - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - ) - endif() + if(ONNXRUNTIME_FOUND) + message(STATUS "Found system installed ONNX Runtime") + + message(STATUS "Found ONNX Include Dirs: ${ONNXRUNTIME_INCLUDE_DIRS}") + message(STATUS "Found ONNX Library: ${ONNXRUNTIME_LIBRARIES}") + + target_include_directories(SerialProgramsLib PUBLIC ${ONNXRUNTIME_INCLUDE_DIRS}) + target_link_libraries(SerialProgramsLib PUBLIC ${ONNXRUNTIME_LIBRARIES}) - # Update the ONNX_ROOT_PATH to point to our downloaded version, force ensures we update it - set(ONNX_ROOT_PATH "${EXTRACTED_ONNX_PATH}" CACHE PATH "Path to the downloaded ONNX Runtime" FORCE) - message(STATUS "Using downloaded ONNX Runtime at: ${ONNX_ROOT_PATH}") else() - message(STATUS "Using user-provided ONNX Runtime at: ${ONNX_ROOT_PATH}") - endif() + set(ONNXRUNTIME_VERSION "1.23.0") + set(ONNXRUNTIME_ARCH "x64") + set(ONNXRUNTIME_DIR_NAME "onnxruntime-linux-${ONNXRUNTIME_ARCH}-${ONNXRUNTIME_VERSION}") + set(ONNXRUNTIME_TGZ_NAME "${ONNXRUNTIME_DIR_NAME}.tgz") + # build up the URL based on the version and name + set(ONNXRUNTIME_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/${ONNXRUNTIME_TGZ_NAME}") + + # Cache variable is editable by users *and* is properly stored + set(ONNX_ROOT_PATH "" CACHE PATH "Path to the root of a pre-installed ONNX Runtime (e.g., /path/to/onnxruntime-linux-x64-1.23.0)") + + # Download the onnxruntime if it doesn't exist or if the path is bad + if(NOT ONNX_ROOT_PATH OR NOT EXISTS "${ONNX_ROOT_PATH}/include/onnxruntime_cxx_api.h") + set(EXTRACTED_ONNX_PATH "${CMAKE_BINARY_DIR}/${ONNXRUNTIME_DIR_NAME}") + + if(NOT IS_DIRECTORY ${EXTRACTED_ONNX_PATH}) + message(STATUS "ONNX_ROOT_PATH not found. Downloading ONNX Runtime v${ONNXRUNTIME_VERSION}...") + + # logic for downloading... + set(DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/${ONNXRUNTIME_TGZ_NAME}") + file(DOWNLOAD + ${ONNXRUNTIME_URL} + ${DOWNLOAD_LOCATION} + SHOW_PROGRESS + STATUS download_status + ) + list(GET download_status 0 error_code) + if(NOT error_code EQUAL 0) + list(GET download_status 1 error_message) + message(FATAL_ERROR "Failed to download ONNX Runtime: ${error_message}") + endif() + + # logic for extracting the tarball to our working directory + message(STATUS "Extracting ONNX Runtime...") + execute_process( + COMMAND ${CMAKE_COMMAND} -E tar xzf "${DOWNLOAD_LOCATION}" + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + endif() - # configure and find headers from the onnxruntime - find_path(ONNX_INCLUDE_DIRS - NAMES onnxruntime_cxx_api.h - HINTS "${ONNX_ROOT_PATH}/include" - REQUIRED - ) + # Update the ONNX_ROOT_PATH to point to our downloaded version, force ensures we update it + set(ONNX_ROOT_PATH "${EXTRACTED_ONNX_PATH}" CACHE PATH "Path to the downloaded ONNX Runtime" FORCE) + message(STATUS "Using downloaded ONNX Runtime at: ${ONNX_ROOT_PATH}") + else() + message(STATUS "Using user-provided ONNX Runtime at: ${ONNX_ROOT_PATH}") + endif() - find_library(ONNX_RUNTIME_LIB - NAMES onnxruntime - HINTS "${ONNX_ROOT_PATH}/lib" - REQUIRED - ) + # configure and find headers from the onnxruntime + find_path(ONNX_INCLUDE_DIRS + NAMES onnxruntime_cxx_api.h + HINTS "${ONNX_ROOT_PATH}/include" + REQUIRED + ) - if(ONNX_INCLUDE_DIRS AND ONNX_RUNTIME_LIB) - # only link if it was found, otherwise we error out - message(STATUS "Found ONNX Include Dirs: ${ONNX_INCLUDE_DIRS}") - message(STATUS "Found ONNX Library: ${ONNX_RUNTIME_LIB}") - target_include_directories(SerialProgramsLib PUBLIC "${ONNX_INCLUDE_DIRS}") - target_link_libraries(SerialProgramsLib PUBLIC "${ONNX_RUNTIME_LIB}") - else() - message(FATAL_ERROR "Could not find ONNX Runtime headers or library.") - endif() + find_library(ONNX_RUNTIME_LIB + NAMES onnxruntime + HINTS "${ONNX_ROOT_PATH}/lib" + REQUIRED + ) + + if(ONNX_INCLUDE_DIRS AND ONNX_RUNTIME_LIB) + # only link if it was found, otherwise we error out + message(STATUS "Found ONNX Include Dirs: ${ONNX_INCLUDE_DIRS}") + message(STATUS "Found ONNX Library: ${ONNX_RUNTIME_LIB}") + target_include_directories(SerialProgramsLib PUBLIC "${ONNX_INCLUDE_DIRS}") + target_link_libraries(SerialProgramsLib PUBLIC "${ONNX_RUNTIME_LIB}") + else() + message(FATAL_ERROR "Could not find ONNX Runtime headers or library.") + endif() + endif () endif() # end Linux # Find OpenCV From bab93f6d18474a4d5eba2cc98ccca96fd88aea27 Mon Sep 17 00:00:00 2001 From: ConnorC432 Date: Fri, 13 Mar 2026 17:24:46 +0000 Subject: [PATCH 2/7] fix compile using onnxruntime 1.24.x --- SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp index 2171a6ea8..822345f76 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -203,10 +203,7 @@ void print_model_input_output_info(const Ort::Session& session){ } Ort::Env create_ORT_env(){ - if (Ort::Global::api_ == nullptr){ - throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Onnx API returned a null pointer."); - } - + // Bit redundant now, but still might be useful to add logging or other init? return Ort::Env(); } From 84639559cbdd8e1342e3b1d9eac4426b03aba130 Mon Sep 17 00:00:00 2001 From: ConnorC432 Date: Fri, 13 Mar 2026 17:44:24 +0000 Subject: [PATCH 3/7] onnx gpu hw accel for linux --- .../ML/Models/ML_ONNXRuntimeHelpers.cpp | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp index 822345f76..ce1447b6b 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -86,6 +86,28 @@ if (use_gpu){ std::cout << "DirectML execution provider not available, falling back to CPU: " << e.what() << std::endl; } } +#elif defined(__linux__) + // Try CUDA first for NVIDIA GPUs (best performance) + // See: https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html + try { + OrtCUDAProviderOptions cuda_options{}; + cuda_options.device_id = 0; + so.AppendExecutionProvider_CUDA(cuda_options); + std::cout << "Using CUDA execution provider for GPU acceleration" << std::endl; + } catch (const Ort::Exception& e) { + std::cout << "CUDA execution provider not available, trying ROCm: " << e.what() << std::endl; + + // Try ROCm next for AMD GPUs + // See: https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html + try { + OrtROCMProviderOptions rocm_options{}; + rocm_options.device_id = 0; + so.AppendExecutionProvider_ROCM(rocm_options); + std::cout << "Using ROCm execution provider for GPU acceleration" << std::endl; + } catch (const Ort::Exception& e) { + std::cout << "ROCm execution provider not available, falling back to CPU: " << e.what() << std::endl; + } + } #endif } From 5c8fd80ea1cd661cf7335e77a2b0d1e186117f76 Mon Sep 17 00:00:00 2001 From: ConnorC432 Date: Fri, 20 Mar 2026 10:50:20 +0000 Subject: [PATCH 4/7] allow null pointer check for ONNX Runtime < 24 --- SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp index ce1447b6b..c0f144106 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -225,6 +225,11 @@ void print_model_input_output_info(const Ort::Session& session){ } Ort::Env create_ORT_env(){ +#if ORT_API_VERSION < 24 // Removed in ONNX Runtime 1.24.0 + if (Ort::Global::api_ == nullptr){ + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Onnx API returned a null pointer."); + } +#endif // Bit redundant now, but still might be useful to add logging or other init? return Ort::Env(); } From 7b7e13302221667af455d7ba995bcffc23c695c4 Mon Sep 17 00:00:00 2001 From: ConnorC432 Date: Fri, 20 Mar 2026 10:53:59 +0000 Subject: [PATCH 5/7] rocm on windows --- .../Source/ML/Models/ML_ONNXRuntimeHelpers.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp index c0f144106..eb71de110 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -75,10 +75,25 @@ if (use_gpu){ std::cout << "CUDA execution provider not available: " << e.what() << std::endl; } + if (!cuda_available) { + bool rocm_available = false; + // Try ROCm next for AMD GPUs + // See: https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html + try { + OrtROCMProviderOptions rocm_options{}; + rocm_options.device_id = 0; + so.AppendExecutionProvider_ROCM(rocm_options); + std::cout << "Using ROCm execution provider for GPU acceleration" << std::endl; + rocm_available = true; + } catch (const Ort::Exception& e) { + std::cout << "ROCm execution provider not available, falling back to CPU: " << e.what() << std::endl; + } + } + // Fallback to DirectML for all GPU vendors (NVIDIA, AMD, Intel) // DirectML is built into Windows 10 1903+ and requires no additional runtime installation // See: https://onnxruntime.ai/docs/execution-providers/DirectML-ExecutionProvider.html - if (!cuda_available){ + if (!cuda_available and !rocm_available){ try { so.AppendExecutionProvider("DML"); std::cout << "Using DirectML execution provider for GPU acceleration" << std::endl; From e7cb109d325fb687fa13c1399da975a9ed72d0e4 Mon Sep 17 00:00:00 2001 From: ConnorC432 Date: Fri, 20 Mar 2026 11:02:45 +0000 Subject: [PATCH 6/7] removed old comment --- SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp index eb71de110..371c61b23 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -245,7 +245,6 @@ Ort::Env create_ORT_env(){ throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Onnx API returned a null pointer."); } #endif - // Bit redundant now, but still might be useful to add logging or other init? return Ort::Env(); } From a05f708ed6026f591326d72c7399771797607773 Mon Sep 17 00:00:00 2001 From: ConnorC432 Date: Fri, 20 Mar 2026 14:47:45 +0000 Subject: [PATCH 7/7] fix --- SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp index 371c61b23..708d53971 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -75,8 +75,8 @@ if (use_gpu){ std::cout << "CUDA execution provider not available: " << e.what() << std::endl; } + bool rocm_available = false; if (!cuda_available) { - bool rocm_available = false; // Try ROCm next for AMD GPUs // See: https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html try {