From 8ee8f4c5e5c55f58b582f5700e00dff4f2db2b46 Mon Sep 17 00:00:00 2001 From: jw098 Date: Mon, 24 Nov 2025 23:14:21 -0800 Subject: [PATCH 1/9] Add option to use CPU instead of GPU for SAM embedder session. Fall back on CPU if GPU fails. --- .../DataLabeling/ML_SegmentAnythingModel.cpp | 39 +++++++++++++++---- .../ML/DataLabeling/ML_SegmentAnythingModel.h | 6 +-- .../ML/Models/ML_ONNXRuntimeHelpers.cpp | 7 ++-- .../Source/ML/Models/ML_ONNXRuntimeHelpers.h | 2 +- .../Source/ML/Models/ML_YOLOv5Model.cpp | 2 +- .../Source/ML/Programs/ML_LabelImages.cpp | 6 ++- .../Source/ML/Programs/ML_LabelImages.h | 3 ++ .../ML/Programs/ML_LabelImagesWidget.cpp | 6 +++ 8 files changed, 53 insertions(+), 18 deletions(-) diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp index 60b9a63dec..7065a0a8d2 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp @@ -24,8 +24,8 @@ namespace PokemonAutomation{ namespace ML{ -SAMEmbedderSession::SAMEmbedderSession(const std::string& model_path) - : m_session_options{create_session_options(ML_MODEL_CACHE_PATH() + "SAMEmbedder/")} +SAMEmbedderSession::SAMEmbedderSession(const std::string& model_path, bool use_gpu) + : m_session_options{create_session_options(ML_MODEL_CACHE_PATH() + "SAMEmbedder/", use_gpu)} , session{create_session(m_env, m_session_options, model_path, ML_MODEL_CACHE_PATH() + "SAMEmbedder/")} , memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)} , input_names{session.GetInputNames()} @@ -64,8 +64,8 @@ void SAMEmbedderSession::run(cv::Mat& input_image, std::vector& model_out } -SAMSession::SAMSession(const std::string& model_path) - : m_session_options{create_session_options(ML_MODEL_CACHE_PATH() + "SAM/")} +SAMSession::SAMSession(const std::string& model_path, bool use_gpu) + : m_session_options{create_session_options(ML_MODEL_CACHE_PATH() + "SAM/", use_gpu)} , session{create_session(m_env, m_session_options, model_path, ML_MODEL_CACHE_PATH() + "SAM/")} , memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)} , input_names{session.GetInputNames()} @@ -176,7 +176,7 @@ void SAMSession::run( } -void compute_embeddings_for_folder(const std::string& embedding_model_path, const std::string& image_folder_path){ +void compute_embeddings_for_folder(const std::string& embedding_model_path, const std::string& image_folder_path, bool use_gpu_for_embedder_session){ const bool recursive_search = true; std::vector all_image_paths = find_images_in_folder(image_folder_path, recursive_search); if (all_image_paths.size() == 0){ @@ -200,7 +200,8 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons return; } - SAMEmbedderSession embedding_session(embedding_model_path); + bool use_gpu = use_gpu_for_embedder_session; + std::unique_ptr embedding_session = make_unique(embedding_model_path, use_gpu); std::vector output_image_embedding; for (size_t i = 0; i < all_image_paths.size(); i++){ const auto& image_path = all_image_paths[i]; @@ -236,7 +237,31 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons cv::resize(image, resized_mat, cv::Size(SAM_EMBEDDER_INPUT_IMAGE_WIDTH, SAM_EMBEDDER_INPUT_IMAGE_HEIGHT)); output_image_embedding.clear(); - embedding_session.run(resized_mat, output_image_embedding); + while (true){ + try{ + embedding_session->run(resized_mat, output_image_embedding); + break; + }catch(Ort::Exception& e){ + if (use_gpu){ + std::cerr << "Error: Embedding session failed using the GPU. Fall back to the CPU.\n" << e.what() << std::endl; + use_gpu = false; + embedding_session = make_unique(embedding_model_path, use_gpu); + }else{ + std::cerr << "Error: Embedding session failed even when using the CPU.\n" << e.what() << std::endl; + QMessageBox box; + box.warning(nullptr, "Error:", + QString::fromStdString("Error: Embedding session failed.")); + return; + } + }catch(...){ + std::cerr << "Error: Unknown error." << std::endl; + QMessageBox box; + box.warning(nullptr, "Error:", + QString::fromStdString("Error: Unknown error. Embedding session failed.")); + return; + + } + } save_image_embedding_to_disk(image_path, output_image_embedding); } std::cout << "Done computing embeddings for images in folder " << image_folder_path << "." << std::endl; diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.h b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.h index e4f285740a..4dae45eaff 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.h +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.h @@ -23,12 +23,12 @@ namespace ML{ // Compute embeddings for all images in a folder. Only support .png, .jpg and .jpeg filename extensions so far. // This can be very slow! -void compute_embeddings_for_folder(const std::string& embedding_model_path, const std::string& image_folder_path); +void compute_embeddings_for_folder(const std::string& embedding_model_path, const std::string& image_folder_path, bool use_gpu_for_embedder_session); class SAMEmbedderSession{ public: - SAMEmbedderSession(const std::string& model_path); + SAMEmbedderSession(const std::string& model_path, bool use_gpu); // Given an image of shape SAM_EMBEDDER_INPUT_IMAGE_WIDTH x SAM_EMBEDDER_INPUT_IMAGE_HEIGHT, RGB channel order, // compute its image embedding as a vector of size [SAM_EMBEDDER_OUTPUT_SIZE] @@ -52,7 +52,7 @@ class SAMEmbedderSession{ // Run Segment Anything Model in an ONNX session. class SAMSession{ public: - SAMSession(const std::string& model_path); + SAMSession(const std::string& model_path, bool use_gpu); // embedding: input embedding // input_points: input point coordinates (x, y) in pixel units. [p0_x, p0_y, p1_x, p1_y, p2_x, ...]. diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp index 8d09788905..beaf244876 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -41,7 +41,7 @@ std::string create_file_hash(const std::string& filepath){ } -Ort::SessionOptions create_session_options(const std::string& model_cache_path){ +Ort::SessionOptions create_session_options(const std::string& model_cache_path, bool use_gpu){ Ort::SessionOptions so; std::cout << "Set potential model cache path in session options: " << model_cache_path << std::endl; #if __APPLE__ @@ -58,8 +58,7 @@ Ort::SessionOptions create_session_options(const std::string& model_cache_path){ so.AppendExecutionProvider("CoreML", provider_options); std::cout << "Using CoreML execution provider for GPU acceleration" << std::endl; #elif _WIN32 -#define ENABLE_WIN_GPU -#ifdef ENABLE_WIN_GPU +if (use_gpu){ // Try CUDA first for NVIDIA GPUs (best performance) // CUDA requires NVIDIA GPU and CUDA runtime installation // See: https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html @@ -85,7 +84,7 @@ Ort::SessionOptions create_session_options(const std::string& model_cache_path){ std::cout << "DirectML execution provider not available, falling back to CPU: " << e.what() << std::endl; } } -#endif // ENABLE_WIN_GPU +} #endif // CPU fallback is always available diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h index b2dac93d0f..22181257d6 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h @@ -23,7 +23,7 @@ namespace ML{ // // model_cache_path: the path to store model caches. This path is better // to be unique for each model for easier file management. -Ort::SessionOptions create_session_options(const std::string& model_cache_path); +Ort::SessionOptions create_session_options(const std::string& model_cache_path, bool use_gpu); // Create an ONNX Session. It will also update the model cache on macOS if necessary. diff --git a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp index 5cbaf11fd6..5594f5f2f7 100644 --- a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp +++ b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp @@ -60,7 +60,7 @@ std::tuple resize_image_with_border( YOLOv5Session::YOLOv5Session(const std::string& model_path, std::vector label_names) : m_label_names(std::move(label_names)) -, m_session_options(create_session_options(ML_MODEL_CACHE_PATH() + "YOLOv5")) +, m_session_options(create_session_options(ML_MODEL_CACHE_PATH() + "YOLOv5", true)) , m_session{create_session(m_env, m_session_options, model_path, ML_MODEL_CACHE_PATH() + "YOLOv5")} , m_memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)} , m_input_names{m_session.GetInputNames()} diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp index bf0c80b7ac..67b307c6cd 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp @@ -65,11 +65,13 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor) , CUSTOM_LABEL_DATABASE(create_string_select_database({"mc"})) // mc for "main character" , CUSTOM_SET_LABEL(CUSTOM_LABEL_DATABASE, LockMode::UNLOCK_WHILE_RUNNING, 0) , MANUAL_LABEL(false, LockMode::UNLOCK_WHILE_RUNNING, "", "Custom Label", true) + , USE_GPU_FOR_EMBEDDER_SESSION("Enable GPU for Embedder session:", LockMode::LOCK_WHILE_RUNNING, true) { ADD_OPTION(LABEL_TYPE); ADD_OPTION(FORM_LABEL); ADD_OPTION(CUSTOM_SET_LABEL); ADD_OPTION(MANUAL_LABEL); + ADD_OPTION(USE_GPU_FOR_EMBEDDER_SESSION); X.add_listener(*this); Y.add_listener(*this); @@ -85,7 +87,7 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor) // , m_sam_session{RESOURCE_PATH() + "ML/sam_cpu.onnx"} const std::string sam_model_path = RESOURCE_PATH() + "ML/sam_cpu.onnx"; if (std::filesystem::exists(sam_model_path)){ - m_sam_session = std::make_unique(sam_model_path); + m_sam_session = std::make_unique(sam_model_path, true); } else{ std::cerr << "Error: no such SAM model path " << sam_model_path << "." << std::endl; QMessageBox box; @@ -418,7 +420,7 @@ void LabelImages::remove_segmentation_exclusion_point(double x, double y){ void LabelImages::compute_embeddings_for_folder(const std::string& image_folder_path){ std::string embedding_model_path = RESOURCE_PATH() + "ML/sam_embedder_cpu.onnx"; std::cout << "Use SAM Embedding model " << embedding_model_path << std::endl; - ML::compute_embeddings_for_folder(embedding_model_path, image_folder_path); + ML::compute_embeddings_for_folder(embedding_model_path, image_folder_path, USE_GPU_FOR_EMBEDDER_SESSION); } void LabelImages::delete_selected_annotation(){ diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.h b/SerialPrograms/Source/ML/Programs/ML_LabelImages.h index 41318c4c8c..66cf644080 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.h +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.h @@ -13,6 +13,7 @@ #include "Common/Cpp/Options/EnumDropdownOption.h" #include "Common/Cpp/Options/EnumDropdownDatabase.h" #include "Common/Cpp/Options/StringOption.h" +#include "Common/Cpp/Options/BooleanCheckBoxOption.h" #include "CommonFramework/Panels/PanelInstance.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "Pokemon/Options/Pokemon_HomeSpriteSelectOption.h" @@ -158,6 +159,8 @@ class LabelImages : public PanelInstance, public ConfigOption::Listener { // source 3: editable text input StringCell MANUAL_LABEL; + BooleanCheckBoxOption USE_GPU_FOR_EMBEDDER_SESSION; + size_t source_image_height = 0; size_t source_image_width = 0; std::vector m_image_embedding; diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImagesWidget.cpp b/SerialPrograms/Source/ML/Programs/ML_LabelImagesWidget.cpp index b567289711..7ccb5c56e2 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImagesWidget.cpp +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImagesWidget.cpp @@ -125,6 +125,12 @@ LabelImages_Widget::LabelImages_Widget( annotation_row->addWidget(load_custom_set_button, 2); annotation_row->addWidget(new QLabel(scroll_inner), 10); // an empty label to push other UIs to the left + // add GPU checkbox row + QHBoxLayout* use_gpu_row = new QHBoxLayout(); + scroll_layout->addLayout(use_gpu_row); + ConfigWidget* gpu_checkbox_widget = program.USE_GPU_FOR_EMBEDDER_SESSION.make_QtWidget(*scroll_inner); + use_gpu_row->addWidget(&gpu_checkbox_widget->widget(), 2); + // add compute embedding button QHBoxLayout* external_action_row = new QHBoxLayout(); From 99805723ba811ee9a8a798027bbd0d036e8c4cc5 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 25 Nov 2025 12:40:24 -0800 Subject: [PATCH 2/9] add CPU fallback for SAM and YOLOv5 sessions. --- .../DataLabeling/ML_SegmentAnythingModel.cpp | 3 +- .../Source/ML/Inference/ML_YOLOv5Detector.cpp | 37 +++++++++-- .../Source/ML/Inference/ML_YOLOv5Detector.h | 2 + .../Source/ML/Programs/ML_LabelImages.cpp | 65 ++++++++++++++----- .../Source/ML/Programs/ML_LabelImages.h | 2 + 5 files changed, 88 insertions(+), 21 deletions(-) diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp index 7065a0a8d2..01b7ee808d 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp @@ -239,11 +239,12 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons output_image_embedding.clear(); while (true){ try{ + // If fails with GPU, fall back to CPU. embedding_session->run(resized_mat, output_image_embedding); break; }catch(Ort::Exception& e){ if (use_gpu){ - std::cerr << "Error: Embedding session failed using the GPU. Fall back to the CPU.\n" << e.what() << std::endl; + std::cerr << "Warning: Embedding session failed using the GPU. Will reattenpt with the CPU.\n" << e.what() << std::endl; use_gpu = false; embedding_session = make_unique(embedding_model_path, use_gpu); }else{ diff --git a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp index 8a74038c32..0a1139580c 100644 --- a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp @@ -31,6 +31,7 @@ YOLOv5Detector::~YOLOv5Detector() = default; YOLOv5Detector::YOLOv5Detector(const std::string& model_path) + : m_model_path(model_path) { if (!model_path.ends_with(".onnx")){ throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, @@ -44,7 +45,6 @@ YOLOv5Detector::YOLOv5Detector(const std::string& model_path) } std::string label_file_path = model_path.substr(0, model_path.size() - 5) + "_label.txt"; - std::vector labels; if (!std::filesystem::exists(label_file_path)){ throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Error: YOLOv5 label file path " + label_file_path + " does not exist."); @@ -72,11 +72,11 @@ YOLOv5Detector::YOLOv5Detector(const std::string& model_path) if (line.empty() || line[0] == '#'){ continue; } - labels.push_back(line); + m_labels.push_back(line); } label_file.close(); - m_yolo_session = std::make_unique(model_path, std::move(labels)); + m_yolo_session = std::make_unique(m_model_path, m_labels); } bool YOLOv5Detector::detect(const ImageViewRGB32& screen){ @@ -89,7 +89,36 @@ bool YOLOv5Detector::detect(const ImageViewRGB32& screen){ cv::cvtColor(frame_mat_bgra, frame_mat_rgb, cv::COLOR_BGRA2RGB); m_output_boxes.clear(); - m_yolo_session->run(frame_mat_rgb, m_output_boxes); + + // fall back to CPU if fails with GPU. + bool use_gpu = true; + for(size_t i = 0;;i++){ + try{ + // if (i >= 0){ throw Ort::Exception("Testing.", ORT_FAIL); } // to simulate GPU/CPU failure + // If fails with GPU, fall back to CPU. + m_yolo_session->run(frame_mat_rgb, m_output_boxes); + }catch(Ort::Exception& e){ + if (use_gpu){ + std::cerr << "Warning: YOLO session failed using the GPU. Will reattenpt with the CPU.\n" << e.what() << std::endl; + use_gpu = false; + m_yolo_session = std::make_unique(m_model_path, m_labels); + }else{ + std::cerr << "Error: YOLO session failed even when using the CPU.\n" << e.what() << std::endl; + QMessageBox box; + box.warning(nullptr, "Error:", + QString::fromStdString("Error: YOLO session failed.")); + return false; + } + }catch(...){ + std::cerr << "Error: Unknown error." << std::endl; + QMessageBox box; + box.warning(nullptr, "Error:", + QString::fromStdString("Error: Unknown error. YOLO session failed.")); + return false; + + } + } + return m_output_boxes.size() > 0; } diff --git a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h index 2be057b579..c324f1d86e 100644 --- a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h +++ b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h @@ -41,6 +41,8 @@ class YOLOv5Detector : public StaticScreenDetector{ const std::unique_ptr& session() const { return m_yolo_session; } protected: + std::string m_model_path; + std::vector m_labels; std::unique_ptr m_yolo_session; std::vector m_output_boxes; }; diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp index 67b307c6cd..5c19680a43 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp @@ -84,16 +84,7 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor) - // , m_sam_session{RESOURCE_PATH() + "ML/sam_cpu.onnx"} - const std::string sam_model_path = RESOURCE_PATH() + "ML/sam_cpu.onnx"; - if (std::filesystem::exists(sam_model_path)){ - m_sam_session = std::make_unique(sam_model_path, true); - } else{ - std::cerr << "Error: no such SAM model path " << sam_model_path << "." << std::endl; - QMessageBox box; - box.critical(nullptr, "SAM Model Does Not Exist", - QString::fromStdString("SAM model path" + sam_model_path + " does not exist.")); - } + init_sam_session(true); m_overlay_manager = new LabelImages_OverlayManager(*this); } @@ -136,6 +127,21 @@ JsonValue LabelImages::to_json() const{ return obj; } +void LabelImages::init_sam_session(bool use_gpu){ + // , m_sam_session{RESOURCE_PATH() + "ML/sam_cpu.onnx"} + + const std::string sam_model_path = RESOURCE_PATH() + "ML/sam_cpu.onnx"; + if (std::filesystem::exists(sam_model_path)){ + m_sam_session = std::make_unique(sam_model_path, use_gpu); + } else{ + std::cerr << "Error: no such SAM model path " << sam_model_path << "." << std::endl; + QMessageBox box; + box.critical(nullptr, "SAM Model Does Not Exist", + QString::fromStdString("SAM model path" + sam_model_path + " does not exist.")); + } + +} + void LabelImages::save_annotation_to_file() const{ if (m_annotation_file_path.size() == 0 || m_fail_to_load_annotation_file){ return; @@ -308,13 +314,40 @@ bool LabelImages::run_sam_to_create_annotation( input_point_labels[inclusion_points.size() + i] = 0; } - m_sam_session->run( - m_image_embedding, - (int)source_height, (int)source_width, input_points, input_point_labels, - {static_cast(user_box.min_x), static_cast(user_box.min_y), static_cast(user_box.max_x)-1, static_cast(user_box.max_y)-1}, - m_output_boolean_mask - ); + // fall back to CPU if fails with GPU. + bool use_gpu = true; + for(size_t i = 0;;i++){ + try{ + // if (i >= 0){ throw Ort::Exception("Testing.", ORT_FAIL); } // to simulate GPU/CPU failure + m_sam_session->run( + m_image_embedding, + (int)source_height, (int)source_width, input_points, input_point_labels, + {static_cast(user_box.min_x), static_cast(user_box.min_y), static_cast(user_box.max_x)-1, static_cast(user_box.max_y)-1}, + m_output_boolean_mask + ); + break; + }catch(Ort::Exception& e){ + if (use_gpu){ + std::cerr << "Warning: SAM session failed using the GPU. Will reattenpt with the CPU.\n" << e.what() << std::endl; + use_gpu = false; + init_sam_session(use_gpu); + }else{ + std::cerr << "Error: SAM session failed even when using the CPU.\n" << e.what() << std::endl; + QMessageBox box; + box.warning(nullptr, "Error:", + QString::fromStdString("Error: SAM session failed.")); + return false; + } + }catch(...){ + std::cerr << "Error: Unknown error." << std::endl; + QMessageBox box; + box.warning(nullptr, "Error:", + QString::fromStdString("Error: Unknown error. SAM session failed.")); + return false; + } + + } size_t min_mask_x = INT_MAX, max_mask_x = 0; size_t min_mask_y = INT_MAX, max_mask_y = 0; for (size_t y = 0; y < source_height; y++){ diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.h b/SerialPrograms/Source/ML/Programs/ML_LabelImages.h index 66cf644080..13cb2b1aa1 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.h +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.h @@ -58,6 +58,8 @@ class LabelImages : public PanelInstance, public ConfigOption::Listener { virtual void from_json(const JsonValue& json) override; virtual JsonValue to_json() const override; + void init_sam_session(bool use_gpu); + void save_annotation_to_file() const; // called after loading a new image, clean up all internal data From e8f8bde33bbb8ad5055f4ef170e24a7d6c7bbf87 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 25 Nov 2025 12:50:41 -0800 Subject: [PATCH 3/9] updates to YoloDetector --- .../Source/ML/Inference/ML_YOLOv5Detector.cpp | 19 ++++++++----------- .../Source/ML/Inference/ML_YOLOv5Detector.h | 2 +- .../Source/ML/Models/ML_YOLOv5Model.h | 2 ++ 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp index 0a1139580c..3ea37fc16e 100644 --- a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp @@ -45,6 +45,7 @@ YOLOv5Detector::YOLOv5Detector(const std::string& model_path) } std::string label_file_path = model_path.substr(0, model_path.size() - 5) + "_label.txt"; + std::vector labels; if (!std::filesystem::exists(label_file_path)){ throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Error: YOLOv5 label file path " + label_file_path + " does not exist."); @@ -72,11 +73,11 @@ YOLOv5Detector::YOLOv5Detector(const std::string& model_path) if (line.empty() || line[0] == '#'){ continue; } - m_labels.push_back(line); + labels.push_back(line); } label_file.close(); - m_yolo_session = std::make_unique(m_model_path, m_labels); + m_yolo_session = std::make_unique(m_model_path, std::move(labels)); } bool YOLOv5Detector::detect(const ImageViewRGB32& screen){ @@ -97,24 +98,20 @@ bool YOLOv5Detector::detect(const ImageViewRGB32& screen){ // if (i >= 0){ throw Ort::Exception("Testing.", ORT_FAIL); } // to simulate GPU/CPU failure // If fails with GPU, fall back to CPU. m_yolo_session->run(frame_mat_rgb, m_output_boxes); + break; }catch(Ort::Exception& e){ if (use_gpu){ std::cerr << "Warning: YOLO session failed using the GPU. Will reattenpt with the CPU.\n" << e.what() << std::endl; use_gpu = false; - m_yolo_session = std::make_unique(m_model_path, m_labels); + std::vector labels = m_yolo_session->get_label_names(); + m_yolo_session = std::make_unique(m_model_path, std::move(labels)); }else{ std::cerr << "Error: YOLO session failed even when using the CPU.\n" << e.what() << std::endl; - QMessageBox box; - box.warning(nullptr, "Error:", - QString::fromStdString("Error: YOLO session failed.")); - return false; + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Error: YOLO session failed."); } }catch(...){ std::cerr << "Error: Unknown error." << std::endl; - QMessageBox box; - box.warning(nullptr, "Error:", - QString::fromStdString("Error: Unknown error. YOLO session failed.")); - return false; + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown error: YOLO session failed."); } } diff --git a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h index c324f1d86e..c39cf7741c 100644 --- a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h +++ b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h @@ -42,7 +42,7 @@ class YOLOv5Detector : public StaticScreenDetector{ protected: std::string m_model_path; - std::vector m_labels; + // std::vector m_labels; std::unique_ptr m_yolo_session; std::vector m_output_boxes; }; diff --git a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h index d8a247cb2f..c34d57e3d4 100644 --- a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h +++ b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h @@ -33,6 +33,8 @@ class YOLOv5Session{ void run(const cv::Mat& input_image, std::vector& detections); const std::string& label_name(size_t idx) const { return m_label_names[idx]; } + + std::vector get_label_names() const { return m_label_names; } private: const int YOLO5_INPUT_IMAGE_SIZE = 640; From 921e447762aa631062def098e88a79583e1a7441 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 25 Nov 2025 13:34:51 -0800 Subject: [PATCH 4/9] LabelImages: allow user to select color of selected annotation. --- .../Source/ML/Programs/ML_LabelImages.cpp | 30 +++++++++++++++++++ .../Source/ML/Programs/ML_LabelImages.h | 13 ++++++++ .../Programs/ML_LabelImagesOverlayManager.cpp | 2 +- .../ML/Programs/ML_LabelImagesWidget.cpp | 6 ++++ 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp index 5c19680a43..fc29d58dce 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp @@ -66,12 +66,24 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor) , CUSTOM_SET_LABEL(CUSTOM_LABEL_DATABASE, LockMode::UNLOCK_WHILE_RUNNING, 0) , MANUAL_LABEL(false, LockMode::UNLOCK_WHILE_RUNNING, "", "Custom Label", true) , USE_GPU_FOR_EMBEDDER_SESSION("Enable GPU for Embedder session:", LockMode::LOCK_WHILE_RUNNING, true) + , SELECTED_ANNO_COLOR( + "Color of selected annotation:", + { + {ColorChoice::BLACK, "black", "Black"}, + {ColorChoice::GREEN, "green", "Green"}, + {ColorChoice::ORANGE, "orange", "Orange"}, + {ColorChoice::MAGENTA, "magenta", "Magenta"}, + }, + LockMode::LOCK_WHILE_RUNNING, + ColorChoice::BLACK + ) { ADD_OPTION(LABEL_TYPE); ADD_OPTION(FORM_LABEL); ADD_OPTION(CUSTOM_SET_LABEL); ADD_OPTION(MANUAL_LABEL); ADD_OPTION(USE_GPU_FOR_EMBEDDER_SESSION); + ADD_OPTION(SELECTED_ANNO_COLOR); X.add_listener(*this); Y.add_listener(*this); @@ -691,6 +703,24 @@ void LabelImages::export_to_yolov5_dataset(const std::string& image_folder_path, export_image_annotations_to_yolo_dataset(image_folder_path, image_folder_path, dataset_path); // image_folder_path, ML_ANNOTATION_PATH(), dataset_path); } +Color enum_to_color(ColorChoice color_choice){ + switch(color_choice){ + case ColorChoice::BLACK: + return COLOR_BLACK; + case ColorChoice::GREEN: + return COLOR_GREEN; + case ColorChoice::ORANGE: + return COLOR_ORANGE; + case ColorChoice::MAGENTA: + return COLOR_MAGENTA; + default: + QMessageBox box; + box.warning(nullptr, "Error:", + QString::fromStdString("Error: Unknown color selected.")); + return COLOR_BLACK; + + } +} } } diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.h b/SerialPrograms/Source/ML/Programs/ML_LabelImages.h index 13cb2b1aa1..ae65d97119 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.h +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.h @@ -14,6 +14,7 @@ #include "Common/Cpp/Options/EnumDropdownDatabase.h" #include "Common/Cpp/Options/StringOption.h" #include "Common/Cpp/Options/BooleanCheckBoxOption.h" +#include "Common/Cpp/Color.h" #include "CommonFramework/Panels/PanelInstance.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "Pokemon/Options/Pokemon_HomeSpriteSelectOption.h" @@ -40,6 +41,15 @@ class LabelImages_Widget; class LabelImages_OverlayManager; +enum class ColorChoice{ + BLACK, + GREEN, + ORANGE, + MAGENTA, +}; + +Color enum_to_color(ColorChoice color_choice); + class LabelImages_Descriptor : public PanelDescriptor{ public: LabelImages_Descriptor(); @@ -163,6 +173,9 @@ class LabelImages : public PanelInstance, public ConfigOption::Listener { BooleanCheckBoxOption USE_GPU_FOR_EMBEDDER_SESSION; + + EnumDropdownOption SELECTED_ANNO_COLOR; + size_t source_image_height = 0; size_t source_image_width = 0; std::vector m_image_embedding; diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImagesOverlayManager.cpp b/SerialPrograms/Source/ML/Programs/ML_LabelImagesOverlayManager.cpp index 1977f86e4d..258235e0c7 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImagesOverlayManager.cpp +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImagesOverlayManager.cpp @@ -105,7 +105,7 @@ void LabelImages_OverlayManager::update_rendered_annotations(){ if (form != nullptr){ label = form->display_name(); } - Color mask_box_color = (i_obj == m_selected) ? COLOR_BLACK : COLOR_BLUE; + Color mask_box_color = (i_obj == m_selected) ? enum_to_color(m_program.SELECTED_ANNO_COLOR) : COLOR_BLUE; m_overlay_set.add(mask_box_color, mask_float_box, label); size_t mask_width = obj.mask_box.width(); size_t mask_height = obj.mask_box.height(); diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImagesWidget.cpp b/SerialPrograms/Source/ML/Programs/ML_LabelImagesWidget.cpp index 7ccb5c56e2..fe91ec2747 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImagesWidget.cpp +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImagesWidget.cpp @@ -131,6 +131,12 @@ LabelImages_Widget::LabelImages_Widget( ConfigWidget* gpu_checkbox_widget = program.USE_GPU_FOR_EMBEDDER_SESSION.make_QtWidget(*scroll_inner); use_gpu_row->addWidget(&gpu_checkbox_widget->widget(), 2); + // add Color selection dropdown + QHBoxLayout* color_choice_row = new QHBoxLayout(); + scroll_layout->addLayout(color_choice_row); + ConfigWidget* color_choice_widget = program.SELECTED_ANNO_COLOR.make_QtWidget(*scroll_inner); + color_choice_row->addWidget(&color_choice_widget->widget(), 2); + // add compute embedding button QHBoxLayout* external_action_row = new QHBoxLayout(); From 37bc98a7f56f84e14ec2139627c96567b2ae5a22 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 25 Nov 2025 17:52:04 -0800 Subject: [PATCH 5/9] minor changes --- .../ML/DataLabeling/ML_SegmentAnythingModel.cpp | 2 +- .../Source/ML/Inference/ML_YOLOv5Detector.cpp | 10 +++++----- .../Source/ML/Inference/ML_YOLOv5Detector.h | 1 + SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp | 12 ++++++------ SerialPrograms/Source/ML/Programs/ML_LabelImages.h | 2 ++ 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp index 01b7ee808d..492555faa0 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp @@ -244,7 +244,7 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons break; }catch(Ort::Exception& e){ if (use_gpu){ - std::cerr << "Warning: Embedding session failed using the GPU. Will reattenpt with the CPU.\n" << e.what() << std::endl; + std::cerr << "Warning: Embedding session failed using the GPU. Will reattempt with the CPU.\n" << e.what() << std::endl; use_gpu = false; embedding_session = make_unique(embedding_model_path, use_gpu); }else{ diff --git a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp index 3ea37fc16e..69419b1e2e 100644 --- a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp @@ -32,6 +32,7 @@ YOLOv5Detector::~YOLOv5Detector() = default; YOLOv5Detector::YOLOv5Detector(const std::string& model_path) : m_model_path(model_path) + , m_use_gpu(true) { if (!model_path.ends_with(".onnx")){ throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, @@ -92,17 +93,16 @@ bool YOLOv5Detector::detect(const ImageViewRGB32& screen){ m_output_boxes.clear(); // fall back to CPU if fails with GPU. - bool use_gpu = true; for(size_t i = 0;;i++){ try{ - // if (i >= 0){ throw Ort::Exception("Testing.", ORT_FAIL); } // to simulate GPU/CPU failure + // if (m_use_gpu){ throw Ort::Exception("Testing.", ORT_FAIL); } // to simulate GPU/CPU failure // If fails with GPU, fall back to CPU. m_yolo_session->run(frame_mat_rgb, m_output_boxes); break; }catch(Ort::Exception& e){ - if (use_gpu){ - std::cerr << "Warning: YOLO session failed using the GPU. Will reattenpt with the CPU.\n" << e.what() << std::endl; - use_gpu = false; + if (m_use_gpu){ + std::cerr << "Warning: YOLO session failed using the GPU. Will reattempt with the CPU.\n" << e.what() << std::endl; + m_use_gpu = false; std::vector labels = m_yolo_session->get_label_names(); m_yolo_session = std::make_unique(m_model_path, std::move(labels)); }else{ diff --git a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h index c39cf7741c..51537633c1 100644 --- a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h +++ b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.h @@ -42,6 +42,7 @@ class YOLOv5Detector : public StaticScreenDetector{ protected: std::string m_model_path; + bool m_use_gpu; // std::vector m_labels; std::unique_ptr m_yolo_session; std::vector m_output_boxes; diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp index fc29d58dce..f5388263ac 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp @@ -55,6 +55,7 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor) : PanelInstance(descriptor) , m_display_session(m_display_option) , m_options(LockMode::UNLOCK_WHILE_RUNNING) + , m_use_gpu_for_sam_anno(true) , X("X Coordinate:", LockMode::UNLOCK_WHILE_RUNNING, 0.3, 0.0, 1.0) , Y("Y Coordinate:", LockMode::UNLOCK_WHILE_RUNNING, 0.3, 0.0, 1.0) , WIDTH("Width:", LockMode::UNLOCK_WHILE_RUNNING, 0.4, 0.0, 1.0) @@ -327,10 +328,9 @@ bool LabelImages::run_sam_to_create_annotation( } // fall back to CPU if fails with GPU. - bool use_gpu = true; for(size_t i = 0;;i++){ try{ - // if (i >= 0){ throw Ort::Exception("Testing.", ORT_FAIL); } // to simulate GPU/CPU failure + // if (m_use_gpu_for_sam_anno){ throw Ort::Exception("Testing.", ORT_FAIL); } // to simulate GPU/CPU failure m_sam_session->run( m_image_embedding, (int)source_height, (int)source_width, input_points, input_point_labels, @@ -339,10 +339,10 @@ bool LabelImages::run_sam_to_create_annotation( ); break; }catch(Ort::Exception& e){ - if (use_gpu){ - std::cerr << "Warning: SAM session failed using the GPU. Will reattenpt with the CPU.\n" << e.what() << std::endl; - use_gpu = false; - init_sam_session(use_gpu); + if (m_use_gpu_for_sam_anno){ + std::cerr << "Warning: SAM session failed using the GPU. Will reattempt with the CPU.\n" << e.what() << std::endl; + m_use_gpu_for_sam_anno = false; + init_sam_session(m_use_gpu_for_sam_anno); }else{ std::cerr << "Error: SAM session failed even when using the CPU.\n" << e.what() << std::endl; QMessageBox box; diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.h b/SerialPrograms/Source/ML/Programs/ML_LabelImages.h index ae65d97119..7407ac964f 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.h +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.h @@ -153,6 +153,8 @@ class LabelImages : public PanelInstance, public ConfigOption::Listener { // the group option that holds rest of the options defined below: BatchOption m_options; + bool m_use_gpu_for_sam_anno; + FloatingPointOption X; FloatingPointOption Y; FloatingPointOption WIDTH; From f2aed77c6ecb86ced86fda1b7fc8981ec076c560 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 25 Nov 2025 17:59:31 -0800 Subject: [PATCH 6/9] actually enable CPU fallback for YoloSession. --- SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp | 4 ++-- SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp | 4 ++-- SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp index 69419b1e2e..87a4e6e519 100644 --- a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp @@ -78,7 +78,7 @@ YOLOv5Detector::YOLOv5Detector(const std::string& model_path) } label_file.close(); - m_yolo_session = std::make_unique(m_model_path, std::move(labels)); + m_yolo_session = std::make_unique(m_model_path, std::move(labels), m_use_gpu); } bool YOLOv5Detector::detect(const ImageViewRGB32& screen){ @@ -104,7 +104,7 @@ bool YOLOv5Detector::detect(const ImageViewRGB32& screen){ std::cerr << "Warning: YOLO session failed using the GPU. Will reattempt with the CPU.\n" << e.what() << std::endl; m_use_gpu = false; std::vector labels = m_yolo_session->get_label_names(); - m_yolo_session = std::make_unique(m_model_path, std::move(labels)); + m_yolo_session = std::make_unique(m_model_path, std::move(labels), m_use_gpu); }else{ std::cerr << "Error: YOLO session failed even when using the CPU.\n" << e.what() << std::endl; throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Error: YOLO session failed."); diff --git a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp index 5594f5f2f7..3929704426 100644 --- a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp +++ b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp @@ -58,9 +58,9 @@ std::tuple resize_image_with_border( } -YOLOv5Session::YOLOv5Session(const std::string& model_path, std::vector label_names) +YOLOv5Session::YOLOv5Session(const std::string& model_path, std::vector label_names, bool use_gpu) : m_label_names(std::move(label_names)) -, m_session_options(create_session_options(ML_MODEL_CACHE_PATH() + "YOLOv5", true)) +, m_session_options(create_session_options(ML_MODEL_CACHE_PATH() + "YOLOv5", use_gpu)) , m_session{create_session(m_env, m_session_options, model_path, ML_MODEL_CACHE_PATH() + "YOLOv5")} , m_memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)} , m_input_names{m_session.GetInputNames()} diff --git a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h index c34d57e3d4..72cac8f538 100644 --- a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h +++ b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h @@ -28,7 +28,7 @@ class YOLOv5Session{ size_t label_idx; }; - YOLOv5Session(const std::string& model_path, std::vector label_names); + YOLOv5Session(const std::string& model_path, std::vector label_names, bool use_gpu); void run(const cv::Mat& input_image, std::vector& detections); From 47a901de6328294fcbb9d7d3377541ec8a454e5d Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 25 Nov 2025 21:28:13 -0800 Subject: [PATCH 7/9] Ensure no infinite loop when falling back from GPU to CPU. Ensure Apple's ML implementation can also fall back to CPU. --- .../ML/DataLabeling/ML_SegmentAnythingModel.cpp | 12 +++++++++++- .../Source/ML/Inference/ML_YOLOv5Detector.cpp | 6 +++++- .../Source/ML/Models/ML_ONNXRuntimeHelpers.cpp | 5 +++-- SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp | 10 +++++++++- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp index 492555faa0..c177a44905 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp @@ -237,7 +237,9 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons cv::resize(image, resized_mat, cv::Size(SAM_EMBEDDER_INPUT_IMAGE_WIDTH, SAM_EMBEDDER_INPUT_IMAGE_HEIGHT)); output_image_embedding.clear(); - while (true){ + + // fall back to CPU if fails with GPU. + for(size_t i = 0; i < 2; i++){ try{ // If fails with GPU, fall back to CPU. embedding_session->run(resized_mat, output_image_embedding); @@ -262,6 +264,14 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons return; } + + if (i > 0){ + std::cerr << "Internal Program Error: This section of code shouldn't be reachable." << std::endl; + QMessageBox box; + box.warning(nullptr, "Error:", + QString::fromStdString("Internal Program Error: This section of code shouldn't be reachable.")); + return false; + } } save_image_embedding_to_disk(image_path, output_image_embedding); } diff --git a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp index 87a4e6e519..5aac5b5555 100644 --- a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp @@ -93,7 +93,7 @@ bool YOLOv5Detector::detect(const ImageViewRGB32& screen){ m_output_boxes.clear(); // fall back to CPU if fails with GPU. - for(size_t i = 0;;i++){ + for(size_t i = 0; i < 2; i++){ try{ // if (m_use_gpu){ throw Ort::Exception("Testing.", ORT_FAIL); } // to simulate GPU/CPU failure // If fails with GPU, fall back to CPU. @@ -114,6 +114,10 @@ bool YOLOv5Detector::detect(const ImageViewRGB32& screen){ throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown error: YOLO session failed."); } + + if (i > 0){ + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Internal Program Error: This section of code shouldn't be reachable."); + } } diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp index beaf244876..32ab505e6b 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -44,6 +44,8 @@ std::string create_file_hash(const std::string& filepath){ Ort::SessionOptions create_session_options(const std::string& model_cache_path, bool use_gpu){ Ort::SessionOptions so; std::cout << "Set potential model cache path in session options: " << model_cache_path << std::endl; + +if (use_gpu){ #if __APPLE__ // create session using Apple ML acceleration library CoreML std::unordered_map provider_options; @@ -58,7 +60,6 @@ Ort::SessionOptions create_session_options(const std::string& model_cache_path, so.AppendExecutionProvider("CoreML", provider_options); std::cout << "Using CoreML execution provider for GPU acceleration" << std::endl; #elif _WIN32 -if (use_gpu){ // Try CUDA first for NVIDIA GPUs (best performance) // CUDA requires NVIDIA GPU and CUDA runtime installation // See: https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html @@ -84,8 +85,8 @@ if (use_gpu){ std::cout << "DirectML execution provider not available, falling back to CPU: " << e.what() << std::endl; } } -} #endif +} // CPU fallback is always available return so; diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp index f5388263ac..36aeeb1746 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp @@ -328,7 +328,7 @@ bool LabelImages::run_sam_to_create_annotation( } // fall back to CPU if fails with GPU. - for(size_t i = 0;;i++){ + for(size_t i = 0; i < 2; i++){ try{ // if (m_use_gpu_for_sam_anno){ throw Ort::Exception("Testing.", ORT_FAIL); } // to simulate GPU/CPU failure m_sam_session->run( @@ -358,6 +358,14 @@ bool LabelImages::run_sam_to_create_annotation( return false; } + + if (i > 0){ + std::cerr << "Internal Program Error: This section of code shouldn't be reachable." << std::endl; + QMessageBox box; + box.warning(nullptr, "Error:", + QString::fromStdString("Internal Program Error: This section of code shouldn't be reachable.")); + return false; + } } size_t min_mask_x = INT_MAX, max_mask_x = 0; From 1e33b96b008046d83635bc44d301ae293ad8db05 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 25 Nov 2025 21:30:08 -0800 Subject: [PATCH 8/9] fix build --- .../Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp index c177a44905..ea1e8b389e 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp @@ -239,7 +239,7 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons output_image_embedding.clear(); // fall back to CPU if fails with GPU. - for(size_t i = 0; i < 2; i++){ + for(size_t j = 0; j < 2; j++){ try{ // If fails with GPU, fall back to CPU. embedding_session->run(resized_mat, output_image_embedding); @@ -265,12 +265,12 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons } - if (i > 0){ + if (j > 0){ std::cerr << "Internal Program Error: This section of code shouldn't be reachable." << std::endl; QMessageBox box; box.warning(nullptr, "Error:", QString::fromStdString("Internal Program Error: This section of code shouldn't be reachable.")); - return false; + return; } } save_image_embedding_to_disk(image_path, output_image_embedding); From 3398446e25b16264d00b3a1237018a8468650921 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 25 Nov 2025 21:32:06 -0800 Subject: [PATCH 9/9] minor --- .../Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp | 3 ++- SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp index ea1e8b389e..13cf30ba93 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp @@ -242,6 +242,7 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons for(size_t j = 0; j < 2; j++){ try{ // If fails with GPU, fall back to CPU. + // throw Ort::Exception("Testing.", ORT_FAIL); // to simulate GPU/CPU failure embedding_session->run(resized_mat, output_image_embedding); break; }catch(Ort::Exception& e){ @@ -257,7 +258,7 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons return; } }catch(...){ - std::cerr << "Error: Unknown error." << std::endl; + std::cerr << "Error: Unknown error. Embedding session failed." << std::endl; QMessageBox box; box.warning(nullptr, "Error:", QString::fromStdString("Error: Unknown error. Embedding session failed.")); diff --git a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp index 5aac5b5555..9f7ee55db4 100644 --- a/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_YOLOv5Detector.cpp @@ -106,11 +106,9 @@ bool YOLOv5Detector::detect(const ImageViewRGB32& screen){ std::vector labels = m_yolo_session->get_label_names(); m_yolo_session = std::make_unique(m_model_path, std::move(labels), m_use_gpu); }else{ - std::cerr << "Error: YOLO session failed even when using the CPU.\n" << e.what() << std::endl; - throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Error: YOLO session failed."); + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Error: YOLO session failed even when using the CPU." + std::string(e.what())); } }catch(...){ - std::cerr << "Error: Unknown error." << std::endl; throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Unknown error: YOLO session failed."); }