From a33eb990af1f59b584c7f2585ef58b4c5a7f2415 Mon Sep 17 00:00:00 2001 From: Gin <> Date: Sun, 10 Aug 2025 11:38:39 -0700 Subject: [PATCH 1/4] add YOLO model --- SerialPrograms/CMakeLists.txt | 6 + .../ImageTypes/ImageViewRGB32.h | 1 + .../DataLabeling/ML_SegmentAnythingModel.cpp | 19 +- SerialPrograms/Source/ML/ML_Panels.cpp | 5 +- .../ML/Models/ML_ONNXRuntimeHelpers.cpp | 64 +++++++ .../Source/ML/Models/ML_ONNXRuntimeHelpers.h | 49 +++++ .../Source/ML/Models/ML_YOLOv5Model.cpp | 181 ++++++++++++++++++ .../Source/ML/Models/ML_YOLOv5Model.h | 56 ++++++ .../Source/ML/Programs/ML_LabelImages.cpp | 4 +- .../Source/ML/Programs/ML_RunYOLO.cpp | 79 ++++++++ .../Source/ML/Programs/ML_RunYOLO.h | 43 +++++ 11 files changed, 487 insertions(+), 20 deletions(-) create mode 100644 SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp create mode 100644 SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h create mode 100644 SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp create mode 100644 SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h create mode 100644 SerialPrograms/Source/ML/Programs/ML_RunYOLO.cpp create mode 100644 SerialPrograms/Source/ML/Programs/ML_RunYOLO.h diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 134128b8a8..4c14c70f7d 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -903,12 +903,18 @@ file(GLOB MAIN_SOURCES Source/ML/DataLabeling/ML_SegmentAnythingModelConstants.h Source/ML/ML_Panels.cpp Source/ML/ML_Panels.h + Source/ML/Models/ML_ONNXRuntimeHelpers.cpp + Source/ML/Models/ML_ONNXRuntimeHelpers.h + Source/ML/Models/ML_YOLOv5Model.cpp + Source/ML/Models/ML_YOLOv5Model.h Source/ML/Programs/ML_LabelImages.cpp Source/ML/Programs/ML_LabelImages.h Source/ML/Programs/ML_LabelImagesOverlayManager.cpp Source/ML/Programs/ML_LabelImagesOverlayManager.h Source/ML/Programs/ML_LabelImagesWidget.cpp Source/ML/Programs/ML_LabelImagesWidget.h + Source/ML/Programs/ML_RunYOLO.cpp + Source/ML/Programs/ML_RunYOLO.h Source/ML/UI/ML_ImageAnnotationCommandRow.cpp Source/ML/UI/ML_ImageAnnotationCommandRow.h Source/ML/UI/ML_ImageAnnotationDisplayOption.cpp diff --git a/SerialPrograms/Source/CommonFramework/ImageTypes/ImageViewRGB32.h b/SerialPrograms/Source/CommonFramework/ImageTypes/ImageViewRGB32.h index e9da7a2611..5a9995b288 100644 --- a/SerialPrograms/Source/CommonFramework/ImageTypes/ImageViewRGB32.h +++ b/SerialPrograms/Source/CommonFramework/ImageTypes/ImageViewRGB32.h @@ -57,6 +57,7 @@ class ImageViewRGB32 : public ImageViewPlanar32{ QImage to_QImage_owning() const; // Return a copy that owns its own buffer. (slow) QImage scaled_to_QImage(size_t width, size_t height) const; + // convert to cv::Mat with BGRA color channel order cv::Mat to_opencv_Mat() const; private: diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp index 5130600d26..2532624f91 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp @@ -14,6 +14,7 @@ #include #include #include "3rdParty/ONNX/OnnxToolsPA.h" +#include "ML/Models/ML_ONNXRuntimeHelpers.h" #include "ML_SegmentAnythingModelConstants.h" #include "ML_SegmentAnythingModel.h" #include "ML_AnnotationIO.h" @@ -21,24 +22,6 @@ namespace PokemonAutomation{ namespace ML{ -Ort::SessionOptions create_session_option(){ - return Ort::SessionOptions{}; - - // create session using Apple ML - - // Ort::SessionOptions so; - // std::unordered_map provider_options; - // provider_options["ModelFormat"] = "NeuralNetwork"; - // so.AppendExecutionProvider("CoreML", provider_options); - // return so; -} - - -template Ort::Value create_tensor(const OrtMemoryInfo* memory_info, Buffer& buffer, const Shape& shape){ - return Ort::Value::CreateTensor(memory_info, buffer.data(), buffer.size(), - shape.data(), shape.size()); -} - SAMEmbedderSession::SAMEmbedderSession(const std::string& model_path) : session_options(create_session_option()) diff --git a/SerialPrograms/Source/ML/ML_Panels.cpp b/SerialPrograms/Source/ML/ML_Panels.cpp index 3128d36502..d67a721587 100644 --- a/SerialPrograms/Source/ML/ML_Panels.cpp +++ b/SerialPrograms/Source/ML/ML_Panels.cpp @@ -8,7 +8,8 @@ #include "CommonFramework/Panels/PanelTools.h" #include "Pokemon/Pokemon_Strings.h" #include "Programs/ML_LabelImages.h" - +#include "Programs/ML_RunYOLO.h" +#include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" #include "ML_Panels.h" @@ -26,6 +27,8 @@ std::vector PanelListFactory::make_panels() const{ if (PreloadSettings::instance().DEVELOPER_MODE){ ret.emplace_back("---- Developer Tools ----"); ret.emplace_back(make_panel()); + // ret.emplace_back(make_panel()); + ret.emplace_back(NintendoSwitch::make_single_switch_program()); // ret.emplace_back(make_single_switch_program()); } diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp new file mode 100644 index 0000000000..f11b328fde --- /dev/null +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -0,0 +1,64 @@ +/* ML ONNX Runtime Helpers + * + * From: https://github.com/PokemonAutomation/ + * + * Helper functions to work with ONNX Runtime library + */ + +#include +#include +#include "ML_ONNXRuntimeHelpers.h" + +namespace PokemonAutomation{ +namespace ML{ + +Ort::SessionOptions create_session_option(){ + return Ort::SessionOptions{}; + + // create session using Apple ML + + // Ort::SessionOptions so; + // std::unordered_map provider_options; + // provider_options["ModelFormat"] = "NeuralNetwork"; + // so.AppendExecutionProvider("CoreML", provider_options); + // return so; +} + + +void print_model_input_output_info(const Ort::Session& session){ + const auto input_names = session.GetInputNames(); + const auto output_names = session.GetOutputNames(); + + for (size_t i = 0; i < input_names.size(); ++i) { + Ort::TypeInfo type_info = session.GetInputTypeInfo(i); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + std::vector input_dims = tensor_info.GetShape(); + + std::cout << "Input " << i << ": " << input_names[i] << " Type " << tensor_info.GetElementType() << " Shape: ["; + for (size_t j = 0; j < input_dims.size(); ++j) { + std::cout << input_dims[j]; + if (j < input_dims.size() - 1) { + std::cout << ", "; + } + } + std::cout << "]" << std::endl; + } + + for (size_t i = 0; i < output_names.size(); ++i) { + Ort::TypeInfo type_info = session.GetOutputTypeInfo(i); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + std::vector output_dims = tensor_info.GetShape(); + + std::cout << "Output " << i << ": " << input_names[i] << " Type " << tensor_info.GetElementType() << " Shape: ["; + for (size_t j = 0; j < output_dims.size(); ++j) { + std::cout << output_dims[j]; + if (j < output_dims.size() - 1) { + std::cout << ", "; + } + } + std::cout << "]" << std::endl; + } +} + +} +} \ No newline at end of file diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h new file mode 100644 index 0000000000..dc6556789c --- /dev/null +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h @@ -0,0 +1,49 @@ +/* ML ONNX Runtime Helpers + * + * From: https://github.com/PokemonAutomation/ + * + * Helper functions to work with ONNX Runtime library + */ + +#ifndef PokemonAutomation_ML_ONNXRuntimeHelpers_H +#define PokemonAutomation_ML_ONNXRuntimeHelpers_H + + +#include +#include +#include + +namespace PokemonAutomation{ +namespace ML{ + + +// Create an ONNX Runtiem session options object. +// For now it only sets the session to be CPU. In future we can create options for GPU or macOS MPS +Ort::SessionOptions create_session_option(); + +// Handy function to create an ONNX Runtime tensor view class from a vector-like `buffer` object holding +// the tensor data and an array-like `shape` object that represents the dimension of the tensor. +template +Ort::Value create_tensor(const OrtMemoryInfo* memory_info, Buffer& buffer, const Shape& shape){ + return Ort::Value::CreateTensor(memory_info, buffer.data(), buffer.size(), + shape.data(), shape.size()); +} + + +// Print vector as std::string. Useful for printing debugging info on tensor shapes +template +std::string to_string(std::vector& vec){ + std::ostringstream os; + os << "["; + std::copy(vec.begin(), vec.end(), std::ostream_iterator(os, ", ")); + os << "]"; + return os.str(); +} + +// Print model input and output types and shapes to cout. Useful for debugging. +void print_model_input_output_info(const Ort::Session& session); + + +} +} +#endif \ No newline at end of file diff --git a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp new file mode 100644 index 0000000000..6547e98acd --- /dev/null +++ b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp @@ -0,0 +1,181 @@ +/* ML YOLOv5 Model + * + * From: https://github.com/PokemonAutomation/ + * + * Run YOLOv5 model for object detection + */ + + +#include +#include +#include +#include +#include "ML/Models/ML_ONNXRuntimeHelpers.h" +#include "ML_YOLOv5Model.h" + +namespace PokemonAutomation{ +namespace ML{ + +std::tuple resize_image_with_border( + const cv::Mat& input_image, + cv::Mat& output_image, + int target_width, int target_height, + cv::Scalar border_color = cv::Scalar(0,0,0) +){ + int original_width = input_image.cols; + int original_height = input_image.rows; + + double scale_x = static_cast(target_width) / original_width; + double scale_y = static_cast(target_height) / original_height; + double scale = std::min(scale_x, scale_y); + + int new_width = static_cast(original_width * scale); + int new_height = static_cast(original_height * scale); + new_width = std::min(new_width, target_width); + new_height = std::min(new_height, target_height); + + if (new_width == 0 || new_height == 0){ + throw std::runtime_error("Input Image too small: " + std::to_string(original_width) + " x " + std::to_string(original_height)); + } + + cv::Mat resized_image; + cv::resize(input_image, resized_image, cv::Size(new_width, new_height), 0, 0, cv::INTER_LINEAR); // INTER_AREA for shrinking + + int border_top = (target_height - new_height) / 2; + int border_bottom = target_height - new_height - border_top; + int border_left = (target_width - new_width) / 2; + int border_right = target_width - new_width - border_left; + + cv::copyMakeBorder(resized_image, output_image, border_top, border_bottom, border_left, border_right, cv::BORDER_CONSTANT, border_color); + + return std::make_tuple( + border_left, border_top, + 1.0 / new_width, 1.0 / new_height + ); +} + + +YOLOv5Session::YOLOv5Session(const std::string& model_path, std::vector label_names) +: m_label_names(std::move(label_names)) +, m_session_options(create_session_option()) +, m_session{m_env, model_path.c_str(), m_session_options} +, m_memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)} +, m_input_names{m_session.GetInputNames()} +, m_output_names{m_session.GetOutputNames()} +, m_model_input(3*YOLO5_INPUT_IMAGE_SIZE*YOLO5_INPUT_IMAGE_SIZE) +{ + if (m_session.GetOutputCount() != 1){ + throw std::runtime_error("YOLOv5 model does not have the correct output count, found count " + std::to_string(m_session.GetOutputCount())); + } + + std::vector output_dims = m_session.GetOutputTypeInfo(0).GetTensorTypeAndShapeInfo().GetShape(); + if (output_dims.size() != 3 || output_dims[2] <= 5){ + throw std::runtime_error("YOLOv5 model does not have the correct output dimension, found shape " + to_string(output_dims)); + } + m_output_shape[2] = output_dims[2]; + if (output_dims[2] - 5 != static_cast(m_label_names.size())){ + throw std::runtime_error( + "YOLOv5 model has " + std::to_string(output_dims[2]-5) + + " output labels but YOLOv5Session was initialized with " + std::to_string(label_names.size()) + " labels" + ); + } + m_model_output.resize(YOLO5_NUM_CANDIDATES * m_output_shape[2]); +} + +// input: rgb color order +void YOLOv5Session::run(const cv::Mat& input_image, std::vector& output_boxes){ + CV_Assert(input_image.depth() == CV_8U); + CV_Assert(input_image.channels() == 3); + + cv::Mat image_resized; + + int x_shift = 0, y_shift = 0; + double x_scale = 1.0, y_scale = 1.0; + std::tie(x_shift, y_shift, x_scale, y_scale) = resize_image_with_border(input_image, image_resized, + YOLO5_INPUT_IMAGE_SIZE, YOLO5_INPUT_IMAGE_SIZE, cv::Scalar(114, 114, 114)); + + // Declare a destination Mat for float32 + cv::Mat image_float; + + // Convert the uint8_image to image_float + // The third argument (alpha) is a scaling factor. + // For normalization to [0.0, 1.0], use 1.0 / 255.0. + // For retaining original values (0-255), use 1.0. + image_resized.convertTo(image_float, CV_32F, 1.0 / 255.0); + + for (int c = 0, i = 0; c < 3; c++) { + for (int row = 0; row < image_float.rows; row++) { + for (int col = 0; col < image_float.cols; col++) { + float pixel_value = image_float.at(row, col)[c]; + m_model_input[i++] = pixel_value; + } + } + } + + auto input_tensor = create_tensor(m_memory_info, m_model_input, m_input_shape); + auto output_tensor = create_tensor(m_memory_info, m_model_output, m_output_shape); + + const char* input_name_c = m_input_names[0].data(); + const char* output_name_c = m_output_names[0].data(); + // auto start = std::chrono::steady_clock::now(); + m_session.Run(m_run_options, &input_name_c, &input_tensor, 1, &output_name_c, &output_tensor, 1); + // auto end = std::chrono::steady_clock::now(); + // auto milliseconds = std::chrono::duration_cast(end - start).count(); + // std::cout << "Yolov5 inference time: " << milliseconds << " ms" << std::endl; + + const size_t cand_size = m_label_names.size() + 5; + + std::vector pixel_boxes; + std::vector indices; + std::vector scores; + std::vector labels; + + for(int i = 0; i < YOLO5_NUM_CANDIDATES; i++){ + float cx = m_model_output[cand_size*i]; + float cy = m_model_output[cand_size*i+1]; + float w = m_model_output[cand_size*i+2]; + float h = m_model_output[cand_size*i+3]; + float sc = m_model_output[cand_size*i+4]; + + float max_score = 0.0; + size_t pred_label = 0; // predicted label + for(size_t j_label = 0; j_label < m_label_names.size(); j_label++){ + float score = m_model_output[cand_size*i+5+j_label]; + if (score > max_score){ + max_score = score; + pred_label = j_label; + } + } + scores.push_back(max_score * sc); // sc is like a global confidence scale? + pixel_boxes.emplace_back((int)(cx - w / 2 + 0.5), (int)(cy - h / 2 + 0.5), int(w + 0.5), int(h + 0.5)); + indices.push_back(i); + labels.push_back(pred_label); + } + + cv::dnn::NMSBoxes(pixel_boxes, scores, 0.2f, 0.45f, indices); + + // std::cout << "num found pixel_boxes " << indices.size() << std::endl; + // return; + + for (int index : indices) + { + // Note the model predicts on (640x640) images, we need to convert the detected pixel_boxes back to + // the full frame dimension. + double x = (pixel_boxes[index].x - x_shift) * x_scale; + double y = (pixel_boxes[index].y - y_shift) * y_scale; + double w = pixel_boxes[index].width * x_scale; + double h = pixel_boxes[index].height * y_scale; + // std::cout << scores[index] << " " << x << " " << y << " " << w << " " << h << std::endl; + + YOLOv5Session::DetectionBox b; + b.box = ImageFloatBox(x, y, w, h); + b.score = scores[index]; + b.label_idx = labels[index]; + output_boxes.push_back(b); + } +} + + + +} +} \ No newline at end of file diff --git a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h new file mode 100644 index 0000000000..b01e93a789 --- /dev/null +++ b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.h @@ -0,0 +1,56 @@ +/* ML YOLOv5 Model + * + * From: https://github.com/PokemonAutomation/ + * + * Run YOLOv5 model for object detection + */ + +#ifndef PokemonAutomation_ML_YOLOv5Model_H +#define PokemonAutomation_ML_YOLOv5Model_H + + +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" + +namespace PokemonAutomation{ +namespace ML{ + + +class YOLOv5Session{ +public: + struct DetectionBox{ + double score; + ImageFloatBox box; + size_t label_idx; + }; + + YOLOv5Session(const std::string& model_path, std::vector label_names); + + void run(const cv::Mat& input_image, std::vector& detections); + + const std::string& label_name(size_t idx) const { return m_label_names[idx]; } + +private: + const int YOLO5_INPUT_IMAGE_SIZE = 640; + const int YOLO5_NUM_CANDIDATES = 25200; + + std::vector m_label_names; + + Ort::Env m_env; + Ort::SessionOptions m_session_options; + Ort::Session m_session; + Ort::MemoryInfo m_memory_info; + Ort::RunOptions m_run_options; + std::vector m_input_names, m_output_names; + + const std::array m_input_shape{1, 3, YOLO5_INPUT_IMAGE_SIZE, YOLO5_INPUT_IMAGE_SIZE}; + std::array m_output_shape{1, YOLO5_NUM_CANDIDATES, 0}; + + std::vector m_model_input; + std::vector m_model_output; +}; + + +} +} +#endif diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp index 4a49154d21..a906c405bf 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp @@ -70,7 +70,7 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor) ADD_OPTION(FORM_LABEL); ADD_OPTION(CUSTOM_SET_LABEL); ADD_OPTION(MANUAL_LABEL); - + X.add_listener(*this); Y.add_listener(*this); WIDTH.add_listener(*this); @@ -80,6 +80,8 @@ LabelImages::LabelImages(const LabelImages_Descriptor& descriptor) CUSTOM_SET_LABEL.add_listener(*this); MANUAL_LABEL.add_listener(*this); + + // , m_sam_session{RESOURCE_PATH() + "ML/sam_cpu.onnx"} const std::string sam_model_path = RESOURCE_PATH() + "ML/sam_cpu.onnx"; if (std::filesystem::exists(sam_model_path)){ diff --git a/SerialPrograms/Source/ML/Programs/ML_RunYOLO.cpp b/SerialPrograms/Source/ML/Programs/ML_RunYOLO.cpp new file mode 100644 index 0000000000..9c8a824c8d --- /dev/null +++ b/SerialPrograms/Source/ML/Programs/ML_RunYOLO.cpp @@ -0,0 +1,79 @@ +/* ML Run YOLO Program + * + * From: https://github.com/PokemonAutomation/ + * + * Test YOLO detection model on Switch streams. + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/Globals.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "ML_RunYOLO.h" + +namespace PokemonAutomation{ +namespace ML{ + + +RunYOLO_Descriptor::RunYOLO_Descriptor() + : SingleSwitchProgramDescriptor( + "NintendoSwitch:RunYOLO", + "Nintendo Switch", "Run YOLO", + "ComputerControl/blob/master/Wiki/Programs/NintendoSwitch/RunYOLO.md", + "Run YOLO object detection model.", + FeedbackType::NONE, + AllowCommandsWhenRunning::ENABLE_COMMANDS, + {} + ) +{} + + + +RunYOLO::RunYOLO() +{ + const std::string sam_model_path = RESOURCE_PATH() + "ML/yolov5_cpu.onnx"; + std::vector labels = {"Bidoof"}; + if (std::filesystem::exists(sam_model_path)){ + m_yolo_session = std::make_unique(sam_model_path, labels); + } else{ + std::cerr << "Error: no such YOLOv5 model path " << sam_model_path << "." << std::endl; + QMessageBox box; + box.critical(nullptr, "YOLOv5 Model Does Not Exist", + QString::fromStdString("YOLOv5 model path" + sam_model_path + " does not exist.")); + } + +} + +void RunYOLO::program(NintendoSwitch::SingleSwitchProgramEnvironment& env, NintendoSwitch::ProControllerContext& context){ + if (!m_yolo_session){ + return; + } + + VideoOverlaySet overlay_set(env.console.overlay()); + + std::vector output_boxes; + while (true){ + VideoSnapshot last = env.console.video().snapshot(); + cv::Mat frame_mat_bgra = last.frame->to_opencv_Mat(); + cv::Mat frame_mat_rgb; + cv::cvtColor(frame_mat_bgra, frame_mat_rgb, cv::COLOR_BGRA2RGB); + + output_boxes.clear(); + m_yolo_session->run(frame_mat_rgb, output_boxes); + overlay_set.clear(); + for(const auto& box : output_boxes){ + overlay_set.add(COLOR_RED, box.box, m_yolo_session->label_name(box.label_idx)); + } + // context.wait_until(last.timestamp + std::chrono::milliseconds(PERIOD_MILLISECONDS)); + } +} + + +} +} + diff --git a/SerialPrograms/Source/ML/Programs/ML_RunYOLO.h b/SerialPrograms/Source/ML/Programs/ML_RunYOLO.h new file mode 100644 index 0000000000..a547fdaf97 --- /dev/null +++ b/SerialPrograms/Source/ML/Programs/ML_RunYOLO.h @@ -0,0 +1,43 @@ +/* ML Run YOLO Program + * + * From: https://github.com/PokemonAutomation/ + * + * Test YOLO detection model on Switch streams. + */ + +#ifndef PokemonAutomation_ML_RunYOLO_H +#define PokemonAutomation_ML_RunYOLO_H + +#include +#include "Common/Cpp/Options/SimpleIntegerOption.h" +#include "Common/Cpp/Options/EnumDropdownOption.h" +#include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" +#include "ML/Models/ML_YOLOv5Model.h" + +namespace PokemonAutomation{ +namespace ML{ + + +class RunYOLO_Descriptor : public NintendoSwitch::SingleSwitchProgramDescriptor{ +public: + RunYOLO_Descriptor(); +}; + + +class RunYOLO : public NintendoSwitch::SingleSwitchProgramInstance{ +public: + RunYOLO(); + + virtual void program(NintendoSwitch::SingleSwitchProgramEnvironment& env, NintendoSwitch::ProControllerContext& context) override; + +private: + std::unique_ptr m_yolo_session; +}; + + +} +} +#endif + + + From 694ea914109b10e3fc5eeaa8a25d0d2da6d92057 Mon Sep 17 00:00:00 2001 From: Alexander Yee Date: Sun, 10 Aug 2025 21:06:05 -0700 Subject: [PATCH 2/4] Fix build. --- SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp index 6547e98acd..3a7fd90d11 100644 --- a/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp +++ b/SerialPrograms/Source/ML/Models/ML_YOLOv5Model.cpp @@ -7,9 +7,10 @@ #include -#include +//#include #include #include +#include "3rdParty/ONNX/OnnxToolsPA.h" #include "ML/Models/ML_ONNXRuntimeHelpers.h" #include "ML_YOLOv5Model.h" @@ -58,7 +59,7 @@ std::tuple resize_image_with_border( YOLOv5Session::YOLOv5Session(const std::string& model_path, std::vector label_names) : m_label_names(std::move(label_names)) , m_session_options(create_session_option()) -, m_session{m_env, model_path.c_str(), m_session_options} +, m_session{m_env, str_to_onnx_str(model_path).c_str(), m_session_options} , m_memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)} , m_input_names{m_session.GetInputNames()} , m_output_names{m_session.GetOutputNames()} @@ -178,4 +179,4 @@ void YOLOv5Session::run(const cv::Mat& input_image, std::vector Date: Mon, 11 Aug 2025 02:29:52 -0700 Subject: [PATCH 3/4] Minor refactor for firmware. --- .../NintendoSwitch2_WiredController_State.c | 26 ++++++++ .../NintendoSwitch2_WiredController_State.h | 41 ++++++++++++ .../NintendoSwitch_WirelessController_State.c | 30 +++++++++ .../NintendoSwitch_WirelessController_State.h | 64 +++++++++++++++++++ .../Concurrency/ComputationThreadPoolCore.h | 2 +- .../SerialPABotBase_Messages_ESP32.h | 30 +-------- ...alPABotBase_Messages_NS2_WiredController.h | 12 +--- SerialPrograms/CMakeLists.txt | 2 + .../SerialPABotBase_Routines_ESP32.h | 13 ++-- ...tch_SerialPABotBase_WirelessController.cpp | 12 ++-- ...witch_SerialPABotBase_WirelessController.h | 10 +-- ...oSwitch_SerialPABotBase_WirelessJoycon.cpp | 8 +-- ..._SerialPABotBase_WirelessProController.cpp | 4 +- 13 files changed, 194 insertions(+), 60 deletions(-) create mode 100644 Common/Controllers/NintendoSwitch2_WiredController_State.c create mode 100644 Common/Controllers/NintendoSwitch2_WiredController_State.h create mode 100644 Common/Controllers/NintendoSwitch_WirelessController_State.c create mode 100644 Common/Controllers/NintendoSwitch_WirelessController_State.h diff --git a/Common/Controllers/NintendoSwitch2_WiredController_State.c b/Common/Controllers/NintendoSwitch2_WiredController_State.c new file mode 100644 index 0000000000..61fb3bf5c9 --- /dev/null +++ b/Common/Controllers/NintendoSwitch2_WiredController_State.c @@ -0,0 +1,26 @@ +/* Nintendo Switch 2 - Wired Controller State + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "NintendoSwitch2_WiredController_State.h" + + +const pa_NintendoSwitch2_WiredController_State pa_NintendoSwitch2_WiredController_State_NEUTRAL_STATE = { + .buttons0 = 0, + .buttons1 = 0, + .dpad_byte = 8, + .left_joystick_x = 128, + .left_joystick_y = 128, + .right_joystick_x = 128, + .right_joystick_y = 128, +}; + +bool pa_NintendoSwitch2_WiredController_State_equals( + const pa_NintendoSwitch2_WiredController_State* state0, + const pa_NintendoSwitch2_WiredController_State* state1 +){ + return memcmp(state0, state1, sizeof(pa_NintendoSwitch2_WiredController_State)) == 0; +} diff --git a/Common/Controllers/NintendoSwitch2_WiredController_State.h b/Common/Controllers/NintendoSwitch2_WiredController_State.h new file mode 100644 index 0000000000..d03deef92d --- /dev/null +++ b/Common/Controllers/NintendoSwitch2_WiredController_State.h @@ -0,0 +1,41 @@ +/* Nintendo Switch 2 - Wired Controller State + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_NintendoSwitch2_WiredController_State_H +#define PokemonAutomation_NintendoSwitch2_WiredController_State_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + + +typedef struct{ + uint8_t buttons0; + uint8_t buttons1; + uint8_t dpad_byte; + uint8_t left_joystick_x; + uint8_t left_joystick_y; + uint8_t right_joystick_x; + uint8_t right_joystick_y; +} pa_NintendoSwitch2_WiredController_State; + +extern const pa_NintendoSwitch2_WiredController_State pa_NintendoSwitch2_WiredController_State_NEUTRAL_STATE; + +bool pa_NintendoSwitch2_WiredController_State_equals( + const pa_NintendoSwitch2_WiredController_State* state0, + const pa_NintendoSwitch2_WiredController_State* state1 +); + + + +#ifdef __cplusplus +} +#endif +#endif diff --git a/Common/Controllers/NintendoSwitch_WirelessController_State.c b/Common/Controllers/NintendoSwitch_WirelessController_State.c new file mode 100644 index 0000000000..14fc3dd56d --- /dev/null +++ b/Common/Controllers/NintendoSwitch_WirelessController_State.c @@ -0,0 +1,30 @@ +/* Nintendo Switch - Wireless Controller State + * + * From: https://github.com/PokemonAutomation/ + * + */ + + +#include +#include "NintendoSwitch_WirelessController_State.h" + + +const pa_NintendoSwitch_WirelessController_State0x30 pa_NintendoSwitch_WirelessController_State0x30_NEUTRAL_STATE = { + .buttons = { + .button3 = 0, + .button4 = 0, + .button5 = 0, + .left_joystick = {0x00, 0x08, 0x80}, + .right_joystick = {0x00, 0x08, 0x80}, + .vibrator = 0x80, + }, + .gyro = {}, +}; + +bool pa_NintendoSwitch_WirelessController_State0x30_equals( + const pa_NintendoSwitch_WirelessController_State0x30* state0, + const pa_NintendoSwitch_WirelessController_State0x30* state1 +){ + return memcmp(state0, state1, sizeof(pa_NintendoSwitch_WirelessController_State0x30)) == 0; +} + diff --git a/Common/Controllers/NintendoSwitch_WirelessController_State.h b/Common/Controllers/NintendoSwitch_WirelessController_State.h new file mode 100644 index 0000000000..a06dabdc1d --- /dev/null +++ b/Common/Controllers/NintendoSwitch_WirelessController_State.h @@ -0,0 +1,64 @@ +/* Nintendo Switch - Wireless Controller State + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_WirelessController_State_H +#define PokemonAutomation_NintendoSwitch_WirelessController_State_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + + + +typedef struct{ + uint8_t button3; + uint8_t button4; + uint8_t button5; + uint8_t left_joystick[3]; + uint8_t right_joystick[3]; + uint8_t vibrator; +} pa_NintendoSwitch_WirelessController_State0x30_Buttons; + +typedef struct{ + int16_t accel_x; + int16_t accel_y; + int16_t accel_z; + int16_t rotation_x; + int16_t rotation_y; + int16_t rotation_z; +} pa_NintendoSwitch_WirelessController_State0x30_Gyro; + +typedef struct{ + pa_NintendoSwitch_WirelessController_State0x30_Gyro time0; + pa_NintendoSwitch_WirelessController_State0x30_Gyro time1; + pa_NintendoSwitch_WirelessController_State0x30_Gyro time2; +} pa_NintendoSwitch_WirelessController_State0x30_GyroX3; + +typedef struct{ + pa_NintendoSwitch_WirelessController_State0x30_Buttons buttons; + pa_NintendoSwitch_WirelessController_State0x30_GyroX3 gyro; +} pa_NintendoSwitch_WirelessController_State0x30; + + + + +extern const pa_NintendoSwitch_WirelessController_State0x30 pa_NintendoSwitch_WirelessController_State0x30_NEUTRAL_STATE; + +bool pa_NintendoSwitch_WirelessController_State0x30_equals( + const pa_NintendoSwitch_WirelessController_State0x30* state0, + const pa_NintendoSwitch_WirelessController_State0x30* state1 +); + + + +#ifdef __cplusplus +} +#endif +#endif diff --git a/Common/Cpp/Concurrency/ComputationThreadPoolCore.h b/Common/Cpp/Concurrency/ComputationThreadPoolCore.h index a262b83c36..6112c72f7b 100644 --- a/Common/Cpp/Concurrency/ComputationThreadPoolCore.h +++ b/Common/Cpp/Concurrency/ComputationThreadPoolCore.h @@ -17,7 +17,7 @@ #define PokemonAutomation_ComputationThreadPoolCore_H #include -#include +//#include #include #include #include "Common/Cpp/CpuUtilization/CpuUtilization.h" diff --git a/Common/SerialPABotBase/SerialPABotBase_Messages_ESP32.h b/Common/SerialPABotBase/SerialPABotBase_Messages_ESP32.h index 7fe3b30e8a..d13ccb253c 100644 --- a/Common/SerialPABotBase/SerialPABotBase_Messages_ESP32.h +++ b/Common/SerialPABotBase/SerialPABotBase_Messages_ESP32.h @@ -7,6 +7,7 @@ #ifndef PokemonAutomation_SerialPABotBase_Messages_ESP32_H #define PokemonAutomation_SerialPABotBase_Messages_ESP32_H +#include "../Controllers/NintendoSwitch_WirelessController_State.h" #include "SerialPABotBase_Protocol.h" #if _WIN32 @@ -32,30 +33,6 @@ typedef struct{ uint8_t right_grip[3]; } PABB_NintendoSwitch_ControllerColors; -typedef struct{ - uint8_t button3; - uint8_t button4; - uint8_t button5; - uint8_t left_joystick[3]; - uint8_t right_joystick[3]; - uint8_t vibrator; -} PABB_NintendoSwitch_ButtonState; - -typedef struct{ - int16_t accel_x; - int16_t accel_y; - int16_t accel_z; - int16_t rotation_x; - int16_t rotation_y; - int16_t rotation_z; -} PABB_NintendoSwitch_GyroState; - -typedef struct{ - PABB_NintendoSwitch_GyroState time0; - PABB_NintendoSwitch_GyroState time1; - PABB_NintendoSwitch_GyroState time2; -} PABB_NintendoSwitch_GyroStateX3; - @@ -82,7 +59,7 @@ typedef struct{ typedef struct{ seqnum_t seqnum; uint16_t milliseconds; - PABB_NintendoSwitch_ButtonState buttons; + pa_NintendoSwitch_WirelessController_State0x30_Buttons buttons; } PABB_PACK pabb_Message_ESP32_CommandButtonState; @@ -90,8 +67,7 @@ typedef struct{ typedef struct{ seqnum_t seqnum; uint16_t milliseconds; - PABB_NintendoSwitch_ButtonState buttons; - PABB_NintendoSwitch_GyroStateX3 gyro; + pa_NintendoSwitch_WirelessController_State0x30 state; } PABB_PACK pabb_Message_ESP32_CommandFullState; diff --git a/Common/SerialPABotBase/SerialPABotBase_Messages_NS2_WiredController.h b/Common/SerialPABotBase/SerialPABotBase_Messages_NS2_WiredController.h index cd21ccd2ad..d64089311e 100644 --- a/Common/SerialPABotBase/SerialPABotBase_Messages_NS2_WiredController.h +++ b/Common/SerialPABotBase/SerialPABotBase_Messages_NS2_WiredController.h @@ -7,6 +7,7 @@ #ifndef PokemonAutomation_SerialPABotBase_Messages_NS2_WiredController_H #define PokemonAutomation_SerialPABotBase_Messages_NS2_WiredController_H +#include "../Controllers/NintendoSwitch2_WiredController_State.h" #include "SerialPABotBase_Protocol.h" #if _WIN32 @@ -26,19 +27,10 @@ namespace SerialPABotBase{ #define PABB_MSG_REPORT_NS2_WIRED_CONTROLLER 0x90 -typedef struct{ - uint8_t buttons0; - uint8_t buttons1; - uint8_t dpad_byte; - uint8_t left_joystick_x; - uint8_t left_joystick_y; - uint8_t right_joystick_x; - uint8_t right_joystick_y; -} pabb_Report_NS2_WiredController; typedef struct{ seqnum_t seqnum; uint16_t milliseconds; - pabb_Report_NS2_WiredController report; + pa_NintendoSwitch2_WiredController_State report; } PABB_PACK pabb_Message_Report_NS2_WiredController; diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 4c14c70f7d..84b535911b 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -62,6 +62,8 @@ file(GLOB MAIN_SOURCES ../Common/CRC32.cpp ../Common/CRC32.h ../Common/Compiler.h + ../Common/Controllers/NintendoSwitch2_WiredController_State.h + ../Common/Controllers/NintendoSwitch_WirelessController_State.h ../Common/Cpp/AbstractLogger.h ../Common/Cpp/BitmapConversion.cpp ../Common/Cpp/BitmapConversion.h diff --git a/SerialPrograms/Source/Controllers/SerialPABotBase/SerialPABotBase_Routines_ESP32.h b/SerialPrograms/Source/Controllers/SerialPABotBase/SerialPABotBase_Routines_ESP32.h index 64e6d04e98..8cf24f6a13 100644 --- a/SerialPrograms/Source/Controllers/SerialPABotBase/SerialPABotBase_Routines_ESP32.h +++ b/SerialPrograms/Source/Controllers/SerialPABotBase/SerialPABotBase_Routines_ESP32.h @@ -84,7 +84,10 @@ class MessageControllerWriteSpi : public BotBaseRequest{ class MessageControllerStateButtons : public BotBaseRequest{ public: pabb_Message_ESP32_CommandButtonState params; - MessageControllerStateButtons(uint16_t milliseconds, const PABB_NintendoSwitch_ButtonState& state) + MessageControllerStateButtons( + uint16_t milliseconds, + const pa_NintendoSwitch_WirelessController_State0x30_Buttons& state + ) : BotBaseRequest(true) { params.seqnum = 0; @@ -100,15 +103,15 @@ class MessageControllerStateFull : public BotBaseRequest{ pabb_Message_ESP32_CommandFullState params; MessageControllerStateFull( uint16_t milliseconds, - const PABB_NintendoSwitch_ButtonState& buttons, - const PABB_NintendoSwitch_GyroStateX3& gyro + const pa_NintendoSwitch_WirelessController_State0x30_Buttons& buttons, + const pa_NintendoSwitch_WirelessController_State0x30_GyroX3& gyro ) : BotBaseRequest(true) { params.seqnum = 0; params.milliseconds = milliseconds; - params.buttons = buttons; - params.gyro = gyro; + params.state.buttons = buttons; + params.state.gyro = gyro; } virtual BotBaseMessage message() const override{ return BotBaseMessage(PABB_MSG_ESP32_CONTROLLER_STATE_FULL, params); diff --git a/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessController.cpp b/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessController.cpp index c751318703..55a916b448 100644 --- a/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessController.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessController.cpp @@ -64,7 +64,7 @@ void SerialPABotBase_WirelessController::stop(){ -Button SerialPABotBase_WirelessController::populate_report_buttons(PABB_NintendoSwitch_ButtonState& buttons){ +Button SerialPABotBase_WirelessController::populate_report_buttons(pa_NintendoSwitch_WirelessController_State0x30_Buttons& buttons){ // https://github.com/dekuNukem/Nintendo_Switch_Reverse_Engineering/blob/master/bluetooth_hid_notes.md Button all_buttons = BUTTON_NONE; @@ -108,7 +108,7 @@ Button SerialPABotBase_WirelessController::populate_report_buttons(PABB_Nintendo } return all_buttons; } -bool SerialPABotBase_WirelessController::populate_report_gyro(PABB_NintendoSwitch_GyroState& gyro){ +bool SerialPABotBase_WirelessController::populate_report_gyro(pa_NintendoSwitch_WirelessController_State0x30_Gyro& gyro){ bool gyro_active = false; { if (m_accel_x.is_busy()){ @@ -143,7 +143,7 @@ bool SerialPABotBase_WirelessController::populate_report_gyro(PABB_NintendoSwitc void SerialPABotBase_WirelessController::issue_report( const Cancellable* cancellable, WallDuration duration, - const PABB_NintendoSwitch_ButtonState& buttons + const pa_NintendoSwitch_WirelessController_State0x30_Buttons& buttons ){ // Release the state lock since we are no longer touching state. // This loop can block indefinitely if the command queue is full. @@ -170,15 +170,15 @@ void SerialPABotBase_WirelessController::issue_report( void SerialPABotBase_WirelessController::issue_report( const Cancellable* cancellable, WallDuration duration, - const PABB_NintendoSwitch_ButtonState& buttons, - const PABB_NintendoSwitch_GyroState& gyro + const pa_NintendoSwitch_WirelessController_State0x30_Buttons& buttons, + const pa_NintendoSwitch_WirelessController_State0x30_Gyro& gyro ){ // Release the state lock since we are no longer touching state. // This loop can block indefinitely if the command queue is full. ReverseLockGuard lg(m_state_lock); // TODO: For now we duplicate the gyro data to all 3 5ms segments. - PABB_NintendoSwitch_GyroStateX3 gyro3{ + pa_NintendoSwitch_WirelessController_State0x30_GyroX3 gyro3{ gyro, gyro, gyro }; diff --git a/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessController.h b/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessController.h index 1cf6222c82..38e51c45d8 100644 --- a/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessController.h +++ b/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessController.h @@ -98,19 +98,19 @@ class SerialPABotBase_WirelessController : public SerialPABotBase_Controller{ data[2] = (uint8_t)(wy >> 4); } - Button populate_report_buttons(PABB_NintendoSwitch_ButtonState& buttons); - bool populate_report_gyro(PABB_NintendoSwitch_GyroState& gyro); + Button populate_report_buttons(pa_NintendoSwitch_WirelessController_State0x30_Buttons& buttons); + bool populate_report_gyro(pa_NintendoSwitch_WirelessController_State0x30_Gyro& gyro); void issue_report( const Cancellable* cancellable, WallDuration duration, - const PABB_NintendoSwitch_ButtonState& buttons + const pa_NintendoSwitch_WirelessController_State0x30_Buttons& buttons ); void issue_report( const Cancellable* cancellable, WallDuration duration, - const PABB_NintendoSwitch_ButtonState& buttons, - const PABB_NintendoSwitch_GyroState& gyro + const pa_NintendoSwitch_WirelessController_State0x30_Buttons& buttons, + const pa_NintendoSwitch_WirelessController_State0x30_Gyro& gyro ); diff --git a/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessJoycon.cpp b/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessJoycon.cpp index 745c187db4..446df5ee4a 100644 --- a/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessJoycon.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessJoycon.cpp @@ -127,7 +127,7 @@ void SerialPABotBase_WirelessJoycon::issue_mash_button( void SerialPABotBase_WirelessJoycon::push_state_left_joycon(const Cancellable* cancellable, WallDuration duration){ - PABB_NintendoSwitch_ButtonState buttons{ + pa_NintendoSwitch_WirelessController_State0x30_Buttons buttons{ .button3 = 0, .button4 = 0, .button5 = 0, @@ -146,7 +146,7 @@ void SerialPABotBase_WirelessJoycon::push_state_left_joycon(const Cancellable* c ); } - PABB_NintendoSwitch_GyroState gyro{}; + pa_NintendoSwitch_WirelessController_State0x30_Gyro gyro{}; bool gyro_active = populate_report_gyro(gyro); if (!gyro_active){ @@ -156,7 +156,7 @@ void SerialPABotBase_WirelessJoycon::push_state_left_joycon(const Cancellable* c } } void SerialPABotBase_WirelessJoycon::push_state_right_joycon(const Cancellable* cancellable, WallDuration duration){ - PABB_NintendoSwitch_ButtonState buttons{ + pa_NintendoSwitch_WirelessController_State0x30_Buttons buttons{ .button3 = 0, .button4 = 0, .button5 = 0, @@ -177,7 +177,7 @@ void SerialPABotBase_WirelessJoycon::push_state_right_joycon(const Cancellable* // cout << (int)m_right_joystick.x << " - " << (int)m_right_joystick.y << ": " << std::chrono::duration_cast(duration).count() << endl; - PABB_NintendoSwitch_GyroState gyro{}; + pa_NintendoSwitch_WirelessController_State0x30_Gyro gyro{}; bool gyro_active = populate_report_gyro(gyro); if (!gyro_active){ diff --git a/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessProController.cpp b/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessProController.cpp index 90b6903745..547a0c8ba7 100644 --- a/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessProController.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Controllers/SerialPABotBase/NintendoSwitch_SerialPABotBase_WirelessProController.cpp @@ -37,7 +37,7 @@ SerialPABotBase_WirelessProController::~SerialPABotBase_WirelessProController(){ void SerialPABotBase_WirelessProController::push_state(const Cancellable* cancellable, WallDuration duration){ // https://github.com/dekuNukem/Nintendo_Switch_Reverse_Engineering/blob/master/bluetooth_hid_notes.md - PABB_NintendoSwitch_ButtonState buttons{ + pa_NintendoSwitch_WirelessController_State0x30_Buttons buttons{ .button3 = 0, .button4 = 0, .button5 = 0, @@ -73,7 +73,7 @@ void SerialPABotBase_WirelessProController::push_state(const Cancellable* cancel ); } - PABB_NintendoSwitch_GyroState gyro{ + pa_NintendoSwitch_WirelessController_State0x30_Gyro gyro{ 0x0000, 0x0000, 0x0000, From 804625a36b7f4d3671f436f127f980600ad518c2 Mon Sep 17 00:00:00 2001 From: Alexander Yee Date: Mon, 11 Aug 2025 22:36:57 -0700 Subject: [PATCH 4/4] Allow 2 sliders to count as Switch 2 international. News page can be black at the bottom. --- .../Source/CommonFramework/Globals.cpp | 2 +- .../DevPrograms/TestProgramSwitch.cpp | 11 ++++++++++- .../NintendoSwitch2_HomeToDateTime.cpp | 1 + .../Programs/PokemonSV_ConnectToInternet.cpp | 19 +++++++++++++------ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Globals.cpp b/SerialPrograms/Source/CommonFramework/Globals.cpp index 06afe9c498..c7704217df 100644 --- a/SerialPrograms/Source/CommonFramework/Globals.cpp +++ b/SerialPrograms/Source/CommonFramework/Globals.cpp @@ -26,7 +26,7 @@ namespace PokemonAutomation{ const bool IS_BETA_VERSION = true; const int PROGRAM_VERSION_MAJOR = 0; const int PROGRAM_VERSION_MINOR = 55; -const int PROGRAM_VERSION_PATCH = 2; +const int PROGRAM_VERSION_PATCH = 3; const std::string PROGRAM_VERSION_BASE = "v" + std::to_string(PROGRAM_VERSION_MAJOR) + diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index e05f679cf6..3523d982f5 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -141,6 +141,8 @@ #include "NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h" #include "PokemonSV/Inference/Boxes/PokemonSV_BoxShinyDetector.h" #include "PokemonSwSh/Inference/PokemonSwSh_DialogBoxDetector.h" +#include "CommonTools/Images/SolidColorTest.h" + #include #include @@ -262,10 +264,17 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& auto screenshot = feed.snapshot(); +#if 0 + NewsDetector detector; + detector.make_overlays(overlays); + cout << detector.detect(screenshot) << endl; +#endif + +#if 0 PokemonSwSh::BlackDialogBoxDetector detector(true); detector.make_overlays(overlays); cout << detector.process_frame(screenshot, current_time()) << endl; - +#endif // PokemonSV::BoxShinyDetector detector; diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch2_HomeToDateTime.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch2_HomeToDateTime.cpp index 8fa27e53c6..9345025d47 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch2_HomeToDateTime.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch2_HomeToDateTime.cpp @@ -50,6 +50,7 @@ ConsoleType settings_detect_console_type( size_t sliders = detector.detect(snapshot).size(); switch (sliders){ case 1: + case 2: console.state().set_console_type(console, ConsoleType::Switch2_FW20_International); break; case 3: diff --git a/SerialPrograms/Source/PokemonSV/Programs/PokemonSV_ConnectToInternet.cpp b/SerialPrograms/Source/PokemonSV/Programs/PokemonSV_ConnectToInternet.cpp index 6909ec113f..13fab40d0e 100644 --- a/SerialPrograms/Source/PokemonSV/Programs/PokemonSV_ConnectToInternet.cpp +++ b/SerialPrograms/Source/PokemonSV/Programs/PokemonSV_ConnectToInternet.cpp @@ -34,21 +34,21 @@ class NewsDetector : public StaticScreenDetector{ private: Color m_color; - ImageFloatBox m_bottom_white; + ImageFloatBox m_bottom_solid; ImageFloatBox m_bottom_buttons; }; NewsDetector::NewsDetector(Color color) : m_color(color) - , m_bottom_white(0.15, 0.92, 0.20, 0.06) + , m_bottom_solid(0.15, 0.92, 0.20, 0.06) , m_bottom_buttons(0.40, 0.92, 0.58, 0.06) {} void NewsDetector::make_overlays(VideoOverlaySet& items) const{ - items.add(m_color, m_bottom_white); + items.add(m_color, m_bottom_solid); items.add(m_color, m_bottom_buttons); } bool NewsDetector::detect(const ImageViewRGB32& screen){ - ImageStats bottom_white = image_stats(extract_box_reference(screen, m_bottom_white)); - if (!is_white(bottom_white)){ + ImageStats bottom_solid = image_stats(extract_box_reference(screen, m_bottom_solid)); + if (!is_white(bottom_solid) && !is_black(bottom_solid)){ return false; } @@ -161,7 +161,14 @@ void connect_to_internet_from_overworld(const ProgramInfo& info, VideoStream& st int ret = wait_until( stream, context, std::chrono::seconds(60), - {overworld, main_menu, dialog, prompt, news, battle_menu} + { + overworld, + main_menu, + dialog, + prompt, + news, + battle_menu, + } ); context.wait_for(std::chrono::milliseconds(100)); switch (ret){