Add box cursor detection whild holding a pokemon

Author: Gin

SerialPrograms/Source/PokemonLZA/Inference/Boxes/PokemonLZA_BoxDetection.cpp

@@ -6,15 +6,21 @@
 
 #include "Common/Cpp/Exceptions.h"
 #include "CommonFramework/Exceptions/FatalProgramException.h"
+#include "CommonFramework/Globals.h"
+#include "CommonFramework/GlobalSettingsPanel.h"
 #include "CommonFramework/ImageTools/ImageStats.h"
+#include "CommonFramework/ImageTypes/ImageRGB32.h"
+#include "CommonFramework/ImageTypes/ImageHSV32.h"
 #include "CommonFramework/VideoPipeline/VideoOverlay.h"
 #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
 #include "CommonFramework/Tools/ErrorDumper.h"
 #include "CommonFramework/Tools/VideoStream.h"
 #include "CommonFramework/VideoPipeline/VideoFeed.h"
 #include "CommonTools/Images/SolidColorTest.h"
 #include "CommonTools/ImageMatch/SubObjectTemplateMatcher.h"
+#include "CommonTools/ImageMatch/WaterfillTemplateMatcher.h"
 #include "CommonTools/Images/BinaryImage_FilterRgb32.h"
+#include "CommonTools/Images/WaterfillUtilities.h"
 #include "Kernels/Waterfill/Kernels_Waterfill.h"
 #include "Kernels/Waterfill/Kernels_Waterfill_Session.h"
 #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h"
@@ -36,6 +42,7 @@ namespace{
     bool debug_switch = false;
 }
 
+
 // match the downward arrow (green border, white interior) in the box system view
 class BoxCellSelectionArrowMatcher : public ImageMatch::SubObjectTemplateMatcher{
 public:
@@ -58,19 +65,101 @@ class BoxCellSelectionArrowMatcher : public ImageMatch::SubObjectTemplateMatcher
                 m_path
             );
         }
-        
+
         set_subobject(objects[0]);
 
+        // Analyze template HSV values to determine green hue range
+        // analyze_template_hsv();
+
 //        extract_box_reference(m_matcher.image_template(), objects[0]).save("template.png");
     }
 
+    // Optional custom image validation hook called before rmsd checks.
+    // Check if the input image contains green pixels with hue close to 51
+    virtual bool check_image(const ImageViewRGB32& input_image) const override{
+        // Convert RGB to HSV
+        ImageHSV32 input_hsv(input_image);
+        ImageViewHSV32 hsv_view(input_hsv);
+
+        // Target hue for green arrow border
+        const uint32_t target_hue = 50;
+        const uint32_t hue_tolerance = 10;
+        const uint32_t min_hue = target_hue - hue_tolerance;
+        const uint32_t max_hue = target_hue + hue_tolerance;
+
+        // Minimum saturation and value to avoid detecting grayish/dark pixels as green
+        const uint32_t min_saturation = 30;  // At least 30/255 saturation
+        const uint32_t min_value = 30;        // At least 30/255 brightness
+
+        // Count pixels with green hue
+        size_t green_pixel_count = 0;
+        const size_t total_pixels = input_image.width() * input_image.height();
+
+        for (size_t y = 0; y < hsv_view.height(); y++){
+            for (size_t x = 0; x < hsv_view.width(); x++){
+                uint32_t hsv_pixel = hsv_view.pixel(x, y);
+                uint32_t hue = (hsv_pixel >> 16) & 0xFF;
+                uint32_t sat = (hsv_pixel >> 8) & 0xFF;
+                uint32_t val = hsv_pixel & 0xFF;
+
+                // Check if pixel is green with target hue
+                if (hue >= min_hue && hue <= max_hue &&
+                    sat >= min_saturation &&
+                    val >= min_value){
+                    green_pixel_count++;
+                }
+            }
+        }
+
+        // Require at least 33% of pixels to be green
+        const size_t min_required_green_pixels = total_pixels / 3;
+        const bool has_green = green_pixel_count >= min_required_green_pixels;
+
+        if (debug_switch){
+            static int counter = 0;
+            cout << "check_image() input image: check_image_input_" << counter << ".png" << endl;
+            input_image.save("check_image_input_" + std::to_string(counter++) + ".png");
+            cout << "check_image() HSV validation:" << endl;
+            cout << "  Green pixels found: " << green_pixel_count << " / " << total_pixels
+                 << " (" << (100.0 * green_pixel_count / total_pixels) << "%)" << endl;
+            cout << "  Required minimum: " << min_required_green_pixels
+                 << " (" << (100.0 * min_required_green_pixels / total_pixels) << "%)" << endl;
+            cout << "  Hue range: [" << min_hue << ", " << max_hue << "]" << endl;
+            cout << "  Result: " << (has_green ? "PASS" : "FAIL") << endl;
+        }
+
+        return has_green;
+    };
+
     static const BoxCellSelectionArrowMatcher& matcher(){
         static BoxCellSelectionArrowMatcher matcher;
         return matcher;
     }
 };
 
 
+class BoxCellSelectionArrowGreenPartMatcher : public ImageMatch::WaterfillTemplateMatcher{
+public:
+    BoxCellSelectionArrowGreenPartMatcher();
+
+    static const BoxCellSelectionArrowGreenPartMatcher& instance();
+};
+
+BoxCellSelectionArrowGreenPartMatcher::BoxCellSelectionArrowGreenPartMatcher()
+    : WaterfillTemplateMatcher(
+        "PokemonLZA/SelectionArrowDown.png",
+        Color(150, 200, 20),
+        Color(200, 250, 50),
+        100
+    )
+{}
+
+const BoxCellSelectionArrowGreenPartMatcher& BoxCellSelectionArrowGreenPartMatcher::instance(){
+    static BoxCellSelectionArrowGreenPartMatcher matcher;
+    return matcher;
+}
+
+
 BoxDetector::BoxDetector(Color color, VideoOverlay* overlay)
     : m_color(color)
     , m_plus_button(color, ButtonType::ButtonPlus, {0.581, 0.940, 0.310, 0.046}, overlay)
@@ -80,6 +169,7 @@ BoxDetector::BoxDetector(Color color, VideoOverlay* overlay)
         for(size_t col = 0; col < 6; col++){
             double x = 0.058 + col * (0.386 - 0.059)/5.0;
             m_arrow_boxes.emplace_back(x, y, 0.018, 0.026);
+            m_lifted_arrow_boxes.emplace_back(x+0.011, y-0.010, 0.023, 0.032);
         }
     }
 }
@@ -91,11 +181,46 @@ void BoxDetector::make_overlays(VideoOverlaySet& items) const{
     }
 }
 
+
 // detect arrow's white interior first
 // then use subobject template matcher BoxCellSelectionArrowMatcher to detect the whole arrow
-bool BoxDetector::detect_at_cell(const ImageViewRGB32& image_crop){
-    // the arrow's white interior has color between rgb [220, 220, 220] to [255, 255, 255]
-    // the arrow's green border has color between rgb [170, 230, 50] to [190, 255, 80]
+bool BoxDetector::detect_at_cell(const Resolution& screen_resolution, const ImageViewRGB32& image_crop){
+    if (m_holding_pokemon){
+        // If the box cursor is holding a pokemon, it will be a green downard arrow with white interior.
+        // This green arrow may appear on top of the bottom part of a pokemon on the upper row of the box.
+        // If the pokemon above has white bottom part, like a regular-color Spewpa, the arrrow white interior
+        // blends in with the Spewpa white color and make it impossible to find the arrow interior using
+        // waterfill. So we do a waterfill for the green part of the arrow against non-green background here.
+        const std::vector<std::pair<uint32_t, uint32_t>> FILTERS = {
+            {combine_rgb(170, 230, 50), combine_rgb(200, 255, 100)},
+            {combine_rgb(140, 180, 0), combine_rgb(200, 255, 100)},
+        };
+        const double screen_rel_size = (screen_resolution.height / 1080.0);
+        const size_t min_area = static_cast<size_t>(250 * screen_rel_size * screen_rel_size);
+        const double rmsd_threshold = 70.0;
+        const bool detected = match_template_by_waterfill(
+            screen_resolution,
+            image_crop,
+            BoxCellSelectionArrowGreenPartMatcher::instance(),
+            FILTERS,
+            {min_area, SIZE_MAX},
+            rmsd_threshold,
+            [&](Kernels::Waterfill::WaterfillObject& object) -> bool {
+                return true;
+            }
+        );
+        if (detected){
+            return true;
+        }
+    }
+
+    // The box curosr either is not holding a pokemon, or it is holding a pokemon but the background of the cursor
+    // happens to be green (bottom part of a green pokemon) failing the arrow green part waterfill detection.
+    // In both cases, we try to use waterfill to find the white interior of the arrow and then match the full arrow
+    // tempalte image around the found white interior: 
+
+    // The arrow's white interior has color between rgb [220, 220, 220] to [255, 255, 255]
+    // The arrow's green border has color between rgb [170, 230, 50] to [190, 255, 80]
     std::vector<std::pair<uint32_t, uint32_t>> filters = {
         {0xff808080, 0xffffffff},
         {0xff909090, 0xffffffff},
@@ -147,9 +272,9 @@ bool BoxDetector::detect_at_cell(const ImageViewRGB32& image_crop){
             }
 
             if (debug_switch){
-                cout << "Find object: area " << object.area << ", save to found_object" << saved_object_id << ".png" << endl;
+                cout << "Find object: area " << object.area << ", save to found_subobject" << saved_object_id << ".png" << endl;
                 ImagePixelBox box(object);
-                extract_box_reference(image_crop, box).save("found_object" + std::to_string(saved_object_id++) + ".png");
+                extract_box_reference(image_crop, box).save("found_subobject" + std::to_string(saved_object_id++) + ".png");
             }
             if (object.area > max_area){
                 continue;
@@ -186,6 +311,7 @@ bool BoxDetector::detect_at_cell(const ImageViewRGB32& image_crop){
     return detected;
 }
 
+
 bool BoxDetector::detect(const ImageViewRGB32& screen){
     if (!m_plus_button.detect(screen)){
         return false;
@@ -201,16 +327,19 @@ bool BoxDetector::detect(const ImageViewRGB32& screen){
             }
             cout << "row = " << (int)row << ", col = " << (int)col << endl;
 #endif
-            ImageViewRGB32 image_crop = extract_box_reference(screen, m_arrow_boxes[cell_idx]);
+            const auto& box = (m_holding_pokemon ? m_lifted_arrow_boxes[cell_idx] : m_arrow_boxes[cell_idx]);
+            ImageViewRGB32 image_crop = extract_box_reference(screen, box);
             // image_crop.save("cell_" + std::to_string(row) + "_" + std::to_string(col) + ".png");
             const uint8_t debug_cell_row = 255, debug_cell_col = 255;
             if (row == debug_cell_row && col == debug_cell_col){
                 debug_switch = true;
                 cout << "start debugging switch at " << int(row) << ", " << int(col) << endl;
+                PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING = true;
             }
-            bool detected = detect_at_cell(image_crop);
+            const bool detected = detect_at_cell(screen.size(), image_crop);
             if (row == debug_cell_row && col == debug_cell_col){
                 debug_switch = false;
+                PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING = false;
             }
             if (detected){
                 if (arrow_found && m_debug_mode){
@@ -244,12 +373,13 @@ BoxCursorCoordinates BoxDetector::detected_location() const{
 
 void BoxDetector::move_cursor(
     const ProgramInfo& info, VideoStream& stream, ProControllerContext& context,
-    uint8_t row, uint8_t col
+    uint8_t row, uint8_t col, bool holding_pokemon
 ){
     if (row >= 6 || col >= 6){
         throw InternalProgramError(&stream.logger(), "BoxDetector::move_cursor",
             "row or col out of range: " + std::to_string(row) + ", " + std::to_string(col));
     }
+    this->holding_pokemon(holding_pokemon);
 
     WallClock start = current_time();
     while (true){

SerialPrograms/Source/PokemonLZA/Inference/Boxes/PokemonLZA_BoxDetection.h

@@ -43,16 +43,23 @@ class BoxDetector : public StaticScreenDetector{
 public:
     BoxDetector(Color color = COLOR_RED, VideoOverlay* overlay = nullptr);
 
+    // Set whether the game is currently holding a pokemon to move around in box view.
+    // The box cursor detection functionality inside BoxDetector needs to know this info.
+    void holding_pokemon(bool holding_pokemon) { m_holding_pokemon = holding_pokemon; }
+
     virtual void make_overlays(VideoOverlaySet& items) const override;
     virtual bool detect(const ImageViewRGB32& screen) override;
 
     // return detected location fround by calling `detect()`
     BoxCursorCoordinates detected_location() const;
 
     // While in the box system view, move the cursor to the desired slot.
+    // - holding_pokemon: whether the movement is with a held pokemon
+    // This function will call `holding_pokemon()` to change internal detection based on
+    // whether the cursor is holding a pokemon or not.
     void move_cursor(
         const ProgramInfo& info, VideoStream& stream, ProControllerContext& context,
-        uint8_t row, uint8_t col
+        uint8_t row, uint8_t col, bool holding_pokemon = false
     );
 
     // Under debug mode, will throw FatalProgramException when more than one box cell
@@ -61,12 +68,14 @@ class BoxDetector : public StaticScreenDetector{
 
 private:
     // called for each box cell to check if the selection arrow is above that cell
-    bool detect_at_cell(const ImageViewRGB32& image_crop);
+    bool detect_at_cell(const Resolution& screen_resolution, const ImageViewRGB32& image_crop);
 
+    bool m_holding_pokemon = false;
     Color m_color;
 
     ButtonDetector m_plus_button;
     std::vector<ImageFloatBox> m_arrow_boxes;  // all 6 x 6 potential locations of the arrow interiors on box view
+    std::vector<ImageFloatBox> m_lifted_arrow_boxes;  // all 6 x 6 potential locations of the arrow interiors on box view when lifting/holding a pokemon
     uint8_t m_found_row = 0;
     uint8_t m_found_col = 0;
     bool m_debug_mode = false;