fix box arrow holding pokemon

Author: Gin

SerialPrograms/Scripts/check_detector_regions.py

@@ -136,14 +136,16 @@
 
 # ==================================================================
 # LZA box system cursor detection
-# triangle_upper_edge_width = 0.008
-# traingle_full_height = 0.026
-# for row in range(6):
-# 	y = 0.122 if row == 0 else 0.333 + (0.797 - 0.331)/ 4.0 * (row-1)
-# 	for col in range(6):
-# 		x = 0.058 + col*(0.386 - 0.059)/5.0
-# 		# add_infer_box_to_image(raw_image, x, y, 0.018, traingle_full_height, image)
-# 		add_infer_box_to_image(raw_image, x+0.011, y-0.009, 0.023, 0.031, image)
+for row in range(6):
+	y = 0.122 if row == 0 else (0.333 + (0.797 - 0.331)/ 4.0 * (row-1))
+	for col in range(6):
+		x = 0.058 + col*(0.386 - 0.059)/5.0
+		# non-held positions
+		# add_infer_box_to_image(raw_image, x, y, 0.018, 0.026, image)
+		# held positions
+		# add_infer_box_to_image(raw_image, x+0.011, y-0.010, 0.023, 0.032, image)
+		# narrow gap to detect holding
+		add_infer_box_to_image(raw_image, x+0.011, y+0.010, 0.023, 0.004, image)
 
 # Pixel (x,y) = (775, 136), (0.404, 0.126), rgb=[255,188,245,71] hsv=[ 80 181 245]
 # Pixel (x,y) = (759, 135), (0.395, 0.125), rgb=[255,182,233,70] hsv=[ 81 178 233]

SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp

@@ -90,6 +90,8 @@ void PreloadSettings::load(const JsonValue& json){
         debug_obj->read_boolean(DEBUG.COLOR_CHECK, "COLOR_CHECK");
         debug_obj->read_boolean(DEBUG.IMAGE_TEMPLATE_MATCHING, "IMAGE_TEMPLATE_MATCHING");
         debug_obj->read_boolean(DEBUG.IMAGE_DICTIONARY_MATCHING, "IMAGE_DICTIONARY_MATCHING");
+        debug_obj->read_integer(DEBUG.BOX_SYSTEM_CELL_ROW, "BOX_SYSTEM_CELL_ROW");
+        debug_obj->read_integer(DEBUG.BOX_SYSTEM_CELL_COL, "BOX_SYSTEM_CELL_COL");
     }
 }
 
@@ -397,6 +399,8 @@ JsonValue GlobalSettings::to_json() const{
     debug_obj["COLOR_CHECK"] = debug_settings.COLOR_CHECK;
     debug_obj["IMAGE_TEMPLATE_MATCHING"] = debug_settings.IMAGE_TEMPLATE_MATCHING;
     debug_obj["IMAGE_DICTIONARY_MATCHING"] = debug_settings.IMAGE_DICTIONARY_MATCHING;
+    debug_obj["BOX_SYSTEM_CELL_ROW"] = debug_settings.BOX_SYSTEM_CELL_ROW;
+    debug_obj["BOX_SYSTEM_CELL_COL"] = debug_settings.BOX_SYSTEM_CELL_COL;
     obj["DEBUG"] = std::move(debug_obj);
 
     return obj;

SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h

@@ -54,9 +54,19 @@ class FolderInputOption : public StringOption{
 
 
 struct DebugSettings{
+    // Debug color stats like those in:
+    // - CommonFramework/ImageTools/ImageStats.cpp
     bool COLOR_CHECK = false;
+    // Debug image tempalte matching like those in:
+    // - CommonTools/ImageMatch/WaterfillTemplateMatcher.cpp
     bool IMAGE_TEMPLATE_MATCHING = false;
+    // Debug image dictionary matching like those in:
+    // - CommonTools/ImageMatch/CroppedImageDictionaryMatcher.cpp
     bool IMAGE_DICTIONARY_MATCHING = false;
+    // Debug box system detection to only debug on a box cell at row:
+    int8_t BOX_SYSTEM_CELL_ROW = -1;
+    // Debug box system detection to only debug on a box cell at col:
+    int8_t BOX_SYSTEM_CELL_COL = -1;
 };
 
 

SerialPrograms/Source/CommonTools/ImageMatch/ExactImageMatcher.h

@@ -16,7 +16,7 @@ namespace ImageMatch{
 
 //  Match images against a template image.
 //  Before matching, resize the input image to the template shape and scale template brightness to
-//  match the input image. Alpha channels as used as masks in matching.
+//  match the input image. The template alpha channel is used as masks in matching.
 //  No other treatment like tolerating translation or scaling based on stddevs, hence the name "Exact".
 class ExactImageMatcher{
     ExactImageMatcher(ExactImageMatcher&&) = default;

SerialPrograms/Source/PokemonLZA/Inference/Boxes/PokemonLZA_BoxDetection.cpp

@@ -165,11 +165,12 @@ BoxDetector::BoxDetector(Color color, VideoOverlay* overlay)
     , m_plus_button(color, ButtonType::ButtonPlus, {0.581, 0.940, 0.310, 0.046}, overlay)
 {
     for(size_t row = 0; row < 6; row++){
-         double y = (row == 0 ? 0.122 : 0.333 + (0.797 - 0.331)/ 4.0 * (row-1));
+        double y = (row == 0 ? 0.122 : 0.333 + (0.797 - 0.331)/ 4.0 * (row-1));
         for(size_t col = 0; col < 6; col++){
             double x = 0.058 + col * (0.386 - 0.059)/5.0;
             m_arrow_boxes.emplace_back(x, y, 0.018, 0.026);
-            m_lifted_arrow_boxes.emplace_back(x+0.011, y-0.010, 0.023, 0.032);
+            // m_lifted_arrow_boxes.emplace_back(x+0.011, y-0.010, 0.023, 0.032);
+            m_gaps_for_lifted.emplace_back(x+0.011, y+0.010, 0.023, 0.004);
         }
     }
 }
@@ -184,7 +185,11 @@ void BoxDetector::make_overlays(VideoOverlaySet& items) const{
 
 // detect arrow's white interior first
 // then use subobject template matcher BoxCellSelectionArrowMatcher to detect the whole arrow
-bool BoxDetector::detect_at_cell(const Resolution& screen_resolution, const ImageViewRGB32& image_crop){
+bool BoxDetector::detect_at_cell(uint8_t cell_idx, const ImageViewRGB32& screen){
+#if 0
+    const auto& box = (m_holding_pokemon ? m_lifted_arrow_boxes[cell_idx] : m_arrow_boxes[cell_idx]);
+    ImageViewRGB32 image_crop = extract_box_reference(screen, box);
+
     if (m_holding_pokemon){
         // If the box cursor is holding a pokemon, it will be a green downard arrow with white interior.
         // This green arrow may appear on top of the bottom part of a pokemon on the upper row of the box.
@@ -195,7 +200,7 @@ bool BoxDetector::detect_at_cell(const Resolution& screen_resolution, const Imag
             {combine_rgb(170, 230, 50), combine_rgb(200, 255, 100)},
             {combine_rgb(140, 180, 0), combine_rgb(200, 255, 100)},
         };
-        const double screen_rel_size = (screen_resolution.height / 1080.0);
+        const double screen_rel_size = (screen.height() / 1080.0);
         const size_t min_area = static_cast<size_t>(250 * screen_rel_size * screen_rel_size);
         const double rmsd_threshold = 70.0;
         const bool detected = match_template_by_waterfill(
@@ -213,6 +218,25 @@ bool BoxDetector::detect_at_cell(const Resolution& screen_resolution, const Imag
             return true;
         }
     }
+#else
+    if (m_holding_pokemon){
+        // If the box cursor is holding a pokemon, it will be a green downward arrow with white interior.
+        // This green arrow may appear on top of the bottom part of a pokemon on the upper row of the box.
+        // If the pokemon above has white bottom part, like a regular-color Spewpa, the arrrow white interior
+        // blends in with the Spewpa white color and make it impossible to find the arrow interior using
+        // waterfill. If the held pokemon is green (e.g. shiny Scizor) it will blend with the green part of
+        // the arrow, making waterfill impossible. So we have to use this alternative detection:
+        // Use the stddev of a cross section of the green-white arrow.
+        // If there is an arrow, the bright green and white color makes the stddev quite high (>= 100)
+        // while without the arrow, it is the background, which are the blurred overworld view which
+        // typically has stddev lower than 50.
+        ImageViewRGB32 image_crop = extract_box_reference(screen, m_gaps_for_lifted[cell_idx]);
+        const auto stats = image_stats(image_crop);
+        return stats.stddev.sum() > 80;
+    }
+
+    ImageViewRGB32 image_crop = extract_box_reference(screen, m_arrow_boxes[cell_idx]);
+#endif
 
     // The box curosr either is not holding a pokemon, or it is holding a pokemon but the background of the cursor
     // happens to be green (bottom part of a green pokemon) failing the arrow green part waterfill detection.
@@ -327,16 +351,15 @@ bool BoxDetector::detect(const ImageViewRGB32& screen){
             }
             cout << "row = " << (int)row << ", col = " << (int)col << endl;
 #endif
-            const auto& box = (m_holding_pokemon ? m_lifted_arrow_boxes[cell_idx] : m_arrow_boxes[cell_idx]);
-            ImageViewRGB32 image_crop = extract_box_reference(screen, box);
             // image_crop.save("cell_" + std::to_string(row) + "_" + std::to_string(col) + ".png");
-            const uint8_t debug_cell_row = 255, debug_cell_col = 255;
+            const uint8_t debug_cell_row = PreloadSettings::debug().BOX_SYSTEM_CELL_ROW;
+            const uint8_t debug_cell_col = PreloadSettings::debug().BOX_SYSTEM_CELL_COL;
             if (row == debug_cell_row && col == debug_cell_col){
                 debug_switch = true;
                 cout << "start debugging switch at " << int(row) << ", " << int(col) << endl;
                 PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING = true;
             }
-            const bool detected = detect_at_cell(screen.size(), image_crop);
+            const bool detected = detect_at_cell(cell_idx, screen);
             if (row == debug_cell_row && col == debug_cell_col){
                 debug_switch = false;
                 PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING = false;

SerialPrograms/Source/PokemonLZA/Inference/Boxes/PokemonLZA_BoxDetection.h

@@ -68,14 +68,18 @@ class BoxDetector : public StaticScreenDetector{
 
 private:
     // called for each box cell to check if the selection arrow is above that cell
-    bool detect_at_cell(const Resolution& screen_resolution, const ImageViewRGB32& image_crop);
+    bool detect_at_cell(uint8_t cell_idx, const ImageViewRGB32& screen);
 
     bool m_holding_pokemon = false;
     Color m_color;
 
     ButtonDetector m_plus_button;
-    std::vector<ImageFloatBox> m_arrow_boxes;  // all 6 x 6 potential locations of the arrow interiors on box view
-    std::vector<ImageFloatBox> m_lifted_arrow_boxes;  // all 6 x 6 potential locations of the arrow interiors on box view when lifting/holding a pokemon
+    // all 6 x 6 potential locations of the arrow interiors on box view
+    std::vector<ImageFloatBox> m_arrow_boxes;
+    // all 6 x 6 potential locations of the arrow interiors on box view when lifting/holding a pokemon
+    // std::vector<ImageFloatBox> m_lifted_arrow_boxes;
+    // all 6 x 6 gaps between box rows to detect when lifting/holding a pokemon
+    std::vector<ImageFloatBox> m_gaps_for_lifted;
     uint8_t m_found_row = 0;
     uint8_t m_found_col = 0;
     bool m_debug_mode = false;