Add dummy LZA dialog box detection

Author: Gin

SerialPrograms/CMakeLists.txt

@@ -1432,6 +1432,8 @@ file(GLOB MAIN_SOURCES
     Source/PokemonLGPE/Programs/ShinyHunting/PokemonLGPE_LegendaryReset.h
     Source/PokemonLGPE/Programs/TestPrograms/PokemonLGPE_SoundListener.cpp
     Source/PokemonLGPE/Programs/TestPrograms/PokemonLGPE_SoundListener.h
+    Source/PokemonLZA/Inference/PokemonLZA_DialogDetector.cpp
+    Source/PokemonLZA/Inference/PokemonLZA_DialogDetector.h
     Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.cpp
     Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.h
     Source/PokemonRSE/Inference/PokemonRSE_ShinyNumberDetector.cpp
@@ -2220,6 +2222,8 @@ file(GLOB MAIN_SOURCES
     Source/Tests/NintendoSwitch_Tests.h
     Source/Tests/PokemonLA_Tests.cpp
     Source/Tests/PokemonLA_Tests.h
+    Source/Tests/PokemonLZA_Tests.cpp
+    Source/Tests/PokemonLZA_Tests.h
     Source/Tests/PokemonSV_Tests.cpp
     Source/Tests/PokemonSV_Tests.h
     Source/Tests/PokemonSwSh_Tests.cpp

SerialPrograms/Scripts/add_new_file.py

@@ -25,6 +25,7 @@
 target_path = ""
 if len (sys.argv) > 1:
 	target_path = sys.argv[1]
+	print(f"Adding a new file path {target_path}")
 
 # Sanitize target path:
 SerialPrograms_prefix = "SerialPrograms/"
@@ -76,26 +77,33 @@ def get_code_file_range(file_lines: List[str], starting_line: str, ending_line:
 
 cmakelists_path = code_root_path + os.sep + "SerialPrograms" + os.sep + "CMakeLists.txt"
 pro_path = code_root_path + os.sep + "SerialPrograms" + os.sep + "SerialPrograms.pro"
+print(f"Found CMakeLists path {cmakelists_path}")
+print(f"Found QT Pro project file path {pro_path}")
 
 file_lines = read_lines(cmakelists_path)
+old_file_lines = file_lines
 
 code_file_start, code_file_end = get_code_file_range(file_lines, "file(GLOB MAIN_SOURCES", ")")
 code_file_lines = set(file_lines[code_file_start:code_file_end])
+old_num_code_files = len(code_file_lines)
+print(f"{old_num_code_files} lines from CMakeLists are for code file paths")
 
 # Add new file
 if len(target_path) > 0:
 	code_file_lines.add("    " + target_path)
 
 code_file_lines = list(code_file_lines)
 code_file_lines.sort()
+if len(code_file_lines) == old_num_code_files:
+	print("Warning: you are adding an existing file! The added file is ignored")
 
 # print(f"\"{code_file_lines[0]}\"")
 
 file_lines = file_lines[0:code_file_start] + code_file_lines + file_lines[code_file_end:]
 file_lines = [line + "\r\n" for line in file_lines]
-
 with open(cmakelists_path, "w") as f:
 	f.writelines(file_lines)
+print(f"Writed changes back to {cmakelists_path}")
 
 
 file_lines = read_lines(pro_path)
@@ -148,6 +156,7 @@ def process_pro_lines(file_lines: List[str], starting_line: str, new_path: str)
 
 with open(pro_path, "w") as f:
 	f.writelines(file_lines)
+print(f"Writed changes back to {pro_path}")
 
 
 

SerialPrograms/Scripts/build_code_from_template.py

@@ -16,6 +16,14 @@
 
 from typing import Dict
 
+if len(sys.argv) == 1:
+	print(
+		"Usage:\n"
+		f"python3 {sys.argv[0]} VisualDetector PokemonSV Map PokeCenter Icon"
+		"  - Generates PokemonSV_MapPokeCenterIconDetector.h and PokemonSV_MapPokeCenterIconDetector.cpp"
+	)
+	exit(0)
+
 cur_folder_path = os.path.abspath(os.getcwd())
 
 split_path = cur_folder_path.split(os.sep)

SerialPrograms/Scripts/image_viewer.py

@@ -220,7 +220,7 @@ def run(self):
 				if len(self.highlight_list) > 0:
 					self.cur_highlight_index = -1 if self.cur_highlight_index == len(self.highlight_list)-1 else self.cur_highlight_index+1
 					print(f"Change highlight index to {self.cur_highlight_index}/{len(self.highlight_list)}")
-			elif key == 127: # backspace, remove selected rectangle
+			elif key == 127 or key == 8: # DEL or backspace (BS), remove selected rectangle
 				mouse_down = False
 				mouse_move_counter = 0
 				if len(self.rects) > 0:

SerialPrograms/SerialPrograms.pro

@@ -1090,11 +1090,13 @@ SOURCES += \
     Source/PokemonSwSh/Resources/PokemonSwSh_TypeMatchup.cpp \
     Source/PokemonSwSh/Resources/PokemonSwSh_TypeSprites.cpp \
     Source/PokemonSwSh/ShinyHuntTracker.cpp \
+    Source/PokemonZLA/Inference/PokemonZLA_DialogDetector.cpp \
     Source/Tests/CommandLineTests.cpp \
     Source/Tests/CommonFramework_Tests.cpp \
     Source/Tests/Kernels_Tests.cpp \
     Source/Tests/NintendoSwitch_Tests.cpp \
     Source/Tests/PokemonLA_Tests.cpp \
+    Source/Tests/PokemonLZA_Tests.cpp \
     Source/Tests/PokemonSV_Tests.cpp \
     Source/Tests/PokemonSwSh_Tests.cpp \
     Source/Tests/TestMap.cpp \
@@ -2310,11 +2312,13 @@ HEADERS += \
     Source/PokemonSwSh/Resources/PokemonSwSh_TypeMatchup.h \
     Source/PokemonSwSh/Resources/PokemonSwSh_TypeSprites.h \
     Source/PokemonSwSh/ShinyHuntTracker.h \
+    Source/PokemonZLA/Inference/PokemonZLA_DialogDetector.h \
     Source/Tests/CommandLineTests.h \
     Source/Tests/CommonFramework_Tests.h \
     Source/Tests/Kernels_Tests.h \
     Source/Tests/NintendoSwitch_Tests.h \
     Source/Tests/PokemonLA_Tests.h \
+    Source/Tests/PokemonLZA_Tests.h \
     Source/Tests/PokemonSV_Tests.h \
     Source/Tests/PokemonSwSh_Tests.h \
     Source/Tests/TestMap.h \

SerialPrograms/Source/CommonTools/ImageMatch/WaterfillTemplateMatcher.cpp

@@ -40,7 +40,7 @@ WaterfillTemplateMatcher::WaterfillTemplateMatcher(
         dump_debug_image(
             global_logger_command_line(), 
             "CommonFramework/WaterfillTemplateMatcher", 
-            "binary_image", 
+            "waterfill_template_image_black_background", 
             binaryImage);
     }
     std::vector<WaterfillObject> objects = find_objects_inplace(matrix, min_area);
@@ -66,7 +66,11 @@ WaterfillTemplateMatcher::WaterfillTemplateMatcher(
         const auto exact_image = extract_box_reference(reference, *best);
         cout << "Build waterfil template matcher from " << full_path << ", W x H: " << exact_image.width()
              << " x " << exact_image.height() <<  ", area ratio: " << m_area_ratio << ", Object area: " << best->area << endl;
-        dump_debug_image(global_logger_command_line(), "CommonFramework/WaterfillTemplateMatcher", "matcher_exact_image", exact_image);
+        dump_debug_image(
+            global_logger_command_line(),
+            "CommonFramework/WaterfillTemplateMatcher",
+            "waterfill_template_matcher_reference_image",
+        exact_image);
     }
 }
 
@@ -150,7 +154,12 @@ double WaterfillTemplateMatcher::rmsd_original(const ImageViewRGB32& original_im
 
     if (PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING){
         cout << "rmsd_original()" << endl;
-        dump_debug_image(global_logger_command_line(), "CommonFramework/WaterfillTemplateMatcher", "rmsd_original_input", extract_box_reference(original_image, object));
+        dump_debug_image(
+            global_logger_command_line(),
+            "CommonFramework/WaterfillTemplateMatcher",
+            "waterfill_template_matcher_rmsd_original_input",
+            extract_box_reference(original_image, object)
+        );
     }
 
     if (!check_aspect_ratio(object.width(), object.height())){

SerialPrograms/Source/CommonTools/ImageMatch/WaterfillTemplateMatcher.h

@@ -69,6 +69,9 @@ class WaterfillTemplateMatcher{
     //  See `double rmsd(const ImageViewRGB32& image) const` on the details of comparing the image against the template.
     virtual double rmsd_original(const ImageViewRGB32& original_image, const WaterfillObject& object) const;
 
+    //  Return the image template mesh
+    const ImageRGB32& image_template() const { return m_matcher->image_template(); }
+
 protected:
     virtual bool check_image(const ImageViewRGB32& image) const{ return true; };
     bool check_aspect_ratio(size_t candidate_width, size_t candidate_height) const;

SerialPrograms/Source/CommonTools/Images/WaterfillUtilities.cpp

@@ -80,6 +80,12 @@ bool match_template_by_waterfill(
     std::function<bool(Kernels::Waterfill::WaterfillObject& object)> check_matched_object)
 {
     if (PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING){
+        dump_debug_image(
+            global_logger_command_line(), 
+            "CommonFramework/WaterfillTemplateMatcher", 
+            "match_template_by_waterfill_input_image", 
+            image
+        );
         std::cout << "Match template by waterfill, " << filters.size() << " filter(s), size range ("
                   << area_thresholds.first << ", ";
         if (area_thresholds.second == SIZE_MAX){
@@ -89,20 +95,22 @@ bool match_template_by_waterfill(
         }
         std::cout << ")" << std::endl;
     }
-    auto matrices = compress_rgb32_to_binary_range(image, filters);
+    std::vector<PokemonAutomation::PackedBinaryMatrix> matrices = compress_rgb32_to_binary_range(image, filters);
 
     bool detected = false;
     bool stop_match = false;
-    for (PokemonAutomation::PackedBinaryMatrix &matrix : matrices){
+    for (size_t i_matrix = 0; i_matrix < matrices.size(); i_matrix++){
+        PackedBinaryMatrix& matrix = matrices[i_matrix];
         if (PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING){
             ImageRGB32 binaryImage = image.copy();
             filter_by_mask(matrix, binaryImage, Color(COLOR_BLACK), true);
             //filter_by_mask(matrix, binaryImage, Color(COLOR_WHITE), true);
             dump_debug_image(
                 global_logger_command_line(), 
                 "CommonFramework/WaterfillTemplateMatcher", 
-                "binary_image", 
-                binaryImage);
+                "match_template_by_waterfill_filtered_image_" + std::to_string(i_matrix), 
+                binaryImage
+            );
         }
 
         std::unique_ptr<Kernels::Waterfill::WaterfillSession> session = Kernels::Waterfill::make_WaterfillSession();

SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_DialogDetector.cpp

@@ -0,0 +1,168 @@
+/*  Dialog Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include "CommonFramework/ImageTools/ImageStats.h"
+#include "CommonTools/Images/SolidColorTest.h"
+#include "PokemonLZA_DialogDetector.h"
+#include "Common/Cpp/AbstractLogger.h"
+#include "CommonTools/ImageMatch/WaterfillTemplateMatcher.h"
+#include "CommonTools/Images/WaterfillUtilities.h"
+#include "Kernels/Waterfill/Kernels_Waterfill_Types.h"
+#include "CommonFramework/VideoPipeline/VideoOverlay.h"
+#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+
+//#include <iostream>
+//using std::cout;
+//using std::endl;
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonLZA{
+
+
+class DialogTitleGreenLineMatcher : public ImageMatch::WaterfillTemplateMatcher{
+public:
+    // The white background for the template file is of color range [r=240, g=255, b=230] to [255, 255, 255]
+    // the green line is of color range [r=180,g=200,b=75] to [190, 214, 110]
+    DialogTitleGreenLineMatcher() : WaterfillTemplateMatcher(
+        "PokemonLZA/DialogBox/DialogBoxTitleGreenLine-Template.png", Color(180,200,70), Color(200, 220, 115), 50
+    ) {
+        m_aspect_ratio_lower = 0.9;
+        m_aspect_ratio_upper = 1.1;
+        m_area_ratio_lower = 0.8;
+        m_area_ratio_upper = 1.1;
+    }
+
+    static const ImageMatch::WaterfillTemplateMatcher& instance() {
+        static DialogTitleGreenLineMatcher matcher;
+        return matcher;
+    }
+};
+
+class DialogBlackArrowMatcher : public ImageMatch::WaterfillTemplateMatcher{
+public:
+    // The white background for the template file is of color range [r=240, g=255, b=230] to [255, 255, 255]
+    // the black arrow color is about [r=36,g=38,b=51]
+    DialogBlackArrowMatcher() : WaterfillTemplateMatcher(
+        "PokemonLZA/DialogBox/DialogBoxBlackArrow-Template.png", Color(0,0,0), Color(90, 90, 90), 50
+    ) {
+        m_aspect_ratio_lower = 0.9;
+        m_aspect_ratio_upper = 1.1;
+        m_area_ratio_lower = 0.85;
+        m_area_ratio_upper = 1.1;
+    }
+
+    static const ImageMatch::WaterfillTemplateMatcher& instance() {
+        static DialogBlackArrowMatcher matcher;
+        return matcher;
+    }
+};
+
+
+ 
+NormalDialogDetector::NormalDialogDetector(Logger& logger, VideoOverlay& overlay, bool stop_on_detected)
+    : VisualInferenceCallback("NormalDialogDetector")
+    , m_stop_on_detected(stop_on_detected)
+    , m_detected(false)
+    , m_title_green_line_box(0.224, 0.727, 0.016, 0.056)
+    , m_black_arrow_box (0.727, 0.868, 0.037, 0.086)
+{}
+void NormalDialogDetector::make_overlays(VideoOverlaySet& items) const{
+    items.add(COLOR_RED, m_title_green_line_box);
+    items.add(COLOR_RED, m_black_arrow_box);
+}
+bool NormalDialogDetector::process_frame(const ImageViewRGB32& frame, WallClock timestamp){
+    const double screen_rel_size = (frame.height() / 1080.0);
+    const double screen_rel_size_2 = screen_rel_size * screen_rel_size;
+
+    bool found_green_title_line = false;
+    bool found_black_arrow = false;
+
+    // Example green pixels from screenshots:
+    // 194,230,70
+    // 212,235,127
+    // 176,212,62
+    const std::vector<std::pair<uint32_t, uint32_t>> green_line_filters = {
+        {combine_rgb(160,200,55), combine_rgb(220, 240, 130)}
+    };
+  
+    const double min_green_line_size_1080P = 100.0;
+    const double green_line_rmsd_threshold = 50.0;
+    const size_t min_green_line_size = size_t(screen_rel_size_2 * min_green_line_size_1080P);
+    match_template_by_waterfill(
+        extract_box_reference(frame, m_title_green_line_box), 
+        DialogTitleGreenLineMatcher::instance(),
+        green_line_filters,
+        {min_green_line_size, SIZE_MAX},
+        green_line_rmsd_threshold,
+        [&](Kernels::Waterfill::WaterfillObject& object) -> bool {
+            found_green_title_line = true;
+            return true;
+        }
+    );
+
+    // Example green pixels from screenshots:
+    // [37,34,6] [20,15,55]
+    // [39,42,58] [60,56,74] [50,49,63]
+    const std::vector<std::pair<uint32_t, uint32_t>> black_arrow_filters = {
+        {combine_rgb(0,0,0), combine_rgb(100, 100, 100)}
+    };
+  
+    const double min_black_arrow_size_1080P = 150.0;
+    const double black_arrow_rmsd_threshold = 120.0;
+    const size_t min_black_arrow_size = size_t(screen_rel_size_2 * min_black_arrow_size_1080P);
+    match_template_by_waterfill(
+        extract_box_reference(frame, m_black_arrow_box), 
+        DialogBlackArrowMatcher::instance(),
+        black_arrow_filters,
+        {min_black_arrow_size, SIZE_MAX},
+        black_arrow_rmsd_threshold,
+        [&](Kernels::Waterfill::WaterfillObject& object) -> bool {
+            found_black_arrow = true;
+            return true;
+        }
+    );
+
+    bool is_dialog_box = found_green_title_line & found_black_arrow;
+
+    if (is_dialog_box){
+        m_detected.store(is_dialog_box);
+    }
+
+    return is_dialog_box & m_stop_on_detected;
+
+
+    // size_t hits = 0;
+
+    // const ImageStats title_top = image_stats(extract_box_reference(frame, m_title_top));
+    // const ImageStats title_bottom = image_stats(extract_box_reference(frame, m_title_bottom));
+    // const ImageStats title_left = image_stats(extract_box_reference(frame, m_title_left));
+    // const ImageStats title_right = image_stats(extract_box_reference(frame, m_title_right));
+
+    // ImageStats top_white = image_stats(extract_box_reference(frame, m_top_white));
+    // hits += is_white(top_white, 480, 30) ? 1 : 0;
+
+    // ImageStats bottom_white = image_stats(extract_box_reference(frame, m_bottom_white));
+    // hits += is_white(bottom_white, 480, 30) ? 1 : 0;
+
+    // ImageStats left_white = image_stats(extract_box_reference(frame, m_left_white));
+    // hits += is_white(left_white, 480, 30) ? 1 : 0;
+
+    // ImageStats right_white = image_stats(extract_box_reference(frame, m_right_white));
+    // hits += is_white(right_white, 480, 30) ? 1 : 0;
+
+    // bool detected = hits == 5;
+    // m_detected.store(detected, std::memory_order_release);
+
+    // return detected && m_stop_on_detected;
+}
+
+
+
+
+}
+}
+}