improve reliability of Item printer material detector (#662)

* improve reliability of Item printer material detector. use string OCR directly instead of number OCR, then string OCR.

* minor refactor to ItemPrinterMaterialDetector::detect_material_quantity()

Author: jw098

SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp

@@ -254,14 +254,28 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
     ProControllerContext context(scope, console.pro_controller());
     VideoOverlaySet overlays(overlay);
 
+#if 1
+    ItemPrinterMaterialDetector detector(COLOR_RED, LANGUAGE);
+    // detector.make_overlays(overlays);
+    // cout << (int)detector.find_happiny_dust_row_index(console, context) << endl;
+
+    ImageRGB32 image(IMAGE_PATH);
+    // auto image = feed.snapshot();
 
+    for (int i = 0; i < 10; i++){
+        cout << detector.detect_material_name(console, image, context, (int8_t)i) << endl;
+        cout << detector.detect_material_quantity(console, image, context, (int8_t)i) << endl;
+    }
+#endif
+
+#if 0
     auto screenshot = feed.snapshot();
 
 
     DateReader reader(console);
     reader.make_overlays(overlays);
     reader.read_date(logger, screenshot);
-
+#endif
 
 #if 0
     BinarySliderDetector detector(COLOR_BLUE, {0.836431, 0.097521, 0.069703, 0.796694});

SerialPrograms/Source/PokemonSV/Inference/ItemPrinter/PokemonSV_ItemPrinterMaterialDetector.cpp

@@ -159,43 +159,51 @@ int16_t ItemPrinterMaterialDetector::read_number(
 int8_t ItemPrinterMaterialDetector::find_happiny_dust_row_index(
     VideoStream& stream, ProControllerContext& context
 ) const{
-    int8_t value_68_row_index;
+    int8_t happiny_dust_row_index = -1;
     for (size_t c = 0; c < 30; c++){
         context.wait_for_all_requests();
-        std::vector<int8_t> value_68_row_index_list = find_material_value_row_index(stream, context, 68);
-        for (size_t i = 0; i < value_68_row_index_list.size(); i++){
-            value_68_row_index = value_68_row_index_list[i];
-            if (detect_material_name(stream, context, value_68_row_index) == "happiny-dust"){
-                // found screen and row number with Happiny dust.
-                // std::cout << "Happiny dust found. Row number: " << std::to_string(value_68_row_index) << std::endl;
-                return value_68_row_index;
-            }
-        }
+        VideoSnapshot snapshot = stream.video().snapshot();
+        int8_t total_rows = 10;
 
+        SpinLock lock;
+        GlobalThreadPools::normal_inference().run_in_parallel(
+            [&](size_t index){
+                std::string material_name = detect_material_name(stream, snapshot, context, (int8_t)index);
+                if ("happiny-dust" == material_name){
+                    WriteSpinLock lg(lock);
+                    happiny_dust_row_index = (int8_t)index;
+                }
+            },
+            0, total_rows, 1
+        );
+
+        if (happiny_dust_row_index != -1){
+            // found screen and row number with Happiny dust.
+            // std::cout << "Happiny dust found. Row number: " << std::to_string(value_68_row_index) << std::endl;
+            return happiny_dust_row_index;
+        }
+
         // keep searching for Happiny dust
         pbf_press_dpad(context, DPAD_RIGHT, 20, 30);
     }
 
     OperationFailedException::fire(
         ErrorReport::SEND_ERROR_REPORT,
-        "Failed to find Happiny dust after 10 tries.",
+        "Failed to find Happiny dust after multiple attempts.",
         stream
     );
 
 }
 
-// detects the material name at the given row_index
-// MaterialNameReader has a very limited dictionary,
-// so it can only reliably read the material names with 68% value
-// (i.e. Ditto Goo, Happiny Dust, Magby Hair, Beldum Claw)
 std::string ItemPrinterMaterialDetector::detect_material_name(
     VideoStream& stream,
+    const ImageViewRGB32& screen,
     ProControllerContext& context,
     int8_t row_index
 ) const{
-    VideoSnapshot snapshot = stream.video().snapshot();
+    
     ImageFloatBox material_name_box = m_box_mat_name[row_index];
-    ImageViewRGB32 material_name_image = extract_box_reference(snapshot, material_name_box);
+    ImageViewRGB32 material_name_image = extract_box_reference(screen, material_name_box);
     const auto ocr_result = MaterialNameReader::instance().read_substring(
         stream.logger(), m_language,
         material_name_image, OCR::BLACK_OR_WHITE_TEXT_FILTERS()
@@ -250,15 +258,13 @@ std::vector<int8_t> ItemPrinterMaterialDetector::find_material_value_row_index(
 
 }
 
-// detect the quantity of material at the given row number
 int16_t ItemPrinterMaterialDetector::detect_material_quantity(
     VideoStream& stream,
+    const ImageViewRGB32& screen,
     ProControllerContext& context,
     int8_t row_index
 ) const{
-    context.wait_for_all_requests();
-    VideoSnapshot snapshot = stream.video().snapshot();
-    int16_t value = read_number(stream.logger(), snapshot, m_box_mat_quantity[row_index], row_index);
+    int16_t value = read_number(stream.logger(), screen, m_box_mat_quantity[row_index], row_index);
     return value;
 }
 

SerialPrograms/Source/PokemonSV/Inference/ItemPrinter/PokemonSV_ItemPrinterMaterialDetector.h

@@ -58,14 +58,18 @@ class ItemPrinterMaterialDetector{
         int16_t material_value
     ) const;
 
+    // detect the quantity of material at the given row number
     int16_t detect_material_quantity(
         VideoStream& stream,
+        const ImageViewRGB32& screen,
         ProControllerContext& context,
         int8_t row_index
     ) const;
 
+    // detects the material name at the given row_index, given the screen
     std::string detect_material_name(
         VideoStream& stream,
+        const ImageViewRGB32& screen,
         ProControllerContext& context,
         int8_t row_index
     ) const;    

SerialPrograms/Source/PokemonSV/Programs/ItemPrinter/PokemonSV_ItemPrinterRNG.cpp

@@ -993,8 +993,11 @@ uint32_t ItemPrinterRNG::check_num_happiny_dust(
             int8_t happiny_dust_row_num = detector.find_happiny_dust_row_index(
                 env.console, context
             );
+
+            context.wait_for_all_requests();
+            VideoSnapshot snapshot = env.console.video().snapshot();
             num_happiny_dust = detector.detect_material_quantity(
-                env.console, context,
+                env.console, snapshot, context,
                 happiny_dust_row_num
             );
             pbf_mash_button(context, BUTTON_B, 100);