Improve type detection. Improve reliability of exit sequence. Show type labels on path.

Author: Mysticial

SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp

@@ -118,7 +118,6 @@
 #include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_NumberCodeEntry.h"
 #include "PokemonSV/Inference/ItemPrinter/PokemonSV_ItemPrinterMenuDetector.h"
 #include "PokemonSV/Inference/Picnics/PokemonSV_SandwichHandDetector.h"
-#include "PokemonSV/Inference/Picnics/PokemonSV_SandwichPlateDetector.h"
 #include "PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_PokemonSwapMenu.h"
 #include "CommonTools/Images/ImageFilter.h"
 #include "NintendoSwitch/Options/NintendoSwitch_ModelType.h"
@@ -138,6 +137,7 @@
 #include "PokemonLA/Inference/Map/PokemonLA_SelectedRegionDetector.h"
 #include "PokemonHome/Inference/PokemonHome_BallReader.h"
 #include "PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_PathSide.h"
+#include "PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_PathMap.h"
 
 #include <QPixmap>
 #include <QVideoFrame>
@@ -256,10 +256,29 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
     VideoOverlaySet overlays(overlay);
 
 
+#if 0
+    auto screenshot = feed.snapshot();
+
+    OcrFailureWatchdog watchdog(logger);
+    MaxLairInternal::BattleMenuReader reader(overlay, Language::ChineseTraditional, watchdog);
+    reader.read_opponent_in_summary(logger, screenshot);
+#endif
+
 #if 1
-    ImageRGB32 image(IMAGE_PATH);
-    SandwichPlateDetector middle_plate_detector(logger, COLOR_RED, LANGUAGE, SandwichPlateDetector::Side::MIDDLE);
-    cout << middle_plate_detector.detect_filling_name(image) << endl;
+    auto screenshot = feed.snapshot();
+
+    std::deque<OverlayBoxScope> hits;
+    Pokemon::PokemonType type[4];
+
+    bool ret = MaxLairInternal::read_type_array(
+        console,
+        screenshot,
+        ImageFloatBox{0.150, 0.020, 0.800, 0.780},
+        hits,
+        4, type, nullptr
+    );
+
+    cout << "ret = " << ret << endl;
 #endif
 
 #if 0
@@ -652,13 +671,13 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
     ItemPrinterMaterialDetector detector(COLOR_RED, Language::English);
 
     std::vector<ImageFloatBox> boxes = {
-        // {0.485,0.176758,0.037,0.05}, {0.485,0.250977,0.037,0.05}, {0.485,0.325196,0.037,0.05}, {0.485,0.399415,0.037,0.05}, {0.485,0.473634,0.037,0.05}, {0.485,0.547853,0.037,0.05}, {0.485,0.622072,0.037,0.05}, {0.485,0.696291,0.037,0.05}, {0.485,0.77051,0.037,0.05}, {0.485,0.844729,0.037,0.05}, 
-        {0.39,0.176758,0.025,0.05}, {0.39,0.250977,0.025,0.05}, {0.39,0.325196,0.025,0.05}, {0.39,0.399415,0.025,0.05}, {0.39,0.473634,0.025,0.05}, {0.39,0.547853,0.025,0.05}, {0.39,0.622072,0.025,0.05}, {0.39,0.696291,0.025,0.05}, {0.39,0.77051,0.025,0.05}, {0.39,0.844729,0.025,0.05}, 
+        // {0.485,0.176758,0.037,0.05}, {0.485,0.250977,0.037,0.05}, {0.485,0.325196,0.037,0.05}, {0.485,0.399415,0.037,0.05}, {0.485,0.473634,0.037,0.05}, {0.485,0.547853,0.037,0.05}, {0.485,0.622072,0.037,0.05}, {0.485,0.696291,0.037,0.05}, {0.485,0.77051,0.037,0.05}, {0.485,0.844729,0.037,0.05},
+        {0.39,0.176758,0.025,0.05}, {0.39,0.250977,0.025,0.05}, {0.39,0.325196,0.025,0.05}, {0.39,0.399415,0.025,0.05}, {0.39,0.473634,0.025,0.05}, {0.39,0.547853,0.025,0.05}, {0.39,0.622072,0.025,0.05}, {0.39,0.696291,0.025,0.05}, {0.39,0.77051,0.025,0.05}, {0.39,0.844729,0.025,0.05},
     };
     for (ImageFloatBox box : boxes){
         detector.read_number(console.logger(), env.inference_dispatcher(), image, box);
     }
-    
+
 #endif
 #endif
 
@@ -919,7 +938,7 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
 
     VideoSnapshot image = feed.snapshot();
     DirectionDetector detector;
-    
+
     // ImageRGB32 image("MaterialFarmer-1.png");
     // ImageRGB32 image("dark-capture-card_1.png");
     // DirectionDetector detector(COLOR_BLUE, ImageFloatBox(0,0,1,1));
@@ -937,7 +956,7 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
     // cout << (int)detector.detect_material_quantity(env.inference_dispatcher(), console, context, 2) << endl;
 
     // test OCR for number 1 -> 999. for black text on light background.
-    // increasing quantity of materials to sell. 
+    // increasing quantity of materials to sell.
     for (int i = 1; i < 1000; i++){
        context.wait_for_all_requests();
         if (i != (int)detector.detect_material_quantity(env.inference_dispatcher(), console, context, 2)){

SerialPrograms/Source/PokemonSwSh/Inference/PokemonSwSh_TypeSymbolFinder.cpp

@@ -49,6 +49,7 @@ size_t distance_sqr(const ImagePixelBox& a, const ImagePixelBox& b){
     return dist_x*dist_x + dist_y*dist_y;
 }
 
+//bool print = false;
 
 std::pair<double, PokemonType> match_type_symbol(const ImageViewRGB32& image){
     size_t width = image.width();
@@ -64,6 +65,9 @@ std::pair<double, PokemonType> match_type_symbol(const ImageViewRGB32& image){
     }
     ImageStats stats = image_stats(image);
     if (stats.stddev.sum() < 50){
+//        if (print){
+//            cout << "stats.stddev.sum() = " << stats.stddev.sum() << endl;
+//        }
         return {1.0, PokemonType::NONE};
     }
 
@@ -73,7 +77,7 @@ std::pair<double, PokemonType> match_type_symbol(const ImageViewRGB32& image){
 //    image.save("test-" + std::to_string(threshold) + "-" + std::to_string(c++) + ".png");
 
 //    std::map<double, PokemonType> rank;
-    double best_score = 0.4;
+    double best_score = 0.45;
     PokemonType best_type = PokemonType::NONE;
     for (const auto& item : all_type_sprites()){
 //        if (threshold != 700 || id != 55){
@@ -83,10 +87,12 @@ std::pair<double, PokemonType> match_type_symbol(const ImageViewRGB32& image){
         double expected_aspect_ratio = item.second.aspect_ratio();
         double ratio = aspect_ratio / expected_aspect_ratio;
 #if 0
-        cout << item.second.slug()
-             << " : expected = " << expected_aspect_ratio
-             << ", actual = " << aspect_ratio
-             << ", ratio = " << ratio << endl;
+        if (print){
+            cout << item.second.slug()
+                 << " : expected = " << expected_aspect_ratio
+                 << ", actual = " << aspect_ratio
+                 << ", ratio = " << ratio << endl;
+        }
 #endif
         if (std::abs(ratio - 1) > 0.2){
             continue;
@@ -95,7 +101,9 @@ std::pair<double, PokemonType> match_type_symbol(const ImageViewRGB32& image){
         double rmsd_alpha = item.second.matcher().diff(image);
 
 //        item.second.matcher().m_image.save("sprite.png");
-//        cout << item.second.slug() << ": " << rmsd_alpha << endl;
+//        if (print){
+//            cout << item.second.slug() << ": " << rmsd_alpha << endl;
+//        }
 
 #if 0
         //  Handicap fairy due to white and pink being too similar in color and
@@ -135,7 +143,7 @@ void find_type_symbol_candidates(
         (size_t)(20. * original_screen.total_pixels() / (1920*1080))
     );
 
-//    static int index = 0;
+    static int index = 0;
 
     std::map<size_t, WaterfillObject> objmap;
     for (size_t c = 0; c < objects.size(); c++){
@@ -144,9 +152,9 @@ void find_type_symbol_candidates(
         }
         objmap[c] = objects[c];
 
-//        image.copy(
-//            objects[c].min_x, objects[c].min_y, objects[c].width(), objects[c].height()
-//        ).save("test-" + std::to_string(index++) + ".png");
+#if 0
+        extract_box_reference(image, ImagePixelBox(objects[c])).save("test-" + std::to_string(index++) + ".png");
+#endif
     }
 
 //    cout << "begin = " << objmap.size() << endl;
@@ -183,7 +191,12 @@ void find_type_symbol_candidates(
     //  Identify objects.
     for (const auto& item : objmap){
         ImageViewRGB32 img = extract_box_reference(image, item.second);
+
+//        print = index == 137;
+//        img.save("test-" + std::to_string(index++) + ".png");
+
         std::pair<double, PokemonType> result = match_type_symbol(img);
+//        cout << "result = " << POKEMON_TYPE_SLUGS().get_string(result.second) << ": " << result.first << endl;
         if (result.second != PokemonType::NONE){
             const WaterfillObject& obj = item.second;
             candidates.emplace(
@@ -211,6 +224,11 @@ std::multimap<double, std::pair<PokemonType, ImagePixelBox>> find_type_symbols(
         std::vector<PackedBinaryMatrix> matrices = compress_rgb32_to_binary_range(
             image,
             {
+                {0xff808060, 0xffffffff},
+                {0xffa0a060, 0xffffffff},
+                {0xff606060, 0xffffffff},
+                {0xff707070, 0xffffffff},
+                {0xff808080, 0xffffffff},
                 {0xff909090, 0xffffffff},
                 {0xffa0a0a0, 0xffffffff},
                 {0xffb0b0b0, 0xffffffff},
@@ -224,9 +242,11 @@ std::multimap<double, std::pair<PokemonType, ImagePixelBox>> find_type_symbols(
         }
     }
 
+//    cout << "-------------" << endl;
+
     std::multimap<double, std::pair<PokemonType, ImagePixelBox>> filtered;
     for (const auto& candidate : candidates){
-//        cout << get_type_slug(candidate.second.first) << ": " << candidate.first << endl;
+//        cout << POKEMON_TYPE_SLUGS().get_string(candidate.second.first) << ": " << candidate.first << endl;
 //        hits.emplace_back(overlay, translate_to_parent(screen, box, candidate.second.second.box), COLOR_GREEN);
 
         bool is_dupe = false;
@@ -242,14 +262,15 @@ std::multimap<double, std::pair<PokemonType, ImagePixelBox>> find_type_symbols(
     }
 
 #if 0
+    static int c = 0;
     for (const auto& item : filtered){
 //        cout << get_type_slug(item.second.first) << ": " << item.first << " - [" << item.second.second.center_x() << "," << item.second.second.center_y() << "]" << endl;
         const ImagePixelBox& box = item.second.second;
         ImageViewRGB32 img = image.sub_image(
             box.min_x, box.min_y,
             box.width(), box.height()
         );
-//        img.save("test-" + std::to_string(c++) + ".png");
+        img.save("test-" + std::to_string(c++) + ".png");
     }
 #endif
 

SerialPrograms/Source/PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_BattleMenu.cpp

@@ -333,12 +333,14 @@ std::set<std::string> BattleMenuReader::read_opponent_in_summary(Logger& logger,
         }
     }
 
-//    for (const auto& slug : allowed_slugs){
-//        cout << "allowed_slugs = " << slug << endl;
-//    }
-//    for (const auto& slug : name_slugs){
-//        cout << "name_slugs = " << slug << endl;
-//    }
+#if 0
+    for (const auto& slug : allowed_slugs){
+        cout << "allowed_slugs = " << slug << endl;
+    }
+    for (const auto& slug : name_slugs){
+        cout << "name_slugs = " << slug << endl;
+    }
+#endif
 
     if (common_slugs.size() == 1){
         logger.log("Disambiguation succeeded: " + *common_slugs.begin(), COLOR_BLUE);

SerialPrograms/Source/PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_PathMap.cpp

@@ -55,7 +55,12 @@ bool read_type_array(
     hits.clear();
     size_t c = 0;
     for (const auto& item : candidates){
-        hits.emplace_back(stream.overlay(), translate_to_parent(screen, box, item.second.second), COLOR_GREEN);
+        hits.emplace_back(
+            stream.overlay(),
+            COLOR_GREEN,
+            translate_to_parent(screen, box, item.second.second),
+            POKEMON_TYPE_SLUGS().get_string(item.second.first)
+        );
         sorted.emplace(item.second.second.min_x, &item.second);
         c++;
         if (c >= count){

SerialPrograms/Source/PokemonSwSh/MaxLair/Program/PokemonSwSh_MaxLair_Run_CaughtScreen.cpp

@@ -223,6 +223,7 @@ StateMachineAction run_caught_screen(
             synchronize_caught_screen(console_index, console, context, state_tracker);
             pbf_press_dpad(context, DPAD_DOWN, 10, 50);
             pbf_press_button(context, BUTTON_B, 10, TICKS_PER_SECOND);
+            pbf_press_button(context, BUTTON_A, 10, 115);
             return mash_A_to_entrance(runtime, console, context, entrance);
         }else{
             console.log("Taking non-shiny boss and returning to entrance...", COLOR_BLUE);