use inference to confirm we are in the Mystery Gift window

Author: jw098

SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp

@@ -138,6 +138,7 @@
 #include "PokemonHome/Inference/PokemonHome_BallReader.h"
 #include "PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_PathSide.h"
 #include "PokemonSwSh/MaxLair/Inference/PokemonSwSh_MaxLair_Detect_PathMap.h"
+#include "NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h"
 
 #include <QPixmap>
 #include <QVideoFrame>
@@ -256,6 +257,36 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
     VideoOverlaySet overlays(overlay);
 
 
+
+#if 0
+    bool switch2 = true;
+    ImageFloatBox key1_box; 
+    ImageFloatBox other_setting1; 
+    ImageFloatBox other_setting2; 
+
+    if (!switch2){
+        key1_box = {0.037322, 0.451172, 0.009879, 0.113281};
+        other_setting1 = {0.01, 0.451172, 0.009879, 0.113281};
+        other_setting2 = {0.02, 0.451172, 0.009879, 0.113281};
+    }else if (switch2){
+        key1_box = {0.062706, 0.510763, 0.009901, 0.097847};
+        other_setting1 = {0.02, 0.510763, 0.009901, 0.097847};
+        other_setting2 = {0.04, 0.510763, 0.009901, 0.097847};            
+    }     
+    
+    SelectedSettingWatcher key1_selected(key1_box, other_setting1, other_setting2, other_setting1);
+    int ret = wait_until(
+        console, context,
+        Milliseconds(5000),
+        {key1_selected}
+    );
+    if (ret < 0){  // failed to detect Key 1 being highlighted. Reset game and re-try
+        console.log("claim_mystery_gift: Failed to detect the Mystery Gift window. Reset game and re-try.", COLOR_YELLOW);
+        reset_game(env.program_info(), console, context);
+    }   
+
+#endif
+
 #if 0
     auto screenshot = feed.snapshot();
 
@@ -342,7 +373,7 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
 #endif
 
 
-#if 1
+#if 0
     auto screenshot = feed.snapshot();
 
     PokemonHome::BallReader reader(console);
@@ -607,7 +638,7 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
 //    context->issue_gyro_rotate_x(&scope, duration, duration, 0s, 0x1000);
 //    context->issue_nop(&scope, 60h);
 
-#if 1
+#if 0
     auto duration = 15ms;
     for (size_t c = 0; c < 65536; c += 1){
         context->issue_gyro_accel_x(&scope, 0s, duration, 0s, (uint16_t)(688 + 0*c % 2));
@@ -655,7 +686,7 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
 #if 0
     // ImageRGB32 image(IMAGE_PATH);
     auto image = feed.snapshot();
-#if 1
+#if 0
     ImageRGB32 image(IMAGE_PATH);
     // auto image = feed.snapshot();
 

SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.cpp

@@ -9,9 +9,9 @@
 #include "NintendoSwitch_SelectedSettingDetector.h"
 
 //
-//#include <iostream>
-//using std::cout;
-//using std::endl;
+// #include <iostream>
+// using std::cout;
+// using std::endl;
 
 namespace PokemonAutomation{
 namespace NintendoSwitch{
@@ -21,24 +21,26 @@ SelectedSettingWatcher::~SelectedSettingWatcher() = default;
 SelectedSettingWatcher::SelectedSettingWatcher(
     ImageFloatBox selected_box,
     ImageFloatBox not_selected_box1,
-    ImageFloatBox not_selected_box2
+    ImageFloatBox not_selected_box2,
+    ImageFloatBox representative_background
 )
     : VisualInferenceCallback("SelectedSettingWatcher")
     , m_selected_box(selected_box)
     , m_not_selected_box1(not_selected_box1)
     , m_not_selected_box2(not_selected_box2)
+    , m_representative_background(representative_background)
 {}
 
 void SelectedSettingWatcher::make_overlays(VideoOverlaySet& items) const{
     items.add(COLOR_RED, m_selected_box);
     items.add(COLOR_BLUE, m_not_selected_box1);
     items.add(COLOR_BLUE, m_not_selected_box2);
+    items.add(COLOR_GREEN, m_representative_background);
 }
 
-bool is_white_theme(const ImageViewRGB32& screen){
-    ImageFloatBox window_top(0.60, 0.02, 0.35, 0.05);
-    ImageStats stats_window_top = image_stats(extract_box_reference(screen, window_top));
-    bool white_theme = stats_window_top.average.sum() > 500;
+bool SelectedSettingWatcher::is_white_theme(const ImageViewRGB32& screen){
+    ImageStats stats_window_background = image_stats(extract_box_reference(screen, m_representative_background));
+    bool white_theme = stats_window_background.average.sum() > 500;
     return white_theme;
 }
 

SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h

@@ -20,16 +20,20 @@ class SelectedSettingWatcher : public VisualInferenceCallback{
     SelectedSettingWatcher(
         ImageFloatBox selected_box,
         ImageFloatBox not_selected_box1,
-        ImageFloatBox not_selected_box2
+        ImageFloatBox not_selected_box2,
+        ImageFloatBox representative_background = {0.60, 0.02, 0.35, 0.05}
     );
     virtual ~SelectedSettingWatcher();
 
     virtual void make_overlays(VideoOverlaySet& items) const override;
 
+    bool is_white_theme(const ImageViewRGB32& screen);
+
     // return true if the area within the selected_box is highlighted, compared with the area within unselected_box
     // This compares the brightness of the selected_box with the unselected_box.
     // selected_box: the box where we expect the screen should be highlighted
     // not_selected_box 1 and 2: the boxes where we expect the screen should NOT be highlighted. These acts as the control, for comparison.
+    // representative_background: box around an area that would represent the background. for determining whether this is white or dark mode.
     // the average sum of selected_box should be greater than the absolute difference of average sum between unselected_box 1 and 2.
     virtual bool process_frame(const ImageViewRGB32& screen, WallClock timestamp) override;
 
@@ -38,6 +42,7 @@ class SelectedSettingWatcher : public VisualInferenceCallback{
     ImageFloatBox m_selected_box;
     ImageFloatBox m_not_selected_box1;
     ImageFloatBox m_not_selected_box2;
+    ImageFloatBox m_representative_background;
 };
 
 

SerialPrograms/Source/PokemonSV/Programs/Farming/PokemonSV_ClaimMysteryGift.cpp

@@ -4,6 +4,7 @@
  *
  */
 
+ #include "CommonTools/Async/InferenceRoutines.h"
 #include "CommonFramework/GlobalSettingsPanel.h"
 #include "CommonFramework/Notifications/ProgramNotifications.h"
 #include "CommonFramework/VideoPipeline/VideoFeed.h"
@@ -12,9 +13,12 @@
 #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h"
 #include "Pokemon/Pokemon_Strings.h"
 #include "PokemonSwSh/Inference/PokemonSwSh_IvJudgeReader.h"
+#include "NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h"
+#include "NintendoSwitch/Inference/NintendoSwitch_ConsoleTypeDetector.h"
 #include "PokemonSV/Inference/PokemonSV_MainMenuDetector.h"
 #include "PokemonSV/Inference/Overworld/PokemonSV_DirectionDetector.h"
 #include "PokemonSV/Programs/PokemonSV_GameEntry.h"
+#include "PokemonSV/Programs/PokemonSV_SaveGame.h"
 #include "PokemonSV/Programs/PokemonSV_MenuNavigation.h"
 #include "PokemonSV/Programs/PokemonSV_WorldNavigation.h"
 #include "PokemonSV/Programs/AutoStory/PokemonSV_AutoStory.h"
@@ -133,17 +137,68 @@ void ClaimMysteryGift::enter_mystery_gift_code(SingleSwitchProgramEnvironment& e
 }
 
 void ClaimMysteryGift::claim_mystery_gift(SingleSwitchProgramEnvironment& env, ProControllerContext& context, int menu_index){
-
-    enter_menu_from_overworld(env.program_info(), env.console, context, menu_index);
-    pbf_press_button(context, BUTTON_A, 20, 4 * TICKS_PER_SECOND);
-    pbf_press_dpad(context, DPAD_UP, 20, 105);
-    pbf_press_button(context, BUTTON_A, 20, 4 * TICKS_PER_SECOND);
-    pbf_press_dpad(context, DPAD_DOWN, 20, 105);
-    pbf_press_button(context, BUTTON_A, 20, 4 * TICKS_PER_SECOND);
-    pbf_press_button(context, BUTTON_A, 20, 10 * TICKS_PER_SECOND);
-    clear_dialog(env.console, context, ClearDialogMode::STOP_TIMEOUT, 10);  
-
-    enter_mystery_gift_code(env, context);
+    save_game_from_menu_or_overworld(env.program_info(), env.console, context, false);
+
+    size_t max_attempts = 5;
+    for (size_t i = 0; i < max_attempts; i++){
+        enter_menu_from_overworld(env.program_info(), env.console, context, menu_index);
+        pbf_press_button(context, BUTTON_A, 20, 4 * TICKS_PER_SECOND);
+        pbf_press_dpad(context, DPAD_UP, 20, 105);
+        pbf_press_button(context, BUTTON_A, 20, 4 * TICKS_PER_SECOND);
+        pbf_press_dpad(context, DPAD_DOWN, 20, 105);
+        pbf_press_button(context, BUTTON_A, 20, 4 * TICKS_PER_SECOND);
+        pbf_press_button(context, BUTTON_A, 20, 10 * TICKS_PER_SECOND);
+        clear_dialog(env.console, context, ClearDialogMode::STOP_TIMEOUT, 10, {CallbackEnum::PROMPT_DIALOG});
+
+        context.wait_for_all_requests();
+        context.wait_for(Milliseconds(300));
+        // we expect to be within Mystery Gift window, with the keyboard visible and "1" being highlighted
+        // 
+
+        ConsoleType console_type = env.console.state().console_type();
+                
+        if (console_type == ConsoleType::Unknown){
+            env.console.log("Unknown Switch type. Try to detect.");
+            console_type = detect_console_type_from_in_game(env.console, context);
+        }
+
+        ImageFloatBox key1_box; 
+        ImageFloatBox other_setting1; 
+        ImageFloatBox other_setting2; 
+        ImageFloatBox background; 
+
+        if (is_switch1(console_type)){
+            key1_box = {0.037322, 0.451172, 0.009879, 0.113281};
+            other_setting1 = {0.01, 0.451172, 0.009879, 0.113281};
+            other_setting2 = {0.02, 0.451172, 0.009879, 0.113281};
+            background = {0.0, 0.451172, 0.009879, 0.113281};
+        }else if (is_switch2(console_type)){
+            key1_box = {0.062706, 0.510763, 0.009901, 0.097847};
+            other_setting1 = {0.02, 0.510763, 0.009901, 0.097847};
+            other_setting2 = {0.04, 0.510763, 0.009901, 0.097847};   
+            background = {0.0, 0.510763, 0.009901, 0.097847};
+        }else{
+            throw UserSetupError(
+                env.console,
+                "Please select a valid Switch console type."
+            );
+        }        
+        
+        SelectedSettingWatcher key1_selected(key1_box, other_setting1, other_setting2, background);
+        int ret = wait_until(
+            env.console, context,
+            Milliseconds(5000),
+            {key1_selected}
+        );
+        if (ret < 0){  // failed to detect Key 1 being highlighted. Reset game and re-try
+            env.console.log("claim_mystery_gift: Failed to detect the Mystery Gift window. Reset game and re-try.", COLOR_YELLOW);
+            reset_game(env.program_info(), env.console, context);
+            continue;
+        }       
+
+        enter_mystery_gift_code(env, context);
+        return;
+    }
 }
 
 void ClaimMysteryGift::run_autostory_until_pokeportal_unlocked(SingleSwitchProgramEnvironment& env, ProControllerContext& context){