more dialog infra, improve audio starter reset

FeedbackType update, finally got audio working

Author: kichithewolf

SerialPrograms/Source/CommonFramework/Globals.h

@@ -69,8 +69,8 @@ enum class FeedbackType{
     NONE,
     OPTIONAL_,  //  Naming conflict with macro.
     REQUIRED,
-    AUDIO,
     VIDEO_AUDIO,
+    VIDEO_AUDIO_GBA,
 };
 
 

SerialPrograms/Source/CommonFramework/Panels/UI/PanelElements.cpp

@@ -80,13 +80,8 @@ CollapsibleGroupBox* make_panel_header(
             header
         );
         break;
-    case FeedbackType::AUDIO:
-        text = new QLabel(
-            "<font color=\"green\">(This program requires audio feedback. Please make sure you choose the correct audio device.)</font>",
-            header
-        );
-        break;
     case FeedbackType::VIDEO_AUDIO:
+    case FeedbackType::VIDEO_AUDIO_GBA:
         text = new QLabel(
             "<font color=\"green\">(This program requires video and audio feedback. Please make sure you choose the correct capture device, as well as the correct audio device.)</font>",
             header

SerialPrograms/Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.cpp

@@ -5,9 +5,18 @@
  */
 
 #include "CommonFramework/ImageTools/SolidColorTest.h"
+#include "CommonFramework/ImageTools/ImageBoxes.h"
+#include "CommonFramework/ImageTools/ImageFilter.h"
+#include "CommonFramework/ImageTypes/ImageRGB32.h"
+#include "CommonFramework/ImageTools/ImageStats.h"
+#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
 #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
 #include "PokemonRSE_DialogDetector.h"
 
+#include <iostream>
+using std::cout;
+using std::endl;
+
 namespace PokemonAutomation{
 namespace NintendoSwitch{
 namespace PokemonRSE{
@@ -29,11 +38,11 @@ bool DialogDetector::detect(const ImageViewRGB32& screen) const{
     }
     return false;
 }
-
+*/
 
 BattleDialogDetector::BattleDialogDetector(Color color)
-    : m_left_box(0.155, 0.727, 0.015, 0.168)
-    , m_right_box(0.837, 0.729, 0.008, 0.161)
+    : m_left_box(0.158, 0.725, 0.011, 0.176)
+    , m_right_box(0.827, 0.722, 0.013, 0.178)
 {}
 void BattleDialogDetector::make_overlays(VideoOverlaySet& items) const{
     items.add(COLOR_RED, m_left_box);
@@ -42,12 +51,12 @@ void BattleDialogDetector::make_overlays(VideoOverlaySet& items) const{
 bool BattleDialogDetector::detect(const ImageViewRGB32& screen) const{
     ImageViewRGB32 left_image = extract_box_reference(screen, m_left_box);
     ImageViewRGB32 right_image = extract_box_reference(screen, m_right_box);
-    if (is_solid(left_image, { 0.25, 0.38, 0.369 }) && is_solid(right_image, { 0.25, 0.38, 0.369 })){
+    if (is_solid(left_image, { 0.335, 0.331, 0.332 }) && is_solid(right_image, { 0.335, 0.331, 0.332 })){
         return true;
     }
     return false;
 }
-*/
+
 
 BattleMenuDetector::BattleMenuDetector(Color color)
     : m_left_box(0.155, 0.727, 0.015, 0.168)
@@ -67,6 +76,36 @@ bool BattleMenuDetector::detect(const ImageViewRGB32& screen) const{
 }
 
 
+AdvanceDialogDetector::AdvanceDialogDetector(Color color)
+    : m_dialog_box(0.156, 0.715, 0.686, 0.193)
+{}
+void AdvanceDialogDetector::make_overlays(VideoOverlaySet& items) const{
+    items.add(COLOR_RED, m_dialog_box);
+}
+bool AdvanceDialogDetector::detect(const ImageViewRGB32& screen) const{
+    const bool replace_color_within_range = false;
+
+    //Filter out background
+    ImageRGB32 filtered_region = filter_rgb32_range(
+        extract_box_reference(screen, m_dialog_box),
+        combine_rgb(185, 0, 1), combine_rgb(255, 32, 33), Color(0), replace_color_within_range
+    );
+    ImageStats stats = image_stats(filtered_region);
+
+    /*
+    filtered_region.save("./filtered_only.png");
+    cout << stats.average.r << endl;
+    cout << stats.average.g << endl;
+    cout << stats.average.b << endl;
+    */
+
+    if ((stats.average.r > stats.average.b + 200) && (stats.average.r > stats.average.g + 200)){
+        return true;
+    }
+    return false;
+}
+
+
 
 }
 }

SerialPrograms/Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.h

@@ -41,8 +41,7 @@ class DialogWatcher : public DetectorToFinder<DialogDetector>{
         : DetectorToFinder("DialogWatcher", std::chrono::milliseconds(250), color)
     {}
 };
-
-
+*/
 
 // Battle dialog boxes are teal
 class BattleDialogDetector : public StaticScreenDetector{
@@ -62,10 +61,9 @@ class BattleDialogWatcher : public DetectorToFinder<BattleDialogDetector>{
         : DetectorToFinder("BattleDialogWatcher", std::chrono::milliseconds(250), color)
     {}
 };
-*/
 
 
-// Battle menu is up when it is white on the left and teal on the right
+// Battle menu is up when it is white on the right and teal on the left
 class BattleMenuDetector : public StaticScreenDetector{
 public:
     BattleMenuDetector(Color color);
@@ -86,10 +84,27 @@ class BattleMenuWatcher : public DetectorToFinder<BattleMenuDetector>{
 
 
 
-// advancedialogdetector Detect that the dialog arrow is in the dialog box by filtering for the red arrow
+// Detect the red advancement arrow by filtering for red.
+// This works for now, I don't think there's colored text?
+// TODO: Change this to detect that the dialog arrow is in the dialog box by filtering for the red arrow
+class AdvanceDialogDetector : public StaticScreenDetector{
+public:
+    AdvanceDialogDetector(Color color);
 
-// when given a choice popup, there is no advance arrow
+    virtual void make_overlays(VideoOverlaySet& items) const override;
+    virtual bool detect(const ImageViewRGB32& screen) const override;
 
+private:
+    ImageFloatBox m_dialog_box;
+};
+class AdvanceDialogWatcher : public DetectorToFinder<AdvanceDialogDetector>{
+public:
+    AdvanceDialogWatcher(Color color)
+        : DetectorToFinder("AdvanceDialogWatcher", std::chrono::milliseconds(250), color)
+    {}
+};
+
+// when given a choice popup, there is no advance arrow
 
 
 }

SerialPrograms/Source/PokemonRSE/Inference/PokemonRSE_ShinyNumberDetector.cpp

@@ -13,18 +13,20 @@
 #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
 #include "PokemonRSE_ShinyNumberDetector.h"
 
-#include <iostream>
-using std::cout;
-using std::endl;
+//#include <iostream>
+//using std::cout;
+//using std::endl;
 
 namespace PokemonAutomation{
 namespace NintendoSwitch{
 namespace PokemonRSE{
 
-ShinyNumberDetector::ShinyNumberDetector()
+ShinyNumberDetector::ShinyNumberDetector(Color color)
     : m_box_number(0.136, 0.156, 0.123, 0.072)
 {}
-
+void ShinyNumberDetector::make_overlays(VideoOverlaySet& items) const{
+    items.add(COLOR_RED, m_box_number);
+}
 bool ShinyNumberDetector::read(Logger& logger, const ImageViewRGB32& frame){
     const bool replace_color_within_range = true;
 

SerialPrograms/Source/PokemonRSE/Inference/PokemonRSE_ShinyNumberDetector.h

@@ -19,8 +19,9 @@ namespace PokemonRSE{
 // Number is easier to check as background as scan lines.
 class ShinyNumberDetector{
 public:
-    ShinyNumberDetector();
+    ShinyNumberDetector(Color color);
 
+    virtual void make_overlays(VideoOverlaySet& items) const;
     bool read(Logger& logger, const ImageViewRGB32& frame);
 
 private:

SerialPrograms/Source/PokemonRSE/PokemonRSE_Panels.cpp

@@ -34,12 +34,10 @@ std::vector<PanelEntry> PanelListFactory::make_panels() const{
 
     ret.emplace_back("---- Shiny Hunting ----");
     ret.emplace_back(make_single_switch_program<StarterReset_Descriptor, StarterReset>());
+    ret.emplace_back(make_single_switch_program<AudioStarterReset_Descriptor, AudioStarterReset>());
 
 
     if (PreloadSettings::instance().DEVELOPER_MODE){
-        ret.emplace_back("---- WIP: Shiny Hunting (Audio only) ----");
-        ret.emplace_back(make_single_switch_program<AudioStarterReset_Descriptor, AudioStarterReset>());
-
         ret.emplace_back("---- Developer Tools ----");
         ret.emplace_back(make_single_switch_program<SoundListener_Descriptor, SoundListener>());
     }

SerialPrograms/Source/PokemonRSE/Programs/ShinyHunting/PokemonRSE_AudioStarterReset.cpp

@@ -12,6 +12,7 @@
 #include "CommonFramework/Tools/StatsTracking.h"
 #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h"
 #include "NintendoSwitch/Commands/NintendoSwitch_Commands_Superscalar.h"
+#include "PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.h"
 #include "PokemonRSE/Inference/Sounds/PokemonRSE_ShinySoundDetector.h"
 #include "PokemonRSE/PokemonRSE_Navigation.h"
 #include "PokemonRSE_AudioStarterReset.h"
@@ -23,10 +24,10 @@ namespace PokemonRSE{
 AudioStarterReset_Descriptor::AudioStarterReset_Descriptor()
     : SingleSwitchProgramDescriptor(
         "PokemonRSE:AudioStarterReset",
-        "Pokemon RSE", "[RS] Starter Reset - Audio only",
+        "Pokemon RSE", "[RS] Starter Reset",
         "ComputerControl/blob/master/Wiki/Programs/PokemonRSE/AudioStarterReset.md",
-        "Soft reset for a shiny starter. Ruby and Sapphire only. WIP, audio recognition does not work well.",
-        FeedbackType::AUDIO,
+        "Soft reset for a shiny starter. Ruby and Sapphire only.",
+        FeedbackType::VIDEO_AUDIO_GBA,
         AllowCommandsWhenRunning::DISABLE_COMMANDS,
         PABotBaseLevel::PABOTBASE_12KB
     )
@@ -101,8 +102,12 @@ void AudioStarterReset::program(SingleSwitchProgramEnvironment& env, BotBaseCont
 
     /*
     * Settings: Text Speed fast.
+    * Full screen, no filter? The device I'm using to test has similar looking output, but I don't have switch online+.
+    * If on a retro handheld, make sure the screen matches that of NSO+ and that there is an overlay to avoid the black border check.
+    * 
     * Setup: Stand in front of the Professor's bag and save the game.
     * 
+    * 
     * Required to fight, so have to do the SR method instead of run away
     * Soft reset programs are only for Ruby/Sapphire, as Emerald has the 0 seed issue.
     * 
@@ -137,29 +142,33 @@ void AudioStarterReset::program(SingleSwitchProgramEnvironment& env, BotBaseCont
             break;
         }
         pbf_mash_button(context, BUTTON_A, 540);
+        context.wait_for_all_requests();
         env.log("Starter selected. Checking for shiny Poochyena.");
 
-
-        int ret = run_until(
+        AdvanceDialogWatcher pooch_appeared(COLOR_YELLOW);
+        int res = run_until(
             env.console, context,
-            [&](BotBaseContext& context){
-                //Wait for battle to start and for Pooch battle cry
-                pbf_wait(context, POOCH_WAIT);
-
-                context.wait_for_all_requests();
-
+            [&](BotBaseContext& context) {
+                int ret = wait_until(
+                    env.console, context,
+                    std::chrono::seconds(20),
+                    {{pooch_detector}}
+                );
+                pooch_detector.throw_if_no_sound();
+                if (ret == 0){
+                    env.log("Shiny Poochyena detected!");
+                    stats.poochyena++;
+                    send_program_status_notification(env, NOTIFICATION_SHINY_POOCH, "Shiny Poochyena found.");
+                }
+                else {
+                    env.log("Poochyena is not shiny.");
+                }
             },
-            {{pooch_detector}}
+            {{pooch_appeared}}
         );
-        pooch_detector.throw_if_no_sound();
-        if (ret == 0){
-            env.log("Shiny Poochyena detected!");
-            stats.poochyena++;
-            send_program_status_notification(env, NOTIFICATION_SHINY_POOCH, "Shiny Poochyena found.");
-        }
-        else {
-            env.log("Poochyena is not shiny.");
-        }
+        if (res == 0) {
+            env.log("Advance arrow detected. Pressing A.");
+        } //res != if pooch is shiny
 
         ShinySoundDetector starter_detector(env.console, [&](float error_coefficient) -> bool{
             return true;
@@ -201,7 +210,7 @@ void AudioStarterReset::program(SingleSwitchProgramEnvironment& env, BotBaseCont
         stats.resets++;
     }
 
-    //TODO: if system set to nintendo switch, have go home when done option
+    //if system set to nintendo switch, have go home when done option?
 
     send_program_finished_notification(env, NOTIFICATION_PROGRAM_FINISH);
 }

SerialPrograms/Source/PokemonRSE/Programs/ShinyHunting/PokemonRSE_StarterReset.cpp

@@ -25,10 +25,10 @@ namespace PokemonRSE{
 StarterReset_Descriptor::StarterReset_Descriptor()
     : SingleSwitchProgramDescriptor(
         "PokemonRSE:StarterReset",
-        "Pokemon RSE", "[RS] Starter Reset",
+        "Pokemon RSE", "[RS] Starter Reset - Video only",
         "ComputerControl/blob/master/Wiki/Programs/PokemonRSE/StarterReset.md",
         "Soft reset for a shiny starter. Ruby and Sapphire only.",
-        FeedbackType::REQUIRED,
+        FeedbackType::VIDEO_AUDIO_GBA,
         AllowCommandsWhenRunning::DISABLE_COMMANDS,
         PABotBaseLevel::PABOTBASE_12KB
     )
@@ -88,6 +88,9 @@ void StarterReset::program(SingleSwitchProgramEnvironment& env, BotBaseContext&
 
     /*
     * Settings: Text Speed fast.
+    * Full screen, no filter? The device I'm using to test has similar looking output, but I don't have switch online+.
+    * If on a retro handheld, make sure the screen matches that of NSO+ and that there is an overlay to avoid the black border check.
+    * 
     * Setup: Stand in front of the Professor's bag and save the game.
     * 
     * Required to fight, so have to do the SR method instead of run away
@@ -147,13 +150,14 @@ void StarterReset::program(SingleSwitchProgramEnvironment& env, BotBaseContext&
         pbf_press_dpad(context, DPAD_DOWN, 40, 80);
         pbf_press_button(context, BUTTON_A, 40, 80);
 
+        //Check second party member - used for testing with hacked in shiny starter
         //pbf_press_dpad(context, DPAD_DOWN, 40, 80);
 
         pbf_wait(context, 125);
         context.wait_for_all_requests();
 
         VideoSnapshot screen = env.console.video().snapshot();
-        ShinyNumberDetector shiny_checker;
+        ShinyNumberDetector shiny_checker(COLOR_YELLOW);
         shiny_starter = shiny_checker.read(env.console.logger(), screen);
 
         if (shiny_starter) {
@@ -173,7 +177,7 @@ void StarterReset::program(SingleSwitchProgramEnvironment& env, BotBaseContext&
         }
     }
 
-    //TODO: if system set to nintendo switch, have go home when done option
+    //if system set to nintendo switch, have go home when done option?
 
     send_program_finished_notification(env, NOTIFICATION_PROGRAM_FINISH);
 }