Autostory: Player/Camera movement based on YOLO. Arrive at Area Zero Station 1. (#816)

* add extra segments/checkpoints

* checkpoint 93: Entered Area Zero Gate. Flew down to Area Zero from the platform.

* minor update to checkpoint 93

* add functions that move player forward and move the camera, based on yolo object detection

* for routines that move the player/camera based on yolo, throw exception if caught in battle. adjust scale factors for move_camera_yolo

* add TEST_YOLO_BOX

* add bag detection for run_battle_press_A()

* clear YOLO overlays when caught in battle

* add move_forward_until_yolo_object_not_detected(). change color of overlays if it's the target object.

* add do_action_and_monitor_for_battles_early(), which uses NoMinimapWatcher

* add times_not_seen_threshold to move_forward_until_yolo_object_not_detected()

* partial checkpoint_94: arrive at rock-3

* add move_player_to_realign_via_yolo()

* adjust recovery routine after battle to use move_player_to_realign_via_yolo()

* add move_camera_until_yolo_object_detected()

* add do_action_until_dialog(). adjust move_player_forward() to include a recovery_action

* partial checkpoint_94: arrive at station-1 dialog

* add recovery action to move_camera_yolo()

* minor tweaks to station 1 navigation routine.

* adjust close_game_from_home()

* more changes to station 1 nav routine

* adjust target x values when recovering from battles within move_camera_yolo: X.

* when moving forward, don't pause to use Let's Go.

* more changes to station 1 nav routine

* complete checkpoint_94

* adjust recovery_action for move_camera_yolo: X. Don't move the player after a battle. just adjust the camera.

* more adjustments to checkpoint_94

* yolo-based camera/player movement routines: rename variables. throw exception if never detected yolo object. throw exception if exceed max rounds. reduce delays after joystick movement.

* more checkpoint_94 adjustments

* minor changes to close_game_from_home

* move_camera_yolo: increase max_attempts

* updated move_player_forward() and walk_forward_until_dialog()

* preliminary checkpoint 95: entering station 1

* more tidy up

Author: jw098

SerialPrograms/Source/NintendoSwitch/Programs/NintendoSwitch_GameEntry.cpp

@@ -70,6 +70,7 @@ void ensure_at_home(ConsoleHandle& console, ControllerContext& context){
             {home_menu}
         );
         if (ret == 0){
+            console.log("Home detected.");
             //  While we're on the Home screen, we might as well read the
             //  console type as well.
             if (console.state().console_type_confirmed()){
@@ -104,6 +105,7 @@ void ensure_at_home(ConsoleHandle& console, JoyconContext& context){
 //
 
 void close_game_from_home(ConsoleHandle& console, ProControllerContext& context){
+    console.log("close_game_from_home");
     ensure_at_home(console, context);
 
     //  Use mashing to ensure that the X press succeeds. If it fails, the SR
@@ -124,6 +126,7 @@ void close_game_from_home(ConsoleHandle& console, ProControllerContext& context)
     pbf_mash_button(context, BUTTON_B, 350);
 }
 void close_game_from_home(ConsoleHandle& console, JoyconContext& context){
+    console.log("close_game_from_home");
     ensure_at_home(console, context);
     //  Use mashing to ensure that the X press succeeds. If it fails, the SR
     //  will fail and can kill a den for the autohosts.

SerialPrograms/Source/PokemonSV/Programs/AutoStory/PokemonSV_AutoStory.cpp

@@ -3,6 +3,13 @@
  *  From: https://github.com/PokemonAutomation/
  *
  */
+
+
+#include "ML/Inference/ML_YOLOv5Detector.h"
+#include "CommonFramework/VideoPipeline/VideoFeed.h"
+#include "CommonFramework/Exceptions/UnexpectedBattleException.h"
+
+
 #include "CommonFramework/Exceptions/OperationFailedException.h"
 #include "CommonTools/Async/InferenceRoutines.h"
 
@@ -57,6 +64,11 @@
 #include "PokemonSV_AutoStory_Segment_33.h"
 #include "PokemonSV_AutoStory_Segment_34.h"
 #include "PokemonSV_AutoStory_Segment_35.h"
+#include "PokemonSV_AutoStory_Segment_36.h"
+#include "PokemonSV_AutoStory_Segment_37.h"
+#include "PokemonSV_AutoStory_Segment_38.h"
+#include "PokemonSV_AutoStory_Segment_39.h"
+#include "PokemonSV_AutoStory_Segment_40.h"
 #include "PokemonSV_AutoStory.h"
 
 #include <iostream>
@@ -70,6 +82,7 @@ namespace NintendoSwitch{
 namespace PokemonSV{
 
 using namespace Pokemon;
+using namespace ML;
 
 static constexpr size_t INDEX_OF_LAST_TUTORIAL_SEGMENT = 9;
 static constexpr size_t INDEX_OF_LAST_TUTORIAL_CHECKPOINT = 20;
@@ -112,8 +125,15 @@ std::vector<std::unique_ptr<AutoStory_Segment>> make_autoStory_segment_list(){
     segment_list.emplace_back(std::make_unique<AutoStory_Segment_32>());
     segment_list.emplace_back(std::make_unique<AutoStory_Segment_33>());
     segment_list.emplace_back(std::make_unique<AutoStory_Segment_34>());
-    // segment_list.emplace_back(std::make_unique<AutoStory_Segment_35>());
 
+    if (PreloadSettings::instance().DEVELOPER_MODE){
+    segment_list.emplace_back(std::make_unique<AutoStory_Segment_35>());
+    // segment_list.emplace_back(std::make_unique<AutoStory_Segment_36>());
+    // segment_list.emplace_back(std::make_unique<AutoStory_Segment_37>());
+    // segment_list.emplace_back(std::make_unique<AutoStory_Segment_38>());
+    // segment_list.emplace_back(std::make_unique<AutoStory_Segment_39>());
+    // segment_list.emplace_back(std::make_unique<AutoStory_Segment_40>());
+    }
     return segment_list;
 };
 
@@ -276,6 +296,15 @@ std::vector<std::unique_ptr<AutoStory_Checkpoint>> make_autoStory_checkpoint_lis
     checkpoint_list.emplace_back(std::make_unique<AutoStory_Checkpoint_91>());
     checkpoint_list.emplace_back(std::make_unique<AutoStory_Checkpoint_92>());
 
+    if (PreloadSettings::instance().DEVELOPER_MODE){
+    checkpoint_list.emplace_back(std::make_unique<AutoStory_Checkpoint_93>());
+    checkpoint_list.emplace_back(std::make_unique<AutoStory_Checkpoint_94>());
+    checkpoint_list.emplace_back(std::make_unique<AutoStory_Checkpoint_95>());
+    // checkpoint_list.emplace_back(std::make_unique<AutoStory_Checkpoint_96>());
+    // checkpoint_list.emplace_back(std::make_unique<AutoStory_Checkpoint_97>());
+    // checkpoint_list.emplace_back(std::make_unique<AutoStory_Checkpoint_98>());
+    // checkpoint_list.emplace_back(std::make_unique<AutoStory_Checkpoint_99>());
+    }
 
     return checkpoint_list;
 };
@@ -658,11 +687,34 @@ AutoStory::AutoStory()
         LockMode::UNLOCK_WHILE_RUNNING,
         0
     )
+    , TEST_YOLO_BOX(
+        "<b>TEST: get_yolo_box():</b>",
+        LockMode::UNLOCK_WHILE_RUNNING,
+        false
+    )
+    , YOLO_PATH(
+        false,
+        "<b>YOLO Path:</b>", 
+        LockMode::LOCK_WHILE_RUNNING, 
+        "PokemonSV/YOLO/yolo_area0_station1.onnx",
+        "<.onnx file>"
+    )
+    , TARGET_LABEL(
+        false,
+        "<b>YOLO Object Label:</b>", 
+        LockMode::LOCK_WHILE_RUNNING, 
+        "rock-1",
+        "<target label>"
+    )
 {
 
     if (PreloadSettings::instance().DEVELOPER_MODE){
         PA_ADD_OPTION(m_advanced_options);
 
+        PA_ADD_OPTION(TEST_YOLO_BOX);
+        PA_ADD_OPTION(YOLO_PATH);
+        PA_ADD_OPTION(TARGET_LABEL);
+
         PA_ADD_OPTION(FLYPOINT_TYPE);
         PA_ADD_OPTION(TEST_FLYPOINT_LOCATIONS);
         PA_ADD_OPTION(TEST_MOVE_CURSOR_OFFSET_FROM_FLYPOINT);
@@ -1137,6 +1189,18 @@ void AutoStory::run_autostory(SingleSwitchProgramEnvironment& env, ProController
 
 void AutoStory::test_code(SingleSwitchProgramEnvironment& env, ProControllerContext& context){
 
+
+    if (TEST_YOLO_BOX){
+        VideoOverlaySet overlays(env.console.overlay());
+        YOLOv5Detector yolo_detector(RESOURCE_PATH() + std::string(YOLO_PATH));
+        
+        ImageFloatBox target_box = get_yolo_box(env, context, overlays, yolo_detector, TARGET_LABEL);
+
+        context.wait_for(Milliseconds(1000));
+        return;
+    }
+
+
     if (TEST_FLYPOINT_LOCATIONS){
         print_flypoint_location(env.program_info(), env.console, context, FLYPOINT_TYPE);
         // print_flypoint_location(env.program_info(), env.console, context, FlyPoint::FAST_TRAVEL);
@@ -1200,6 +1264,11 @@ void AutoStory::test_code(SingleSwitchProgramEnvironment& env, ProControllerCont
         DirectionDetector direction;
 
 
+        YOLOv5Detector yolo_detector(RESOURCE_PATH() + "PokemonSV/YOLO/yolo_area0_station1.onnx");
+        // move_camera_yolo(env, context, CameraAxis::Y, yolo_detector, "tree-tera", 0.294444);
+        // move_camera_yolo(env, context, CameraAxis::X, yolo_detector, "tree-tera", 0.604688);
+
+
 
         return;
     }

SerialPrograms/Source/PokemonSV/Programs/AutoStory/PokemonSV_AutoStory.h

@@ -11,6 +11,7 @@
 #include "Common/Cpp/Options/SimpleIntegerOption.h"
 #include "Common/Cpp/Options/FloatingPointOption.h"
 #include "Common/Cpp/Options/EnumDropdownOption.h"
+#include "Common/Cpp/Options/StringOption.h"
 #include "CommonFramework/Notifications/EventNotificationsTable.h"
 #include "CommonTools/Options/StringSelectOption.h"
 #include "CommonTools/Options/LanguageOCROption.h"
@@ -147,6 +148,12 @@ class AutoStory : public SingleSwitchProgramInstance, public ConfigOption::Liste
     BooleanCheckBoxOption TEST_MOVE_CURSOR_OFFSET_FROM_FLYPOINT;
     FloatingPointOption X_OFFSET;
     FloatingPointOption Y_OFFSET;
+
+    BooleanCheckBoxOption TEST_YOLO_BOX;
+    StringOption YOLO_PATH;
+    StringOption TARGET_LABEL;
+    
+
 };
 
 const std::vector<std::unique_ptr<AutoStory_Segment>>& ALL_AUTO_STORY_SEGMENT_LIST();