Skip to content

Commit adf6dd3

Browse files
jw098jw098
committed
create SelectedSettingWatcher
1 parent 623c9b0 commit adf6dd3

File tree

5 files changed

+167
-63
lines changed

5 files changed

+167
-63
lines changed

‎SerialPrograms/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -957,6 +957,8 @@ file(GLOB MAIN_SOURCES
957957
Source/NintendoSwitch/Inference/NintendoSwitch_DateReader.h
958958
Source/NintendoSwitch/Inference/NintendoSwitch_DetectHome.cpp
959959
Source/NintendoSwitch/Inference/NintendoSwitch_DetectHome.h
960+
Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.cpp
961+
Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h
960962
Source/NintendoSwitch/NintendoSwitch_ConsoleHandle.cpp
961963
Source/NintendoSwitch/NintendoSwitch_ConsoleHandle.h
962964
Source/NintendoSwitch/NintendoSwitch_MultiSwitchProgram.cpp

‎SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.cpp‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/* Detect Home
2+
*
3+
* From: https://github.com/PokemonAutomation/
4+
*
5+
*/
6+
7+
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
8+
#include "CommonTools/Images/SolidColorTest.h"
9+
#include "NintendoSwitch_SelectedSettingDetector.h"
10+
11+
#include <iostream>
12+
using std::cout;
13+
using std::endl;
14+
15+
namespace PokemonAutomation{
16+
namespace NintendoSwitch{
17+
18+
SelectedSettingWatcher::~SelectedSettingWatcher() = default;
19+
20+
SelectedSettingWatcher::SelectedSettingWatcher(ImageFloatBox selected_box, ImageFloatBox not_selected_box1, ImageFloatBox not_selected_box2)
21+
: VisualInferenceCallback("SelectedSettingWatcher")
22+
, m_selected_box(selected_box)
23+
, m_not_selected_box1(not_selected_box1)
24+
, m_not_selected_box2(not_selected_box2)
25+
{}
26+
27+
void SelectedSettingWatcher::make_overlays(VideoOverlaySet& items) const{
28+
items.add(COLOR_RED, m_selected_box);
29+
items.add(COLOR_BLUE, m_not_selected_box1);
30+
items.add(COLOR_BLUE, m_not_selected_box2);
31+
}
32+
33+
bool is_white_theme(const ImageViewRGB32& screen){
34+
ImageFloatBox window_top(0.60, 0.02, 0.35, 0.05);
35+
ImageStats stats_window_top = image_stats(extract_box_reference(screen, window_top));
36+
bool white_theme = stats_window_top.average.sum() > 600;
37+
return white_theme;
38+
}
39+
40+
bool SelectedSettingWatcher::process_frame(const ImageViewRGB32& screen, WallClock timestamp){
41+
42+
ImageStats stats_unselected_box1 = image_stats(extract_box_reference(screen, m_not_selected_box1));
43+
double unselected1_average_sum = stats_unselected_box1.average.sum();
44+
cout << "unselected_average_sum1: " << std::to_string(unselected1_average_sum) << endl;
45+
46+
ImageStats stats_unselected_box2 = image_stats(extract_box_reference(screen, m_not_selected_box2));
47+
double unselected2_average_sum = stats_unselected_box2.average.sum();
48+
cout << "unselected_average_sum2: " << std::to_string(unselected2_average_sum) << endl;
49+
50+
double average_sum_unselected_diff = std::abs(unselected1_average_sum - unselected2_average_sum);
51+
52+
ImageStats stats_selected_box = image_stats(extract_box_reference(screen, m_selected_box));
53+
double selected_average_sum = stats_selected_box.average.sum();
54+
cout << "selected_average_sum: " << std::to_string(selected_average_sum) << endl;
55+
56+
bool is_selected = false;
57+
if (is_white_theme(screen)){ // light mode
58+
// unselected should be brighter than selected
59+
is_selected = selected_average_sum < std::min(unselected1_average_sum, unselected2_average_sum) - average_sum_unselected_diff - 20 ;
60+
}else{ // dark mode
61+
// selected should be brighter than unselected
62+
is_selected = selected_average_sum > std::max(unselected1_average_sum, unselected2_average_sum) + average_sum_unselected_diff + 20;
63+
}
64+
65+
return is_selected;
66+
}
67+
68+
69+
70+
71+
72+
73+
}
74+
}

‎SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/* Selected Setting Detector
2+
*
3+
* From: https://github.com/PokemonAutomation/
4+
*
5+
*/
6+
7+
#ifndef PokemonAutomation_NintendoSwitch_SelectedSettingDetector_H
8+
#define PokemonAutomation_NintendoSwitch_SelectedSettingDetector_H
9+
10+
#include "Common/Cpp/Color.h"
11+
#include "CommonFramework/ImageTools/ImageBoxes.h"
12+
#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h"
13+
#include "CommonTools/VisualDetector.h"
14+
15+
namespace PokemonAutomation{
16+
namespace NintendoSwitch{
17+
18+
19+
class SelectedSettingWatcher : public VisualInferenceCallback{
20+
public:
21+
SelectedSettingWatcher(ImageFloatBox selected_box, ImageFloatBox not_selected_box1, ImageFloatBox not_selected_box2);
22+
virtual ~SelectedSettingWatcher();
23+
24+
virtual void make_overlays(VideoOverlaySet& items) const override;
25+
26+
// return true if the area within the selected_box is highlighted, compared with the area within unselected_box
27+
// This compares the brightness of the selected_box with the unselected_box.
28+
// selected_box: the box where we expect the screen should be highlighted
29+
// not_selected_box 1 and 2: the boxes where we expect the screen should NOT be highlighted. These acts as the control, for comparison.
30+
// the average sum of selected_box should be greater than the absolute difference of average sum between unselected_box 1 and 2.
31+
virtual bool process_frame(const ImageViewRGB32& screen, WallClock timestamp) override;
32+
33+
34+
protected:
35+
ImageFloatBox m_selected_box;
36+
ImageFloatBox m_not_selected_box1;
37+
ImageFloatBox m_not_selected_box2;
38+
};
39+
40+
41+
}
42+
}
43+
#endif
44+

‎SerialPrograms/Source/NintendoSwitch/Programs/NintendoSwitch_Navigation.cpp‎

Lines changed: 46 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
#include "CommonFramework/Exceptions/OperationFailedException.h"
99
#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h"
1010
#include "NintendoSwitch/Commands/NintendoSwitch_Commands_Superscalar.h"
11+
#include "NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h"
12+
#include "CommonTools/Async/InferenceRoutines.h"
1113
#include "CommonFramework/ImageTools/ImageStats.h"
1214
#include "CommonFramework/VideoPipeline/VideoFeed.h"
1315
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
@@ -72,6 +74,8 @@ bool is_setting_selected(VideoStream& stream, ProControllerContext& context, Ima
7274
}
7375

7476
void home_to_date_time(VideoStream& stream, ProControllerContext& context, bool to_date_change, bool fast){
77+
size_t max_attempts = 5;
78+
for (size_t i = 0; i < max_attempts; i++){
7579
switch (context->performance_class()){
7680
case ControllerPerformanceClass::SerialPABotBase_Wired_125Hz:{
7781
ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 4);
@@ -84,31 +88,9 @@ void home_to_date_time(VideoStream& stream, ProControllerContext& context, bool
8488

8589
ssf_issue_scroll(context, SSF_SCROLL_LEFT, 0);
8690

87-
// we expect to be at Settings icon
88-
ImageFloatBox system_settings(0.685, 0.69, 0.05, 0.03);
89-
ImageFloatBox other_setting1(0.615, 0.69, 0.05, 0.03);
90-
ImageFloatBox other_setting2(0.545, 0.69, 0.05, 0.03);
91-
if (!is_setting_selected(stream, context, system_settings, other_setting1, other_setting2)){
92-
// not at Settings Icon
93-
// mash down, mash right. then left one
94-
size_t iterations = Milliseconds(1200) / 24ms + 1;
95-
for (size_t i = 0; i < iterations; i++){
96-
ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms);
97-
}
98-
99-
for (size_t i = 0; i < iterations; i++){
100-
ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 24ms);
101-
}
102-
ssf_issue_scroll(context, SSF_SCROLL_LEFT, 0);
103-
104-
if (!is_setting_selected(stream, context, system_settings, other_setting1, other_setting2)){
105-
OperationFailedException::fire(
106-
ErrorReport::SEND_ERROR_REPORT,
107-
"home_to_date_time(): Failed to reach settings gear on Home page after 2 attempts.",
108-
stream
109-
);
110-
}
111-
}
91+
// ImageFloatBox system_icon(0.685, 0.69, 0.05, 0.03);
92+
// ImageFloatBox other_setting1(0.615, 0.69, 0.05, 0.03);
93+
// ImageFloatBox other_setting2(0.545, 0.69, 0.05, 0.03);
11294

11395
// Two A presses in case we drop the 1st one.
11496
ssf_press_button(context, BUTTON_A, 3);
@@ -122,6 +104,25 @@ void home_to_date_time(VideoStream& stream, ProControllerContext& context, bool
122104
}while (--iterations);
123105
}
124106

107+
// Should now be in System Settings, with System highlighted
108+
ImageFloatBox system_setting_box(0.056, 0.74, 0.01, 0.1);
109+
ImageFloatBox other_setting1(0.04, 0.74, 0.01, 0.1);
110+
ImageFloatBox other_setting2(0.02, 0.74, 0.01, 0.1);
111+
SelectedSettingWatcher system_setting(system_setting_box, other_setting1, other_setting2);
112+
int ret = run_until<ProControllerContext>(
113+
stream, context,
114+
[](ProControllerContext& context){
115+
for (int i = 0; i < 10; i++){
116+
ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms);
117+
}
118+
},
119+
{system_setting}
120+
);
121+
if (ret < 0){ // failed to detect System highlighted. press home and re-try
122+
pbf_press_button(context, BUTTON_HOME, 100ms, 100ms);
123+
continue;
124+
}
125+
125126
// Scroll left and press A to exit the sleep menu if we happened to
126127
// land there.
127128
ssf_issue_scroll(context, SSF_SCROLL_LEFT, 3);
@@ -148,39 +149,20 @@ void home_to_date_time(VideoStream& stream, ProControllerContext& context, bool
148149

149150
context.wait_for_all_requests();
150151
// we expect to be within "Date and Time", with "Synchronize Clock via Internet" being highlighted
151-
ImageFloatBox sync_clock(0.168, 0.185, 0.01, 0.1);
152+
ImageFloatBox sync_clock_box(0.168, 0.185, 0.01, 0.1);
152153
ImageFloatBox other_setting3(0.1, 0.185, 0.01, 0.1);
153154
ImageFloatBox other_setting4(0.05, 0.185, 0.01, 0.1);
154-
if (!is_setting_selected(stream, context, sync_clock, other_setting3, other_setting4)){
155-
// press B once, then mash Up. then try again to scroll down to Date and Time
156-
pbf_press_button(context, BUTTON_B, 100ms, 100ms);
157-
pbf_move_left_joystick(context, 128, 0, 3000ms, 100ms);
158-
159-
size_t iterations = Milliseconds(1200) / 24ms + 1;
160-
for (size_t i = 0; i < iterations; i++){
161-
ssf_issue_scroll(context, SSF_SCROLL_UP, 24ms);
162-
}
163-
164-
ssf_issue_scroll(context, SSF_SCROLL_DOWN, 3);
165-
ssf_issue_scroll(context, SSF_SCROLL_DOWN, 3);
166-
ssf_issue_scroll(context, SSF_SCROLL_DOWN, 10);
167-
ssf_press_dpad(context, DPAD_DOWN, 45, 40);
168-
ssf_issue_scroll(context, SSF_SCROLL_DOWN, 3);
169-
ssf_issue_scroll(context, SSF_SCROLL_DOWN, 3);
170-
171-
// double up this A press to make sure it gets through.
172-
ssf_press_button(context, BUTTON_A, 3);
173-
ssf_press_button(context, BUTTON_A, 3);
174-
175-
if (!is_setting_selected(stream, context, sync_clock, other_setting3, other_setting4)){
176-
OperationFailedException::fire(
177-
ErrorReport::SEND_ERROR_REPORT,
178-
"home_to_date_time(): Failed to reach 'Synchronize Clock via Internet', within 'Date and Time', after 2 attempts.",
179-
stream
180-
);
181-
}
155+
SelectedSettingWatcher sync_clock(sync_clock_box, other_setting3, other_setting4);
156+
ret = wait_until(
157+
stream, context,
158+
Milliseconds(2000),
159+
{sync_clock}
160+
);
161+
if (ret < 0){ // failed to detect System highlighted. press home and re-try
162+
pbf_press_button(context, BUTTON_HOME, 100ms, 100ms);
163+
continue;
164+
}
182165

183-
}
184166

185167
if (!to_date_change){
186168
return;
@@ -198,7 +180,7 @@ void home_to_date_time(VideoStream& stream, ProControllerContext& context, bool
198180
// confirmation menus.
199181
ssf_issue_scroll(context, SSF_SCROLL_LEFT, 0ms);
200182

201-
break;
183+
return;
202184
}
203185
case ControllerPerformanceClass::SerialPABotBase_Wireless_ESP32:{
204186
Milliseconds tv = context->timing_variation();
@@ -269,7 +251,7 @@ void home_to_date_time(VideoStream& stream, ProControllerContext& context, bool
269251
// confirmation menus.
270252
ssf_issue_scroll(context, SSF_SCROLL_LEFT, 0ms, 2*unit, unit);
271253

272-
break;
254+
return;
273255
}
274256
default:{
275257
// Slow version for tick-imprecise controllers.
@@ -308,10 +290,16 @@ void home_to_date_time(VideoStream& stream, ProControllerContext& context, bool
308290
ssf_press_button_ptv(context, BUTTON_A, 1000ms);
309291
ssf_issue_scroll_ptv(context, SSF_SCROLL_DOWN);
310292
ssf_issue_scroll_ptv(context, SSF_SCROLL_DOWN);
293+
return;
294+
}
311295
}
312296
}
313297

314-
298+
OperationFailedException::fire(
299+
ErrorReport::SEND_ERROR_REPORT,
300+
"home_to_date_time(): Failed to reach Date and Time after several attempts.",
301+
stream
302+
);
315303

316304
}
317305

‎SerialPrograms/Source/NintendoSwitch/Programs/NintendoSwitch_Navigation.h‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,7 @@ namespace NintendoSwitch{
1919
// assumes we are currently in the Switch Settings.
2020
bool is_white_theme(VideoStream& stream, ProControllerContext& context);
2121

22-
// return true if the area within the selected_box is highlighted, compared with the area within unselected_box
23-
// This compares the brightness of the selected_box with the unselected_box.
24-
// selected_box: the box where we expect the screen should be highlighted
25-
// unselected_box 1 and 2: the boxes where we expect the screen should NOT be highlighted. These acts as the control, for comparison.
26-
// the average sum of selected_box should be greater than the absolute difference of average sum between unselected_box 1 and 2.
22+
2723
bool is_setting_selected(VideoStream& stream, ProControllerContext& context, ImageFloatBox selected_box, ImageFloatBox unselected_box1, ImageFloatBox unselected_box2);
2824

2925
void home_to_date_time(VideoStream& stream, ProControllerContext& context, bool to_date_change, bool fast);

0 commit comments

Comments
 (0)