Skip to content

Commit 5f44ba2

Browse files
author
Gin
committed
fix button dpad up down detector
1 parent c5c9666 commit 5f44ba2

File tree

4 files changed

+98
-62
lines changed

4 files changed

+98
-62
lines changed

‎SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_ButtonDetector.cpp‎

Lines changed: 65 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,15 @@
55
*/
66

77
#include "Kernels/Waterfill/Kernels_Waterfill_Types.h"
8+
#include "CommonFramework/GlobalSettingsPanel.h"
89
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
910
#include "CommonTools/Images/WaterfillUtilities.h"
1011
#include "CommonTools/ImageMatch/WaterfillTemplateMatcher.h"
1112
#include "PokemonLZA_ButtonDetector.h"
1213

13-
//#include <iostream>
14-
//using std::cout;
15-
//using std::endl;
14+
// #include <iostream>
15+
// using std::cout;
16+
// using std::endl;
1617

1718
namespace PokemonAutomation{
1819
namespace NintendoSwitch{
@@ -25,7 +26,7 @@ class ButtonMatcher : public ImageMatch::WaterfillTemplateMatcher{
2526
// image template matcher for buttons
2627
// - min_width: candidate image min width if video stream is 4k
2728
// - min_height: candidate image min height if video stream is 4k
28-
ButtonMatcher(ButtonType type, size_t min_width, size_t min_height, double max_rmsd);
29+
ButtonMatcher(ButtonType type, size_t min_width, size_t min_height, double max_rmsd, bool match_black_interior = false);
2930
static const ButtonMatcher& A(){
3031
static ButtonMatcher matcher(ButtonType::ButtonA, 50, 50, 70);
3132
return matcher;
@@ -74,12 +75,25 @@ class ButtonMatcher : public ImageMatch::WaterfillTemplateMatcher{
7475
static ButtonMatcher matcher(ButtonType::ButtonDpadDown, 30, 30, 80);
7576
return matcher;
7677
}
78+
static const ButtonMatcher& DpadUpInterior(){
79+
const bool match_black_interior = true;
80+
static ButtonMatcher matcher(ButtonType::ButtonDpadUpInterior, 10, 10, 80, match_black_interior);
81+
return matcher;
82+
}
83+
static const ButtonMatcher& DpadDownInterior(){
84+
const bool match_black_interior = true;
85+
static ButtonMatcher matcher(ButtonType::ButtonDpadDownInterior, 10, 10, 80, match_black_interior);
86+
return matcher;
87+
}
7788

7889
virtual bool check_image(Resolution input_resolution, const ImageViewRGB32& image) const override{
7990
size_t min_width = m_min_width * input_resolution.width / 3840;
8091
size_t min_height = m_min_height * input_resolution.height / 2160;
81-
// cout << "???? check_image() ???? min size " << min_width << " x " << min_height
82-
// << " got " << image.width() << " x " << image.height() << endl;
92+
93+
if (PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING){
94+
cout << "???? ButtonMatcher::check_image() ???? min size " << min_width << " x " << min_height
95+
<< " got " << image.width() << " x " << image.height() << endl;
96+
}
8397
return image.width() >= min_width && image.height() >= min_height;
8498
};
8599

@@ -89,11 +103,6 @@ class ButtonMatcher : public ImageMatch::WaterfillTemplateMatcher{
89103
};
90104

91105

92-
93-
94-
95-
96-
97106
const char* template_path(ButtonType type){
98107
switch (type){
99108
case ButtonType::ButtonA:
@@ -117,45 +126,16 @@ const char* template_path(ButtonType type){
117126
case ButtonType::RightStickUpDown:
118127
return "PokemonLZA/Buttons/RightStickUpDown.png";
119128
case ButtonType::ButtonDpadUp:
129+
case ButtonType::ButtonDpadUpInterior:
120130
return "PokemonLZA/Buttons/ButtonDpadUp.png";
121131
case ButtonType::ButtonDpadDown:
132+
case ButtonType::ButtonDpadDownInterior:
122133
return "PokemonLZA/Buttons/ButtonDpadDown.png";
123134
default:
124135
return "";
125136
}
126137
}
127138

128-
const char* button_name(ButtonType type){
129-
switch (type){
130-
case ButtonType::ButtonA:
131-
return "ButtonA";
132-
case ButtonType::ButtonB:
133-
return "ButtonB";
134-
case ButtonType::ButtonX:
135-
return "ButtonX";
136-
case ButtonType::ButtonY:
137-
return "ButtonY";
138-
case ButtonType::ButtonL:
139-
return "ButtonL";
140-
case ButtonType::ButtonR:
141-
return "ButtonR";
142-
case ButtonType::ButtonPlus:
143-
return "Button+";
144-
case ButtonType::ButtonMinus:
145-
return "Button-";
146-
case ButtonType::ButtonRight:
147-
return "ButtonR";
148-
case ButtonType::RightStickUpDown:
149-
return "RightStickUpDown";
150-
case ButtonType::ButtonDpadUp:
151-
return "ButtonDpadUp";
152-
case ButtonType::ButtonDpadDown:
153-
return "ButtonDpadDown";
154-
default:
155-
return "";
156-
}
157-
}
158-
159139
const ButtonMatcher& get_button_matcher(ButtonType type){
160140
switch (type){
161141
case ButtonType::ButtonA:
@@ -182,20 +162,25 @@ const ButtonMatcher& get_button_matcher(ButtonType type){
182162
return ButtonMatcher::DpadUp();
183163
case ButtonType::ButtonDpadDown:
184164
return ButtonMatcher::DpadDown();
165+
case ButtonType::ButtonDpadUpInterior:
166+
return ButtonMatcher::DpadUpInterior();
167+
case ButtonType::ButtonDpadDownInterior:
168+
return ButtonMatcher::DpadDownInterior();
185169
default:
186170
throw std::runtime_error("No corresponding ButtonMatcher for ButtonType");
187171
}
188172
}
189173

174+
bool match_button_black_interior(ButtonType button_type){
175+
return button_type == ButtonType::ButtonDpadUpInterior || button_type == ButtonType::ButtonDpadDownInterior;
176+
}
190177

191-
192-
193-
ButtonMatcher::ButtonMatcher(ButtonType type, size_t min_width, size_t min_height, double max_rmsd)
178+
ButtonMatcher::ButtonMatcher(ButtonType type, size_t min_width, size_t min_height, double max_rmsd, bool match_black_interior)
194179
: WaterfillTemplateMatcher(
195180
template_path(type),
196-
Color(0xff808008),
197-
Color(0xffffffff),
198-
100
181+
match_black_interior ? Color(0xff000000) : Color(0xff808008),
182+
match_black_interior ? Color(0xff404040) : Color(0xffffffff),
183+
match_black_interior ? 50 : 100
199184
)
200185
, m_min_width(min_width)
201186
, m_min_height(min_height)
@@ -210,7 +195,8 @@ ButtonDetector::ButtonDetector(
210195
const ImageFloatBox& box,
211196
VideoOverlay* overlay
212197
)
213-
: m_color(color)
198+
: m_button_type(button_type)
199+
, m_color(color)
214200
, m_matcher(get_button_matcher(button_type))
215201
, m_box(box)
216202
, m_overlay(overlay)
@@ -225,20 +211,34 @@ bool ButtonDetector::detect(const ImageViewRGB32& screen){
225211
double screen_rel_size = (screen.height() / 2160.0);
226212
double screen_rel_size_2 = screen_rel_size * screen_rel_size;
227213

228-
double min_area_3840p = 1000.0;
214+
double min_area_3840p = match_button_black_interior(m_button_type) ? 100.0 : 1000.0;
229215
// double rmsd_threshold = 120.0;
230216
size_t min_area = size_t(screen_rel_size_2 * min_area_3840p);
231217

232-
const std::vector<std::pair<uint32_t, uint32_t>> FILTERS = {
233-
{0xff808080, 0xffffffff},
234-
{0xff909090, 0xffffffff},
235-
{0xffa0a0a0, 0xffffffff},
236-
{0xffb0b0b0, 0xffffffff},
237-
{0xffc0c0c0, 0xffffffff},
238-
{0xffd0d0d0, 0xffffffff},
239-
{0xffe0e0e0, 0xffffffff},
240-
{0xfff0f0f0, 0xffffffff},
241-
};
218+
const std::vector<std::pair<uint32_t, uint32_t>> FILTERS = [&]() -> std::vector<std::pair<uint32_t, uint32_t>>{
219+
if (match_button_black_interior(m_button_type)){
220+
return {
221+
{0xff000000, 0xff202020},
222+
{0xff000000, 0xff404040},
223+
{0xff000000, 0xff606060},
224+
};
225+
}
226+
return {
227+
{0xff808080, 0xffffffff},
228+
{0xff909090, 0xffffffff},
229+
{0xffa0a0a0, 0xffffffff},
230+
{0xffb0b0b0, 0xffffffff},
231+
{0xffc0c0c0, 0xffffffff},
232+
{0xffd0d0d0, 0xffffffff},
233+
{0xffe0e0e0, 0xffffffff},
234+
{0xfff0f0f0, 0xffffffff},
235+
};
236+
}();
237+
238+
// if (m_button_type == ButtonType::ButtonDpadUpInterior){
239+
// cout << "Start debugging ButtonDpadUpInterior" << endl;
240+
// PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING = true;
241+
// }
242242

243243
bool found = match_template_by_waterfill(
244244
screen.size(),
@@ -262,6 +262,11 @@ bool ButtonDetector::detect(const ImageViewRGB32& screen){
262262
}
263263
}
264264

265+
// if (m_button_type == ButtonType::ButtonDpadUpInterior){
266+
// cout << "End debugging ButtonDpadUpInterior" << endl;
267+
// PreloadSettings::debug().IMAGE_TEMPLATE_MATCHING = false;
268+
// }
269+
265270
return found;
266271
}
267272

‎SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_ButtonDetector.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ enum class ButtonType{
3131
RightStickUpDown,
3232
ButtonDpadUp,
3333
ButtonDpadDown,
34+
ButtonDpadUpInterior, // Used to create button detector that detects the black triangle inside the DPAD_UP button
35+
ButtonDpadDownInterior, // Used to create button detector that detects the black triangle inside the DPAD_DOWN button
3436
};
3537

3638
class ButtonMatcher;
@@ -51,7 +53,10 @@ class ButtonDetector : public StaticScreenDetector{
5153

5254
virtual void reset_state() override { m_last_detected_box.reset(); }
5355

56+
ButtonType button_type() const { return m_button_type; }
57+
5458
private:
59+
ButtonType m_button_type;
5560
Color m_color;
5661
const ButtonMatcher& m_matcher;
5762
ImageFloatBox m_box;

‎SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_OverworldPartySelectionDetector.cpp‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,22 @@ OverworldPartySelectionDetector::OverworldPartySelectionDetector(Color color, Vi
5454
ButtonDetector(color, ButtonType::ButtonDpadDown, BOXES()[4], overlay),
5555
ButtonDetector(color, ButtonType::ButtonDpadDown, BOXES()[5], overlay)
5656
}
57+
, m_dpad_up_interiors{
58+
ButtonDetector(color, ButtonType::ButtonDpadUpInterior, BOXES()[0]),
59+
ButtonDetector(color, ButtonType::ButtonDpadUpInterior, BOXES()[1]),
60+
ButtonDetector(color, ButtonType::ButtonDpadUpInterior, BOXES()[2]),
61+
ButtonDetector(color, ButtonType::ButtonDpadUpInterior, BOXES()[3]),
62+
ButtonDetector(color, ButtonType::ButtonDpadUpInterior, BOXES()[4]),
63+
ButtonDetector(color, ButtonType::ButtonDpadUpInterior, BOXES()[5])
64+
}
65+
, m_dpad_down_interiors{
66+
ButtonDetector(color, ButtonType::ButtonDpadDownInterior, BOXES()[0]),
67+
ButtonDetector(color, ButtonType::ButtonDpadDownInterior, BOXES()[1]),
68+
ButtonDetector(color, ButtonType::ButtonDpadDownInterior, BOXES()[2]),
69+
ButtonDetector(color, ButtonType::ButtonDpadDownInterior, BOXES()[3]),
70+
ButtonDetector(color, ButtonType::ButtonDpadDownInterior, BOXES()[4]),
71+
ButtonDetector(color, ButtonType::ButtonDpadDownInterior, BOXES()[5])
72+
}
5773
{}
5874

5975
void OverworldPartySelectionDetector::make_overlays(VideoOverlaySet& items) const{
@@ -67,7 +83,10 @@ bool OverworldPartySelectionDetector::detect(const ImageViewRGB32& screen){
6783
m_detected_up_idx = INVALID_PARTY_IDX, m_detected_down_idx = INVALID_PARTY_IDX;
6884
for(uint8_t i = 0; i < 6; i++){
6985
if (m_debug_mode || m_detected_up_idx == INVALID_PARTY_IDX){
70-
if (m_dpad_ups[i].detect(screen)){
86+
if (m_dpad_ups[i].detect(screen) || m_dpad_up_interiors[i].detect(screen)){
87+
if (m_debug_mode && m_dpad_up_interiors[i].detect(screen)){
88+
cout << "DPAD UP Interior detected at index " << i << endl;
89+
}
7190
if (m_debug_mode && m_detected_up_idx != INVALID_PARTY_IDX){
7291
cout << "Multiple dpad up buttons detected! First detection " << int(m_detected_up_idx)
7392
<< " second detection (" << int(i) << endl;
@@ -78,7 +97,10 @@ bool OverworldPartySelectionDetector::detect(const ImageViewRGB32& screen){
7897
}
7998
}
8099
if (m_debug_mode || m_detected_down_idx == INVALID_PARTY_IDX){
81-
if (m_dpad_downs[i].detect(screen)){
100+
if (m_dpad_downs[i].detect(screen) || m_dpad_down_interiors[i].detect(screen)){
101+
if (m_debug_mode && m_dpad_down_interiors[i].detect(screen)){
102+
cout << "DPAD DOWN Interior detected at index " << i << endl;
103+
}
82104
if (m_debug_mode && m_detected_down_idx != INVALID_PARTY_IDX){
83105
cout << "Multiple dpad down buttons detected! First detection " << int(m_detected_down_idx)
84106
<< " second detection (" << int(i) << endl;

‎SerialPrograms/Source/PokemonLZA/Inference/PokemonLZA_OverworldPartySelectionDetector.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ class OverworldPartySelectionDetector : public StaticScreenDetector{
6464
std::array<ButtonDetector, 6> m_dpad_ups;
6565
std::array<ButtonDetector, 6> m_dpad_downs;
6666

67+
// 6 regions, one for each party pokemon, to detect the black triangles inside dpad buttons
68+
std::array<ButtonDetector, 6> m_dpad_up_interiors;
69+
std::array<ButtonDetector, 6> m_dpad_down_interiors;
70+
6771
uint8_t m_detected_up_idx = INVALID_PARTY_IDX;
6872
uint8_t m_detected_down_idx = INVALID_PARTY_IDX;
6973

0 commit comments

Comments
 (0)