Skip to content

Commit 92a83a9

Browse files
kichithewolfkichithewolf
authored
RSE detectors (#529)
* shiny sound detection * shiny dex num detector * dialog detection * Update PokemonRSE_SoundListener.cpp
1 parent 77dfb5b commit 92a83a9

File tree

10 files changed

+582
-2
lines changed

10 files changed

+582
-2
lines changed

‎SerialPrograms/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1313,6 +1313,14 @@ file(GLOB MAIN_SOURCES
13131313
Source/PokemonLA/Resources/PokemonLA_PokemonSprites.h
13141314
Source/PokemonLA/Resources/PokemonLA_WeatherAndTimeIcons.cpp
13151315
Source/PokemonLA/Resources/PokemonLA_WeatherAndTimeIcons.h
1316+
Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.cpp
1317+
Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.h
1318+
Source/PokemonRSE/Inference/Sounds/PokemonRSE_ShinySoundDetector.cpp
1319+
Source/PokemonRSE/Inference/Sounds/PokemonRSE_ShinySoundDetector.h
1320+
Source/PokemonRSE/Inference/PokemonRSE_ShinyNumberDetector.cpp
1321+
Source/PokemonRSE/Inference/PokemonRSE_ShinyNumberDetector.h
1322+
Source/PokemonRSE/Programs/TestPrograms/PokemonRSE_SoundListener.cpp
1323+
Source/PokemonRSE/Programs/TestPrograms/PokemonRSE_SoundListener.h
13161324
Source/PokemonRSE/PokemonRSE_Panels.cpp
13171325
Source/PokemonRSE/PokemonRSE_Panels.h
13181326
Source/PokemonRSE/PokemonRSE_Settings.cpp

‎SerialPrograms/Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.cpp‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/* Dialog Detector
2+
*
3+
* From: https://github.com/PokemonAutomation/Arduino-Source
4+
*
5+
*/
6+
7+
#include "CommonTools/Images/SolidColorTest.h"
8+
#include "CommonTools/Images/ImageFilter.h"
9+
#include "CommonFramework/ImageTools/ImageBoxes.h"
10+
#include "CommonFramework/ImageTypes/ImageRGB32.h"
11+
#include "CommonFramework/ImageTools/ImageStats.h"
12+
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
13+
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
14+
#include "PokemonRSE_DialogDetector.h"
15+
16+
#include <iostream>
17+
using std::cout;
18+
using std::endl;
19+
20+
namespace PokemonAutomation{
21+
namespace NintendoSwitch{
22+
namespace PokemonRSE{
23+
24+
/*
25+
DialogDetector::DialogDetector(Color color)
26+
: m_left_box(0.155, 0.727, 0.015, 0.168)
27+
, m_right_box(0.837, 0.729, 0.008, 0.161)
28+
{}
29+
void DialogDetector::make_overlays(VideoOverlaySet& items) const{
30+
items.add(COLOR_RED, m_left_box);
31+
items.add(COLOR_RED, m_right_box);
32+
}
33+
bool DialogDetector::detect(const ImageViewRGB32& screen) const{
34+
ImageViewRGB32 left_image = extract_box_reference(screen, m_left_box);
35+
ImageViewRGB32 right_image = extract_box_reference(screen, m_right_box);
36+
if (is_solid(left_image, { 0.335, 0.331, 0.332 }) && is_solid(right_image, { 0.335, 0.331, 0.332 })){
37+
return true;
38+
}
39+
return false;
40+
}
41+
*/
42+
43+
BattleDialogDetector::BattleDialogDetector(Color color)
44+
: m_left_box(0.158, 0.725, 0.011, 0.176)
45+
, m_right_box(0.827, 0.722, 0.013, 0.178)
46+
{}
47+
void BattleDialogDetector::make_overlays(VideoOverlaySet& items) const{
48+
items.add(COLOR_RED, m_left_box);
49+
items.add(COLOR_RED, m_right_box);
50+
}
51+
bool BattleDialogDetector::detect(const ImageViewRGB32& screen) const{
52+
ImageViewRGB32 left_image = extract_box_reference(screen, m_left_box);
53+
ImageViewRGB32 right_image = extract_box_reference(screen, m_right_box);
54+
if (is_solid(left_image, { 0.335, 0.331, 0.332 }) && is_solid(right_image, { 0.335, 0.331, 0.332 })){
55+
return true;
56+
}
57+
return false;
58+
}
59+
60+
61+
BattleMenuDetector::BattleMenuDetector(Color color)
62+
: m_left_box(0.439, 0.717, 0.021, 0.192)
63+
, m_right_box(0.821, 0.725, 0.030, 0.181)
64+
{}
65+
void BattleMenuDetector::make_overlays(VideoOverlaySet& items) const{
66+
items.add(COLOR_RED, m_left_box);
67+
items.add(COLOR_RED, m_right_box);
68+
}
69+
bool BattleMenuDetector::detect(const ImageViewRGB32& screen) const{
70+
ImageViewRGB32 left_image = extract_box_reference(screen, m_left_box);
71+
ImageViewRGB32 right_image = extract_box_reference(screen, m_right_box);
72+
if (is_solid(left_image, { 0.335, 0.331, 0.332 }) && is_solid(right_image, { 0.25, 0.38, 0.369 })){
73+
return true;
74+
}
75+
return false;
76+
}
77+
78+
79+
AdvanceBattleDialogDetector::AdvanceBattleDialogDetector(Color color)
80+
: m_dialog_box(0.156, 0.715, 0.686, 0.193)
81+
, m_left_box(0.156, 0.724, 0.010, 0.176)
82+
, m_right_box(0.836, 0.717, 0.008, 0.189)
83+
{}
84+
void AdvanceBattleDialogDetector::make_overlays(VideoOverlaySet& items) const{
85+
items.add(COLOR_RED, m_dialog_box);
86+
items.add(COLOR_RED, m_left_box);
87+
items.add(COLOR_RED, m_right_box);
88+
}
89+
bool AdvanceBattleDialogDetector::detect(const ImageViewRGB32& screen) const{
90+
const bool replace_color_within_range = false;
91+
92+
//Filter out background
93+
ImageRGB32 filtered_region = filter_rgb32_range(
94+
extract_box_reference(screen, m_dialog_box),
95+
combine_rgb(185, 0, 1), combine_rgb(255, 32, 33), Color(0), replace_color_within_range
96+
);
97+
ImageStats stats = image_stats(filtered_region);
98+
99+
/*
100+
filtered_region.save("./filtered_only.png");
101+
cout << stats.average.r << endl;
102+
cout << stats.average.g << endl;
103+
cout << stats.average.b << endl;
104+
*/
105+
106+
ImageViewRGB32 left_image = extract_box_reference(screen, m_left_box);
107+
ImageViewRGB32 right_image = extract_box_reference(screen, m_right_box);
108+
109+
if (is_solid(left_image, { 0.335, 0.331, 0.332 })
110+
&& is_solid(right_image, { 0.335, 0.331, 0.332 })
111+
&& (stats.average.r > stats.average.b + 200)
112+
&& (stats.average.r > stats.average.g + 200)
113+
)
114+
{
115+
return true;
116+
}
117+
return false;
118+
}
119+
120+
121+
122+
}
123+
}
124+
}

‎SerialPrograms/Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.h‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/* Dialog Detector
2+
*
3+
* From: https://github.com/PokemonAutomation/Arduino-Source
4+
*
5+
*/
6+
7+
#ifndef PokemonAutomation_PokemonRSE_DialogDetector_H
8+
#define PokemonAutomation_PokemonRSE_DialogDetector_H
9+
10+
#include <chrono>
11+
#include <atomic>
12+
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
13+
#include "Common/Cpp/Color.h"
14+
#include "CommonFramework/ImageTools/ImageBoxes.h"
15+
#include "CommonTools/VisualDetector.h"
16+
#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h"
17+
18+
namespace PokemonAutomation{
19+
class CancellableScope;
20+
class VideoFeed;
21+
namespace NintendoSwitch{
22+
namespace PokemonRSE{
23+
24+
/*
25+
// TODO: Detect that a dialog box is on screen by looking for the white of the box
26+
class DialogDetector : public StaticScreenDetector{
27+
public:
28+
DialogDetector(Color color);
29+
30+
virtual void make_overlays(VideoOverlaySet& items) const override;
31+
virtual bool detect(const ImageViewRGB32& screen) const override;
32+
33+
private:
34+
ImageFloatBox m_left_box;
35+
ImageFloatBox m_right_box;
36+
};
37+
class DialogWatcher : public DetectorToFinder<DialogDetector>{
38+
public:
39+
DialogWatcher(Color color)
40+
: DetectorToFinder("DialogWatcher", std::chrono::milliseconds(250), color)
41+
{}
42+
};
43+
*/
44+
45+
// Battle dialog boxes are teal
46+
class BattleDialogDetector : public StaticScreenDetector{
47+
public:
48+
BattleDialogDetector(Color color);
49+
50+
virtual void make_overlays(VideoOverlaySet& items) const override;
51+
virtual bool detect(const ImageViewRGB32& screen) const override;
52+
53+
private:
54+
ImageFloatBox m_left_box;
55+
ImageFloatBox m_right_box;
56+
};
57+
class BattleDialogWatcher : public DetectorToFinder<BattleDialogDetector>{
58+
public:
59+
BattleDialogWatcher(Color color)
60+
: DetectorToFinder("BattleDialogWatcher", std::chrono::milliseconds(250), color)
61+
{}
62+
};
63+
64+
65+
// Battle menu is up when it is white on the right and teal on the left
66+
// For emerald the text is more flush to the left, so we target the right of the battle menu box
67+
class BattleMenuDetector : public StaticScreenDetector{
68+
public:
69+
BattleMenuDetector(Color color);
70+
71+
virtual void make_overlays(VideoOverlaySet& items) const override;
72+
virtual bool detect(const ImageViewRGB32& screen) const override;
73+
74+
private:
75+
ImageFloatBox m_left_box;
76+
ImageFloatBox m_right_box;
77+
};
78+
class BattleMenuWatcher : public DetectorToFinder<BattleMenuDetector>{
79+
public:
80+
BattleMenuWatcher(Color color)
81+
: DetectorToFinder("BattleMenuWatcher", std::chrono::milliseconds(250), color)
82+
{}
83+
};
84+
85+
86+
87+
// Detect the red advancement arrow by filtering for red.
88+
// This works for now, I don't think there's colored text?
89+
// TODO: Change this to detect that the dialog arrow is in the dialog box by filtering for the red arrow
90+
class AdvanceBattleDialogDetector : public StaticScreenDetector{
91+
public:
92+
AdvanceBattleDialogDetector(Color color);
93+
94+
virtual void make_overlays(VideoOverlaySet& items) const override;
95+
virtual bool detect(const ImageViewRGB32& screen) const override;
96+
97+
private:
98+
ImageFloatBox m_dialog_box;
99+
ImageFloatBox m_left_box;
100+
ImageFloatBox m_right_box;
101+
};
102+
class AdvanceBattleDialogWatcher : public DetectorToFinder<AdvanceBattleDialogDetector>{
103+
public:
104+
AdvanceBattleDialogWatcher(Color color)
105+
: DetectorToFinder("AdvanceBattleDialogWatcher", std::chrono::milliseconds(250), color)
106+
{}
107+
};
108+
109+
// Future note: when given a choice popup, there is no advance arrow.
110+
// FRLG doesn't have an advance arrow, that's a future issue.
111+
112+
113+
}
114+
}
115+
}
116+
117+
#endif

‎SerialPrograms/Source/PokemonRSE/Inference/PokemonRSE_ShinyNumberDetector.cpp‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/* Shiny Number Detector
2+
*
3+
* From: https://github.com/PokemonAutomation/Arduino-Source
4+
*
5+
*/
6+
7+
#include "Common/Cpp/Color.h"
8+
#include "CommonFramework/ImageTools/ImageBoxes.h"
9+
#include "CommonFramework/ImageTools/ImageStats.h"
10+
#include "CommonFramework/ImageTypes/ImageRGB32.h"
11+
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
12+
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
13+
#include "CommonTools/Images/ImageFilter.h"
14+
#include "PokemonRSE_ShinyNumberDetector.h"
15+
16+
//#include <iostream>
17+
//using std::cout;
18+
//using std::endl;
19+
20+
namespace PokemonAutomation{
21+
namespace NintendoSwitch{
22+
namespace PokemonRSE{
23+
24+
ShinyNumberDetector::ShinyNumberDetector(Color color)
25+
: m_box_number(0.136, 0.156, 0.123, 0.072)
26+
{}
27+
void ShinyNumberDetector::make_overlays(VideoOverlaySet& items) const{
28+
items.add(COLOR_RED, m_box_number);
29+
}
30+
bool ShinyNumberDetector::read(Logger& logger, const ImageViewRGB32& frame){
31+
const bool replace_color_within_range = true;
32+
33+
//Filter out background
34+
ImageRGB32 filtered_region = filter_rgb32_range(
35+
extract_box_reference(frame, m_box_number),
36+
combine_rgb(138, 97, 221), combine_rgb(200, 181, 239), Color(0), replace_color_within_range
37+
);
38+
ImageStats stats = image_stats(filtered_region);
39+
40+
/*
41+
filtered_region.save("./filtered_only.png");
42+
cout << stats.average.r << endl;
43+
cout << stats.average.g << endl;
44+
cout << stats.average.b << endl;
45+
*/
46+
47+
/*
48+
Shiny:
49+
R: 196.632, G: 196.771, B: 145.863
50+
Not shiny:
51+
R: 181.862, G: 180.686, B: 193.999
52+
*/
53+
54+
if (stats.average.b + 20 < stats.average.r){
55+
return true;
56+
}
57+
return false;
58+
}
59+
60+
61+
}
62+
}
63+
}
64+

‎SerialPrograms/Source/PokemonRSE/Inference/PokemonRSE_ShinyNumberDetector.h‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/* Shiny Number Detector
2+
*
3+
* From: https://github.com/PokemonAutomation/Arduino-Source
4+
*
5+
*/
6+
7+
#ifndef PokemonAutomation_PokemonRSE_ShinyNumberDetector_H
8+
#define PokemonAutomation_PokemonRSE_ShinyNumberDetector_H
9+
10+
#include "Common/Cpp/AbstractLogger.h"
11+
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
12+
13+
namespace PokemonAutomation{
14+
namespace NintendoSwitch{
15+
namespace PokemonRSE{
16+
17+
// In the summary screen, the dex number will be yellow if a shiny, white if not.
18+
// Additionally, the background behind the sprite will be white if shiny, grey if not.
19+
// Number is easier to check as the background is scan lines.
20+
class ShinyNumberDetector{
21+
public:
22+
ShinyNumberDetector(Color color);
23+
24+
virtual void make_overlays(VideoOverlaySet& items) const;
25+
bool read(Logger& logger, const ImageViewRGB32& frame);
26+
27+
private:
28+
ImageFloatBox m_box_number;
29+
};
30+
31+
32+
33+
}
34+
}
35+
}
36+
#endif

0 commit comments

Comments
 (0)