Skip to content

Commit 7f2898b

Browse files
kichithewolfkichithewolf
committed
dialog detection
1 parent b0eee5f commit 7f2898b

File tree

4 files changed

+251
-2
lines changed

4 files changed

+251
-2
lines changed

‎SerialPrograms/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1306,6 +1306,14 @@ file(GLOB MAIN_SOURCES
13061306
Source/PokemonLA/Resources/PokemonLA_PokemonSprites.h
13071307
Source/PokemonLA/Resources/PokemonLA_WeatherAndTimeIcons.cpp
13081308
Source/PokemonLA/Resources/PokemonLA_WeatherAndTimeIcons.h
1309+
Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.cpp
1310+
Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.h
1311+
Source/PokemonRSE/Inference/Sounds/PokemonRSE_ShinySoundDetector.cpp
1312+
Source/PokemonRSE/Inference/Sounds/PokemonRSE_ShinySoundDetector.h
1313+
Source/PokemonRSE/Inference/PokemonRSE_ShinyNumberDetector.cpp
1314+
Source/PokemonRSE/Inference/PokemonRSE_ShinyNumberDetector.h
1315+
Source/PokemonRSE/Programs/TestPrograms/PokemonRSE_SoundListener.cpp
1316+
Source/PokemonRSE/Programs/TestPrograms/PokemonRSE_SoundListener.h
13091317
Source/PokemonRSE/PokemonRSE_Panels.cpp
13101318
Source/PokemonRSE/PokemonRSE_Panels.h
13111319
Source/PokemonRSE/PokemonRSE_Settings.cpp

‎SerialPrograms/Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.cpp‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/* Dialog Detector
2+
*
3+
* From: https://github.com/PokemonAutomation/Arduino-Source
4+
*
5+
*/
6+
7+
#include "CommonTools/Images/SolidColorTest.h"
8+
#include "CommonTools/Images/ImageFilter.h"
9+
#include "CommonFramework/ImageTools/ImageBoxes.h"
10+
#include "CommonFramework/ImageTypes/ImageRGB32.h"
11+
#include "CommonFramework/ImageTools/ImageStats.h"
12+
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
13+
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
14+
#include "PokemonRSE_DialogDetector.h"
15+
16+
#include <iostream>
17+
using std::cout;
18+
using std::endl;
19+
20+
namespace PokemonAutomation{
21+
namespace NintendoSwitch{
22+
namespace PokemonRSE{
23+
24+
/*
25+
DialogDetector::DialogDetector(Color color)
26+
: m_left_box(0.155, 0.727, 0.015, 0.168)
27+
, m_right_box(0.837, 0.729, 0.008, 0.161)
28+
{}
29+
void DialogDetector::make_overlays(VideoOverlaySet& items) const{
30+
items.add(COLOR_RED, m_left_box);
31+
items.add(COLOR_RED, m_right_box);
32+
}
33+
bool DialogDetector::detect(const ImageViewRGB32& screen) const{
34+
ImageViewRGB32 left_image = extract_box_reference(screen, m_left_box);
35+
ImageViewRGB32 right_image = extract_box_reference(screen, m_right_box);
36+
if (is_solid(left_image, { 0.335, 0.331, 0.332 }) && is_solid(right_image, { 0.335, 0.331, 0.332 })){
37+
return true;
38+
}
39+
return false;
40+
}
41+
*/
42+
43+
BattleDialogDetector::BattleDialogDetector(Color color)
44+
: m_left_box(0.158, 0.725, 0.011, 0.176)
45+
, m_right_box(0.827, 0.722, 0.013, 0.178)
46+
{}
47+
void BattleDialogDetector::make_overlays(VideoOverlaySet& items) const{
48+
items.add(COLOR_RED, m_left_box);
49+
items.add(COLOR_RED, m_right_box);
50+
}
51+
bool BattleDialogDetector::detect(const ImageViewRGB32& screen) const{
52+
ImageViewRGB32 left_image = extract_box_reference(screen, m_left_box);
53+
ImageViewRGB32 right_image = extract_box_reference(screen, m_right_box);
54+
if (is_solid(left_image, { 0.335, 0.331, 0.332 }) && is_solid(right_image, { 0.335, 0.331, 0.332 })){
55+
return true;
56+
}
57+
return false;
58+
}
59+
60+
61+
BattleMenuDetector::BattleMenuDetector(Color color)
62+
: m_left_box(0.439, 0.717, 0.021, 0.192)
63+
, m_right_box(0.821, 0.725, 0.030, 0.181)
64+
{}
65+
void BattleMenuDetector::make_overlays(VideoOverlaySet& items) const{
66+
items.add(COLOR_RED, m_left_box);
67+
items.add(COLOR_RED, m_right_box);
68+
}
69+
bool BattleMenuDetector::detect(const ImageViewRGB32& screen) const{
70+
ImageViewRGB32 left_image = extract_box_reference(screen, m_left_box);
71+
ImageViewRGB32 right_image = extract_box_reference(screen, m_right_box);
72+
if (is_solid(left_image, { 0.335, 0.331, 0.332 }) && is_solid(right_image, { 0.25, 0.38, 0.369 })){
73+
return true;
74+
}
75+
return false;
76+
}
77+
78+
79+
AdvanceBattleDialogDetector::AdvanceBattleDialogDetector(Color color)
80+
: m_dialog_box(0.156, 0.715, 0.686, 0.193)
81+
, m_left_box(0.156, 0.724, 0.010, 0.176)
82+
, m_right_box(0.836, 0.717, 0.008, 0.189)
83+
{}
84+
void AdvanceBattleDialogDetector::make_overlays(VideoOverlaySet& items) const{
85+
items.add(COLOR_RED, m_dialog_box);
86+
items.add(COLOR_RED, m_left_box);
87+
items.add(COLOR_RED, m_right_box);
88+
}
89+
bool AdvanceBattleDialogDetector::detect(const ImageViewRGB32& screen) const{
90+
const bool replace_color_within_range = false;
91+
92+
//Filter out background
93+
ImageRGB32 filtered_region = filter_rgb32_range(
94+
extract_box_reference(screen, m_dialog_box),
95+
combine_rgb(185, 0, 1), combine_rgb(255, 32, 33), Color(0), replace_color_within_range
96+
);
97+
ImageStats stats = image_stats(filtered_region);
98+
99+
/*
100+
filtered_region.save("./filtered_only.png");
101+
cout << stats.average.r << endl;
102+
cout << stats.average.g << endl;
103+
cout << stats.average.b << endl;
104+
*/
105+
106+
ImageViewRGB32 left_image = extract_box_reference(screen, m_left_box);
107+
ImageViewRGB32 right_image = extract_box_reference(screen, m_right_box);
108+
109+
if (is_solid(left_image, { 0.335, 0.331, 0.332 })
110+
&& is_solid(right_image, { 0.335, 0.331, 0.332 })
111+
&& (stats.average.r > stats.average.b + 200)
112+
&& (stats.average.r > stats.average.g + 200)
113+
)
114+
{
115+
return true;
116+
}
117+
return false;
118+
}
119+
120+
121+
122+
}
123+
}
124+
}

‎SerialPrograms/Source/PokemonRSE/Inference/Dialogs/PokemonRSE_DialogDetector.h‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/* Dialog Detector
2+
*
3+
* From: https://github.com/PokemonAutomation/Arduino-Source
4+
*
5+
*/
6+
7+
#ifndef PokemonAutomation_PokemonRSE_DialogDetector_H
8+
#define PokemonAutomation_PokemonRSE_DialogDetector_H
9+
10+
#include <chrono>
11+
#include <atomic>
12+
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
13+
#include "Common/Cpp/Color.h"
14+
#include "CommonFramework/ImageTools/ImageBoxes.h"
15+
#include "CommonTools/VisualDetector.h"
16+
#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h"
17+
18+
namespace PokemonAutomation{
19+
class CancellableScope;
20+
class VideoFeed;
21+
namespace NintendoSwitch{
22+
namespace PokemonRSE{
23+
24+
/*
25+
// TODO: Detect that a dialog box is on screen by looking for the white of the box
26+
class DialogDetector : public StaticScreenDetector{
27+
public:
28+
DialogDetector(Color color);
29+
30+
virtual void make_overlays(VideoOverlaySet& items) const override;
31+
virtual bool detect(const ImageViewRGB32& screen) const override;
32+
33+
private:
34+
ImageFloatBox m_left_box;
35+
ImageFloatBox m_right_box;
36+
};
37+
class DialogWatcher : public DetectorToFinder<DialogDetector>{
38+
public:
39+
DialogWatcher(Color color)
40+
: DetectorToFinder("DialogWatcher", std::chrono::milliseconds(250), color)
41+
{}
42+
};
43+
*/
44+
45+
// Battle dialog boxes are teal
46+
class BattleDialogDetector : public StaticScreenDetector{
47+
public:
48+
BattleDialogDetector(Color color);
49+
50+
virtual void make_overlays(VideoOverlaySet& items) const override;
51+
virtual bool detect(const ImageViewRGB32& screen) const override;
52+
53+
private:
54+
ImageFloatBox m_left_box;
55+
ImageFloatBox m_right_box;
56+
};
57+
class BattleDialogWatcher : public DetectorToFinder<BattleDialogDetector>{
58+
public:
59+
BattleDialogWatcher(Color color)
60+
: DetectorToFinder("BattleDialogWatcher", std::chrono::milliseconds(250), color)
61+
{}
62+
};
63+
64+
65+
// Battle menu is up when it is white on the right and teal on the left
66+
// For emerald the text is more flush to the left, so we target the right of the battle menu box
67+
class BattleMenuDetector : public StaticScreenDetector{
68+
public:
69+
BattleMenuDetector(Color color);
70+
71+
virtual void make_overlays(VideoOverlaySet& items) const override;
72+
virtual bool detect(const ImageViewRGB32& screen) const override;
73+
74+
private:
75+
ImageFloatBox m_left_box;
76+
ImageFloatBox m_right_box;
77+
};
78+
class BattleMenuWatcher : public DetectorToFinder<BattleMenuDetector>{
79+
public:
80+
BattleMenuWatcher(Color color)
81+
: DetectorToFinder("BattleMenuWatcher", std::chrono::milliseconds(250), color)
82+
{}
83+
};
84+
85+
86+
87+
// Detect the red advancement arrow by filtering for red.
88+
// This works for now, I don't think there's colored text?
89+
// TODO: Change this to detect that the dialog arrow is in the dialog box by filtering for the red arrow
90+
class AdvanceBattleDialogDetector : public StaticScreenDetector{
91+
public:
92+
AdvanceBattleDialogDetector(Color color);
93+
94+
virtual void make_overlays(VideoOverlaySet& items) const override;
95+
virtual bool detect(const ImageViewRGB32& screen) const override;
96+
97+
private:
98+
ImageFloatBox m_dialog_box;
99+
ImageFloatBox m_left_box;
100+
ImageFloatBox m_right_box;
101+
};
102+
class AdvanceBattleDialogWatcher : public DetectorToFinder<AdvanceBattleDialogDetector>{
103+
public:
104+
AdvanceBattleDialogWatcher(Color color)
105+
: DetectorToFinder("AdvanceBattleDialogWatcher", std::chrono::milliseconds(250), color)
106+
{}
107+
};
108+
109+
// Future note: when given a choice popup, there is no advance arrow.
110+
// FRLG doesn't have an advance arrow, that's a future issue.
111+
112+
113+
}
114+
}
115+
}
116+
117+
#endif

‎SerialPrograms/Source/PokemonRSE/PokemonRSE_Panels.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
//#include "Programs/ShinyHunting/PokemonRSE_AudioStarterReset.h"
1414
//#include "Programs/ShinyHunting/PokemonRSE_StarterReset.h"
15-
//#include "Programs/TestPrograms/PokemonRSE_SoundListener.h"
15+
#include "Programs/TestPrograms/PokemonRSE_SoundListener.h"
1616

1717
namespace PokemonAutomation{
1818
namespace NintendoSwitch{
@@ -41,7 +41,7 @@ std::vector<PanelEntry> PanelListFactory::make_panels() const{
4141
//ret.emplace_back(make_single_switch_program<StarterReset_Descriptor, StarterReset>()); //outdated early test program
4242

4343
ret.emplace_back("---- Developer Tools ----");
44-
//ret.emplace_back(make_single_switch_program<SoundListener_Descriptor, SoundListener>());
44+
ret.emplace_back(make_single_switch_program<SoundListener_Descriptor, SoundListener>());
4545
}
4646

4747
return ret;

0 commit comments

Comments
 (0)