Skip to content

Commit d0dd14d

Browse files
author
Gin
committed
add image display widget for ML
1 parent 305900d commit d0dd14d

File tree

6 files changed

+293
-6
lines changed

6 files changed

+293
-6
lines changed

‎SerialPrograms/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -894,6 +894,8 @@ file(GLOB MAIN_SOURCES
894894
Source/ML/ML_Panels.h
895895
Source/ML/Programs/ML_LabelImages.cpp
896896
Source/ML/Programs/ML_LabelImages.h
897+
Source/ML/UI/ML_ImageDisplayWidget.cpp
898+
Source/ML/UI/ML_ImageDisplayWidget.h
897899
Source/NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.cpp
898900
Source/NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h
899901
Source/NintendoSwitch/Commands/NintendoSwitch_Commands_Routines.cpp

‎SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
#include "Common/Qt/CollapsibleGroupBox.h"
2828
#include "Pokemon/Resources/Pokemon_PokemonForms.h"
2929
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
30-
#include "NintendoSwitch/Framework/UI/NintendoSwitch_SwitchSystemWidget.h"
30+
#include "ML/UI/ML_ImageDisplayWidget.h"
3131
#include "CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h"
3232
#include "CommonFramework/VideoPipeline/VideoSources/VideoSource_StillImage.h"
3333
#include "ML_LabelImages.h"
@@ -437,7 +437,7 @@ LabelImages_Widget::LabelImages_Widget(
437437
QVBoxLayout* scroll_layout = new QVBoxLayout(scroll_inner);
438438
scroll_layout->setAlignment(Qt::AlignTop);
439439

440-
m_switch_widget = new NintendoSwitch::SwitchSystemWidget(*this, m_session, 0);
440+
m_switch_widget = new ImageDisplayWidget(*this, m_session, 0);
441441
scroll_layout->addWidget(m_switch_widget);
442442

443443
QPushButton* button = new QPushButton("Delete Last Mask", scroll_inner);

‎SerialPrograms/Source/ML/Programs/ML_LabelImages.h‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,12 @@ namespace PokemonAutomation{
2828

2929

3030
class ConfigWidget;
31-
namespace NintendoSwitch{
32-
class SwitchSystemWidget;
33-
}
3431

3532

3633
namespace ML{
3734

3835

36+
class ImageDisplayWidget;
3937
class LabelImages_Widget;
4038

4139

@@ -146,7 +144,7 @@ class LabelImages_Widget : public PanelWidget, public ConfigOption::Listener{
146144
private:
147145
LabelImages& m_program;
148146
NintendoSwitch::SwitchSystemSession m_session;
149-
NintendoSwitch::SwitchSystemWidget* m_switch_widget;
147+
ImageDisplayWidget* m_switch_widget;
150148
VideoOverlaySet m_overlay_set;
151149
DrawnBoundingBox m_drawn_box;
152150
ConfigWidget* m_option_widget;

‎SerialPrograms/Source/ML/UI/ML_ImageDisplayWidget.cpp‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
/* ML Image Display Widget
2+
*
3+
* From: https://github.com/PokemonAutomation/
4+
*
5+
*/
6+
7+
#include <QKeyEvent>
8+
#include <QVBoxLayout>
9+
#include <QGroupBox>
10+
#include <QFileDialog>
11+
#include "Common/Cpp/PrettyPrint.h"
12+
#include "Common/Cpp/Concurrency/FireForgetDispatcher.h"
13+
#include "Common/Cpp/Json/JsonValue.h"
14+
#include "Common/Qt/CollapsibleGroupBox.h"
15+
#include "CommonFramework/Globals.h"
16+
#include "CommonFramework/VideoPipeline/UI/VideoSourceSelectorWidget.h"
17+
#include "CommonFramework/VideoPipeline/UI/VideoDisplayWidget.h"
18+
#include "NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.h"
19+
#include "NintendoSwitch/Framework/UI/NintendoSwitch_SwitchSystemWidget.h"
20+
#include "NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.h"
21+
#include "ML_ImageDisplayWidget.h"
22+
23+
//#include <iostream>
24+
//using std::cout;
25+
//using std::endl;
26+
27+
namespace PokemonAutomation{
28+
namespace ML{
29+
30+
31+
ImageDisplayWidget::~ImageDisplayWidget(){
32+
// Delete all the UI elements first since they reference the states.
33+
delete m_video_display;
34+
delete m_video_selector;
35+
}
36+
37+
ImageDisplayWidget::ImageDisplayWidget(
38+
QWidget& parent,
39+
NintendoSwitch::SwitchSystemSession& session,
40+
uint64_t program_id
41+
)
42+
: QWidget(&parent)
43+
, m_session(session)
44+
{
45+
QVBoxLayout* layout = new QVBoxLayout(this);
46+
layout->setContentsMargins(0, 0, 0, 0);
47+
layout->setAlignment(Qt::AlignTop);
48+
49+
m_group_box = new CollapsibleGroupBox(*this, "Image Settings");
50+
layout->addWidget(m_group_box);
51+
52+
QWidget* widget = new QWidget(m_group_box);
53+
m_group_box->set_widget(widget);
54+
{
55+
QVBoxLayout* video_holder = new QVBoxLayout();
56+
layout->addLayout(video_holder);
57+
video_holder->setContentsMargins(0, 0, 0, 0);
58+
59+
m_video_display = new VideoDisplayWidget(
60+
*this, *video_holder,
61+
m_session.console_number(),
62+
*this,
63+
m_session.video_session(),
64+
m_session.overlay_session()
65+
);
66+
video_holder->addWidget(m_video_display);
67+
}
68+
{
69+
QVBoxLayout* group_layout = new QVBoxLayout(widget);
70+
group_layout->setAlignment(Qt::AlignTop);
71+
group_layout->setContentsMargins(0, 0, 0, 0);
72+
73+
m_video_selector = new VideoSourceSelectorWidget(m_session.logger(), m_session.video_session());
74+
group_layout->addWidget(m_video_selector);
75+
76+
m_command = new NintendoSwitch::CommandRow(
77+
*widget,
78+
m_session.controller_session(),
79+
m_session.overlay_session(),
80+
m_session.console_type(),
81+
m_session.allow_commands_while_running()
82+
);
83+
group_layout->addWidget(m_command);
84+
}
85+
86+
setFocusPolicy(Qt::StrongFocus);
87+
88+
connect(
89+
m_command, &NintendoSwitch::CommandRow::load_profile,
90+
m_command, [this](){
91+
std::string path = QFileDialog::getOpenFileName(this, tr("Choose the name of your profile file"), "", tr("JSON files (*.json)")).toStdString();
92+
if (path.empty()){
93+
return;
94+
}
95+
96+
NintendoSwitch::SwitchSystemOption option(
97+
m_session.required_features(),
98+
m_session.allow_commands_while_running()
99+
);
100+
101+
// Deserialize into this local option instance.
102+
option.load_json(load_json_file(path));
103+
104+
m_session.set(option);
105+
}
106+
);
107+
connect(
108+
m_command, &NintendoSwitch::CommandRow::save_profile,
109+
m_command, [this](){
110+
std::string path = QFileDialog::getSaveFileName(this, tr("Choose the name of your profile file"), "", tr("JSON files (*.json)")).toStdString();
111+
if (path.empty()){
112+
return;
113+
}
114+
115+
// Create a copy of option, to be able to serialize it later on
116+
NintendoSwitch::SwitchSystemOption option(
117+
m_session.required_features(),
118+
m_session.allow_commands_while_running()
119+
);
120+
121+
m_session.get(option);
122+
123+
option.to_json().dump(path);
124+
}
125+
);
126+
127+
connect(
128+
m_command, &NintendoSwitch::CommandRow::video_requested,
129+
m_video_display, [this](){
130+
global_dispatcher.dispatch([this]{
131+
std::string filename = SCREENSHOTS_PATH() + "video-" + now_to_filestring() + ".mp4";
132+
m_session.logger().log("Saving screenshot to: " + filename, COLOR_PURPLE);
133+
m_session.save_history(filename);
134+
});
135+
}
136+
);
137+
}
138+
139+
140+
void ImageDisplayWidget::update_ui(ProgramState state){
141+
m_session.controller_session().set_options_locked(state != ProgramState::STOPPED);
142+
if (m_session.allow_commands_while_running()){
143+
m_session.set_allow_user_commands("");
144+
}else{
145+
switch (state){
146+
case ProgramState::NOT_READY:
147+
m_session.set_allow_user_commands("Program is not ready.");
148+
break;
149+
case ProgramState::STOPPED:
150+
m_session.set_allow_user_commands("");
151+
break;
152+
case ProgramState::RUNNING:
153+
case ProgramState::STOPPING:
154+
m_session.set_allow_user_commands("Program is running.");
155+
break;
156+
}
157+
}
158+
m_command->on_state_changed(state);
159+
}
160+
161+
void ImageDisplayWidget::key_press(QKeyEvent* event){
162+
// cout << "press: " << event->nativeVirtualKey() << endl;
163+
m_command->on_key_press(*event);
164+
}
165+
166+
void ImageDisplayWidget::key_release(QKeyEvent* event){
167+
// cout << "release: " << event->nativeVirtualKey() << endl;
168+
m_command->on_key_release(*event);
169+
}
170+
171+
void ImageDisplayWidget::focus_in(QFocusEvent* event){
172+
m_command->set_focus(true);
173+
}
174+
175+
void ImageDisplayWidget::focus_out(QFocusEvent* event){
176+
m_command->set_focus(false);
177+
}
178+
179+
void ImageDisplayWidget::keyPressEvent(QKeyEvent* event){
180+
key_press(event);
181+
}
182+
void ImageDisplayWidget::keyReleaseEvent(QKeyEvent* event){
183+
key_release(event);
184+
}
185+
void ImageDisplayWidget::focusInEvent(QFocusEvent* event){
186+
// cout << "focusInEvent" << endl;
187+
focus_in(event);
188+
QWidget::focusInEvent(event);
189+
}
190+
void ImageDisplayWidget::focusOutEvent(QFocusEvent* event){
191+
// cout << "focusOutEvent" << endl;
192+
focus_out(event);
193+
QWidget::focusOutEvent(event);
194+
}
195+
196+
197+
198+
}
199+
}

‎SerialPrograms/Source/ML/UI/ML_ImageDisplayWidget.h‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/* ML Image Display Widget
2+
*
3+
* From: https://github.com/PokemonAutomation/
4+
*
5+
* This is the Qt Widget for displaying images for ML annotation purposes.
6+
* This is modified from NintendoSwitch/Framework/UI/NintendoSwitch_SwitchSystemWidget.
7+
*/
8+
9+
#ifndef ML_ImageDisplayWidget_H
10+
#define ML_ImageDisplayWidget_H
11+
12+
#include <QWidget>
13+
#include "CommonFramework/VideoPipeline/UI/VideoDisplayWidget.h"
14+
15+
namespace PokemonAutomation{
16+
class CollapsibleGroupBox;
17+
class AudioFeed;
18+
class CameraSelectorWidget;
19+
class VideoSourceSelectorWidget;
20+
class VideoDisplayWidget;
21+
class AudioDisplayWidget;
22+
class AudioSelectorWidget;
23+
class VideoOverlay;
24+
25+
namespace NintendoSwitch{
26+
class CommandRow;
27+
class SwitchSystemSession;
28+
}
29+
30+
namespace ML{
31+
32+
33+
34+
// UI widget for controlling and monitoring a Nintendo Switch.
35+
// It includes:
36+
// - A micro-controller selection UI
37+
// - Video source selection UI
38+
// - Audio source selection UI
39+
// - Audio display
40+
// - Video stream display
41+
// It also owns a SwitchSystemSession that manages the life time of the controller,
42+
// audio and video streams that will be exposed to automation programs.
43+
class ImageDisplayWidget final : public QWidget, public CommandReceiver{
44+
public:
45+
virtual ~ImageDisplayWidget();
46+
ImageDisplayWidget(
47+
QWidget& parent,
48+
NintendoSwitch::SwitchSystemSession& session,
49+
uint64_t program_id
50+
);
51+
52+
public:
53+
void update_ui(ProgramState state);
54+
55+
// The public versions of the private QWidget key and focus event handling functions.
56+
// They are needed to accept key and focus passed from CommonFramework/VideoPipeline/UI:VideoDisplayWindow.
57+
58+
virtual void key_press(QKeyEvent* event) override;
59+
virtual void key_release(QKeyEvent* event) override;
60+
61+
virtual void focus_in(QFocusEvent* event) override;
62+
virtual void focus_out(QFocusEvent* event) override;
63+
64+
private:
65+
virtual void keyPressEvent(QKeyEvent* event) override;
66+
virtual void keyReleaseEvent(QKeyEvent* event) override;
67+
virtual void focusInEvent(QFocusEvent* event) override;
68+
virtual void focusOutEvent(QFocusEvent* event) override;
69+
70+
private:
71+
NintendoSwitch::SwitchSystemSession& m_session;
72+
73+
CollapsibleGroupBox* m_group_box;
74+
75+
VideoDisplayWidget* m_video_display;
76+
77+
NintendoSwitch::CommandRow* m_command;
78+
79+
VideoSourceSelectorWidget* m_video_selector;
80+
};
81+
82+
83+
84+
85+
}
86+
}
87+
#endif

‎SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_SwitchSystemWidget.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ namespace PokemonAutomation{
3030
class VideoSourceSelectorWidget;
3131
class VideoDisplayWidget;
3232
class AudioDisplayWidget;
33+
class AudioSelectorWidget;
3334
class VideoOverlay;
3435

3536
namespace NintendoSwitch{

0 commit comments

Comments
 (0)