Skip to content

Commit d135e08

Browse files
author
Gin
committed
more image UI for ML labeling
1 parent 9d9519a commit d135e08

26 files changed

+570
-299
lines changed

SerialPrograms/CMakeLists.txt

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -894,14 +894,16 @@ file(GLOB MAIN_SOURCES
894894
Source/ML/ML_Panels.h
895895
Source/ML/Programs/ML_LabelImages.cpp
896896
Source/ML/Programs/ML_LabelImages.h
897-
Source/ML/UI/ML_ImageDisplayCommandRow.cpp
898-
Source/ML/UI/ML_ImageDisplayCommandRow.h
899-
Source/ML/UI/ML_ImageDisplayOption.cpp
900-
Source/ML/UI/ML_ImageDisplayOption.h
901-
Source/ML/UI/ML_ImageDisplaySession.cpp
902-
Source/ML/UI/ML_ImageDisplaySession.h
903-
Source/ML/UI/ML_ImageDisplayWidget.cpp
904-
Source/ML/UI/ML_ImageDisplayWidget.h
897+
Source/ML/UI/ML_ImageAnnotationSourceSelectorWidget.cpp
898+
Source/ML/UI/ML_ImageAnnotationSourceSelectorWidget.h
899+
Source/ML/UI/ML_ImageAnnotationCommandRow.cpp
900+
Source/ML/UI/ML_ImageAnnotationCommandRow.h
901+
Source/ML/UI/ML_ImageAnnotationDisplayOption.cpp
902+
Source/ML/UI/ML_ImageAnnotationDisplayOption.h
903+
Source/ML/UI/ML_ImageAnnotationDisplaySession.cpp
904+
Source/ML/UI/ML_ImageAnnotationDisplaySession.h
905+
Source/ML/UI/ML_ImageAnnotationDisplayWidget.cpp
906+
Source/ML/UI/ML_ImageAnnotationDisplayWidget.h
905907
Source/NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.cpp
906908
Source/NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h
907909
Source/NintendoSwitch/Commands/NintendoSwitch_Commands_Routines.cpp

SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoDisplayWidget.h

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,17 @@
1-
/* Video Display
1+
/* Video Display Widget
22
*
33
* From: https://github.com/PokemonAutomation/
44
*
5+
* VideoDisplayWidget renders a window for video stream from a Switch console or
6+
* other sources. Stream control UI is not part of this class.
7+
*
8+
* The actual QWidget that displays the video content is not created by this class.
9+
* But rather, this class receives the content widget from
10+
* VideoSession.current_source()->make_display_QtWidget().
11+
*
12+
* VideoDisplayWidget is repsonsible for handling aspect ratio and passing key events
13+
* around. It also uses VideoDisplayWindow to pop up the stream view into a standalone
14+
* QMainWindow.
515
*/
616

717
#ifndef PokemonAutomation_VideoPipeline_VideoDisplayWidget_H
@@ -19,8 +29,8 @@ namespace PokemonAutomation{
1929
class VideoDisplayWindow;
2030

2131

22-
// Interface for forwarding keyboard commands from the VideoDisplayWidget to
23-
// whatever thing under it handles it.
32+
// Interface for forwarding keyboard and focus events from the VideoDisplayWidget to
33+
// whatever thing inheriting CommandReceiver.
2434
struct CommandReceiver{
2535
virtual void key_press(QKeyEvent* event) = 0;
2636
virtual void key_release(QKeyEvent* event) = 0;
@@ -35,6 +45,7 @@ struct CommandReceiver{
3545

3646
class VideoDisplayWidget;
3747

48+
// Render the FPS of the incoming video stream as a text overlay on the video window
3849
class VideoSourceFPS : public OverlayStat{
3950
public:
4051
VideoSourceFPS(VideoDisplayWidget& parent)
@@ -46,6 +57,7 @@ class VideoSourceFPS : public OverlayStat{
4657
VideoDisplayWidget& m_parent;
4758
};
4859

60+
// Render the FPS of the rendering thread of the video window as a text overlay on the window
4961
class VideoDisplayFPS : public OverlayStat{
5062
public:
5163
VideoDisplayFPS(VideoDisplayWidget& parent)
@@ -59,12 +71,17 @@ class VideoDisplayFPS : public OverlayStat{
5971

6072

6173

62-
// The widget that owns the video window.
63-
// It consists of a VideoWidget that loads the video content from Switch and a
64-
// VideoOverlayWidget that renders inference boxes and other visualizations on
65-
// top of the video content.
74+
// The widget that owns the video stream window.
75+
// It consists of a VideoWidget that loads the video content from Switch or other sources
76+
// and a VideoOverlayWidget that renders text overlays, inference boxes and other visualizations
77+
// on top of the video content.
6678
class VideoDisplayWidget : public WidgetStackFixedAspectRatio, private VideoSession::StateListener{
6779
public:
80+
// holder: parent's layout that holds this VideoDisplayWidget
81+
// id: the ID to differentiate multiple video views. For multi-Switch automation programs, this id
82+
// is the console ID.
83+
// command_receiver: whenever there is a keyboard or focus event, forward the event to this
84+
// command_receiver object.
6885
VideoDisplayWidget(
6986
QWidget& parent, QLayout& holder,
7087
size_t id,

SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoDisplayWindow.cpp

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,19 +22,20 @@ namespace PokemonAutomation{
2222
VideoDisplayWindow::VideoDisplayWindow(VideoDisplayWidget* display_widget)
2323
: m_display_widget(display_widget)
2424
{
25-
// m_display_widget->setParent(this);
2625
setWindowTitle("Console: " + QString::number(m_display_widget->id()));
2726
this->setWindowIcon(QIcon(QString::fromStdString(RESOURCE_PATH() + "icon.png")));
2827

29-
// this->setSizePolicy(QSizePolicy::Minimum, QSizePolicy::Minimum);
28+
// Sets m_display_widget to be the VideoDisplayWindow's central widget.
29+
// VideoDisplayWindow now takes ownership of m_display_widget as a QWidget
3030
this->setCentralWidget(m_display_widget);
31-
// m_display_widget->setAlignment(Qt::AlignCenter);
32-
33-
// cout << "Display Widget: " << m_display_widget->width() << " x " << m_display_widget->height() << endl;
31+
32+
// this->setSizePolicy(QSizePolicy::Minimum, QSizePolicy::Minimum);
33+
// m_display_widget->setAlignment(Qt::AlignCenter);
3434

35+
// cout << "Display Widget: " << m_display_widget->width() << " x " << m_display_widget->height() << endl;
3536
m_normal_size = m_display_widget->size();
3637
this->resize(m_normal_size);
37-
// cout << this->width() << " x " << this->height() << endl;
38+
// cout << this->width() << " x " << this->height() << endl;
3839

3940
this->show();
4041
this->raise(); // bring the window to front on macOS

SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoSourceSelectorWidget.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ VideoSourceSelectorWidget::VideoSourceSelectorWidget(Logger& logger, VideoSessio
5151
update_source_list();
5252
update_resolution_list();
5353

54+
// Set the action for the video source selection box
5455
connect(
5556
m_sources_box, static_cast<void(QComboBox::*)(int)>(&QComboBox::activated),
5657
this, [this](int index){
@@ -62,6 +63,8 @@ VideoSourceSelectorWidget::VideoSourceSelectorWidget(Logger& logger, VideoSessio
6263
}
6364
}
6465
);
66+
67+
// Set the action for the video resolution selection box
6568
connect(
6669
m_resolution_box, static_cast<void(QComboBox::*)(int)>(&QComboBox::activated),
6770
this, [this](int index){
@@ -72,6 +75,8 @@ VideoSourceSelectorWidget::VideoSourceSelectorWidget(Logger& logger, VideoSessio
7275
m_session.set_resolution(resolution);
7376
}
7477
);
78+
79+
// Set the action for the video reset button
7580
connect(
7681
m_reset_button, &QPushButton::clicked,
7782
this, [this](bool){

SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoSourceSelectorWidget.h

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,26 @@ class QPushButton;
1717
namespace PokemonAutomation{
1818

1919

20-
20+
// UI to select what video source to use for an Switch automation program
21+
// This class inherits VideoSession::StateListener so it has callbacks for state change
22+
// in the video session.
2123
class VideoSourceSelectorWidget : public QWidget, public VideoSession::StateListener{
2224
public:
2325
~VideoSourceSelectorWidget();
2426
VideoSourceSelectorWidget(Logger& logger, VideoSession& session);
2527

2628

2729
private:
30+
// Query the OS to get all the possible video sources and update the UI to show them.
2831
void update_source_list();
32+
// Load all possible video resolutions from the current video source.
2933
void update_resolution_list();
3034

35+
// Overwrites VideoSession::StateListener::post_startup().
36+
// This function is called after setting a new video source or resetting the
37+
// existing video source.
38+
// This also includes after the video resolution is changed as we need to rebuild the video
39+
// source with the new resolution.
3140
virtual void post_startup(VideoSource* source) override;
3241

3342

SerialPrograms/Source/CommonFramework/VideoPipeline/VideoFeed.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,11 @@ class VideoFeed{
101101

102102

103103
public:
104-
// Returns the currently measured frames/second for the video source + display.
104+
// Returns the currently measured frames/second for the video source.
105105
// Use this for diagnostic purposes.
106106
virtual double fps_source() const = 0;
107+
// Returns the currently measured frames/second for the video display thread.
108+
// Use this for diagnostic purposes.
107109
virtual double fps_display() const = 0;
108110
};
109111

SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSession.h

Lines changed: 93 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,67 +19,154 @@ namespace PokemonAutomation{
1919

2020

2121
class VideoSession
22-
: public VideoFeed
23-
, private VideoFrameListener
24-
, private VideoSource::RenderedFrameListener
22+
: public VideoFeed // interface class the automation programs have access to to query video snapshots
23+
// and reset the video source
24+
, private VideoFrameListener // listens to incoming frames from video source
25+
, private VideoSource::RenderedFrameListener // listens to newly rendered frame from video source
2526
, private WatchdogCallback
2627
{
2728
public:
29+
// other classes can inherit this struct to react to state changes in the VideoSession.
30+
// You need to call VideoSession::add_state_listener() to add other classes as listeners
31+
// and remove them by VideoSession::remove_state_listener() when they are no longer needed.
32+
// When a state change happens, VideoSession will call the corresponding callback functions
33+
// in the StateListener.
2834
struct StateListener{
35+
// Called whenever the video source is initialized or reset.
36+
// This also includes when the video resolution is changed as we need to rebuild the video
37+
// source with the new resolution.
2938
virtual void post_startup(VideoSource* source){}
3039

31-
// Sent before the video source shuts down. Listeners should drop their
40+
// Sent before the video source resets or shuts down. Listeners should drop their
3241
// references to the internal video source before returning.
42+
// This also includes when the video resolution is changed as we need to rebuild the video
43+
// source with the new resolution.
3344
virtual void pre_shutdown(){}
45+
// Currently not called by VideoSession yet.
3446
virtual void post_shutdown(){}
35-
47+
// Currently not called by VideoSession yet.
3648
virtual void pre_resolution_change(Resolution resolution){}
49+
// Currently not called by VideoSession yet.
3750
virtual void post_resolution_change(Resolution resolution){}
3851
};
3952

53+
// Add a state listener for it to react to state changes in the VideoSession.
54+
// When a state change happens, VideoSession will call the corresponding callback functions
55+
// in the StateListener.
4056
void add_state_listener(StateListener& listener);
57+
// Remove the state listener.
4158
void remove_state_listener(StateListener& listener);
4259

60+
// Implements VideoFeed::add_frame_listener().
61+
// Add a VideoFrameListener for it to react to incoming frames in the video source.
62+
// Whenever the video source sends a frame, VideoSession will call the callback functions
63+
// in added listeners.
4364
virtual void add_frame_listener(VideoFrameListener& listener) override;
65+
// Implements VideoFeed::remove_frame_listener().
66+
// Remove the frame listener.
4467
virtual void remove_frame_listener(VideoFrameListener& listener) override;
4568

4669

4770
public:
4871
~VideoSession();
72+
// Built from a VideoSourceOption.
73+
// VideoSourceOption manages the current VideoSourceDescriptor, which are the descriptors
74+
// to build a VideoSource.
75+
// VideoSession stores a reference of the passed in VideoSourceOption to manipulate the
76+
// option according to user selection.
4977
VideoSession(Logger& logger, VideoSourceOption& option);
5078

5179
Logger& logger(){
5280
return m_logger;
5381
}
5482

83+
// Current video source descriptor.
84+
// This function is thread-safe. It has a lock to prevent concurrent calls
85+
// of other VideoSession functions.
5586
std::shared_ptr<const VideoSourceDescriptor> descriptor() const;
5687

5788
// Warning, not thread-safe with any source changes.
5889
VideoSource* current_source(){
5990
return m_video_source.get();
6091
}
6192

93+
// Return current resolution.
94+
// This function is thread-safe. It has a lock to prevent concurrent calls
95+
// of other VideoSession functions.
6296
Resolution current_resolution();
97+
// Return supported resolutions.
98+
// This function is thread-safe. It has a lock to prevent concurrent calls
99+
// of other VideoSession functions.
63100
std::vector<Resolution> supported_resolutions() const;
64101

102+
// Implements VideoFeed::snapshot() to provide a snapshot from the current video stream.
103+
// It will wait for a latest snapshot, blocking the current thread to wait for it.
104+
// This function is thread-safe. It has a lock to prevent concurrent calls
105+
// of other VideoSession functions.
65106
virtual VideoSnapshot snapshot_latest_blocking() override;
107+
// Implements VideoFeed::snapshot_recent_nonblocking() to provide a snapshot from the
108+
// current video stream.
109+
// It will use the most recent available frame, therefore non-blocking.
110+
// This function is thread-safe. It has a lock to prevent concurrent calls
111+
// of other VideoSession functions.
66112
virtual VideoSnapshot snapshot_recent_nonblocking(WallClock min_time) override;
67113

114+
// Implements VideoFeed::fps_source().
115+
// Returns the currently measured frames/second for the video source.
116+
// Use this for diagnostic purposes.
117+
// This function is thread-safe. It has a lock to prevent concurrent fps calls.
68118
virtual double fps_source() const override;
119+
// Implements VideoFeed::fps_display().
120+
// Returns the currently measured frames/second for the video display thread.
121+
// Use this for diagnostic purposes.
122+
// This function is thread-safe. It has a lock to prevent concurrent fps calls.
69123
virtual double fps_display() const override;
70124

71125

72126
public:
127+
// Get current video source option
73128
void get(VideoSourceOption& option);
129+
// Set a new video source. This will close the old video source.
130+
// This equals to calling VideoSession::set_source(option.descriptor()).
131+
// Change of video source and resolution will be reflected on the internal
132+
// referenced video source option (aka the VideoSourceOption passed to the
133+
// VideoSession constructor).
134+
// The change is dispatched to the Qt main thread to execute.
74135
void set(const VideoSourceOption& option);
75-
136+
// Implements VideoFeed::reset().
137+
// Reset the current video source. This equals to close the old video source
138+
// and reopen it.
139+
// This may also change the internal referenced video source option (aka the
140+
// VideoSourceOption passed to the VideoSession constructor).
141+
// The change is dispatched to the Qt main thread to execute.
76142
virtual void reset() override;
143+
// Set a new video source. This will close the old video source.
144+
// Change of video source and resolution will be reflected on the internal
145+
// referenced video source option (aka the VideoSourceOption passed to the
146+
// VideoSession constructor).
147+
// The change is dispatched to the Qt main thread to execute.
77148
void set_source(const std::shared_ptr<VideoSourceDescriptor>& device);
149+
// Change video resolution.
150+
// This will trigger a video source reset.
151+
// Change of resolution will be reflected on the internal referenced video
152+
// source option (aka the VideoSourceOption passed to the VideoSession
153+
// constructor).
154+
// The change is dispatched to the Qt main thread to execute.
78155
void set_resolution(Resolution resolution);
79156

80157

81158
private:
159+
// Overwrites VideoFrameListener::on_frame()
160+
// When this function is called, the VideoSession will update its internal fps record
161+
// and call frame listeners added by VideoSession::add_frame_listener().
162+
// This function is thread-safe as it has a lock to prevent concurrent fps calls.
163+
// Note the lock does not protect the other frame listeners added by
164+
// VideoSession::add_frame_listener().
82165
virtual void on_frame(std::shared_ptr<const VideoFrame> frame) override;
166+
// Overwrites VideoSource::RenderedFrameListener::on_rendered_frame()
167+
// This function is called when the video source finds a new rendered frame so
168+
// VideoSession can update its internal fps record.
169+
// This function is thread-safe as it has a lock to prevent concurrent fps calls.
83170
virtual void on_rendered_frame(WallClock timestamp) override;
84171

85172
virtual void on_watchdog_timeout() override;

SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSource.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ class VideoSource{
6969
auto scope_check = m_sanitizer.check_scope();
7070
m_source_frame_listeners.run_method_unique(&VideoFrameListener::on_frame, frame);
7171
}
72+
// Called by UI to report a frame is rendered
7273
void report_rendered_frame(WallClock timestamp){
7374
auto scope_check = m_sanitizer.check_scope();
7475
m_rendered_frame_listeners.run_method_unique(&RenderedFrameListener::on_rendered_frame, timestamp);

SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSourceDescriptor.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ const EnumStringMap<VideoSourceType> VIDEO_TYPE_STRINGS{
3030

3131

3232
VideoSourceOption::VideoSourceOption()
33-
: m_descriptor(new VideoSourceDescriptor_Null())
34-
, m_resolution(1920, 1080)
33+
: m_resolution(1920, 1080)
34+
, m_descriptor(new VideoSourceDescriptor_Null())
3535
{}
3636

3737
void VideoSourceOption::set_descriptor(std::shared_ptr<VideoSourceDescriptor> descriptor){

0 commit comments

Comments
 (0)