add shortcut support to label image program

Author: Gin

SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp

@@ -20,6 +20,7 @@
 #include <fstream>
 #include <filesystem>
 #include <cmath>
+#include <iomanip>
 #include "CommonFramework/Globals.h"
 #include "Common/Cpp/BitmapConversion.h"
 #include "Common/Cpp/Json/JsonArray.h"
@@ -108,79 +109,6 @@ JsonObject object_annotation_to_json(const ObjectAnnotation& object_annotation){
 }
 
 
-DrawnBoundingBox::DrawnBoundingBox(LabelImages_Widget& widget, VideoOverlay& overlay)
-    : m_widget(widget)
-    , m_overlay(overlay)
-{
-    auto& program = m_widget.m_program;
-    program.X.add_listener(*this);
-    program.Y.add_listener(*this);
-    program.WIDTH.add_listener(*this);
-    program.HEIGHT.add_listener(*this);
-    overlay.add_listener(*this);
-}
-
-DrawnBoundingBox::~DrawnBoundingBox(){
-    detach();
-}
-
-// called when drawn bounding box changed
-void DrawnBoundingBox::on_config_value_changed(void* object){
-    auto& program = m_widget.m_program;
-    std::lock_guard<std::mutex> lg(m_lock);
-    program.update_rendered_objects(m_widget.m_overlay_set);
-}
-void DrawnBoundingBox::on_mouse_press(double x, double y){
-    auto& program = m_widget.m_program;
-    program.WIDTH.set(0);
-    program.HEIGHT.set(0);
-    program.X.set(x);
-    program.Y.set(y);
-    m_mouse_start.emplace();
-    m_mouse_start->first = x;
-    m_mouse_start->second = y;
-}
-void DrawnBoundingBox::on_mouse_release(double, double){
-    m_mouse_start.reset();
-    auto& m_program = m_widget.m_program;
-    auto& m_overlay_set = m_widget.m_overlay_set;
-
-    m_program.compute_mask(m_overlay_set);
-}
-
-void DrawnBoundingBox::on_mouse_move(double x, double y){
-    auto& program = m_widget.m_program;
-    if (!m_mouse_start){
-        return;
-    }
-
-    double xl = m_mouse_start->first;
-    double xh = x;
-    double yl = m_mouse_start->second;
-    double yh = y;
-
-    if (xl > xh){
-        std::swap(xl, xh);
-    }
-    if (yl > yh){
-        std::swap(yl, yh);
-    }
-
-    program.X.set(xl);
-    program.Y.set(yl);
-    program.WIDTH.set(xh - xl);
-    program.HEIGHT.set(yh - yl);
-}
-
-void DrawnBoundingBox::detach(){
-    auto& program = m_widget.m_program;
-    m_overlay.remove_listener(*this);
-    program.X.remove_listener(*this);
-    program.Y.remove_listener(*this);
-    program.WIDTH.remove_listener(*this);
-    program.HEIGHT.remove_listener(*this);
-}
-
 
 LabelImages_Descriptor::LabelImages_Descriptor()
     : PanelDescriptor(
@@ -444,10 +372,26 @@ void LabelImages::compute_embeddings_for_folder(const std::string& image_folder_
     ML::compute_embeddings_for_folder(embedding_model_path, image_folder_path);
 }
 
+void LabelImages::delete_last_annotation(){
+    if (m_annotations.size() > 0){
+        m_annotations.pop_back();
+    }
+    if (m_annotations.size() > 0){
+        m_last_object_idx = m_annotations.size() - 1;
+    }
+}
 
 
 LabelImages_Widget::~LabelImages_Widget(){
+    m_program.X.remove_listener(*this);
+    m_program.Y.remove_listener(*this);
+    m_program.WIDTH.remove_listener(*this);
+    m_program.HEIGHT.remove_listener(*this);
     m_program.FORM_LABEL.remove_listener(*this);
+
+    m_display_session.overlay().remove_listener(*this);
+    m_display_session.video_session().remove_state_listener(*this);
+
     delete m_image_display_widget;
 }
 LabelImages_Widget::LabelImages_Widget(
@@ -459,9 +403,14 @@ LabelImages_Widget::LabelImages_Widget(
     , m_program(program)
     , m_display_session(m_program.m_display_session)
     , m_overlay_set(m_display_session.overlay())
-    , m_drawn_box(*this, m_display_session.overlay())
 {
+    m_program.X.add_listener(*this);
+    m_program.Y.add_listener(*this);
+    m_program.WIDTH.add_listener(*this);
+    m_program.HEIGHT.add_listener(*this);
     m_program.FORM_LABEL.add_listener(*this);
+
+    m_display_session.overlay().add_listener(*this);
     m_display_session.video_session().add_state_listener(*this);
 
     m_embedding_info_label = new QLabel(this);
@@ -479,7 +428,7 @@ LabelImages_Widget::LabelImages_Widget(
     QVBoxLayout* scroll_layout = new QVBoxLayout(scroll_inner);
     scroll_layout->setAlignment(Qt::AlignTop);
 
-    m_image_display_widget = new ImageAnnotationDisplayWidget(*this, m_display_session, 0);
+    m_image_display_widget = new ImageAnnotationDisplayWidget(*this, m_display_session, this);
     scroll_layout->addWidget(m_image_display_widget);
 
     QHBoxLayout* embedding_info_row = new QHBoxLayout();
@@ -491,12 +440,7 @@ LabelImages_Widget::LabelImages_Widget(
     scroll_layout->addWidget(button);
     connect(button, &QPushButton::clicked, this, [this](bool){
         auto& program = this->m_program;
-        if (program.m_annotations.size() > 0){
-            program.m_annotations.pop_back();
-        }
-        if (program.m_annotations.size() > 0){
-            program.m_last_object_idx = program.m_annotations.size() - 1;
-        }
+        program.delete_last_annotation();
         program.update_rendered_objects(this->m_overlay_set);
     });
 
@@ -527,8 +471,8 @@ void LabelImages_Widget::on_config_value_changed(void* object){
     if (m_program.m_annotations.size() > 0 && m_program.m_last_object_idx < m_program.m_annotations.size()){
         std::string& cur_label = m_program.m_annotations[m_program.m_last_object_idx].label;
         cur_label = m_program.FORM_LABEL.slug();
-        m_program.update_rendered_objects(m_overlay_set);
     }
+    m_program.update_rendered_objects(m_overlay_set);
 }
 
 // This callback function will be called whenever the display source (the image source) is loaded or reloaded:
@@ -565,7 +509,56 @@ void LabelImages_Widget::post_startup(VideoSource* source){
     m_program.update_rendered_objects(m_overlay_set);
 }
 
+void LabelImages_Widget::key_release(QKeyEvent* event){
+    const auto key = Qt::Key(event->key());
+    if (key == Qt::Key::Key_Delete || key == Qt::Key::Key_Backspace){
+        m_program.delete_last_annotation();
+        m_program.update_rendered_objects(m_overlay_set);
+    }
+}
+
+void LabelImages_Widget::on_mouse_press(double x, double y){
+    m_program.WIDTH.set(0);
+    m_program.HEIGHT.set(0);
+    m_program.X.set(x);
+    m_program.Y.set(y);
+    m_mouse_start.emplace();
+    m_mouse_end.emplace();
+    m_mouse_start->first = m_mouse_end->first = x;
+    m_mouse_start->second = m_mouse_end->second = y;
+}
 
+void LabelImages_Widget::on_mouse_release(double x, double y){
+    m_mouse_start.reset();
+    m_mouse_end.reset();
+    m_program.compute_mask(m_overlay_set);
+}
+
+void LabelImages_Widget::on_mouse_move(double x, double y){
+    if (!m_mouse_start){
+        return;
+    }
+
+    m_mouse_end->first = x;
+    m_mouse_end->second = y;
+
+    double xl = m_mouse_start->first;
+    double yl = m_mouse_start->second;
+    double xh = x;
+    double yh = y;
+
+    if (xl > xh){
+        std::swap(xl, xh);
+    }
+    if (yl > yh){
+        std::swap(yl, yh);
+    }
+
+    m_program.X.set(xl);
+    m_program.Y.set(yl);
+    m_program.WIDTH.set(xh - xl);
+    m_program.HEIGHT.set(yh - yl);
+}
 
 }
 }

SerialPrograms/Source/ML/Programs/ML_LabelImages.h

@@ -20,6 +20,7 @@
 #include "NintendoSwitch/Framework/NintendoSwitch_SwitchSystemOption.h"
 #include "NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.h"
 #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+#include "CommonFramework/VideoPipeline/UI/VideoDisplayWidget.h"
 
 #include "ML/DataLabeling/ML_SegmentAnythingModel.h"
 #include "ML/UI/ML_ImageAnnotationDisplayOption.h"
@@ -96,9 +97,11 @@ class LabelImages : public PanelInstance{
     // This can be very slow!
     void compute_embeddings_for_folder(const std::string& image_folder);
 
+    // Delete the last object annotation.
+    void delete_last_annotation();
+
 private:
     friend class LabelImages_Widget;
-    friend class DrawnBoundingBox;
 
     // image display options like what image file is loaded
     ImageAnnotationDisplayOption m_display_option;
@@ -132,28 +135,11 @@ class LabelImages : public PanelInstance{
 };
 
 
-class DrawnBoundingBox : public ConfigOption::Listener, public VideoOverlay::MouseListener{
-public:
-    ~DrawnBoundingBox();
-    DrawnBoundingBox(LabelImages_Widget& widget, VideoOverlay& overlay);
-    virtual void on_config_value_changed(void* object) override;
-    virtual void on_mouse_press(double x, double y) override;
-    virtual void on_mouse_release(double x, double y) override;
-    virtual void on_mouse_move(double x, double y) override;
-
-private:
-    void detach();
-
-private:
-    LabelImages_Widget& m_widget;
-    VideoOverlay& m_overlay;
-    std::mutex m_lock;
-
-    std::optional<std::pair<double, double>> m_mouse_start;
-};
-
-
-class LabelImages_Widget : public PanelWidget, public ConfigOption::Listener, public VideoSession::StateListener{
+class LabelImages_Widget : public PanelWidget,
+                           public ConfigOption::Listener,
+                           public VideoSession::StateListener,
+                           public CommandReceiver,
+                           public VideoOverlay::MouseListener{
 public:
     ~LabelImages_Widget();
     LabelImages_Widget(
@@ -165,24 +151,44 @@ class LabelImages_Widget : public PanelWidget, public ConfigOption::Listener, pu
     // called after loading a new image, clean up all internal data 
     void clear_for_new_image();
 
+    //  Overwrites ConfigOption::Listener::on_config_value_changed().
     virtual void on_config_value_changed(void* object) override;
 
     //  Overwrites VideoSession::StateListener::post_startup().
     virtual void post_startup(VideoSource* source) override;
 
+    //  Overwrites CommandReceiver::key_press().
+    virtual void key_press(QKeyEvent* event) override {}
+    //  Overwrites CommandReceiver::key_release().
+    virtual void key_release(QKeyEvent* event) override;
+    //  Overwrites CommandReceiver::focus_in().
+    virtual void focus_in(QFocusEvent* event) override {}
+    //  Overwrites CommandReceiver::focus_out().
+    virtual void focus_out(QFocusEvent* event) override {}
+
+    //  Overwrites VideoOverlay::MouseListener::on_mouse_press().
+    virtual void on_mouse_press(double x, double y) override;
+    //  Overwrites VideoOverlay::MouseListener::on_mouse_release().
+    virtual void on_mouse_release(double x, double y) override;
+    //  Overwrites VideoOverlay::MouseListener::on_mouse_move().
+    virtual void on_mouse_move(double x, double y) override;
+
 private:
     LabelImages& m_program;
     ImageAnnotationDisplaySession& m_display_session;
 
     ImageAnnotationDisplayWidget* m_image_display_widget;
 
     VideoOverlaySet m_overlay_set;
-    DrawnBoundingBox m_drawn_box;
 
+    // show the info about the loaded image embedding data corresponding to the currently
+    // displayed image
     QLabel* m_embedding_info_label = nullptr;
+    // a UI widget that holds all the editable UI elements defined in LabelImage program.
     ConfigWidget* m_option_widget;
 
-    friend class DrawnBoundingBox;
+    std::optional<std::pair<double, double>> m_mouse_start;
+    std::optional<std::pair<double, double>> m_mouse_end;
 };
 
 

SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplayWidget.cpp

@@ -40,10 +40,11 @@ ImageAnnotationDisplayWidget::~ImageAnnotationDisplayWidget(){
 ImageAnnotationDisplayWidget::ImageAnnotationDisplayWidget(
     QWidget& parent,
     ImageAnnotationDisplaySession& session,
-    uint64_t program_id
+    CommandReceiver* command_receiver
 )
     : QWidget(&parent)
     , m_session(session)
+    , m_command_receiver(command_receiver)
 {
 
     QVBoxLayout* layout = new QVBoxLayout(this);
@@ -95,19 +96,31 @@ void ImageAnnotationDisplayWidget::update_ui(ProgramState state){
 void ImageAnnotationDisplayWidget::key_press(QKeyEvent* event){
 //    cout << "press:   " << event->nativeVirtualKey() << endl;
     m_command->on_key_press(*event);
+    if (m_command_receiver){
+        m_command_receiver->key_press(event);
+    }
 }
 
 void ImageAnnotationDisplayWidget::key_release(QKeyEvent* event){
 //    cout << "release: " << event->nativeVirtualKey() << endl;
     m_command->on_key_release(*event);
+    if (m_command_receiver){
+        m_command_receiver->key_release(event);
+    }
 }
 
 void ImageAnnotationDisplayWidget::focus_in(QFocusEvent* event){
     m_command->set_focus(true);
+    if (m_command_receiver){
+        m_command_receiver->focus_in(event);
+    }
 }
 
 void ImageAnnotationDisplayWidget::focus_out(QFocusEvent* event){
     m_command->set_focus(false);
+    if (m_command_receiver){
+        m_command_receiver->focus_out(event);
+    }
 }
 
 void ImageAnnotationDisplayWidget::keyPressEvent(QKeyEvent* event){