add overlay image UI

Author: Gin

SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.cpp

@@ -9,9 +9,9 @@
 #include "CommonFramework/GlobalServices.h"
 #include "VideoOverlayWidget.h"
 
-//#include <iostream>
-//using std::cout;
-//using std::endl;
+// #include <iostream>
+// using std::cout;
+// using std::endl;
 
 namespace PokemonAutomation{
 
@@ -33,6 +33,7 @@ VideoOverlayWidget::VideoOverlayWidget(QWidget& parent, VideoOverlaySession& ses
     , m_session(session)
     , m_boxes(std::make_shared<std::vector<OverlayBox>>(session.boxes()))
     , m_texts(std::make_shared<std::vector<OverlayText>>(session.texts()))
+    , m_images(std::make_shared<std::vector<OverlayImage>>(session.images()))
     , m_log(std::make_shared<std::vector<OverlayLogLine>>(session.log_texts()))
     , m_stats(nullptr)
 {
@@ -50,38 +51,37 @@ VideoOverlayWidget::VideoOverlayWidget(QWidget& parent, VideoOverlaySession& ses
     }
 }
 
-void VideoOverlayWidget::enabled_boxes(bool enabled){
-    QMetaObject::invokeMethod(this, [this]{ this->update(); });
-}
-void VideoOverlayWidget::enabled_text(bool enabled){
-    QMetaObject::invokeMethod(this, [this]{ this->update(); });
-}
-void VideoOverlayWidget::enabled_log(bool enabled){
-    QMetaObject::invokeMethod(this, [this]{ this->update(); });
-}
-void VideoOverlayWidget::enabled_stats(bool enabled){
+void VideoOverlayWidget::async_update(){
+    // by using QMetaObject::invokeMethod, we can call this->update() on a non-main thread
+    // without trigger Qt crash, as normally this->update() can only be called on the main
+    // thread.
     QMetaObject::invokeMethod(this, [this]{ this->update(); });
 }
 
-void VideoOverlayWidget::update_boxes(const std::shared_ptr<const std::vector<OverlayBox>>& boxes){
+void VideoOverlayWidget::on_overlay_update_boxes(const std::shared_ptr<const std::vector<OverlayBox>>& boxes){
     WriteSpinLock lg(m_lock, "VideoOverlay::update_boxes()");
     m_boxes = boxes;
 }
-void VideoOverlayWidget::update_text(const std::shared_ptr<const std::vector<OverlayText>>& texts){
+void VideoOverlayWidget::on_overlay_update_text(const std::shared_ptr<const std::vector<OverlayText>>& texts){
     WriteSpinLock lg(m_lock, "VideoOverlay::update_text()");
     m_texts = texts;
 }
-void VideoOverlayWidget::update_log(const std::shared_ptr<const std::vector<OverlayLogLine>>& texts){
+void VideoOverlayWidget::on_overlay_update_images(const std::shared_ptr<const std::vector<OverlayImage>>& images){
+    WriteSpinLock lg(m_lock, "VideoOverlay::update_images()");
+    m_images = images;
+}
+
+void VideoOverlayWidget::on_overlay_update_log(const std::shared_ptr<const std::vector<OverlayLogLine>>& logs){
     WriteSpinLock lg(m_lock, "VideoOverlay::update_log_text()");
-    m_log = texts;
+    m_log = logs;
 }
 #if 0
 void VideoOverlayWidget::update_log_background(const std::shared_ptr<const std::vector<VideoOverlaySession::Box>>& bg_boxes){
     WriteSpinLock lg(m_lock, "VideoOverlay::update_log_background()");
     m_log_text_bg_boxes = bg_boxes;
 }
 #endif
-void VideoOverlayWidget::update_stats(const std::list<OverlayStat*>* stats){
+void VideoOverlayWidget::on_overlay_update_stats(const std::list<OverlayStat*>* stats){
     WriteSpinLock lg(m_lock, "VideoOverlay::update_stats()");
     m_stats = stats;
 }
@@ -94,33 +94,37 @@ void VideoOverlayWidget::on_watchdog_timeout(){
 
 
 void VideoOverlayWidget::resizeEvent(QResizeEvent* event){}
+
 void VideoOverlayWidget::paintEvent(QPaintEvent*){
     QPainter painter(this);
 
     {
         WriteSpinLock lg(m_lock, "VideoOverlay::paintEvent()");
 
         if (m_session.enabled_boxes()){
-            update_boxes(painter);
+            render_boxes(painter);
         }
         if (m_session.enabled_text()){
-            update_text(painter);
+            render_text(painter);
+        }
+        if (m_session.enabled_images()){
+            render_images(painter);
         }
         if (m_session.enabled_log()){
-            update_log(painter);
+            render_log(painter);
         }
         if (m_session.enabled_stats() && m_stats){
-            update_stats(painter);
+            render_stats(painter);
         }
     }
 
     global_watchdog().delay(*this);
 }
 
 
-void VideoOverlayWidget::update_boxes(QPainter& painter){
-    int width = this->width();
-    int height = this->height();
+void VideoOverlayWidget::render_boxes(QPainter& painter){
+    const int width = this->width();
+    const int height = this->height();
     for (const auto& item : *m_boxes){
         QColor color = QColor((uint32_t)item.color);
         painter.setPen(color);
@@ -196,9 +200,9 @@ void VideoOverlayWidget::update_boxes(QPainter& painter){
         painter.drawText(QPoint(xmin + padding_width, ymin - 2*padding_height), text);
     }
 }
-void VideoOverlayWidget::update_text(QPainter& painter){
-    int width = this->width();
-    int height = this->height();
+void VideoOverlayWidget::render_text(QPainter& painter){
+    const int width = this->width();
+    const int height = this->height();
     for (const auto& item: *m_texts){
         painter.setPen(QColor((uint32_t)item.color));
         QFont text_font = this->font();
@@ -211,7 +215,25 @@ void VideoOverlayWidget::update_text(QPainter& painter){
         painter.drawText(QPoint(xmin, ymin), QString::fromStdString(item.message));
     }
 }
-void VideoOverlayWidget::update_log(QPainter& painter){
+void VideoOverlayWidget::render_images(QPainter& painter){
+    const double width = static_cast<double>(this->width());
+    const double height = static_cast<double>(this->height());
+
+    for(const auto& image_overlay: *m_images){
+        QImage q_image = image_overlay.image.to_QImage_ref();
+        // source rect is the entire portion of the q_image, in pixel units
+        QRectF source_rect(0.0, 0.0, static_cast<double>(q_image.width()), static_cast<double>(q_image.height()));
+        // build a target_rect. target_rect is what region the overlay image should appear inside the overlay viewport.
+        // target_rect is in pixel units of the viewport
+        const double target_start_x = width * image_overlay.x;
+        const double target_start_y = height * image_overlay.y;
+        const double target_width = width * image_overlay.width;
+        const double target_height = height * image_overlay.height;
+        QRectF target_rect(target_start_x, target_start_y, target_width, target_height);
+        painter.drawImage(target_rect, q_image, source_rect);
+    }
+}
+void VideoOverlayWidget::render_log(QPainter& painter){
     if (m_log->empty()){
         return;
     }
@@ -231,7 +253,9 @@ void VideoOverlayWidget::update_log(QPainter& painter){
 
     //  Draw the box.
     {
-        QColor box_color(10, 10, 10, 200);
+        // set a semi-transparent dark color so that user can see the log lines while can also see
+        // vaguely the video stream content behind it
+        QColor box_color(10, 10, 10, 200); // r=g=b=10, alpha=200
         painter.setPen(box_color);
         const int xmin = std::max((int)(width * LOG_MIN_X + 0.5), 1);
         const int ymin = std::max((int)(height * (LOG_MAX_Y - log_bg_height) + 0.5), 1);
@@ -257,7 +281,7 @@ void VideoOverlayWidget::update_log(QPainter& painter){
         y -= LOG_LINE_SPACING;
     }
 }
-void VideoOverlayWidget::update_stats(QPainter& painter){
+void VideoOverlayWidget::render_stats(QPainter& painter){
     const double TEXT_SIZE = 0.02;
     const double ROW_HEIGHT = 0.03;
 

SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.h

@@ -20,6 +20,7 @@ class VideoOverlayWidget : public QWidget, private VideoOverlaySession::ContentL
 public:
     static constexpr bool DEFAULT_ENABLE_BOXES  = true;
     static constexpr bool DEFAULT_ENABLE_TEXT   = true;
+    static constexpr bool DEFAULT_ENABLE_IMAGES = true;
     static constexpr bool DEFAULT_ENABLE_LOG    = false;
     static constexpr bool DEFAULT_ENABLE_STATS  = true;
 
@@ -32,33 +33,61 @@ class VideoOverlayWidget : public QWidget, private VideoOverlaySession::ContentL
 
     //  Asynchronous changes to the overlays.
 
-    virtual void enabled_boxes(bool enabled) override;
-    virtual void enabled_text (bool enabled) override;
-    virtual void enabled_log  (bool enabled) override;
-    virtual void enabled_stats(bool enabled) override;
-
-    virtual void update_boxes(const std::shared_ptr<const std::vector<OverlayBox>>& boxes) override;
-    virtual void update_text (const std::shared_ptr<const std::vector<OverlayText>>& texts) override;
-    virtual void update_log  (const std::shared_ptr<const std::vector<OverlayLogLine>>& texts) override;
-    virtual void update_stats(const std::list<OverlayStat*>* stats) override;
+    // callback function from VideoOverlaySession on overlay boxes enabled
+    virtual void on_overlay_enabled_boxes (bool enabled) override{async_update();}
+    // callback function from VideoOverlaySession on overlay text enabled
+    virtual void on_overlay_enabled_text  (bool enabled) override{async_update();}
+    // callback function from VideoOverlaySession on overlay images enabled
+    virtual void on_overlay_enabled_images(bool enabled) override{async_update();}
+    // callback function from VideoOverlaySession on overlay log enabled
+    virtual void on_overlay_enabled_log   (bool enabled) override{async_update();}
+    // callback function from VideoOverlaySession on overlay stats enabled
+    virtual void on_overlay_enabled_stats (bool enabled) override{async_update();}
+
+    // callback function from VideoOverlaySession on overlay boxes updated
+    virtual void on_overlay_update_boxes (const std::shared_ptr<const std::vector<OverlayBox>>& boxes) override;
+    // callback function from VideoOverlaySession on overlay text updated
+    virtual void on_overlay_update_text  (const std::shared_ptr<const std::vector<OverlayText>>& texts) override;
+    // callback function from VideoOverlaySession on overlay images updated
+    virtual void on_overlay_update_images(const std::shared_ptr<const std::vector<OverlayImage>>& images) override;
+    // callback function from VideoOverlaySession on overlay images updated
+    virtual void on_overlay_update_log   (const std::shared_ptr<const std::vector<OverlayLogLine>>& logs) override;
+    // callback function from VideoOverlaySession on overlay stats updated
+    virtual void on_overlay_update_stats (const std::list<OverlayStat*>* stats) override;
 
     virtual void on_watchdog_timeout() override;
 
     virtual void resizeEvent(QResizeEvent* event) override;
+    // render video overlay, override QWidget::paintEvent()
     virtual void paintEvent(QPaintEvent*) override;
 
 private:
-    void update_boxes(QPainter& painter);
-    void update_text (QPainter& painter);
-    void update_log  (QPainter& painter);
-    void update_stats(QPainter& painter);
+    // Call QWidget::update() to notify Qt to schedule a re-rendering of the overlay widget.
+    //
+    // Threads other than the main thread may change video overlay, causing this VideoOverlayWidget, which
+    // listens to video overlay change, to get called on the non-main thread.
+    // Since QWidget::update() must be called on the main thread to not crash, we have to use this
+    // async_update() to avoid that.
+    void async_update();
+
+    // render overlay boxes
+    void render_boxes  (QPainter& painter);
+    // render overlay texts
+    void render_text   (QPainter& painter);
+    // render overlay images
+    void render_images (QPainter& painter);
+    // render overlay log lines
+    void render_log    (QPainter& painter);
+    // render overlay stats
+    void render_stats  (QPainter& painter);
 
 private:
     VideoOverlaySession& m_session;
 
     SpinLock m_lock;
     std::shared_ptr<const std::vector<OverlayBox>> m_boxes;
     std::shared_ptr<const std::vector<OverlayText>> m_texts;
+    std::shared_ptr<const std::vector<OverlayImage>> m_images;
     std::shared_ptr<const std::vector<OverlayLogLine>> m_log;
     const std::list<OverlayStat*>* m_stats;
 };

SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlay.h

@@ -15,7 +15,11 @@
 
 namespace PokemonAutomation{
 
-
+// Interface to add overlay objects (e.g. bounding boxes and texts) on top of rendered
+// video stream
+// The implementation is in CommonFramework/VideoPipeline/VideoOverlaySession.h:VideoOverlaySession
+// The reason to create this interface is to reduce the header dependency on other code that relies
+// on the overlay.
 class VideoOverlay{
 public:
     VideoOverlay();
@@ -38,8 +42,8 @@ class VideoOverlay{
 
     // Asychronously, add a text, `OverlayText` as part of the video overlay.
     // Once added, `text` cannot be destroyed until after `VideoOverlay::remove_text()` is called to remove it.
-    // If a `text` with the same address is added, it will override the old text's position, content and color. You only need
-    // to call `remove_text()` once to remove it.
+    // If a `text` with the same address is added, it will override the old text's position, content and color.
+    // You only need to call `remove_text()` once to remove it.
     //
     // Can use `OverlayTextScope: public OverlayText` to handle text removal automatically when it's destroyed.
     virtual void add_text(const OverlayText& text) = 0;
@@ -49,6 +53,17 @@ class VideoOverlay{
     // See `add_text()` for more info on managing texts.
     virtual void remove_text(const OverlayText& text) = 0;
 
+    // Asychronously, add an image, `OverlayImage` as part of the video overlay.
+    // Allow transparency to create mask overlay.
+    // Once added, `image` cannot be destroyed until after `VideoOverlay::remove_image()` is called to remove it.
+    // If an `image` with the same address is added, it will override the old content. You only need
+    // to call `remove_text()` once to remove it.
+    virtual void add_image(const OverlayImage& image) = 0;
+    // Asychronously, remove an added `OverlayImage`.
+    // The OverlayImage must be already added.
+    // See `add_image()` for more info on managing overlay images.
+    virtual void remove_image(const OverlayImage& image) = 0;
+
     // Asynchronously, add a log message to the screen. The older messages added via `add_log_text()`
     // will be placed higher like in the logging window.
     // Use `OverlayLogTextScope` to remove the log messages automatically. 

SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlayOption.cpp

@@ -13,6 +13,7 @@ namespace PokemonAutomation{
 
 const std::string VideoOverlayOption::JSON_BOXES = "Boxes";
 const std::string VideoOverlayOption::JSON_TEXT = "Text";
+const std::string VideoOverlayOption::JSON_IMAGES = "Images";
 const std::string VideoOverlayOption::JSON_LOG = "Log";
 const std::string VideoOverlayOption::JSON_STATS = "Stats";
 
@@ -37,6 +38,12 @@ void VideoOverlayOption::load_json(const JsonValue& json){
     if (obj->read_boolean(tmp, JSON_TEXT)){
         text.store(tmp, std::memory_order_relaxed);
     }
+    if (obj->read_boolean(tmp, JSON_IMAGES)){
+        images.store(tmp, std::memory_order_relaxed);
+    } else{
+        // by default the image overlay is on
+        images.store(true, std::memory_order_relaxed);
+    }
     if (obj->read_boolean(tmp, JSON_LOG)){
         log.store(tmp, std::memory_order_relaxed);
     }
@@ -46,10 +53,11 @@ void VideoOverlayOption::load_json(const JsonValue& json){
 }
 JsonValue VideoOverlayOption::to_json() const{
     JsonObject root;
-    root[JSON_BOXES] = boxes.load(std::memory_order_relaxed);
-    root[JSON_TEXT]  = text.load(std::memory_order_relaxed);
-    root[JSON_LOG]   = log.load(std::memory_order_relaxed);
-    root[JSON_STATS] = stats.load(std::memory_order_relaxed);
+    root[JSON_BOXES]  = boxes.load(std::memory_order_relaxed);
+    root[JSON_TEXT]   = text.load(std::memory_order_relaxed);
+    root[JSON_IMAGES] = images.load(std::memory_order_relaxed);
+    root[JSON_LOG]    = log.load(std::memory_order_relaxed);
+    root[JSON_STATS]  = stats.load(std::memory_order_relaxed);
     return root;
 }
 

SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlayOption.h

@@ -15,9 +15,12 @@ namespace PokemonAutomation{
 class JsonValue;
 
 
+// store data related to the checkerboxes on UI: whether user enabled box overlay,
+// text overlay, etc.
 class VideoOverlayOption{
     static const std::string JSON_BOXES;
     static const std::string JSON_TEXT;
+    static const std::string JSON_IMAGES;
     static const std::string JSON_LOG;
     static const std::string JSON_STATS;
 
@@ -28,10 +31,11 @@ class VideoOverlayOption{
     JsonValue to_json() const;
 
 public:
-    std::atomic<bool> boxes;
-    std::atomic<bool> text;
-    std::atomic<bool> log;
-    std::atomic<bool> stats;
+    std::atomic<bool> boxes;  // whether user enabled box overlay
+    std::atomic<bool> text;   // whether user enabled text overlay
+    std::atomic<bool> images; // whether user enabled image overlay
+    std::atomic<bool> log;    // whether user enabled log overlay
+    std::atomic<bool> stats;  // whether user enabled stats overlay
 };