Video Pipeline Refactor (part 1): Implement new pipeline.

Author: Mysticial

Common/Qt/WidgetStackFixedAspectRatio.cpp

@@ -38,6 +38,11 @@ WidgetStackFixedAspectRatio::WidgetStackFixedAspectRatio(
 void WidgetStackFixedAspectRatio::add_widget(QWidget& widget){
     widget.setParent(m_stack_holder);
     m_widgets.insert(&widget);
+    widget.show();
+}
+void WidgetStackFixedAspectRatio::remove_widget(QWidget* widget){
+    m_widgets.erase(widget);
+    delete widget;
 }
 double WidgetStackFixedAspectRatio::sanitize_aspect_ratio(double aspect_ratio) const{
     if (aspect_ratio == 0 || std::isnan(aspect_ratio)){

Common/Qt/WidgetStackFixedAspectRatio.h

@@ -35,6 +35,7 @@ class WidgetStackFixedAspectRatio : public QWidget{
     void set_all(SizePolicy size_policy, double aspect_ratio);
 
     void add_widget(QWidget& widget);
+    void remove_widget(QWidget* widget);
 
     virtual void resizeEvent(QResizeEvent* event) override;
 

SerialPrograms/CMakeLists.txt

@@ -483,6 +483,8 @@ file(GLOB MAIN_SOURCES
     Source/CommonFramework/VideoPipeline/UI/VideoDisplayWindow.h
     Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.cpp
     Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.h
+    Source/CommonFramework/VideoPipeline/UI/VideoSourceSelectorWidget.cpp
+    Source/CommonFramework/VideoPipeline/UI/VideoSourceSelectorWidget.h
     Source/CommonFramework/VideoPipeline/UI/VideoWidget.h
     Source/CommonFramework/VideoPipeline/VideoFeed.h
     Source/CommonFramework/VideoPipeline/VideoOverlay.cpp
@@ -495,6 +497,16 @@ file(GLOB MAIN_SOURCES
     Source/CommonFramework/VideoPipeline/VideoOverlayTypes.cpp
     Source/CommonFramework/VideoPipeline/VideoOverlayTypes.h
     Source/CommonFramework/VideoPipeline/VideoPipelineOptions.h
+    Source/CommonFramework/VideoPipeline/VideoSession.cpp
+    Source/CommonFramework/VideoPipeline/VideoSession.h
+    Source/CommonFramework/VideoPipeline/VideoSource.cpp
+    Source/CommonFramework/VideoPipeline/VideoSource.h
+    Source/CommonFramework/VideoPipeline/VideoSource_Camera.cpp
+    Source/CommonFramework/VideoPipeline/VideoSource_Camera.h
+    Source/CommonFramework/VideoPipeline/VideoSource_Null.cpp
+    Source/CommonFramework/VideoPipeline/VideoSource_Null.h
+    Source/CommonFramework/VideoPipeline/VideoSourceDescriptor.cpp
+    Source/CommonFramework/VideoPipeline/VideoSourceDescriptor.h
     Source/CommonFramework/Windows/ButtonDiagram.cpp
     Source/CommonFramework/Windows/ButtonDiagram.h
     Source/CommonFramework/Windows/DpiScaler.cpp

SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp

@@ -216,11 +216,10 @@ void StreamHistorySession::post_shutdown(){
     data.m_has_video = false;
     initialize();
 }
-void StreamHistorySession::post_new_source(const CameraInfo& device, Resolution resolution){
-//    cout << "post_new_source()" << endl;
+void StreamHistorySession::post_startup(VideoSource* source){
     Data& data = *m_data;
     WriteSpinLock lg(data.m_lock);
-    data.m_has_video = !device.device_name().empty();
+    data.m_has_video = source != nullptr;
     initialize();
 }
 void StreamHistorySession::pre_resolution_change(Resolution resolution){

SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.h

@@ -12,7 +12,7 @@
 #include "Common/Cpp/AbstractLogger.h"
 #include "Common/Cpp/Containers/Pimpl.h"
 #include "CommonFramework/AudioPipeline/AudioSession.h"
-#include "CommonFramework/VideoPipeline/CameraSession.h"
+#include "CommonFramework/VideoPipeline/VideoSession.h"
 
 namespace PokemonAutomation{
 
@@ -21,7 +21,7 @@ class StreamHistorySession
     : public AudioFloatStreamListener
     , public VideoFrameListener
     , public AudioSession::StateListener
-    , public CameraSession::StateListener
+    , public VideoSession::StateListener
 {
 public:
     ~StreamHistorySession();
@@ -39,7 +39,7 @@ class StreamHistorySession
 
     virtual void pre_shutdown() override;
     virtual void post_shutdown() override;
-    virtual void post_new_source(const CameraInfo& device, Resolution resolution) override;
+    virtual void post_startup(VideoSource* source) override;
     virtual void pre_resolution_change(Resolution resolution) override;
     virtual void post_resolution_change(Resolution resolution) override;
 

SerialPrograms/Source/CommonFramework/Tools/VideoStream.h

@@ -14,8 +14,8 @@
  *
  */
 
-#ifndef PokemonAutomation_VideoStream_H
-#define PokemonAutomation_VideoStream_H
+#ifndef PokemonAutomation_VideoPipeline_VideoStream_H
+#define PokemonAutomation_VideoPipeline_VideoStream_H
 
 #include <string>
 #include "Common/Cpp/AbstractLogger.h"

SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp

@@ -252,11 +252,11 @@ VideoSnapshot CameraSession::snapshot(){
 
     return VideoSnapshot(m_last_image, m_last_image_timestamp);
 }
-double CameraSession::fps_source(){
+double CameraSession::fps_source() const{
     ReadSpinLock lg(m_frame_lock);
     return m_fps_tracker_source.events_per_second();
 }
-double CameraSession::fps_display(){
+double CameraSession::fps_display() const{
     ReadSpinLock lg(m_frame_lock);
     return m_fps_tracker_display.events_per_second();
 }
@@ -612,6 +612,247 @@ void VideoDisplayWidget::paintEvent(QPaintEvent* event){
 
 
 
+CameraVideoSource::~CameraVideoSource(){
+    if (!m_capture_session){
+        return;
+    }
+
+    try{
+        m_logger.log("Stopping Camera...");
+    }catch (...){}
+
+    m_camera->stop();
+    m_capture_session.reset();
+    m_camera.reset();
+}
+CameraVideoSource::CameraVideoSource(
+    Logger& logger,
+    const CameraInfo& info,
+    Resolution desired_resolution
+)
+    : m_logger(logger)
+    , m_last_image_timestamp(WallClock::min())
+    , m_stats_conversion("ConvertFrame", "ms", 1000, std::chrono::seconds(10))
+    , m_last_frame_seqnum(0)
+{
+    if (!info){
+        return;
+    }
+    m_logger.log("Starting Camera: Backend = CameraQt65QMediaCaptureSession");
+
+    auto cameras = QMediaDevices::videoInputs();
+    const QCameraDevice* device = nullptr;
+    for (const auto& camera : cameras){
+        if (camera.id().toStdString() == info.device_name()){
+            device = &camera;
+            break;
+        }
+    }
+    if (device == nullptr){
+        m_logger.log("Camera not found: " + info.device_name(), COLOR_RED);
+        return;
+    }
+    m_logger.log("Camera: " + device->description().toStdString());
+
+    QList<QCameraFormat> formats = device->videoFormats();
+    if (formats.empty()){
+        m_logger.log("No usable resolutions: " + device->description().toStdString(), COLOR_RED);
+        return;
+    }
+
+    std::map<Resolution, const QCameraFormat*> resolution_map;
+    for (const QCameraFormat& format : formats){
+        QSize resolution = format.resolution();
+        resolution_map.emplace(
+            std::piecewise_construct,
+            std::forward_as_tuple(resolution.width(), resolution.height()),
+            std::forward_as_tuple(&format)
+        );
+    }
+
+    const QCameraFormat* format = nullptr;
+    m_resolutions.clear();
+    for (const auto& res : resolution_map){
+        m_resolutions.emplace_back(res.first);
+        if (res.first == desired_resolution){
+            format = res.second;
+        }
+    }
+    if (format == nullptr){
+        format = resolution_map.rbegin()->second;
+    }
+
+    QSize size = format->resolution();
+    m_resolution = Resolution(size.width(), size.height());
+    m_logger.log("Resolution: " + m_resolution.to_string());
+
+    m_camera.reset(new QCamera(*device));
+    m_camera->setCameraFormat(*format);
+
+    m_capture_session.reset(new QMediaCaptureSession());
+    m_capture_session->setCamera(m_camera.get());
+
+    connect(m_camera.get(), &QCamera::errorOccurred, this, [&](){
+        if (m_camera->error() == QCamera::NoError){
+            return;
+        }
+        m_logger.log("QCamera error: " + m_camera->errorString().toStdString(), COLOR_RED);
+    });
+
+    m_camera->start();
+}
+
+
+void CameraVideoSource::set_video_output(QGraphicsVideoItem& item){
+    if (m_capture_session == nullptr){
+        return;
+    }
+    if (m_capture_session->videoSink() == item.videoSink()){
+        return;
+    }
+    m_capture_session->setVideoOutput(&item);
+
+    connect(
+        item.videoSink(), &QVideoSink::videoFrameChanged,
+        m_camera.get(), [&](const QVideoFrame& frame){
+            //  This will be on the main thread. So we waste as little time as
+            //  possible. Shallow-copy the frame, update the listeners, and
+            //  return immediately to unblock the main thread.
+
+            WallClock now = current_time();
+            {
+                WriteSpinLock lg(m_frame_lock);
+
+                //  Skip duplicate frames.
+                if (frame.startTime() != -1 && frame.startTime() <= m_last_frame.startTime()){
+                    return;
+                }
+
+                m_last_frame = frame;
+                m_last_frame_timestamp = now;
+                uint64_t seqnum = m_last_frame_seqnum.load(std::memory_order_relaxed);
+                seqnum++;
+                m_last_frame_seqnum.store(seqnum, std::memory_order_relaxed);
+            }
+
+            report_source_frame(std::make_shared<VideoFrame>(now, frame));
+        },
+        Qt::DirectConnection
+    );
+}
+
+
+
+
+
+Resolution CameraVideoSource::current_resolution() const{
+    //  No to lock since it doesn't change after construction.
+    return m_resolution;
+}
+std::vector<Resolution> CameraVideoSource::supported_resolutions() const{
+    //  No to lock since it doesn't change after construction.
+    return m_resolutions;
+}
+
+VideoSnapshot CameraVideoSource::snapshot(){
+    //  This will be coming in from random threads. (not the main thread)
+    //  So we efficiently grab the last frame to unblock the main thread.
+    //  Then we can do any expensive post-processing as needed.
+
+    std::lock_guard<std::mutex> lg(m_cache_lock);
+
+    //  Check the cached image frame. If it's not stale, return it immediately.
+    uint64_t frame_seqnum = m_last_frame_seqnum.load(std::memory_order_relaxed);
+    if (!m_last_image.isNull() && m_last_image_seqnum == frame_seqnum){
+        return VideoSnapshot(m_last_image, m_last_image_timestamp);
+    }
+
+    //  Cached image is stale. Grab the latest frame.
+    QVideoFrame frame;
+    WallClock frame_timestamp;
+    {
+        ReadSpinLock lg0(m_frame_lock);
+        frame_seqnum = m_last_frame_seqnum.load(std::memory_order_relaxed);
+        frame = m_last_frame;
+        frame_timestamp = m_last_frame_timestamp;
+    }
+
+    if (!frame.isValid()){
+        m_logger.log("QVideoFrame is null.", COLOR_RED);
+        return VideoSnapshot();
+    }
+
+    //  Converting the QVideoFrame to QImage is expensive. Time it and
+    //  report performance.
+    WallClock time0 = current_time();
+
+    QImage image = frame.toImage();
+    QImage::Format format = image.format();
+    if (format != QImage::Format_ARGB32 && format != QImage::Format_RGB32){
+        image = image.convertToFormat(QImage::Format_ARGB32);
+    }
+
+    WallClock time1 = current_time();
+    m_stats_conversion.report_data(m_logger, std::chrono::duration_cast<std::chrono::microseconds>(time1 - time0).count());
+
+    //  Update the cached image.
+    m_last_image = std::move(image);
+    m_last_image_timestamp = frame_timestamp;
+    m_last_image_seqnum = frame_seqnum;
+
+    return VideoSnapshot(m_last_image, m_last_image_timestamp);
+}
+
+QWidget* CameraVideoSource::make_QtWidget(QWidget* parent){
+    return new CameraVideoDisplay(parent, *this);
+}
+
+
+
+
+
+
+
+
+CameraVideoDisplay::CameraVideoDisplay(QWidget* parent, CameraVideoSource& source)
+    : QWidget(parent)
+    , m_source(source)
+    , m_view(new StaticQGraphicsView(this))
+    , m_sanitizer("CameraVideoDisplay")
+{
+    this->setMinimumSize(80, 45);
+    m_view->setFixedSize(this->size());
+    m_view->setScene(&m_scene);
+    m_video.setSize(this->size());
+    m_scene.setSceneRect(QRectF(QPointF(0, 0), this->size()));
+    m_scene.addItem(&m_video);
+    source.set_video_output(m_video);
+
+    connect(
+        &m_scene, &QGraphicsScene::changed,
+        this, [&](const QList<QRectF>&){
+            auto scope_check = m_sanitizer.check_scope();
+            m_source.report_rendered_frame(current_time());
+        }
+    );
+}
+void CameraVideoDisplay::resizeEvent(QResizeEvent* event){
+    auto scope_check = m_sanitizer.check_scope();
+    m_view->setFixedSize(this->size());
+    m_scene.setSceneRect(QRectF(QPointF(0, 0), this->size()));
+    m_video.setSize(this->size());
+}
+
+
+
+
+
+
+
+
+
+
+