Skip to content

Commit 177e8cc

Browse files
MysticialMysticial
committed
Add work-in-progress implementation/experiment of stream history.
1 parent 2beb1d5 commit 177e8cc

17 files changed

+796
-24
lines changed

‎SerialPrograms/CMakeLists.txt‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -514,8 +514,11 @@ file(GLOB MAIN_SOURCES
514514
Source/CommonFramework/PersistentSettings.h
515515
Source/CommonFramework/ProgramSession.cpp
516516
Source/CommonFramework/ProgramSession.h
517-
Source/CommonFramework/Recording/RecentHistory.cpp
518-
Source/CommonFramework/Recording/RecentHistory.h
517+
Source/CommonFramework/Recording/StreamHistorySession.cpp
518+
Source/CommonFramework/Recording/StreamHistorySession.h
519+
Source/CommonFramework/Recording/StreamHistoryTracker_Null.h
520+
Source/CommonFramework/Recording/StreamHistoryTracker_RecordOnTheFly.h
521+
Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h
519522
Source/CommonFramework/Resources/SpriteDatabase.cpp
520523
Source/CommonFramework/Resources/SpriteDatabase.h
521524
Source/CommonFramework/SetupSettings.cpp

‎SerialPrograms/SerialPrograms.pro‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ SOURCES += \
275275
Source/CommonFramework/Panels/UI/SettingsPanelWidget.cpp \
276276
Source/CommonFramework/PersistentSettings.cpp \
277277
Source/CommonFramework/ProgramSession.cpp \
278-
Source/CommonFramework/Recording/RecentHistory.cpp \
278+
Source/CommonFramework/Recording/StreamHistorySession.cpp \
279279
Source/CommonFramework/Resources/SpriteDatabase.cpp \
280280
Source/CommonFramework/SetupSettings.cpp \
281281
Source/CommonFramework/Tools/BlackBorderCheck.cpp \
@@ -1367,7 +1367,9 @@ HEADERS += \
13671367
Source/CommonFramework/Panels/UI/SettingsPanelWidget.h \
13681368
Source/CommonFramework/PersistentSettings.h \
13691369
Source/CommonFramework/ProgramSession.h \
1370-
Source/CommonFramework/Recording/RecentHistory.h \
1370+
Source/CommonFramework/Recording/StreamHistorySession.h \
1371+
Source/CommonFramework/Recording/StreamHistoryTracker_RecordOnTheFly.h \
1372+
Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h \
13711373
Source/CommonFramework/Resources/SpriteDatabase.h \
13721374
Source/CommonFramework/SetupSettings.h \
13731375
Source/CommonFramework/Tools/BlackBorderCheck.h \

‎SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,24 @@ ResolutionOption::ResolutionOption(
5151
PA_ADD_OPTION(HEIGHT);
5252
}
5353

54+
StreamHistoryOption::StreamHistoryOption()
55+
: GroupOption("Stream History", LockMode::LOCK_WHILE_RUNNING, true, false)
56+
, DESCRIPTION(
57+
"Keep a record of this many seconds of video+audio. This will allow video capture for unexpected events.<br>"
58+
"<font color=\"red\">Warning (Developer Only): The current implementation is inefficient and requires "
59+
"10 GB of ram to store just 30 seconds of video. This feature is still a work-in-progress."
60+
"</font>"
61+
)
62+
, VIDEO_HISTORY_SECONDS(
63+
"<b>Video History (seconds):</b><br>"
64+
"Keep this many seconds of video feed for video capture and debugging purposes.",
65+
LockMode::UNLOCK_WHILE_RUNNING,
66+
30
67+
)
68+
{
69+
PA_ADD_STATIC(DESCRIPTION);
70+
PA_ADD_OPTION(VIDEO_HISTORY_SECONDS);
71+
}
5472

5573

5674

@@ -201,12 +219,6 @@ GlobalSettings::GlobalSettings()
201219
true
202220
)
203221
#endif
204-
, VIDEO_HISTORY_SECONDS(
205-
"<b>Video History (seconds):</b><br>"
206-
"Keep this many seconds of video feed for video capture and debugging purposes.",
207-
LockMode::UNLOCK_WHILE_RUNNING,
208-
30
209-
)
210222
, AUTO_RESET_AUDIO_SECONDS(
211223
"<b>Audio Auto-Reset:</b><br>"
212224
"Attempt to reset the audio if this many seconds has elapsed since the last audio frame (in order to fix issues with RDP disconnection, etc).",
@@ -275,7 +287,7 @@ GlobalSettings::GlobalSettings()
275287
#endif
276288
#if QT_VERSION_MAJOR >= 6
277289
if (PreloadSettings::instance().DEVELOPER_MODE){ // REMOVE
278-
PA_ADD_OPTION(VIDEO_HISTORY_SECONDS);
290+
PA_ADD_OPTION(STREAM_HISTORY);
279291
}
280292
#endif
281293

‎SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,18 @@ class ResolutionOption : public GroupOption{
4343
SimpleIntegerOption<uint32_t> WIDTH;
4444
SimpleIntegerOption<uint32_t> HEIGHT;
4545
};
46-
47-
4846
struct DebugSettings{
4947
bool COLOR_CHECK = false;
5048
bool IMAGE_TEMPLATE_MATCHING = false;
5149
bool IMAGE_DICTIONARY_MATCHING = false;
5250
};
51+
class StreamHistoryOption : public GroupOption{
52+
public:
53+
StreamHistoryOption();
54+
55+
StaticTextOption DESCRIPTION;
56+
SimpleIntegerOption<uint16_t> VIDEO_HISTORY_SECONDS;
57+
};
5358

5459

5560

@@ -117,7 +122,8 @@ class GlobalSettings : public BatchOption, private ConfigOption::Listener{
117122
#if QT_VERSION_MAJOR == 5
118123
BooleanCheckBoxOption ENABLE_FRAME_SCREENSHOTS;
119124
#endif
120-
SimpleIntegerOption<uint16_t> VIDEO_HISTORY_SECONDS;
125+
// SimpleIntegerOption<uint16_t> VIDEO_HISTORY_SECONDS;
126+
StreamHistoryOption STREAM_HISTORY;
121127

122128
SimpleIntegerOption<uint8_t> AUTO_RESET_AUDIO_SECONDS;
123129
SimpleIntegerOption<uint8_t> AUTO_RESET_VIDEO_SECONDS;

‎SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
/* Stream History Session
2+
*
3+
* From: https://github.com/PokemonAutomation/Arduino-Source
4+
*
5+
*/
6+
7+
#include "Common/Cpp/Exceptions.h"
8+
#include "Common/Cpp/Containers/Pimpl.tpp"
9+
#include "Common/Cpp/Concurrency/SpinLock.h"
10+
#include "CommonFramework/GlobalSettingsPanel.h"
11+
#include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h"
12+
13+
#if (QT_VERSION_MAJOR == 6) && (QT_VERSION_MINOR >= 8)
14+
#include "StreamHistoryTracker_SaveFrames.h"
15+
//#include "StreamHistoryTracker_RecordOnTheFly.h"
16+
#else
17+
#include "StreamHistoryTracker_Null.h"
18+
#endif
19+
20+
21+
#include "StreamHistorySession.h"
22+
23+
24+
25+
namespace PokemonAutomation{
26+
27+
28+
29+
30+
31+
struct StreamHistorySession::Data{
32+
Logger& m_logger;
33+
mutable SpinLock m_lock;
34+
std::chrono::seconds m_window;
35+
AudioChannelFormat m_audio_format;
36+
std::shared_ptr<StreamHistoryTracker> m_current;
37+
38+
Data(Logger& logger)
39+
: m_logger(logger)
40+
, m_window(GlobalSettings::instance().STREAM_HISTORY.VIDEO_HISTORY_SECONDS)
41+
, m_audio_format(AudioChannelFormat::NONE)
42+
{}
43+
};
44+
45+
46+
47+
48+
StreamHistorySession::StreamHistorySession(Logger& logger)
49+
: AudioFloatStreamListener(1)
50+
, m_data(CONSTRUCT_TOKEN, logger)
51+
{}
52+
void StreamHistorySession::start(AudioChannelFormat format){
53+
Data& data = *m_data;
54+
WriteSpinLock lg(data.m_lock);
55+
if (!data.m_current){
56+
data.m_audio_format = format;
57+
initialize();
58+
}
59+
}
60+
61+
62+
class HistorySaverThread : public QThread{
63+
public:
64+
HistorySaverThread(Logger& logger, const StreamHistoryTracker& tracker, const std::string& filename)
65+
: m_logger(logger)
66+
, m_tracker(tracker)
67+
, m_filename(filename)
68+
{
69+
start();
70+
}
71+
~HistorySaverThread(){
72+
quit();
73+
wait();
74+
}
75+
virtual void run() override{
76+
m_tracker.save(m_logger, m_filename);
77+
}
78+
79+
private:
80+
Logger& m_logger;
81+
const StreamHistoryTracker& m_tracker;
82+
const std::string& m_filename;
83+
};
84+
85+
void StreamHistorySession::save(const std::string& filename) const{
86+
const Data& data = *m_data;
87+
88+
// Get an owning reference to the current tracker.
89+
// This will allow us to promptly release the lock and unblock the UI from
90+
// changing the streams (which will wipe the history).
91+
std::shared_ptr<StreamHistoryTracker> tracker;
92+
{
93+
WriteSpinLock lg(data.m_lock);
94+
if (!data.m_current){
95+
data.m_logger.log("Cannot save stream history: Stream history is not enabled.", COLOR_RED);
96+
return;
97+
}
98+
tracker = data.m_current;
99+
}
100+
101+
// tracker->save(m_logger, filename);
102+
HistorySaverThread saver(data.m_logger, *tracker, filename);
103+
}
104+
void StreamHistorySession::on_samples(const float* samples, size_t frames){
105+
Data& data = *m_data;
106+
WriteSpinLock lg(data.m_lock);
107+
if (data.m_current){
108+
data.m_current->on_samples(samples, frames);
109+
}
110+
}
111+
void StreamHistorySession::on_frame(std::shared_ptr<VideoFrame> frame){
112+
Data& data = *m_data;
113+
WriteSpinLock lg(data.m_lock);
114+
if (data.m_current){
115+
data.m_current->on_frame(std::move(frame));
116+
}
117+
}
118+
119+
120+
void StreamHistorySession::clear(){
121+
// Must call under lock.
122+
Data& data = *m_data;
123+
data.m_logger.log("Clearing stream history...", COLOR_ORANGE);
124+
data.m_current.reset();
125+
expected_samples_per_frame = 0;
126+
data.m_audio_format = AudioChannelFormat::NONE;
127+
}
128+
void StreamHistorySession::initialize(){
129+
if (!GlobalSettings::instance().STREAM_HISTORY.enabled()){
130+
return;
131+
}
132+
133+
// Must call under lock.
134+
Data& data = *m_data;
135+
data.m_logger.log("Starting stream history...", COLOR_ORANGE);
136+
switch (data.m_audio_format){
137+
case AudioChannelFormat::NONE:
138+
expected_samples_per_frame = 0;
139+
data.m_current.reset(new StreamHistoryTracker(0, 0, data.m_window));
140+
return;
141+
case AudioChannelFormat::MONO_48000:
142+
data.m_current.reset(new StreamHistoryTracker(1, 48000, data.m_window));
143+
return;
144+
case AudioChannelFormat::DUAL_44100:
145+
data.m_current.reset(new StreamHistoryTracker(1, 44100, data.m_window));
146+
return;
147+
case AudioChannelFormat::DUAL_48000:
148+
case AudioChannelFormat::MONO_96000:
149+
case AudioChannelFormat::INTERLEAVE_LR_96000:
150+
case AudioChannelFormat::INTERLEAVE_RL_96000:
151+
data.m_current.reset(new StreamHistoryTracker(2, 48000, data.m_window));
152+
return;
153+
default:
154+
throw InternalProgramError(
155+
nullptr, PA_CURRENT_FUNCTION,
156+
"Invalid AudioFormat: " + std::to_string((size_t)data.m_audio_format)
157+
);
158+
}
159+
}
160+
void StreamHistorySession::pre_input_change(){
161+
Data& data = *m_data;
162+
WriteSpinLock lg(data.m_lock);
163+
clear();
164+
}
165+
void StreamHistorySession::post_input_change(const std::string& file, const AudioDeviceInfo& device, AudioChannelFormat format){
166+
Data& data = *m_data;
167+
WriteSpinLock lg(data.m_lock);
168+
data.m_audio_format = format;
169+
initialize();
170+
}
171+
void StreamHistorySession::pre_shutdown(){
172+
Data& data = *m_data;
173+
WriteSpinLock lg(data.m_lock);
174+
clear();
175+
}
176+
void StreamHistorySession::post_new_source(const CameraInfo& device, Resolution resolution){
177+
Data& data = *m_data;
178+
WriteSpinLock lg(data.m_lock);
179+
initialize();
180+
}
181+
void StreamHistorySession::pre_resolution_change(Resolution resolution){
182+
Data& data = *m_data;
183+
WriteSpinLock lg(data.m_lock);
184+
clear();
185+
}
186+
void StreamHistorySession::post_resolution_change(Resolution resolution){
187+
Data& data = *m_data;
188+
WriteSpinLock lg(data.m_lock);
189+
initialize();
190+
}
191+
192+
193+
194+
195+
196+
197+
198+
}

‎SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.h‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/* Stream History Session
2+
*
3+
* From: https://github.com/PokemonAutomation/Arduino-Source
4+
*
5+
* Capture the last X seconds of audio and video.
6+
*
7+
*/
8+
9+
#ifndef PokemonAutomation_StreamHistorySession_H
10+
#define PokemonAutomation_StreamHistorySession_H
11+
12+
#include "Common/Cpp/AbstractLogger.h"
13+
#include "Common/Cpp/Containers/Pimpl.h"
14+
#include "CommonFramework/AudioPipeline/AudioSession.h"
15+
#include "CommonFramework/VideoPipeline/CameraSession.h"
16+
17+
namespace PokemonAutomation{
18+
19+
20+
class StreamHistorySession
21+
: public AudioFloatStreamListener
22+
, public VideoFrameListener
23+
, public AudioSession::StateListener
24+
, public CameraSession::StateListener
25+
{
26+
public:
27+
StreamHistorySession(Logger& logger);
28+
void start(AudioChannelFormat format);
29+
void save(const std::string& filename) const;
30+
31+
public:
32+
virtual void on_samples(const float* data, size_t frames) override;
33+
virtual void on_frame(std::shared_ptr<VideoFrame> frame) override;
34+
35+
public:
36+
virtual void pre_input_change() override;
37+
virtual void post_input_change(const std::string& file, const AudioDeviceInfo& device, AudioChannelFormat format) override;
38+
39+
virtual void pre_shutdown() override;
40+
virtual void post_new_source(const CameraInfo& device, Resolution resolution) override;
41+
virtual void pre_resolution_change(Resolution resolution) override;
42+
virtual void post_resolution_change(Resolution resolution) override;
43+
44+
private:
45+
void clear();
46+
void initialize();
47+
48+
private:
49+
struct Data;
50+
Pimpl<Data> m_data;
51+
};
52+
53+
54+
55+
}
56+
#endif

0 commit comments

Comments
 (0)