diff --git a/.github/workflows/cpp-ci-serial-programs.yml b/.github/workflows/cpp-ci-serial-programs.yml index fa84978aba..d056558dda 100644 --- a/.github/workflows/cpp-ci-serial-programs.yml +++ b/.github/workflows/cpp-ci-serial-programs.yml @@ -10,11 +10,25 @@ jobs: fail-fast: false matrix: os: [windows-2025, macos-13, ubuntu-24.04] + compiler: ['default', 'clang'] qt_version: ['6.9.1'] include: - - qt_version: '6.9.1' - qt_version_major: '6' - qt_modules: 'qtmultimedia qtserialport' + - qt_version: '6.9.1' + qt_version_major: '6' + qt_modules: 'qtmultimedia qtserialport' + + - os: 'windows-2025' + compiler: 'clang' + cmake_additional_param: '-T ClangCL' + + - os: 'ubuntu-24.04' + compiler: 'clang' + cmake_additional_param: '-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' + + exclude: + - os: 'macos-13' + compiler: 'clang' + # Excluded because macos default toolset is already clang steps: - uses: actions/checkout@v4 @@ -44,7 +58,7 @@ jobs: cd Arduino-Source/SerialPrograms mkdir bin cd bin - cmake .. -DQT_MAJOR:STRING=${{ matrix.qt_version_major }} + cmake .. -DQT_MAJOR:STRING=${{ matrix.qt_version_major }} ${{ matrix.cmake_additional_param }} cmake --build . --config Release --parallel 10 - name: Copy resources if: startsWith(matrix.os, 'windows') @@ -66,5 +80,5 @@ jobs: - uses: actions/upload-artifact@v4 if: startsWith(matrix.os, 'windows') with: - name: Serial Programs for windows (${{ matrix.qt_version }}) + name: Serial Programs (os=${{ matrix.os }} - compiler=${{ matrix.compiler }} - qt_version=${{ matrix.qt_version }}) path: Output diff --git a/ClientSource/Connection/MessageLogger.cpp b/ClientSource/Connection/MessageLogger.cpp index 5c608ace08..370bc4153c 100644 --- a/ClientSource/Connection/MessageLogger.cpp +++ b/ClientSource/Connection/MessageLogger.cpp @@ -96,11 +96,6 @@ void SerialLogger::log(const std::string& msg, Color color){ m_logger.log(msg, color); } } -void SerialLogger::log(std::string msg){ - if (ok_to_log()){ - m_logger.log(msg, COLOR_DARKGREEN); - } -} bool SerialLogger::ok_to_log(){ WallClock now = current_time(); diff --git a/ClientSource/Connection/MessageLogger.h b/ClientSource/Connection/MessageLogger.h index c2eee1b4c4..4bb3460f0b 100644 --- a/ClientSource/Connection/MessageLogger.h +++ b/ClientSource/Connection/MessageLogger.h @@ -17,7 +17,7 @@ namespace PokemonAutomation{ -class MessageLogger : public MessageSniffer{ +class MessageLogger : public Logger, public MessageSniffer{ public: MessageLogger(bool log_everything = false) : m_log_everything_owner(log_everything) @@ -29,7 +29,6 @@ class MessageLogger : public MessageSniffer{ {} -// virtual void log(std::string msg) override; virtual void on_send(const BotBaseMessage& message, bool is_retransmit) override; virtual void on_recv(const BotBaseMessage& message) override; @@ -40,13 +39,12 @@ class MessageLogger : public MessageSniffer{ -class SerialLogger : public Logger, public MessageLogger{ +class SerialLogger : public MessageLogger{ public: SerialLogger(Logger& logger, bool log_everything); - virtual void log(const char* msg, Color color = Color()) override; - virtual void log(const std::string& msg, Color color = Color()) override; - virtual void log(std::string msg) override; + virtual void log(const char* msg, Color color = COLOR_DARKGREEN) override; + virtual void log(const std::string& msg, Color color = COLOR_DARKGREEN) override; private: bool ok_to_log(); diff --git a/ClientSource/Connection/MessageSniffer.h b/ClientSource/Connection/MessageSniffer.h index db755e39af..0e43bb03d1 100644 --- a/ClientSource/Connection/MessageSniffer.h +++ b/ClientSource/Connection/MessageSniffer.h @@ -17,7 +17,6 @@ struct BotBaseMessage; class MessageSniffer{ public: - virtual void log(std::string msg){} virtual void on_send(const BotBaseMessage& message, bool is_retransmit){} virtual void on_recv(const BotBaseMessage& message){} }; diff --git a/ClientSource/Connection/PABotBase.cpp b/ClientSource/Connection/PABotBase.cpp index 85cc847e25..eb568d3323 100644 --- a/ClientSource/Connection/PABotBase.cpp +++ b/ClientSource/Connection/PABotBase.cpp @@ -312,7 +312,7 @@ void PABotBase::process_ack_request(BotBaseMessage message){ if constexpr (!variable_length){ if (message.body.size() != sizeof(Params)){ - m_sniffer->log("Ignoring message with invalid size."); + m_logger.log("Ignoring message with invalid size."); return; } } @@ -324,14 +324,14 @@ void PABotBase::process_ack_request(BotBaseMessage message){ WriteSpinLock lg(m_state_lock, "PABotBase::process_ack_request()"); if (m_pending_requests.empty()){ - m_sniffer->log("Unexpected request ack message: seqnum = " + std::to_string(seqnum)); + m_logger.log("Unexpected request ack message: seqnum = " + std::to_string(seqnum)); return; } uint64_t full_seqnum = infer_full_seqnum(m_pending_requests, seqnum); std::map::iterator iter = m_pending_requests.find(full_seqnum); if (iter == m_pending_requests.end()){ - m_sniffer->log("Unexpected request ack message: seqnum = " + std::to_string(seqnum)); + m_logger.log("Unexpected request ack message: seqnum = " + std::to_string(seqnum)); return; } iter->second.sanitizer.check_usage(); @@ -357,10 +357,10 @@ void PABotBase::process_ack_request(BotBaseMessage message){ } return; case AckState::ACKED: - m_sniffer->log("Duplicate request ack message: seqnum = " + std::to_string(seqnum)); + m_logger.log("Duplicate request ack message: seqnum = " + std::to_string(seqnum)); return; case AckState::FINISHED: - m_sniffer->log("Request ack on command finish: seqnum = " + std::to_string(seqnum)); + m_logger.log("Request ack on command finish: seqnum = " + std::to_string(seqnum)); return; } } @@ -369,7 +369,7 @@ void PABotBase::process_ack_command(BotBaseMessage message){ auto scope_check = m_sanitizer.check_scope(); if (message.body.size() != sizeof(Params)){ - m_sniffer->log("Ignoring message with invalid size."); + m_logger.log("Ignoring message with invalid size."); return; } const Params* params = (const Params*)message.body.c_str(); @@ -378,14 +378,14 @@ void PABotBase::process_ack_command(BotBaseMessage message){ WriteSpinLock lg(m_state_lock, "PABotBase::process_ack_command()"); if (m_pending_commands.empty()){ - m_sniffer->log("Unexpected command ack message: seqnum = " + std::to_string(seqnum)); + m_logger.log("Unexpected command ack message: seqnum = " + std::to_string(seqnum)); return; } uint64_t full_seqnum = infer_full_seqnum(m_pending_commands, seqnum); auto iter = m_pending_commands.find(full_seqnum); if (iter == m_pending_commands.end()){ - m_sniffer->log("Unexpected command ack message: seqnum = " + std::to_string(seqnum)); + m_logger.log("Unexpected command ack message: seqnum = " + std::to_string(seqnum)); return; } iter->second.sanitizer.check_usage(); @@ -399,10 +399,10 @@ void PABotBase::process_ack_command(BotBaseMessage message){ iter->second.ack = std::move(message); return; case AckState::ACKED: - m_sniffer->log("Duplicate command ack message: seqnum = " + std::to_string(seqnum)); + m_logger.log("Duplicate command ack message: seqnum = " + std::to_string(seqnum)); return; case AckState::FINISHED: - m_sniffer->log("Command ack on finished command: seqnum = " + std::to_string(seqnum)); + m_logger.log("Command ack on finished command: seqnum = " + std::to_string(seqnum)); return; } } @@ -411,7 +411,7 @@ void PABotBase::process_command_finished(BotBaseMessage message){ auto scope_check = m_sanitizer.check_scope(); if (message.body.size() != sizeof(Params)){ - m_sniffer->log("Ignoring message with invalid size."); + m_logger.log("Ignoring message with invalid size."); return; } const Params* params = (const Params*)message.body.c_str(); @@ -437,7 +437,7 @@ void PABotBase::process_command_finished(BotBaseMessage message){ #endif if (m_pending_commands.empty()){ - m_sniffer->log( + m_logger.log( "Unexpected command finished message: seqnum = " + std::to_string(seqnum) + ", command_seqnum = " + std::to_string(command_seqnum) ); @@ -447,7 +447,7 @@ void PABotBase::process_command_finished(BotBaseMessage message){ uint64_t full_seqnum = infer_full_seqnum(m_pending_commands, command_seqnum); auto iter = m_pending_commands.find(full_seqnum); if (iter == m_pending_commands.end()){ - m_sniffer->log( + m_logger.log( "Unexpected command finished message: seqnum = " + std::to_string(seqnum) + ", command_seqnum = " + std::to_string(command_seqnum) ); @@ -466,7 +466,7 @@ void PABotBase::process_command_finished(BotBaseMessage message){ m_cv.notify_all(); return; case AckState::FINISHED: - m_sniffer->log("Duplicate command finish: seqnum = " + std::to_string(seqnum)); + m_logger.log("Duplicate command finish: seqnum = " + std::to_string(seqnum)); return; } } @@ -494,7 +494,7 @@ void PABotBase::on_recv_message(BotBaseMessage message){ return; case PABB_MSG_ERROR_INVALID_TYPE:{ if (message.body.size() != sizeof(pabb_MsgInfoInvalidType)){ - m_sniffer->log("Ignoring message with invalid size."); + m_logger.log("Ignoring message with invalid size."); return; } const pabb_MsgInfoInvalidType* params = (const pabb_MsgInfoInvalidType*)message.body.c_str(); @@ -509,7 +509,7 @@ void PABotBase::on_recv_message(BotBaseMessage message){ } case PABB_MSG_ERROR_MISSED_REQUEST:{ if (message.body.size() != sizeof(pabb_MsgInfoMissedRequest)){ - m_sniffer->log("Ignoring message with invalid size."); + m_logger.log("Ignoring message with invalid size."); return; } const pabb_MsgInfoMissedRequest* params = (const pabb_MsgInfoMissedRequest*)message.body.c_str(); diff --git a/ClientSource/Connection/PABotBaseConnection.cpp b/ClientSource/Connection/PABotBaseConnection.cpp index dbd941d7c0..0bcd9a284e 100644 --- a/ClientSource/Connection/PABotBaseConnection.cpp +++ b/ClientSource/Connection/PABotBaseConnection.cpp @@ -92,16 +92,16 @@ void PABotBaseConnection::push_error_byte(ErrorBatchType type, char byte){ case ErrorBatchType::NO_ERROR_: break; case ErrorBatchType::ZERO_BYTES: - m_sniffer->log("Skipped " + std::to_string(m_current_error_batch.size()) + " zero byte(s)."); + m_logger.log("Skipped " + std::to_string(m_current_error_batch.size()) + " zero byte(s)."); break; case ErrorBatchType::FF_BYTES: - m_sniffer->log("Skipped " + std::to_string(m_current_error_batch.size()) + " 0xff byte(s)."); + m_logger.log("Skipped " + std::to_string(m_current_error_batch.size()) + " 0xff byte(s)."); break; case ErrorBatchType::ASCII_BYTES: - m_sniffer->log("Received possible ASCII: " + m_current_error_batch); + m_logger.log("Received possible ASCII: " + m_current_error_batch); break; case ErrorBatchType::OTHER: -// m_sniffer->log("Skipped " + std::to_string(m_current_error_batch.size()) + " invalid length byte(s)."); +// m_logger.log("Skipped " + std::to_string(m_current_error_batch.size()) + " invalid length byte(s)."); break; } @@ -119,7 +119,7 @@ void PABotBaseConnection::on_recv(const void* data, size_t bytes){ uint8_t length = ~m_recv_buffer[0]; if (m_recv_buffer[0] == 0){ -// m_sniffer->log("Skipping zero byte."); +// m_logger.log("Skipping zero byte."); push_error_byte(ErrorBatchType::ZERO_BYTES, 0); m_recv_buffer.pop_front(); continue; @@ -130,7 +130,7 @@ void PABotBaseConnection::on_recv(const void* data, size_t bytes){ if (length == 0){ push_error_byte(ErrorBatchType::FF_BYTES, ~length); }else{ - m_sniffer->log("Message is too short: bytes = " + std::to_string(length)); + m_logger.log("Message is too short: bytes = " + std::to_string(length)); push_error_byte(ErrorBatchType::OTHER, ~length); } m_recv_buffer.pop_front(); @@ -143,7 +143,7 @@ void PABotBaseConnection::on_recv(const void* data, size_t bytes){ // std::string text = ascii < 32 // ? ", ascii = " + std::to_string(ascii) // : std::string(", char = ") + ascii; -// m_sniffer->log("Message is too long: bytes = " + std::to_string(length) + text); +// m_logger.log("Message is too long: bytes = " + std::to_string(length) + text); push_error_byte(ErrorBatchType::ASCII_BYTES, ~length); m_recv_buffer.pop_front(); continue; @@ -170,7 +170,7 @@ void PABotBaseConnection::on_recv(const void* data, size_t bytes){ // Compare // std::cout << checksumA << " / " << checksumE << std::endl; if (checksumA != checksumE){ - m_sniffer->log("Invalid Checksum: bytes = " + std::to_string(length)); + m_logger.log("Invalid Checksum: bytes = " + std::to_string(length)); // std::cout << checksumA << " / " << checksumE << std::endl; // log(message_to_string(message[1], &message[2], length - PABB_PROTOCOL_OVERHEAD)); m_recv_buffer.pop_front(); diff --git a/ClientSource/Connection/SerialConnectionWinAPI.h b/ClientSource/Connection/SerialConnectionWinAPI.h index 1b3360184a..676d98344d 100644 --- a/ClientSource/Connection/SerialConnectionWinAPI.h +++ b/ClientSource/Connection/SerialConnectionWinAPI.h @@ -44,7 +44,7 @@ class SerialConnection : public StreamConnection{ throw ConnectionException(nullptr, "Unable to open serial connection (" + name + "). Error = " + std::to_string(error)); } - DCB serial_params{0}; + DCB serial_params{}; serial_params.DCBlength = sizeof(serial_params); if (!GetCommState(m_handle, &serial_params)){ @@ -67,7 +67,7 @@ class SerialConnection : public StreamConnection{ } #if 1 - COMMTIMEOUTS timeouts{0}; + COMMTIMEOUTS timeouts{}; if (!GetCommTimeouts(m_handle, &timeouts)){ DWORD error = GetLastError(); CloseHandle(m_handle); diff --git a/Common/Cpp/CancellableScope.h b/Common/Cpp/CancellableScope.h index 5d2e780cd3..3b6d9b55e5 100644 --- a/Common/Cpp/CancellableScope.h +++ b/Common/Cpp/CancellableScope.h @@ -111,7 +111,7 @@ class CancellableScope : public Cancellable{ public: virtual ~CancellableScope() override; - virtual bool cancel(std::exception_ptr exception) noexcept override; + virtual bool cancel(std::exception_ptr exception = nullptr) noexcept override; void wait_for(std::chrono::milliseconds duration); void wait_until(WallClock stop); diff --git a/Common/Cpp/Options/BoxFloatOption.cpp b/Common/Cpp/Options/BoxFloatOption.cpp new file mode 100644 index 0000000000..c7bfe0e782 --- /dev/null +++ b/Common/Cpp/Options/BoxFloatOption.cpp @@ -0,0 +1,225 @@ +/* Box Float Option + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Common/Cpp/Containers/Pimpl.tpp" +#include "Common/Cpp/Concurrency/SpinLock.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "BoxFloatOption.h" + +namespace PokemonAutomation{ + + + +struct BoxFloatOption::Data{ + std::string m_label; + double m_default_x; + double m_default_y; + double m_default_width; + double m_default_height; + + double m_x; + double m_y; + double m_width; + double m_height; + + mutable SpinLock m_lock; +}; + + + +BoxFloatOption::~BoxFloatOption() = default; +BoxFloatOption::BoxFloatOption( + std::string label, + LockMode lock_while_running, + double default_x, + double default_y, + double default_width, + double default_height +) + : ConfigOption(lock_while_running) + , m_data(CONSTRUCT_TOKEN) +{ + Data& self = *m_data; + + self.m_label = std::move(label); + + set_all(default_x, default_y, default_width, default_height); + + self.m_default_x = self.m_x; + self.m_default_y = self.m_y; + self.m_default_width = self.m_width; + self.m_default_height = self.m_height; +} +const std::string& BoxFloatOption::label() const{ + return m_data->m_label; +} +double BoxFloatOption::x() const{ + return m_data->m_x; +} +double BoxFloatOption::y() const{ + return m_data->m_y; +} +double BoxFloatOption::width() const{ + return m_data->m_width; +} +double BoxFloatOption::height() const{ + return m_data->m_height; +} + + +bool BoxFloatOption::sanitize(double& x, double& y, double& width, double& height) const{ + bool changed = false; + + if (x < 0.0){ + x = 0.0; + changed = true; + } + if (x > 1.0){ + x = 1.0; + changed = true; + } + + if (y < 0.0){ + y = 0.0; + changed = true; + } + if (y > 1.0){ + y = 1.0; + changed = true; + } + + if (width < 0.0){ + width = 0.0; + changed = true; + } + if (width > 1.0){ + width = 1.0; + changed = true; + } + + if (height < 0.0){ + height = 0.0; + changed = true; + } + if (height > 1.0){ + height = 1.0; + changed = true; + } + + if (x + width > 1.00000000001){ + width = 1.0 - x; + changed = true; + } + if (y + height > 1.00000000001){ + height = 1.0 - y; + changed = true; + } + + return changed; +} +void BoxFloatOption::get_all(double& x, double& y, double& width, double& height) const{ + const Data& self = *m_data; + + ReadSpinLock lg(self.m_lock); + x = self.m_x; + y = self.m_y; + width = self.m_width; + height = self.m_height; +} +void BoxFloatOption::set_all(double x, double y, double width, double height){ + sanitize(x, y, width, height); + + Data& self = *m_data; + bool changed = false; + + { + WriteSpinLock lg(self.m_lock); + + changed |= self.m_x != x; + changed |= self.m_y != y; + changed |= self.m_width != width; + changed |= self.m_height != height; + + self.m_x = x; + self.m_y = y; + self.m_width = width; + self.m_height = height; + } + + if (changed){ + report_value_changed(this); + } +} + + +void BoxFloatOption::load_json(const JsonValue& json){ + const JsonObject* obj = json.to_object(); + if (obj == nullptr){ + return; + } + + double x, y, width, height; + get_all(x, y, width, height); + + obj->read_float(x, "x"); + obj->read_float(y, "y"); + obj->read_float(width, "width"); + obj->read_float(height, "height"); + + set_all(x, y, width, height); +} +JsonValue BoxFloatOption::to_json() const{ + double x, y, width, height; + get_all(x, y, width, height); + + JsonObject ret; + ret["x"] = std::to_string(x); + ret["y"] = std::to_string(y); + ret["width"] = std::to_string(width); + ret["height"] = std::to_string(height); + return ret; +} + +std::string BoxFloatOption::check_validity() const{ + double x, y, width, height; + get_all(x, y, width, height); + + if (x < 0 || x > 1.0){ + return "x must be between 0.0 and 1.0."; + } + if (y < 0 || y > 1.0){ + return "y must be between 0.0 and 1.0."; + } + if (width < 0 || width > 1.0){ + return "width must be between 0.0 and 1.0."; + } + if (height < 0 || height > 1.0){ + return "height must be between 0.0 and 1.0."; + } + if (x + width > 1.00000000001){ + return "x + width must be <= 1.0."; + } + if (y + width > 00000000001){ + return "y + width must be <= 1.0."; + } + + return std::string(); +} +void BoxFloatOption::restore_defaults(){ + Data& self = *m_data; + set_all( + self.m_default_x, + self.m_default_y, + self.m_default_width, + self.m_default_height + ); +} + + + + +} + diff --git a/Common/Cpp/Options/BoxFloatOption.h b/Common/Cpp/Options/BoxFloatOption.h new file mode 100644 index 0000000000..25554d04c5 --- /dev/null +++ b/Common/Cpp/Options/BoxFloatOption.h @@ -0,0 +1,57 @@ +/* Box Float Option + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Options_BoxFloatOption_H +#define PokemonAutomation_Options_BoxFloatOption_H + +#include "Common/Cpp/Containers/Pimpl.h" +#include "Common/Cpp/Options/ConfigOption.h" + +namespace PokemonAutomation{ + + +class BoxFloatOption : public ConfigOption{ +public: + ~BoxFloatOption(); + BoxFloatOption( + std::string label, + LockMode lock_while_running, + double default_x, + double default_y, + double default_width, + double default_height + ); + + const std::string& label() const; + double x() const; + double y() const; + double width() const; + double height() const; + + // Returns true if a value was changed. + bool sanitize(double& x, double& y, double& width, double& height) const; + + void get_all(double& x, double& y, double& width, double& height) const; + void set_all(double x, double y, double width, double height); + + virtual void load_json(const JsonValue& json) override; + virtual JsonValue to_json() const override; + + virtual std::string check_validity() const override; + virtual void restore_defaults() override; + + virtual ConfigWidget* make_QtWidget(QWidget& parent) override; + + +private: + struct Data; + Pimpl m_data; +}; + + + +} +#endif diff --git a/Common/Cpp/Options/ConfigOption.h b/Common/Cpp/Options/ConfigOption.h index b90916df6c..d36e6862fa 100644 --- a/Common/Cpp/Options/ConfigOption.h +++ b/Common/Cpp/Options/ConfigOption.h @@ -27,7 +27,7 @@ enum class LockMode{ }; enum class ConfigOptionState{ ENABLED, - DISABLED, + DISABLED, // aka locked HIDDEN, }; @@ -93,6 +93,12 @@ class ConfigOption{ } public: + // Return the lock mode: how locking works on this option. It can be: + // - UNLOCK_WHILE_RUNNING, + // - LOCK_WHILE_RUNNING, + // - READ_ONLY, (aka always locked) + // This value is const throughout the ConfigOption lifetime. It is set + // when constructing the ConfigOption. LockMode lock_mode() const; // Returns error message if invalid. Otherwise returns empty string. @@ -104,7 +110,17 @@ class ConfigOption{ // transient state that the option object may have. virtual void reset_state(){}; + // Thread-safe: return the current visibility state. It can be: + // - ENABLED + // - DISABLED + // - HIDDEN ConfigOptionState visibility() const; + // Thread-safe: set the option's visibility state. It can be: + // - ENABLED + // - DISABLED + // - HIDDEN + // If visibility changed, all attached listeners' on_config_visibility_changed() + // will be called. virtual void set_visibility(ConfigOptionState visibility); diff --git a/Common/Cpp/Options/EnumDropdownDatabase.cpp b/Common/Cpp/Options/EnumDropdownDatabase.cpp index eaaf066bec..81d7ea360b 100644 --- a/Common/Cpp/Options/EnumDropdownDatabase.cpp +++ b/Common/Cpp/Options/EnumDropdownDatabase.cpp @@ -120,7 +120,14 @@ const EnumEntry* IntegerEnumDropdownDatabase::find_display(const std::string& di FixedLimitVector IntegerEnumDropdownDatabase::all_values() const{ return m_core->all_values(); } - +IntegerEnumDropdownDatabase create_integer_enum_dropdown_database(const std::vector& slugs){ + IntegerEnumDropdownDatabase database; + for (size_t i = 0; i < slugs.size(); i++){ + // display name is the same as slug + database.add(i, slugs[i], slugs[i]); + } + return database; +} template class FixedLimitVector; diff --git a/Common/Cpp/Options/EnumDropdownDatabase.h b/Common/Cpp/Options/EnumDropdownDatabase.h index 132e29aa21..4d56b89092 100644 --- a/Common/Cpp/Options/EnumDropdownDatabase.h +++ b/Common/Cpp/Options/EnumDropdownDatabase.h @@ -15,6 +15,7 @@ #define PokemonAutomation_Options_EnumDropdownDatabase_H #include +#include #include "Common/Cpp/Containers/Pimpl.h" #include "Common/Cpp/Containers/FixedLimitVector.h" @@ -47,7 +48,7 @@ class IntegerEnumDropdownDatabase{ IntegerEnumDropdownDatabase(std::initializer_list list); void add(EnumEntry entry); - void add(size_t value, std::string slug, std::string display, bool enabled){ + void add(size_t value, std::string slug, std::string display, bool enabled = true){ add(EnumEntry{value, std::move(slug), std::move(display), enabled}); } @@ -65,6 +66,10 @@ class IntegerEnumDropdownDatabase{ Pimpl m_core; }; +// Create a simple StringSelectDatabase from a list of slugs. +// The display names of each entry will be the same as their slugs and enum_values will be their indices. +IntegerEnumDropdownDatabase create_integer_enum_dropdown_database(const std::vector& slugs); + // This is the type-safe database for your specific enum. diff --git a/Common/Cpp/Options/StringOption.cpp b/Common/Cpp/Options/StringOption.cpp index 9e5b40ebd6..430d50abd7 100644 --- a/Common/Cpp/Options/StringOption.cpp +++ b/Common/Cpp/Options/StringOption.cpp @@ -16,6 +16,7 @@ struct StringCell::Data{ const bool m_is_password; const std::string m_default; const std::string m_placeholder_text; + const bool m_report_all_text_changes; std::atomic m_locked; @@ -25,11 +26,13 @@ struct StringCell::Data{ Data( bool is_password, std::string default_value, - std::string placeholder_text + std::string placeholder_text, + bool report_all_text_changes ) : m_is_password(is_password) , m_default(std::move(default_value)) , m_placeholder_text(std::move(placeholder_text)) + , m_report_all_text_changes(report_all_text_changes) , m_current(m_default) {} }; @@ -40,10 +43,11 @@ StringCell::StringCell( bool is_password, LockMode lock_while_program_is_running, std::string default_value, - std::string placeholder_text + std::string placeholder_text, + bool signal_all_text_changes ) : ConfigOption(lock_while_program_is_running) - , m_data(CONSTRUCT_TOKEN, is_password, std::move(default_value), std::move(placeholder_text)) + , m_data(CONSTRUCT_TOKEN, is_password, std::move(default_value), std::move(placeholder_text), signal_all_text_changes) {} bool StringCell::is_password() const{ @@ -55,6 +59,9 @@ const std::string& StringCell::placeholder_text() const{ const std::string StringCell::default_value() const{ return m_data->m_default; } +bool StringCell::signal_all_text_changes() const{ + return m_data->m_report_all_text_changes; +} bool StringCell::is_locked() const{ @@ -108,9 +115,10 @@ StringOption::StringOption( std::string label, LockMode lock_while_program_is_running, std::string default_value, - std::string placeholder_text + std::string placeholder_text, + bool signal_all_text_changes ) - : StringCell(is_password, lock_while_program_is_running, default_value, placeholder_text) + : StringCell(is_password, lock_while_program_is_running, default_value, placeholder_text, signal_all_text_changes) , m_label(std::move(label)) {} diff --git a/Common/Cpp/Options/StringOption.h b/Common/Cpp/Options/StringOption.h index 1b34f4f496..d86c161742 100644 --- a/Common/Cpp/Options/StringOption.h +++ b/Common/Cpp/Options/StringOption.h @@ -20,12 +20,14 @@ class StringCell : public ConfigOption{ bool is_password, LockMode lock_while_program_is_running, std::string default_value, - std::string placeholder_text + std::string placeholder_text, + bool signal_all_text_changes = false ); bool is_password() const; const std::string& placeholder_text() const; const std::string default_value() const; + bool signal_all_text_changes() const; bool is_locked() const; void set_locked(bool locked); @@ -56,7 +58,8 @@ class StringOption : public StringCell{ std::string label, LockMode lock_while_program_is_running, std::string default_value, - std::string placeholder_text + std::string placeholder_text, + bool signal_all_text_changes = false ); const std::string& label() const{ return m_label; } diff --git a/Common/Cpp/Options/TextEditOption.cpp b/Common/Cpp/Options/TextEditOption.cpp index 8d9e0ba2e5..42682f966d 100644 --- a/Common/Cpp/Options/TextEditOption.cpp +++ b/Common/Cpp/Options/TextEditOption.cpp @@ -37,12 +37,12 @@ struct TextEditOption::Data{ {} }; -void TextEditOption::add_listener(FocusListener& listener){ +void TextEditOption::add_focus_listener(FocusListener& listener){ Data& data = *m_data; WriteSpinLock lg(data.m_lock); data.listeners.insert(&listener); } -void TextEditOption::remove_listener(FocusListener& listener){ +void TextEditOption::remove_focus_listener(FocusListener& listener){ Data& data = *m_data; WriteSpinLock lg(data.m_lock); data.listeners.erase(&listener); diff --git a/Common/Cpp/Options/TextEditOption.h b/Common/Cpp/Options/TextEditOption.h index 5212c60152..246abaf37c 100644 --- a/Common/Cpp/Options/TextEditOption.h +++ b/Common/Cpp/Options/TextEditOption.h @@ -19,8 +19,8 @@ class TextEditOption : public ConfigOption{ struct FocusListener{ virtual void focus_in(){} }; - void add_listener(FocusListener& listener); - void remove_listener(FocusListener& listener); + void add_focus_listener(FocusListener& listener); + void remove_focus_listener(FocusListener& listener); void report_focus_in(); diff --git a/Common/Cpp/Stopwatch.h b/Common/Cpp/Stopwatch.h index b9595e0269..746dcdb2ba 100644 --- a/Common/Cpp/Stopwatch.h +++ b/Common/Cpp/Stopwatch.h @@ -4,8 +4,8 @@ * */ -#ifndef PokemonAutomation_VideoPipeline_SnapshotManager_H -#define PokemonAutomation_VideoPipeline_SnapshotManager_H +#ifndef PokemonAutomation_VideoPipeline_Stopwatch_H +#define PokemonAutomation_VideoPipeline_Stopwatch_H #include "Time.h" diff --git a/Common/Qt/Options/BatchWidget.h b/Common/Qt/Options/BatchWidget.h index b1f8bd0287..765d64d1a7 100644 --- a/Common/Qt/Options/BatchWidget.h +++ b/Common/Qt/Options/BatchWidget.h @@ -14,7 +14,8 @@ namespace PokemonAutomation{ - +// A Widget to hold multiple derived classes of ConfigWidget. +// Construct using a BatchOption. class BatchWidget : public QWidget, public ConfigWidget{ public: ~BatchWidget(); diff --git a/Common/Qt/Options/BoxFloatWidget.cpp b/Common/Qt/Options/BoxFloatWidget.cpp new file mode 100644 index 0000000000..7e48aea3cb --- /dev/null +++ b/Common/Qt/Options/BoxFloatWidget.cpp @@ -0,0 +1,225 @@ +/* Box Float Option + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "BoxFloatWidget.h" + +//#include +//using std::cout; +//using std::endl; + +namespace PokemonAutomation{ + + + +ConfigWidget* BoxFloatOption::make_QtWidget(QWidget& parent){ + return new BoxFloatWidget(parent, *this); +} + + + + +std::vector split(const std::string& str, const std::string& delimiter) { + std::vector tokens; + size_t start = 0; + size_t end = str.find(delimiter); + + while (end != std::string::npos) { + tokens.push_back(str.substr(start, end - start)); + start = end + delimiter.length(); + end = str.find(delimiter, start); + } + + tokens.push_back(str.substr(start)); + return tokens; +} + + + +BoxFloatWidget::~BoxFloatWidget(){ + m_value.remove_listener(*this); +} + +BoxFloatWidget::BoxFloatWidget(QWidget& parent, BoxFloatOption& value) + : QWidget(&parent) + , ConfigWidget(value, *this) + , m_value(value) +{ + QHBoxLayout* layout = new QHBoxLayout(this); + layout->setContentsMargins(0, 0, 0, 0); + + QLabel* text = new QLabel(QString::fromStdString(value.label()), this); + text->setWordWrap(true); + text->setTextFormat(Qt::RichText); + text->setTextInteractionFlags(Qt::TextBrowserInteraction); + text->setOpenExternalLinks(true); + layout->addWidget(text, 1); + + double x, y, width, height; + m_value.get_all(x, y, width, height); + + QVBoxLayout* right_side = new QVBoxLayout(); + layout->addLayout(right_side, 1); + + QHBoxLayout* coordinates = new QHBoxLayout(); + right_side->addLayout(coordinates); + + m_x = new QLineEdit( this); + m_y = new QLineEdit(this); + m_width = new QLineEdit(this); + m_height = new QLineEdit(this); + + coordinates->addWidget(new QLabel("X = ", this)); + coordinates->addWidget(m_x); + coordinates->addWidget(new QLabel("Y = ", this)); + coordinates->addWidget(m_y); + coordinates->addWidget(new QLabel("Width = ", this)); + coordinates->addWidget(m_width); + coordinates->addWidget(new QLabel("Height = ", this)); + coordinates->addWidget(m_height); + + QHBoxLayout* array = new QHBoxLayout(); + right_side->addLayout(array); + + m_array = new QLineEdit(this); + array->addWidget(new QLabel("Array = ", this)); + array->addWidget(m_array); + + BoxFloatWidget::update_value(); + + connect( + m_x, &QLineEdit::editingFinished, + this, [this]{ + bool ok; + double current = m_x->text().toDouble(&ok); + + double x, y, width, height; + m_value.get_all(x, y, width, height); + x = current; + + bool redraw = m_value.sanitize(x, y, width, height); + m_value.set_all(x, y, width, height); + + if (redraw){ + update_value(); + } + } + ); + connect( + m_y, &QLineEdit::editingFinished, + this, [this](){ + bool ok; + double current = m_y->text().toDouble(&ok); + + double x, y, width, height; + m_value.get_all(x, y, width, height); + y = current; + + bool redraw = m_value.sanitize(x, y, width, height); + m_value.set_all(x, y, width, height); + + if (redraw){ + update_value(); + } + } + ); + connect( + m_width, &QLineEdit::editingFinished, + this, [this](){ + bool ok; + double current = m_width->text().toDouble(&ok); + + double x, y, width, height; + m_value.get_all(x, y, width, height); + width = current; + + bool redraw = m_value.sanitize(x, y, width, height); + m_value.set_all(x, y, width, height); + + if (redraw){ + update_value(); + } + } + ); + connect( + m_height, &QLineEdit::editingFinished, + this, [this](){ + bool ok; + double current = m_height->text().toDouble(&ok); + + double x, y, width, height; + m_value.get_all(x, y, width, height); + height = current; + + bool redraw = m_value.sanitize(x, y, width, height); + m_value.set_all(x, y, width, height); + + if (redraw){ + update_value(); + } + } + ); + connect( + m_array, &QLineEdit::editingFinished, + this, [this](){ + std::vector all_coords = split(m_array->text().toStdString(), ", "); + if (all_coords.size() != 4){ + return; + } + + double x = std::stod(all_coords[0]); + double y = std::stod(all_coords[1]); + double width = std::stod(all_coords[2]); + double height = std::stod(all_coords[3]); + + bool redraw = m_value.sanitize(x, y, width, height); + m_value.set_all(x, y, width, height); + + if (redraw){ + update_value(); + } + } + ); + + value.add_listener(*this); +} + + +void BoxFloatWidget::update_value(){ + double x, y, width, height; + m_value.get_all(x, y, width, height); + + std::string str_x = std::to_string(x); + std::string str_y = std::to_string(y); + std::string str_w = std::to_string(width); + std::string str_h = std::to_string(height); + + m_x->setText(QString::fromStdString(str_x)); + m_y->setText(QString::fromStdString(str_y)); + m_width->setText(QString::fromStdString(str_w)); + m_height->setText(QString::fromStdString(str_h)); + + std::string box_coord_string; + box_coord_string += str_x; + box_coord_string += ", "; + box_coord_string += str_y; + box_coord_string += ", "; + box_coord_string += str_w; + box_coord_string += ", "; + box_coord_string += str_h; + m_array->setText(QString::fromStdString(box_coord_string)); +} +void BoxFloatWidget::on_config_value_changed(void* object){ + QMetaObject::invokeMethod(this, [this]{ + update_value(); + }, Qt::QueuedConnection); +} + + + +} diff --git a/Common/Qt/Options/BoxFloatWidget.h b/Common/Qt/Options/BoxFloatWidget.h new file mode 100644 index 0000000000..0d0b9049c6 --- /dev/null +++ b/Common/Qt/Options/BoxFloatWidget.h @@ -0,0 +1,39 @@ +/* Box Float Option + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_BoxFloatWidget_H +#define PokemonAutomation_BoxFloatWidget_H + +#include +#include "Common/Cpp/Options/BoxFloatOption.h" +#include "ConfigWidget.h" + +namespace PokemonAutomation{ + + +class BoxFloatWidget : public QWidget, public ConfigWidget{ +public: + ~BoxFloatWidget(); + BoxFloatWidget(QWidget& parent, BoxFloatOption& value); + + virtual void update_value() override; + virtual void on_config_value_changed(void* object) override; + +private: + BoxFloatOption& m_value; + QLineEdit* m_x; + QLineEdit* m_y; + QLineEdit* m_width; + QLineEdit* m_height; + QLineEdit* m_array; +}; + + + + + +} +#endif diff --git a/Common/Qt/Options/ConfigWidget.cpp b/Common/Qt/Options/ConfigWidget.cpp index c1242475f8..2ff8652d9c 100644 --- a/Common/Qt/Options/ConfigWidget.cpp +++ b/Common/Qt/Options/ConfigWidget.cpp @@ -46,6 +46,11 @@ void ConfigWidget::update_visibility(){ // cout << "ConfigWidget::update_visibility(): " << (int)m_value.visibility() << endl; switch (m_value.visibility()){ case ConfigOptionState::ENABLED: + // setEnable(false) only happens when the lock mode is LOCK_WHILE_RUNNING and the program is running. + // Ideally we should handle lock mode READ_ONLY here too, but m_widget as a QWidget does not have such + // function. Only some of its derived classes, like QLineEdit, can call setReadOnly(). + // So here we treat READ_ONLY as enabled and let derived classes of ConfigWidget that have the + // functionality to set read-only (e.g. StringOptionWidget) to handle READ_ONLY. m_widget->setEnabled(m_value.lock_mode() != LockMode::LOCK_WHILE_RUNNING || !m_program_is_running); m_widget->setVisible(true); break; diff --git a/Common/Qt/Options/ConfigWidget.h b/Common/Qt/Options/ConfigWidget.h index 4d27c505e9..f6da2fa229 100644 --- a/Common/Qt/Options/ConfigWidget.h +++ b/Common/Qt/Options/ConfigWidget.h @@ -12,6 +12,17 @@ namespace PokemonAutomation{ +// Base class for all config widgets that control a program option. +// +// It is intuitive to let ConfigWidget inherit QWidget, base class of Qt UI element, +// But some derived classes of ConfigWidget, e.g. StringCellWidget, needs to inherit +// QWidget's derived classes, e.g. QLineEdit. To avoid nasty diamond inheritance, +// ConfigWidget does not inherit QWidget. User has to call ConfigWidget::widget() +// to get the actual QWidget. +// +// ConfigWidget's derived classes need to inherit a QWidget or its derived class +// and pass *this as the widget in ConfigWidget(m_valuie, widget) so a ConfigWidget +// pointer can get the actual QWidget. class ConfigWidget : protected ConfigOption::Listener{ public: virtual ~ConfigWidget(); @@ -25,11 +36,23 @@ class ConfigWidget : protected ConfigOption::Listener{ // Needs to be called on the UI thread. virtual void update_value(){} + // Needs to be called on the UI thread. Update QWidget's visibility based on + // ConfigOption m_value's visibility setting. + // Note this function is unable to set READ_ONLY visibility state. The derived + // classes of ConfigWidget who have the functionality to set read-only should + // overwrite on_config_visibility_changed() to handle that. virtual void update_visibility(); void update_visibility(bool program_is_running); void update_all(bool program_is_running); protected: + // Overwrite ConfigOption::Listener::on_config_visibility_changed(). + // Called when the listened config option's visibility is changed. + // This function invokes this->update_visibility() on the Qt UI thread. + // Note due to update_visibility() is unable to set READ_ONLY visbility state. + // It is up to the derived classes of ConfigWidget who have the functionality + // to set read-only (e.g. StringOptionWidget) to overwrite this function to + // handle read-only. virtual void on_config_visibility_changed() override; virtual void on_program_state_changed(bool program_is_running) override; diff --git a/Common/Qt/Options/StringWidget.cpp b/Common/Qt/Options/StringWidget.cpp index 34fa56c386..b22ef7ff38 100644 --- a/Common/Qt/Options/StringWidget.cpp +++ b/Common/Qt/Options/StringWidget.cpp @@ -9,9 +9,9 @@ #include #include "StringWidget.h" -//#include -//using std::cout; -//using std::endl; +// #include +// using std::cout; +// using std::endl; namespace PokemonAutomation{ @@ -47,6 +47,19 @@ StringCellWidget::StringCellWidget(QWidget& parent, StringCell& value) m_value.set(this->text().toStdString()); } ); + if (m_value.signal_all_text_changes()){ + connect( + this, &QLineEdit::textChanged, + [this]{ + std::string old_value = (std::string)m_value; + std::string text = this->text().toStdString(); + if (old_value == text){ + return; + } + m_value.set(std::move(text)); + } + ); + } m_value.add_listener(*this); } @@ -59,8 +72,22 @@ void StringCellWidget::on_config_value_changed(void* object){ }, Qt::QueuedConnection); } void StringCellWidget::on_config_visibility_changed(){ + // Overwrite ConfigWidget::on_config_visibility_changed() because ConfigWidget cannot handle + // READ_ONLY state. + this->setEnabled(true); QMetaObject::invokeMethod(this, [this]{ - setReadOnly(m_value.lock_mode() == LockMode::READ_ONLY || m_value.is_locked()); + const ConfigOptionState visibility = m_value.visibility(); + this->setReadOnly(visibility != ConfigOptionState::ENABLED || + m_value.lock_mode() == LockMode::READ_ONLY || m_value.is_locked()); + switch (visibility){ + case ConfigOptionState::ENABLED: + case ConfigOptionState::DISABLED: + this->setVisible(true); + break; + case ConfigOptionState::HIDDEN: + this->setVisible(false); + break; + } }, Qt::QueuedConnection); } @@ -96,6 +123,19 @@ StringOptionWidget::StringOptionWidget(QWidget& parent, StringOption& value) m_value.set(m_box->text().toStdString()); } ); + if (m_value.signal_all_text_changes()){ + connect( + m_box, &QLineEdit::textChanged, + [this]{ + const std::string old_value = (std::string)m_value; + std::string text = m_box->text().toStdString(); + if (old_value == text){ + return; + } + m_value.set(std::move(text)); + } + ); + } m_value.add_listener(*this); } @@ -108,8 +148,22 @@ void StringOptionWidget::on_config_value_changed(void* object){ }, Qt::QueuedConnection); } void StringOptionWidget::on_config_visibility_changed(){ - QMetaObject::invokeMethod(m_box, [this]{ - m_box->setReadOnly(m_value.is_locked()); + // Overwrite ConfigWidget::on_config_visibility_changed() because ConfigWidget cannot handle + // READ_ONLY state. + this->setEnabled(true); + QMetaObject::invokeMethod(this, [this]{ + const ConfigOptionState visibility = m_value.visibility(); + m_box->setReadOnly(visibility != ConfigOptionState::ENABLED || + m_value.lock_mode() == LockMode::READ_ONLY || m_value.is_locked()); + switch (visibility){ + case ConfigOptionState::ENABLED: + case ConfigOptionState::DISABLED: + this->setVisible(true); + break; + case ConfigOptionState::HIDDEN: + this->setVisible(false); + break; + } }, Qt::QueuedConnection); } diff --git a/Common/Qt/Options/TextEditWidget.cpp b/Common/Qt/Options/TextEditWidget.cpp index 6d0595950a..22ed148253 100644 --- a/Common/Qt/Options/TextEditWidget.cpp +++ b/Common/Qt/Options/TextEditWidget.cpp @@ -45,9 +45,9 @@ class TextEditWidget::Box : public QTextEdit{ connect( this, &QTextEdit::textChanged, [this]{ - std::string new_value = (std::string)m_parent.m_value; + const std::string old_value = (std::string)m_parent.m_value; std::string text = this->toPlainText().toStdString(); - if (new_value == text){ + if (old_value == text){ return; } // cout << new_value << " : " << text << endl; diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index d5d5abd040..1dfc479f4e 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -140,6 +140,8 @@ file(GLOB MAIN_SOURCES ../Common/Cpp/Options/BooleanCheckBoxOption.h ../Common/Cpp/Options/ButtonOption.cpp ../Common/Cpp/Options/ButtonOption.h + ../Common/Cpp/Options/BoxFloatOption.cpp + ../Common/Cpp/Options/BoxFloatOption.h ../Common/Cpp/Options/ColorOption.cpp ../Common/Cpp/Options/ColorOption.h ../Common/Cpp/Options/ConfigOption.cpp @@ -226,6 +228,8 @@ file(GLOB MAIN_SOURCES ../Common/Qt/Options/BatchWidget.h ../Common/Qt/Options/BooleanCheckBoxWidget.cpp ../Common/Qt/Options/BooleanCheckBoxWidget.h + ../Common/Qt/Options/BoxFloatWidget.cpp + ../Common/Qt/Options/BoxFloatWidget.h ../Common/Qt/Options/ButtonWidget.cpp ../Common/Qt/Options/ButtonWidget.h ../Common/Qt/Options/ColorWidget.cpp @@ -481,6 +485,7 @@ file(GLOB MAIN_SOURCES Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h Source/CommonFramework/VideoPipeline/Backends/MediaServicesQt6.cpp Source/CommonFramework/VideoPipeline/Backends/MediaServicesQt6.h + Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h Source/CommonFramework/VideoPipeline/Backends/QVideoFrameCache.h Source/CommonFramework/VideoPipeline/Backends/SnapshotManager.cpp Source/CommonFramework/VideoPipeline/Backends/SnapshotManager.h @@ -2414,7 +2419,7 @@ if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../../Internal/SerialPrograms/Internal0. endif() #extract opencv_world4110d.dll from archive on Windows Debug builds -if (MSVC) +if (WIN32) file(ARCHIVE_EXTRACT INPUT ${CMAKE_CURRENT_SOURCE_DIR}/../3rdPartyBinaries/opencv_world4110d.zip DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../3rdPartyBinaries/ @@ -2429,7 +2434,7 @@ target_link_directories(SerialPrograms PRIVATE ../3rdPartyBinaries/) -if (MSVC) +if (WIN32) add_library(OpenCV_lib IMPORTED UNKNOWN) target_include_directories(SerialPrograms SYSTEM PRIVATE ../3rdParty/opencv-4.11.0/) set_target_properties(OpenCV_lib PROPERTIES @@ -2501,6 +2506,15 @@ if (MSVC) set(ARCH_FLAGS_17_Skylake /arch:AVX512) set(ARCH_FLAGS_19_IceLake /arch:AVX512) + if(CMAKE_GENERATOR_TOOLSET MATCHES "ClangCL") + target_compile_options(SerialPrograms PRIVATE -Wno-unused-function) + target_compile_options(SerialPrograms PRIVATE -march=nehalem) + set(ARCH_FLAGS_09_Nehalem -march=nehalem) + set(ARCH_FLAGS_13_Haswell -march=haswell) + set(ARCH_FLAGS_17_Skylake -march=skylake-avx512) + set(ARCH_FLAGS_19_IceLake -march=icelake-client) + endif() + # Run-time ISA dispatching target_compile_definitions(SerialPrograms PRIVATE PA_AutoDispatch_x64_08_Nehalem) target_compile_definitions(SerialPrograms PRIVATE PA_AutoDispatch_x64_13_Haswell) @@ -2615,7 +2629,7 @@ else() # macOS and Linux else() # Intel CPU - target_compile_options(SerialPrograms PRIVATE -msse4.2) + target_compile_options(SerialPrograms PRIVATE -march=nehalem) set(ARCH_FLAGS_09_Nehalem -march=nehalem) set(ARCH_FLAGS_13_Haswell -march=haswell) set(ARCH_FLAGS_17_Skylake -march=skylake-avx512) diff --git a/SerialPrograms/Source/CommonFramework/Globals.cpp b/SerialPrograms/Source/CommonFramework/Globals.cpp index b281d3db39..54fa193d96 100644 --- a/SerialPrograms/Source/CommonFramework/Globals.cpp +++ b/SerialPrograms/Source/CommonFramework/Globals.cpp @@ -26,7 +26,7 @@ namespace PokemonAutomation{ const bool IS_BETA_VERSION = true; const int PROGRAM_VERSION_MAJOR = 0; const int PROGRAM_VERSION_MINOR = 54; -const int PROGRAM_VERSION_PATCH = 26; +const int PROGRAM_VERSION_PATCH = 29; const std::string PROGRAM_VERSION_BASE = "v" + std::to_string(PROGRAM_VERSION_MAJOR) + diff --git a/SerialPrograms/Source/CommonFramework/ImageTools/ImageBoxes.cpp b/SerialPrograms/Source/CommonFramework/ImageTools/ImageBoxes.cpp index b1dca98675..07c0074cb5 100644 --- a/SerialPrograms/Source/CommonFramework/ImageTools/ImageBoxes.cpp +++ b/SerialPrograms/Source/CommonFramework/ImageTools/ImageBoxes.cpp @@ -95,6 +95,34 @@ size_t ImagePixelBox::distance_y(const ImagePixelBox& box) const{ return 0; } +size_t ImagePixelBox::distance_to_point_x(const size_t x) const{ + size_t min_x = std::max(this->min_x, x); + size_t max_x = std::min(this->max_x, x); + return min_x - max_x; +} +// The distance to a point on y axis. If the point is in the box, the distance is 0. +size_t ImagePixelBox::distance_to_point_y(const size_t y) const{ + size_t min_y = std::max(this->min_y, y); + size_t max_y = std::min(this->max_y, y); + return min_y - max_y; +} +// The distance from the box center to a point on x axis +size_t ImagePixelBox::center_distance_to_point_x(const size_t x) const{ + const size_t cx = center_x(); + if (cx >= x){ + return cx - x; + } + return x - cx; +} +// The distance from the box center to a point on y axis +size_t ImagePixelBox::center_distance_to_point_y(const size_t y) const{ + const size_t cy = center_y(); + if (cy >= y){ + return cy - y; + } + return y - cy; +} + ImageViewRGB32 extract_box_reference(const ImageViewRGB32& image, const ImagePixelBox& box){ return image.sub_image(box.min_x, box.min_y, box.width(), box.height()); diff --git a/SerialPrograms/Source/CommonFramework/ImageTools/ImageBoxes.h b/SerialPrograms/Source/CommonFramework/ImageTools/ImageBoxes.h index 44054e1743..46a5b048a2 100644 --- a/SerialPrograms/Source/CommonFramework/ImageTools/ImageBoxes.h +++ b/SerialPrograms/Source/CommonFramework/ImageTools/ImageBoxes.h @@ -56,6 +56,14 @@ struct ImagePixelBox : public Rectangle{ size_t distance_x(const ImagePixelBox& box) const; // The distance to another box on y axis. If two boxes overlap, the distance is 0. size_t distance_y(const ImagePixelBox& box) const; + // The distance to a point on x axis. If the point is in the box, the distance is 0. + size_t distance_to_point_x(const size_t x) const; + // The distance to a point on y axis. If the point is in the box, the distance is 0. + size_t distance_to_point_y(const size_t y) const; + // The distance from the box center to a point on x axis + size_t center_distance_to_point_x(const size_t x) const; + // The distance from the box center to a point on y axis + size_t center_distance_to_point_y(const size_t y) const; }; // An axis aligned box in the normalized image space. Used for getting a crop from an image for various diff --git a/SerialPrograms/Source/CommonFramework/Startup/NewVersionCheck.cpp b/SerialPrograms/Source/CommonFramework/Startup/NewVersionCheck.cpp index 66a847da50..ec326bb632 100644 --- a/SerialPrograms/Source/CommonFramework/Startup/NewVersionCheck.cpp +++ b/SerialPrograms/Source/CommonFramework/Startup/NewVersionCheck.cpp @@ -145,6 +145,9 @@ void show_update_box( QMessageBox box; QPushButton* ok = box.addButton(QMessageBox::Ok); QPushButton* skip = box.addButton("Skip this Version", QMessageBox::NoRole); + box.setEscapeButton(ok); +// cout << "ok = " << ok << endl; +// cout << "skip = " << skip << endl; box.setTextFormat(Qt::RichText); std::string text = header + "
"; @@ -160,6 +163,7 @@ void show_update_box( box.exec(); QAbstractButton* clicked = box.clickedButton(); +// cout << "clicked = " << clicked << endl; if (clicked == ok){ return; } diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp index 992a51daa9..4a1512c796 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp @@ -82,7 +82,7 @@ CameraVideoSource::~CameraVideoSource(){ m_logger.log("Stopping Camera..."); }catch (...){} - m_camera->stop(); +// m_camera->stop(); m_capture_session.reset(); m_camera.reset(); } @@ -91,8 +91,9 @@ CameraVideoSource::CameraVideoSource( const CameraInfo& info, Resolution desired_resolution ) - : VideoSource(true) + : VideoSource(logger, true) , m_logger(logger) + , m_last_frame(logger) , m_snapshot_manager(logger, m_last_frame) { if (!info){ @@ -146,12 +147,15 @@ CameraVideoSource::CameraVideoSource( m_resolution = Resolution(size.width(), size.height()); m_logger.log("Resolution: " + m_resolution.to_string()); - m_camera.reset(new QCamera(*device)); - m_camera->setCameraFormat(*format); + m_camera.reset(new QCameraThread(m_logger, *device, *format)); +// m_camera.reset(new QCamera(*device)); +// m_camera->setCameraFormat(*format); m_capture_session.reset(new QMediaCaptureSession()); - m_capture_session->setCamera(m_camera.get()); +// m_capture_session->setCamera(m_camera.get()); + m_capture_session->setCamera(&m_camera->camera()); +#if 0 connect(m_camera.get(), &QCamera::errorOccurred, this, [&](){ if (m_camera->error() == QCamera::NoError){ return; @@ -160,6 +164,8 @@ CameraVideoSource::CameraVideoSource( }); m_camera->start(); +#endif + } @@ -174,18 +180,16 @@ void CameraVideoSource::set_video_output(QGraphicsVideoItem& item){ connect( item.videoSink(), &QVideoSink::videoFrameChanged, - m_camera.get(), [&](const QVideoFrame& frame){ - // This will be on the main thread. So we waste as little time as - // possible. Shallow-copy the frame, update the listeners, and - // return immediately to unblock the main thread. + &m_camera->camera(), [&](const QVideoFrame& frame){ + // This runs on the QCamera's thread. So it is off the critical path. WallClock now = current_time(); if (!m_last_frame.push_frame(frame, now)){ return; } report_source_frame(std::make_shared(now, frame)); - }, - Qt::DirectConnection + }//, +// Qt::DirectConnection ); } diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h index bb9071682a..3ff139b5f0 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.h @@ -11,7 +11,7 @@ #if QT_VERSION_MAJOR == 6 //#include -#include +//#include #include #include #include @@ -23,6 +23,7 @@ #include "CommonFramework/Tools/StatAccumulator.h" #include "CommonFramework/VideoPipeline/VideoSource.h" #include "CommonFramework/VideoPipeline/CameraInfo.h" +#include "QCameraThread.h" #include "QVideoFrameCache.h" #include "SnapshotManager.h" #include "CameraImplementations.h" @@ -126,7 +127,8 @@ class CameraVideoSource : public QObject, public VideoSource{ Logger& m_logger; Resolution m_resolution; - std::unique_ptr m_camera; +// std::unique_ptr m_camera; + std::unique_ptr m_camera; std::unique_ptr m_video_sink; std::unique_ptr m_capture_session; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp index b275c2c17b..cd2cce5eb7 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp @@ -74,7 +74,7 @@ CameraVideoSource::~CameraVideoSource(){ m_logger.log("Stopping Camera..."); }catch (...){} - m_camera->stop(); +// m_camera->stop(); m_capture.reset(); m_video_sink.reset(); m_camera.reset(); @@ -84,8 +84,9 @@ CameraVideoSource::CameraVideoSource( const CameraInfo& info, Resolution desired_resolution ) - : VideoSource(true) + : VideoSource(logger, true) , m_logger(logger) + , m_last_frame(logger) , m_snapshot_manager(logger, m_last_frame) { if (!info){ @@ -138,24 +139,26 @@ CameraVideoSource::CameraVideoSource( m_resolution = Resolution(size.width(), size.height()); m_logger.log("Resolution: " + m_resolution.to_string()); - m_camera.reset(new QCamera(*device)); - m_camera->setCameraFormat(*format); + m_camera.reset(new QCameraThread(m_logger, *device, *format)); +// m_camera.reset(new QCamera(*device)); +// m_camera->setCameraFormat(*format); m_video_sink.reset(new QVideoSink()); m_capture.reset(new QMediaCaptureSession()); - m_capture->setCamera(m_camera.get()); +// m_capture->setCamera(m_camera.get()); + m_capture->setCamera(&m_camera->camera()); m_capture->setVideoSink(m_video_sink.get()); +#if 0 connect(m_camera.get(), &QCamera::errorOccurred, this, [&](){ if (m_camera->error() != QCamera::NoError){ m_logger.log("QCamera error: " + m_camera->errorString().toStdString()); } }); +#endif connect( m_video_sink.get(), &QVideoSink::videoFrameChanged, - m_camera.get(), [&](const QVideoFrame& frame){ - // This will be on the main thread. So we waste as little time as - // possible. Shallow-copy the frame, update the listeners, and - // return immediately to unblock the main thread. + &m_camera->camera(), [&](const QVideoFrame& frame){ + // This runs on the QCamera's thread. So it is off the critical path. WallClock now = current_time(); if (!m_last_frame.push_frame(frame, now)){ @@ -165,7 +168,7 @@ CameraVideoSource::CameraVideoSource( } ); - m_camera->start(); +// m_camera->start(); } diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h index cde88e30c8..19c7eef2e9 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.h @@ -18,6 +18,7 @@ #include "CommonFramework/Tools/StatAccumulator.h" #include "CommonFramework/VideoPipeline/VideoSource.h" #include "CommonFramework/VideoPipeline/CameraInfo.h" +#include "QCameraThread.h" #include "QVideoFrameCache.h" #include "SnapshotManager.h" #include "CameraImplementations.h" @@ -85,7 +86,8 @@ class CameraVideoSource : public QObject, public VideoSource{ std::mutex m_snapshot_lock; - std::unique_ptr m_camera; +// std::unique_ptr m_camera; + std::unique_ptr m_camera; std::unique_ptr m_video_sink; std::unique_ptr m_capture; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h new file mode 100644 index 0000000000..165f57565c --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h @@ -0,0 +1,91 @@ +/* QCameraThread + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_VideoPipeline_QCameraThread_H +#define PokemonAutomation_VideoPipeline_QCameraThread_H + +#include +#include +#include "Common/Cpp/AbstractLogger.h" + +//#include +//using std::cout; +//using std::endl; + +namespace PokemonAutomation{ + + +class QCameraThread : public QThread{ + Q_OBJECT + +public: + QCameraThread( + Logger& logger, + QCameraDevice device, + QCameraFormat format + ) + : m_logger(logger) + , m_device(std::move(device)) + , m_format(std::move(format)) + , m_camera(nullptr) + { + start(); + + std::unique_lock lg(m_lock); + m_cv.wait(lg, [this]{ return m_camera != nullptr; }); + } + ~QCameraThread(){ + quit(); + wait(); + } + + QCamera& camera() const{ + return *m_camera; + } + + +private: + virtual void run() override{ + QCamera camera(m_device); + m_camera = &camera; + camera.setCameraFormat(m_format); + + connect(&camera, &QCamera::errorOccurred, this, [&](){ + if (camera.error() == QCamera::NoError){ + return; + } + m_logger.log("QCamera error: " + camera.errorString().toStdString(), COLOR_RED); + }); + + camera.start(); + + { + std::lock_guard lg(m_lock); + } + m_cv.notify_all(); + +// cout << "start" << endl; + exec(); +// cout << "end" << endl; + + m_camera = nullptr; + } + + +private: + Logger& m_logger; + QCameraDevice m_device; + QCameraFormat m_format; + QCamera* m_camera; + + std::mutex m_lock; + std::condition_variable m_cv; +}; + + + +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QVideoFrameCache.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QVideoFrameCache.h index c167fc6bc9..575d2d1939 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QVideoFrameCache.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QVideoFrameCache.h @@ -12,6 +12,9 @@ #include #include "Common/Cpp/Time.h" #include "Common/Cpp/Concurrency/SpinLock.h" +#include "CommonFramework/Tools/StatAccumulator.h" + +//#define PA_PROFILE_QVideoFrameCache namespace PokemonAutomation{ @@ -19,45 +22,83 @@ namespace PokemonAutomation{ class QVideoFrameCache{ public: - QVideoFrameCache() + QVideoFrameCache(Logger& logger) +#ifdef PA_PROFILE_QVideoFrameCache + : m_logger(logger) + , m_last_frame_timestamp(WallClock::min()) +#else : m_last_frame_timestamp(WallClock::min()) +#endif , m_last_frame_seqnum(0) +#ifdef PA_PROFILE_QVideoFrameCache + , m_stats_lock("QVideoFrameCache::push_frame()-Lock", "ms", 1000, std::chrono::seconds(10)) + , m_stats_push_frame("QVideoFrameCache::push_frame()-All", "ms", 1000, std::chrono::seconds(10)) +#endif {} uint64_t seqnum() const{ return m_last_frame_seqnum.load(std::memory_order_relaxed); } uint64_t get_latest(QVideoFrame& frame, WallClock& timestamp) const{ - WriteSpinLock lg(m_frame_lock); + WriteSpinLock lg(m_frame_lock, "QVideoFrameCache::get_latest()"); frame = m_last_frame; timestamp = m_last_frame_timestamp; return seqnum(); } bool push_frame(QVideoFrame frame, WallClock timestamp){ - WriteSpinLock lg(m_frame_lock); +#ifdef PA_PROFILE_QVideoFrameCache + WallClock time0 = current_time(); +#endif + + WriteSpinLock lg(m_frame_lock, "QVideoFrameCache::push_frame()"); + +#ifdef PA_PROFILE_QVideoFrameCache + { + WallClock time1 = current_time(); + uint32_t microseconds = (uint32_t)std::chrono::duration_cast(time1 - time0).count(); + m_stats_lock.report_data(m_logger, microseconds); + } +#endif // Skip duplicate frames. if (frame.startTime() != -1 && frame.startTime() <= m_last_frame.startTime()){ return false; } - m_last_frame = frame; + m_last_frame = std::move(frame); m_last_frame_timestamp = timestamp; uint64_t seqnum = m_last_frame_seqnum.load(std::memory_order_relaxed); seqnum++; m_last_frame_seqnum.store(seqnum, std::memory_order_relaxed); +#ifdef PA_PROFILE_QVideoFrameCache + { + WallClock time1 = current_time(); + uint32_t microseconds = (uint32_t)std::chrono::duration_cast(time1 - time0).count(); + m_stats_push_frame.report_data(m_logger, microseconds); + } +#endif + return true; } private: +#ifdef PA_PROFILE_QVideoFrameCache + Logger& m_logger; +#endif + mutable SpinLock m_frame_lock; QVideoFrame m_last_frame; WallClock m_last_frame_timestamp; std::atomic m_last_frame_seqnum; + +#ifdef PA_PROFILE_QVideoFrameCache + PeriodicStatsReporterI32 m_stats_lock; + PeriodicStatsReporterI32 m_stats_push_frame; +#endif }; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.cpp index 45df772ca1..f2ed8fb1c4 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.cpp @@ -35,7 +35,8 @@ VideoOverlayWidget::VideoOverlayWidget(QWidget& parent, VideoOverlaySession& ses , m_texts(std::make_shared>(session.texts())) , m_images(std::make_shared>(session.images())) , m_log(std::make_shared>(session.log_texts())) - , m_stats(nullptr) +// , m_stats(nullptr) + , m_stats_paint("VideoOverlayWidget::paintEvent", "ms", 1000, std::chrono::seconds(10)) { setAttribute(Qt::WA_NoSystemBackground); setAttribute(Qt::WA_TranslucentBackground); @@ -81,10 +82,6 @@ void VideoOverlayWidget::update_log_background(const std::shared_ptr* stats){ - WriteSpinLock lg(m_lock, "VideoOverlay::update_stats()"); - m_stats = stats; -} void VideoOverlayWidget::on_watchdog_timeout(){ QMetaObject::invokeMethod(this, [this]{ this->update(); }); @@ -96,6 +93,8 @@ void VideoOverlayWidget::on_watchdog_timeout(){ void VideoOverlayWidget::resizeEvent(QResizeEvent* event){} void VideoOverlayWidget::paintEvent(QPaintEvent*){ + WallClock time0 = current_time(); + QPainter painter(this); { WriteSpinLock lg(m_lock, "VideoOverlay::paintEvent()"); @@ -112,12 +111,16 @@ void VideoOverlayWidget::paintEvent(QPaintEvent*){ if (m_session.enabled_log()){ render_log(painter); } - if (m_session.enabled_stats() && m_stats){ + if (m_session.enabled_stats()){ render_stats(painter); } } global_watchdog().delay(*this); + + WallClock time1 = current_time(); + uint32_t microseconds = (uint32_t)std::chrono::duration_cast(time1 - time0).count(); + m_stats_paint.report_data(m_session.logger(), microseconds); } @@ -291,14 +294,7 @@ void VideoOverlayWidget::render_stats(QPainter& painter){ int height = this->height(); int start_x = (int)(width * 0.78); - std::vector lines; - for (const auto& stat : *m_stats){ - OverlayStatSnapshot snapshot = stat->get_current(); - if (!snapshot.text.empty()){ - lines.emplace_back(std::move(snapshot)); - } - } - + std::vector lines = m_session.stats(); painter.fillRect( start_x, diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.h index 7174894cfe..9489442ff3 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoOverlayWidget.h @@ -10,6 +10,7 @@ #include #include "Common/Cpp/Concurrency/SpinLock.h" #include "Common/Cpp/Concurrency/Watchdog.h" +#include "CommonFramework/Tools/StatAccumulator.h" #include "CommonFramework/VideoPipeline/VideoOverlaySession.h" namespace PokemonAutomation{ @@ -52,8 +53,6 @@ class VideoOverlayWidget : public QWidget, private VideoOverlaySession::ContentL virtual void on_overlay_update_images(const std::shared_ptr>& images) override; // callback function from VideoOverlaySession on overlay images updated virtual void on_overlay_update_log (const std::shared_ptr>& logs) override; - // callback function from VideoOverlaySession on overlay stats updated - virtual void on_overlay_update_stats (const std::list* stats) override; virtual void on_watchdog_timeout() override; @@ -89,7 +88,8 @@ class VideoOverlayWidget : public QWidget, private VideoOverlaySession::ContentL std::shared_ptr> m_texts; std::shared_ptr> m_images; std::shared_ptr> m_log; - const std::list* m_stats; + + PeriodicStatsReporterI32 m_stats_paint; }; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlaySession.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlaySession.cpp index e1204e0a98..dc50dc46fa 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlaySession.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlaySession.cpp @@ -15,25 +15,25 @@ namespace PokemonAutomation{ void VideoOverlaySession::add_listener(ContentListener& listener){ - WriteSpinLock lg(m_lock); - m_content_listeners.insert(&listener); - listener.on_overlay_update_stats(&m_stats_order); + m_listeners.add(listener); } void VideoOverlaySession::remove_listener(ContentListener& listener){ - WriteSpinLock lg(m_lock); -// listener.on_overlay_update_stats(nullptr); - m_content_listeners.erase(&listener); + m_listeners.remove(listener); } VideoOverlaySession::~VideoOverlaySession(){ - ReadSpinLock lg(m_lock); - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_update_stats(nullptr); + { + std::lock_guard lg(m_stats_lock); + m_stopping = true; } + m_stats_cv.notify_all(); + m_stats_updater.join(); } -VideoOverlaySession::VideoOverlaySession(VideoOverlayOption& option) - : m_option(option) +VideoOverlaySession::VideoOverlaySession(Logger& logger, VideoOverlayOption& option) + : m_logger(logger) + , m_option(option) + , m_stats_updater(&VideoOverlaySession::stats_thread, this) {} @@ -50,7 +50,6 @@ void VideoOverlaySession::get(VideoOverlayOption& option){ option.stats.store(stats, std::memory_order_relaxed); } void VideoOverlaySession::set(const VideoOverlayOption& option){ - WriteSpinLock lg(m_lock, "VideoOverlaySession::set_enabled_boxes()"); bool boxes = option.boxes.load(std::memory_order_relaxed); bool text = option.text.load(std::memory_order_relaxed); bool images = option.images.load(std::memory_order_relaxed); @@ -61,81 +60,87 @@ void VideoOverlaySession::set(const VideoOverlayOption& option){ m_option.images.store(images, std::memory_order_relaxed); m_option.log.store(log, std::memory_order_relaxed); m_option.stats.store(stats, std::memory_order_relaxed); - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_enabled_boxes(boxes); - listener->on_overlay_enabled_text(text); - listener->on_overlay_enabled_images(images); - listener->on_overlay_enabled_log(log); - listener->on_overlay_enabled_stats(stats); + m_listeners.run_method_unique(&ContentListener::on_overlay_enabled_boxes, boxes); + m_listeners.run_method_unique(&ContentListener::on_overlay_enabled_text, text); + m_listeners.run_method_unique(&ContentListener::on_overlay_enabled_images, images); + m_listeners.run_method_unique(&ContentListener::on_overlay_enabled_log, log); + m_listeners.run_method_unique(&ContentListener::on_overlay_enabled_stats, stats); +} + + +void VideoOverlaySession::stats_thread(){ + std::unique_lock lg(m_stats_lock); + while (!m_stopping){ + { + std::vector lines; + ReadSpinLock lg0(m_lock); + for (const auto& stat : m_stats_order){ + OverlayStatSnapshot snapshot = stat->get_current(); + if (!snapshot.text.empty()){ + lines.emplace_back(std::move(snapshot)); + } + } + m_stat_lines = std::move(lines); + } + m_stats_cv.wait_for(lg, std::chrono::milliseconds(100)); } } void VideoOverlaySession::set_enabled_boxes(bool enabled){ m_option.boxes.store(enabled, std::memory_order_relaxed); - ReadSpinLock lg(m_lock, "VideoOverlaySession::set_enabled_boxes()"); - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_enabled_boxes(enabled); - } + m_listeners.run_method_unique(&ContentListener::on_overlay_enabled_boxes, enabled); } void VideoOverlaySession::set_enabled_text(bool enabled){ m_option.text.store(enabled, std::memory_order_relaxed); - ReadSpinLock lg(m_lock, "VideoOverlaySession::set_enabled_text()"); - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_enabled_text(enabled); - } + m_listeners.run_method_unique(&ContentListener::on_overlay_enabled_text, enabled); } void VideoOverlaySession::set_enabled_images(bool enabled){ m_option.images.store(enabled, std::memory_order_relaxed); - ReadSpinLock lg(m_lock, "VideoOverlaySession::set_enabled_images()"); - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_enabled_images(enabled); - } + m_listeners.run_method_unique(&ContentListener::on_overlay_enabled_images, enabled); } void VideoOverlaySession::set_enabled_log(bool enabled){ m_option.log.store(enabled, std::memory_order_relaxed); - ReadSpinLock lg(m_lock, "VideoOverlaySession::set_enabled_log()"); - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_enabled_log(enabled); - } + m_listeners.run_method_unique(&ContentListener::on_overlay_enabled_log, enabled); } void VideoOverlaySession::set_enabled_stats(bool enabled){ m_option.stats.store(enabled, std::memory_order_relaxed); - ReadSpinLock lg(m_lock, "VideoOverlaySession::set_enabled_stats()"); - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_enabled_stats(enabled); - } + m_listeners.run_method_unique(&ContentListener::on_overlay_enabled_stats, enabled); } +// +// Boxes +// void VideoOverlaySession::add_box(const OverlayBox& box){ - WriteSpinLock lg(m_lock, "VideoOverlaySession::add_box()"); - m_boxes.insert(&box); - push_box_update(); + std::shared_ptr> ptr = std::make_shared>(); + { + WriteSpinLock lg(m_lock, "VideoOverlaySession::add_box()"); + m_boxes.insert(&box); + + // We create a newly allocated Box vector to avoid listener accessing + // `m_boxes` asynchronously. + for (const auto& item : m_boxes){ + ptr->emplace_back(*item); + } + } + m_listeners.run_method_unique(&ContentListener::on_overlay_update_boxes, ptr); } void VideoOverlaySession::remove_box(const OverlayBox& box){ - WriteSpinLock lg(m_lock, "VideoOverlaySession::remove_box()"); - m_boxes.erase(&box); - push_box_update(); -} - -void VideoOverlaySession::push_box_update(){ - if (m_content_listeners.empty()){ - return; - } - - // We create a newly allocated Box vector to avoid listener accessing - // `m_boxes` asynchronously. std::shared_ptr> ptr = std::make_shared>(); - for (const auto& item : m_boxes){ - ptr->emplace_back(*item); - } - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_update_boxes(ptr); + { + WriteSpinLock lg(m_lock, "VideoOverlaySession::remove_box()"); + m_boxes.erase(&box); + + // We create a newly allocated Box vector to avoid listener accessing + // `m_boxes` asynchronously. + for (const auto& item : m_boxes){ + ptr->emplace_back(*item); + } } + m_listeners.run_method_unique(&ContentListener::on_overlay_update_boxes, ptr); } - std::vector VideoOverlaySession::boxes() const{ ReadSpinLock lg(m_lock); std::vector ret; @@ -145,33 +150,39 @@ std::vector VideoOverlaySession::boxes() const{ return ret; } -void VideoOverlaySession::add_text(const OverlayText& text){ - WriteSpinLock lg(m_lock, "VideoOverlaySession::add_text()"); - m_texts.insert(&text); - push_text_update(); -} -void VideoOverlaySession::remove_text(const OverlayText& text){ - WriteSpinLock lg(m_lock, "VideoOverlaySession::remove_text()"); - m_texts.erase(&text); - push_text_update(); -} -void VideoOverlaySession::push_text_update(){ - if (m_content_listeners.empty()){ - return; - } +// +// Texts +// - // We create a newly allocated Box vector to avoid listener accessing - // `m_texts` asynchronously. +void VideoOverlaySession::add_text(const OverlayText& text){ std::shared_ptr> ptr = std::make_shared>(); - for (const auto& item : m_texts){ - ptr->emplace_back(*item); + { + WriteSpinLock lg(m_lock, "VideoOverlaySession::add_text()"); + m_texts.insert(&text); + + // We create a newly allocated Box vector to avoid listener accessing + // `m_texts` asynchronously. + for (const auto& item : m_texts){ + ptr->emplace_back(*item); + } } - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_update_text(ptr); + m_listeners.run_method_unique(&ContentListener::on_overlay_update_text, ptr); +} +void VideoOverlaySession::remove_text(const OverlayText& text){ + std::shared_ptr> ptr = std::make_shared>(); + { + WriteSpinLock lg(m_lock, "VideoOverlaySession::remove_text()"); + m_texts.erase(&text); + + // We create a newly allocated Box vector to avoid listener accessing + // `m_texts` asynchronously. + for (const auto& item : m_texts){ + ptr->emplace_back(*item); + } } + m_listeners.run_method_unique(&ContentListener::on_overlay_update_text, ptr); } - std::vector VideoOverlaySession::texts() const{ ReadSpinLock lg(m_lock); std::vector ret; @@ -182,33 +193,38 @@ std::vector VideoOverlaySession::texts() const{ } +// +// Images +// + void VideoOverlaySession::add_image(const OverlayImage& image){ - WriteSpinLock lg(m_lock, "VideoOverlaySession::add_image()"); - m_images.insert(&image); - push_image_update(); + std::shared_ptr> ptr = std::make_shared>(); + { + WriteSpinLock lg(m_lock, "VideoOverlaySession::add_image()"); + m_images.insert(&image); + + // We create a newly allocated Box vector to avoid listener accessing + // `m_images` asynchronously. + for (const auto& item : m_images){ + ptr->emplace_back(*item); + } + } + m_listeners.run_method_unique(&ContentListener::on_overlay_update_images, ptr); } void VideoOverlaySession::remove_image(const OverlayImage& image){ - WriteSpinLock lg(m_lock, "VideoOverlaySession::remove_image()"); - m_images.erase(&image); - push_image_update(); -} - -void VideoOverlaySession::push_image_update(){ - if (m_content_listeners.empty()){ - return; - } - - // We create a newly allocated Box vector to avoid listener accessing - // `m_images` asynchronously. std::shared_ptr> ptr = std::make_shared>(); - for (const auto& item : m_images){ - ptr->emplace_back(*item); - } - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_update_images(ptr); + { + WriteSpinLock lg(m_lock, "VideoOverlaySession::remove_image()"); + m_images.erase(&image); + + // We create a newly allocated Box vector to avoid listener accessing + // `m_images` asynchronously. + for (const auto& item : m_images){ + ptr->emplace_back(*item); + } } + m_listeners.run_method_unique(&ContentListener::on_overlay_update_images, ptr); } - std::vector VideoOverlaySession::images() const{ ReadSpinLock lg(m_lock); std::vector ret; @@ -219,39 +235,42 @@ std::vector VideoOverlaySession::images() const{ } -void VideoOverlaySession::add_log(std::string message, Color color){ - WriteSpinLock lg(m_lock, "VideoOverlaySession::add_log_text()"); - m_log_texts.emplace_front(color, std::move(message)); +// +// Log +// - if (m_log_texts.size() > LOG_MAX_LINES){ - m_log_texts.pop_back(); +void VideoOverlaySession::add_log(std::string message, Color color){ + std::shared_ptr> ptr = std::make_shared>(); + { + WriteSpinLock lg(m_lock, "VideoOverlaySession::add_log_text()"); + m_log_texts.emplace_front(color, std::move(message)); + + if (m_log_texts.size() > LOG_MAX_LINES){ + m_log_texts.pop_back(); + } + + // We create a newly allocated Box vector to avoid listener accessing + // `m_log_texts` asynchronously. + for(const auto& item : m_log_texts){ + ptr->emplace_back(item); + } } - - push_log_text_update(); + m_listeners.run_method_unique(&ContentListener::on_overlay_update_log, ptr); } - void VideoOverlaySession::clear_log(){ - WriteSpinLock lg(m_lock, "VideoOverlaySession::clear_log_texts()"); - m_log_texts.clear(); - push_log_text_update(); -} - -void VideoOverlaySession::push_log_text_update(){ - if (m_content_listeners.empty()){ - return; - } - - // We create a newly allocated Box vector to avoid listener accessing - // `m_log_texts` asynchronously. std::shared_ptr> ptr = std::make_shared>(); - for(const auto& item : m_log_texts){ - ptr->emplace_back(item); - } - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_update_log(ptr); + { + WriteSpinLock lg(m_lock, "VideoOverlaySession::clear_log_texts()"); + m_log_texts.clear(); + + // We create a newly allocated Box vector to avoid listener accessing + // `m_log_texts` asynchronously. + for(const auto& item : m_log_texts){ + ptr->emplace_back(item); + } } + m_listeners.run_method_unique(&ContentListener::on_overlay_update_log, ptr); } - std::vector VideoOverlaySession::log_texts() const{ ReadSpinLock lg(m_lock); std::vector ret; @@ -262,7 +281,9 @@ std::vector VideoOverlaySession::log_texts() const{ } - +// +// Stats +// void VideoOverlaySession::add_stat(OverlayStat& stat){ WriteSpinLock lg(m_lock); @@ -271,11 +292,6 @@ void VideoOverlaySession::add_stat(OverlayStat& stat){ return; } - // Remove all stats so they aren't being referenced. - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_update_stats(nullptr); - } - m_stats_order.emplace_back(&stat); auto list_iter = m_stats_order.end(); --list_iter; @@ -285,11 +301,6 @@ void VideoOverlaySession::add_stat(OverlayStat& stat){ m_stats_order.pop_back(); throw; } - - // Add all the stats back. - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_update_stats(&m_stats_order); - } } void VideoOverlaySession::remove_stat(OverlayStat& stat){ WriteSpinLock lg(m_lock); @@ -298,20 +309,14 @@ void VideoOverlaySession::remove_stat(OverlayStat& stat){ return; } - // Remove all stats so they aren't being referenced. - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_update_stats(nullptr); - } - m_stats_order.erase(iter->second); m_stats.erase(iter); - - // Add all the stats back. - for (ContentListener* listener : m_content_listeners){ - listener->on_overlay_update_stats(&m_stats_order); - } } +std::vector VideoOverlaySession::stats() const{ + ReadSpinLock lg(m_lock); + return m_stat_lines; +} diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlaySession.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlaySession.h index d8212f5155..88def40ef1 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlaySession.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoOverlaySession.h @@ -19,7 +19,12 @@ #include #include #include +#include +#include +#include +#include "Common/Cpp/AbstractLogger.h" #include "Common/Cpp/Color.h" +#include "Common/Cpp/ListenerSet.h" #include "Common/Cpp/Concurrency/SpinLock.h" #include "VideoOverlay.h" #include "VideoOverlayOption.h" @@ -59,16 +64,6 @@ class VideoOverlaySession : public VideoOverlay{ // is modified by VideoOverlaySession::add/remove_log(). virtual void on_overlay_update_log (const std::shared_ptr>& boxes){} - // This one is different from the others. The listeners will store this - // pointer and access it directly and asynchronously. If you need to - // change the structure of the list itself, you must first call this - // with null to remove it from all the listeners. Then add the updated - // one back when you're done. - // This is called immediately when attaching a listener to give the - // current stats. The listener must drop all references to the stats - // before detaching. - virtual void on_overlay_update_stats(const std::list* stats){} - }; // Add a UI class to listen to any overlay change. The UI class needs to inherit Listener. @@ -80,7 +75,9 @@ class VideoOverlaySession : public VideoOverlay{ public: ~VideoOverlaySession(); - VideoOverlaySession(VideoOverlayOption& option); + VideoOverlaySession(Logger& logger, VideoOverlayOption& option); + + Logger& logger() const{ return m_logger; } void get(VideoOverlayOption& option); void set(const VideoOverlayOption& option); @@ -114,6 +111,9 @@ class VideoOverlaySession : public VideoOverlay{ // is modified by VideoOverlaySession::add/remove_log(). std::vector log_texts() const; + std::vector stats() const; + + virtual void add_box(const OverlayBox& box) override; virtual void remove_box(const OverlayBox& box) override; @@ -131,18 +131,15 @@ class VideoOverlaySession : public VideoOverlay{ private: - // Push updates to the various listeners. - void push_box_update(); - void push_text_update(); - void push_image_update(); - void push_log_text_update(); + void stats_thread(); private: - mutable SpinLock m_lock; - + Logger& m_logger; VideoOverlayOption& m_option; + mutable SpinLock m_lock; + std::set m_boxes; std::set m_texts; std::set m_images; @@ -151,7 +148,13 @@ class VideoOverlaySession : public VideoOverlay{ std::list m_stats_order; std::map::iterator> m_stats; - std::set m_content_listeners; + ListenerSet m_listeners; + + bool m_stopping = false; + std::vector m_stat_lines; + std::mutex m_stats_lock; + std::condition_variable m_stats_cv; + std::thread m_stats_updater; }; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSource.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSource.cpp index e3ba271115..001850015d 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSource.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSource.cpp @@ -9,4 +9,33 @@ namespace PokemonAutomation{ + +VideoSource::VideoSource(Logger& m_logger, bool allow_watchdog_reset) + : m_logger(m_logger) + , m_allow_watchdog_reset(allow_watchdog_reset) + , m_stats_report_source_frame("VideoSource::report_source_frame()", "ms", 1000, std::chrono::seconds(60)) + , m_stats_report_rendered_frame("VideoSource::report_rendered_frame()", "ms", 1000, std::chrono::seconds(60)) + , m_sanitizer("VideoSource") +{} + + +void VideoSource::report_source_frame(std::shared_ptr frame){ + auto scope_check = m_sanitizer.check_scope(); + WallClock time0 = current_time(); + m_source_frame_listeners.run_method_unique(&VideoFrameListener::on_frame, frame); + WallClock time1 = current_time(); + auto microseconds = (uint32_t)std::chrono::duration_cast(time1 - time0).count(); + m_stats_report_source_frame.report_data(m_logger, microseconds); +} +void VideoSource::report_rendered_frame(WallClock timestamp){ + auto scope_check = m_sanitizer.check_scope(); + WallClock time0 = current_time(); + m_rendered_frame_listeners.run_method_unique(&RenderedFrameListener::on_rendered_frame, timestamp); + WallClock time1 = current_time(); + auto microseconds = (uint32_t)std::chrono::duration_cast(time1 - time0).count(); + m_stats_report_rendered_frame.report_data(m_logger, microseconds); +} + + + } diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSource.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSource.h index 5e94e66aa6..2abbcbb9a3 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSource.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSource.h @@ -12,6 +12,7 @@ #include "Common/Cpp/ImageResolution.h" #include "Common/Cpp/ListenerSet.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/Tools/StatAccumulator.h" class QWidget; @@ -45,10 +46,7 @@ class VideoSource{ public: - VideoSource(bool allow_watchdog_reset) - : m_allow_watchdog_reset(allow_watchdog_reset) - , m_sanitizer("VideoSource") - {} + VideoSource(Logger& m_logger, bool allow_watchdog_reset); virtual ~VideoSource() = default; @@ -65,15 +63,12 @@ class VideoSource{ protected: - void report_source_frame(std::shared_ptr frame){ - auto scope_check = m_sanitizer.check_scope(); - m_source_frame_listeners.run_method_unique(&VideoFrameListener::on_frame, frame); - } + // These are not thread-safe. + + void report_source_frame(std::shared_ptr frame); + // Called by UI to report a frame is rendered - void report_rendered_frame(WallClock timestamp){ - auto scope_check = m_sanitizer.check_scope(); - m_rendered_frame_listeners.run_method_unique(&RenderedFrameListener::on_rendered_frame, timestamp); - } + void report_rendered_frame(WallClock timestamp); public: @@ -81,11 +76,16 @@ class VideoSource{ private: + Logger& m_logger; + ListenerSet m_source_frame_listeners; ListenerSet m_rendered_frame_listeners; const bool m_allow_watchdog_reset; + PeriodicStatsReporterI32 m_stats_report_source_frame; + PeriodicStatsReporterI32 m_stats_report_rendered_frame; + LifetimeSanitizer m_sanitizer; }; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSources/VideoSource_StillImage.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSources/VideoSource_StillImage.cpp index a2b9c1bd9b..2bfd4e035e 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSources/VideoSource_StillImage.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSources/VideoSource_StillImage.cpp @@ -61,15 +61,15 @@ JsonValue VideoSourceDescriptor_StillImage::to_json() const{ std::unique_ptr VideoSourceDescriptor_StillImage::make_VideoSource(Logger& logger, Resolution resolution) const{ // cout << "make_VideoSource: " << m_path << endl; - return std::make_unique(path(), resolution); + return std::make_unique(logger, path(), resolution); } -VideoSource_StillImage::VideoSource_StillImage(const std::string& path, Resolution resolution) - : VideoSource(false) +VideoSource_StillImage::VideoSource_StillImage(Logger& logger, const std::string& path, Resolution resolution) + : VideoSource(logger, false) , m_original_image(QString::fromStdString(path)) { m_snapshot = VideoSnapshot( diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSources/VideoSource_StillImage.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSources/VideoSource_StillImage.h index 1461db5e1d..a1f36fd470 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSources/VideoSource_StillImage.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSources/VideoSource_StillImage.h @@ -52,7 +52,7 @@ class VideoSourceDescriptor_StillImage : public VideoSourceDescriptor{ class VideoSource_StillImage : public VideoSource{ public: - VideoSource_StillImage(const std::string& path, Resolution resolution); + VideoSource_StillImage(Logger& logger, const std::string& path, Resolution resolution); const std::string path() const{ return m_path; diff --git a/SerialPrograms/Source/CommonTools/Options/ScreenWatchOption.cpp b/SerialPrograms/Source/CommonTools/Options/ScreenWatchOption.cpp index f85fc31d69..e2da3edf85 100644 --- a/SerialPrograms/Source/CommonTools/Options/ScreenWatchOption.cpp +++ b/SerialPrograms/Source/CommonTools/Options/ScreenWatchOption.cpp @@ -10,6 +10,7 @@ #include #include #include "Common/Cpp/Containers/Pimpl.tpp" +#include "CommonFramework/Logging/Logger.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" #include "ScreenWatchOption.h" @@ -60,7 +61,7 @@ ScreenWatchOption::ScreenWatchOption( ) , m_display(*this) , m_buttons(*this) - , m_overlay(m_overlay_option) + , m_overlay(global_logger_tagged(), m_overlay_option) { PA_ADD_OPTION(m_display); PA_ADD_OPTION(MONITOR_INDEX); diff --git a/SerialPrograms/Source/CommonTools/Options/StringSelectOption.cpp b/SerialPrograms/Source/CommonTools/Options/StringSelectOption.cpp index 8f96468bfa..e657f84eaf 100644 --- a/SerialPrograms/Source/CommonTools/Options/StringSelectOption.cpp +++ b/SerialPrograms/Source/CommonTools/Options/StringSelectOption.cpp @@ -12,6 +12,7 @@ #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Containers/Pimpl.tpp" #include "Common/Cpp/Json/JsonValue.h" +#include "Common/Cpp/Json/JsonArray.h" #include "StringSelectOption.h" //#include @@ -74,6 +75,7 @@ struct StringSelectDatabase::Data{ m_longest_text_length = std::max(m_longest_text_length, item.display_name.size()); } + size_t size() const { return m_list.size(); } }; @@ -105,7 +107,35 @@ size_t StringSelectDatabase::search_index_by_name(const std::string& display_nam void StringSelectDatabase::add_entry(StringSelectEntry entry){ m_data->add_entry(std::move(entry)); } +size_t StringSelectDatabase::size() const{ + return m_data->size(); +} +StringSelectDatabase create_string_select_database(const std::vector& slugs){ + StringSelectDatabase database; + for (const std::string& slug : slugs){ + // slug name is also the display name + database.add_entry(StringSelectEntry(slug, slug)); + } + return database; +} + +bool load_json_to_string_select_database(const JsonValue& json, StringSelectDatabase& database){ + const JsonArray* json_array = json.to_array(); + std::vector slugs; + if (json_array == nullptr){ + return false; + } + for (const JsonValue& element : *json_array){ + const std::string* str = element.to_string(); + if (str == nullptr){ + return false; + } + slugs.push_back(*str); + } + database = create_string_select_database(slugs); + return true; +} diff --git a/SerialPrograms/Source/CommonTools/Options/StringSelectOption.h b/SerialPrograms/Source/CommonTools/Options/StringSelectOption.h index c709b4ece5..1d476751c1 100644 --- a/SerialPrograms/Source/CommonTools/Options/StringSelectOption.h +++ b/SerialPrograms/Source/CommonTools/Options/StringSelectOption.h @@ -15,6 +15,7 @@ namespace PokemonAutomation{ +class JsonValue; @@ -50,13 +51,16 @@ class StringSelectDatabase{ public: StringSelectDatabase(); void add_entry(StringSelectEntry entry); + size_t size() const; public: const std::vector& case_list() const; size_t longest_text_length() const; const StringSelectEntry& operator[](size_t index) const; + // if not found, return SIZE_MAX size_t search_index_by_slug(const std::string& slug) const; + // if not found, return SIZE_MAX size_t search_index_by_name(const std::string& display_name) const; private: @@ -64,8 +68,18 @@ class StringSelectDatabase{ Pimpl m_data; }; - - +// Create a simple StringSelectDatabase from a list of slugs. +// The display names of each entry will be the same as their slugs. +StringSelectDatabase create_string_select_database(const std::vector& slugs); +// Load a simple list of JSON strings to a StringSelectDatabase. +// Previous content of the StringSelectDatabase is removed if loading is successful. +// The display names of each loaded entry will be the same as their slugs. +// Return whether we successfully loaded from the JSON. If loading failed, the initial content +// of the database is not removed. +bool load_json_to_string_select_database(const JsonValue& json, StringSelectDatabase& database); + +// Config option that creates a cell where users can select a string from +// its dropdown menu. It is best to put this cell in a table widget. class StringSelectCell : public ConfigOption{ public: ~StringSelectCell(); @@ -117,7 +131,10 @@ class StringSelectCell : public ConfigOption{ }; - +// Config option that creates a dropdown menu for users to select +// a sring. Different from StringSelectCell which is typically used +// in a table, StringSelectOption is considered a standalone option +// that comes with its own label. class StringSelectOption : public StringSelectCell{ public: StringSelectOption( diff --git a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp index dfc29a7a9f..5130600d26 100644 --- a/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp +++ b/SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp @@ -199,6 +199,23 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons return; } + if (!std::filesystem::exists(embedding_model_path)){ + std::cerr << "Error: no such embedding model path " << embedding_model_path << "." << std::endl; + QMessageBox box; + box.critical(nullptr, "Embedding Model Does Not Exist", + QString::fromStdString("Embedding model path" + embedding_model_path + " does not exist.")); + return; + } + // since the embedding model has too many weights, onnx created a .data file to contain weights. + auto embedding_model_data_path = embedding_model_path + ".data"; + if (!std::filesystem::exists(embedding_model_data_path)){ + std::cerr << "Error: no such embedding model data path " << embedding_model_data_path << "." << std::endl; + QMessageBox box; + box.critical(nullptr, "Embedding Model Data File Does Not Exist", + QString::fromStdString("Embedding model data file path" + embedding_model_data_path + " does not exist.")); + return; + } + SAMEmbedderSession embedding_session(embedding_model_path); std::vector output_image_embedding; for (size_t i = 0; i < all_image_paths.size(); i++){ @@ -238,7 +255,7 @@ void compute_embeddings_for_folder(const std::string& embedding_model_path, cons embedding_session.run(resized_mat, output_image_embedding); save_image_embedding_to_disk(image_path, output_image_embedding); } - + std::cout << "Done computing embeddings for images in folder " << image_folder_path << "." << std::endl; } diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp index 1ae25b34b6..68c7dfae95 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.cpp @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -20,6 +23,7 @@ #include #include #include +#include #include "CommonFramework/Globals.h" #include "Common/Cpp/BitmapConversion.h" #include "Common/Cpp/Json/JsonArray.h" @@ -27,6 +31,7 @@ #include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/Json/JsonTools.h" #include "Common/Qt/CollapsibleGroupBox.h" +#include "Pokemon/Pokemon_Strings.h" #include "Pokemon/Resources/Pokemon_PokemonForms.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include "ML/UI/ML_ImageAnnotationDisplayWidget.h" @@ -108,79 +113,6 @@ JsonObject object_annotation_to_json(const ObjectAnnotation& object_annotation){ } -DrawnBoundingBox::DrawnBoundingBox(LabelImages_Widget& widget, VideoOverlay& overlay) - : m_widget(widget) - , m_overlay(overlay) -{ - auto& program = m_widget.m_program; - program.X.add_listener(*this); - program.Y.add_listener(*this); - program.WIDTH.add_listener(*this); - program.HEIGHT.add_listener(*this); - overlay.add_listener(*this); -} - -DrawnBoundingBox::~DrawnBoundingBox(){ - detach(); -} - -// called when drawn bounding box changed -void DrawnBoundingBox::on_config_value_changed(void* object){ - auto& program = m_widget.m_program; - std::lock_guard lg(m_lock); - program.update_rendered_objects(m_widget.m_overlay_set); -} -void DrawnBoundingBox::on_mouse_press(double x, double y){ - auto& program = m_widget.m_program; - program.WIDTH.set(0); - program.HEIGHT.set(0); - program.X.set(x); - program.Y.set(y); - m_mouse_start.emplace(); - m_mouse_start->first = x; - m_mouse_start->second = y; -} -void DrawnBoundingBox::on_mouse_release(double, double){ - m_mouse_start.reset(); - auto& m_program = m_widget.m_program; - auto& m_overlay_set = m_widget.m_overlay_set; - - m_program.compute_mask(m_overlay_set); -} - -void DrawnBoundingBox::on_mouse_move(double x, double y){ - auto& program = m_widget.m_program; - if (!m_mouse_start){ - return; - } - - double xl = m_mouse_start->first; - double xh = x; - double yl = m_mouse_start->second; - double yh = y; - - if (xl > xh){ - std::swap(xl, xh); - } - if (yl > yh){ - std::swap(yl, yh); - } - - program.X.set(xl); - program.Y.set(yl); - program.WIDTH.set(xh - xl); - program.HEIGHT.set(yh - yl); -} - -void DrawnBoundingBox::detach(){ - auto& program = m_widget.m_program; - m_overlay.remove_listener(*this); - program.X.remove_listener(*this); - program.Y.remove_listener(*this); - program.WIDTH.remove_listener(*this); - program.HEIGHT.remove_listener(*this); -} - LabelImages_Descriptor::LabelImages_Descriptor() : PanelDescriptor( @@ -196,23 +128,68 @@ LabelImages_Descriptor::LabelImages_Descriptor() #define ADD_OPTION(x) m_options.add_option(x, #x) +IntegerEnumDropdownDatabase create_label_type_database(){ + IntegerEnumDropdownDatabase database; + database.add(0, "pokemon-form", Pokemon::STRING_POKEMON + " Forms"); + database.add(1, "custom-set", "Custom Set"); + database.add(2, "manual-input", "Manual Input"); + return database; +} + LabelImages::LabelImages(const LabelImages_Descriptor& descriptor) : PanelInstance(descriptor) , m_display_session(m_display_option) + , m_overlay_set(m_display_session.overlay()) , m_options(LockMode::UNLOCK_WHILE_RUNNING) , X("X Coordinate:", LockMode::UNLOCK_WHILE_RUNNING, 0.3, 0.0, 1.0) , Y("Y Coordinate:", LockMode::UNLOCK_WHILE_RUNNING, 0.3, 0.0, 1.0) , WIDTH("Width:", LockMode::UNLOCK_WHILE_RUNNING, 0.4, 0.0, 1.0) , HEIGHT("Height:", LockMode::UNLOCK_WHILE_RUNNING, 0.4, 0.0, 1.0) + , LABEL_TYPE_DATABASE(create_label_type_database()) + , LABEL_TYPE("Select Label:", LABEL_TYPE_DATABASE, LockMode::UNLOCK_WHILE_RUNNING, 0) , FORM_LABEL("bulbasaur") - , m_sam_session{RESOURCE_PATH() + "ML/sam_cpu.onnx"} + , CUSTOM_LABEL_DATABASE(create_string_select_database({"mc"})) // mc for "main character" + , CUSTOM_SET_LABEL(CUSTOM_LABEL_DATABASE, LockMode::UNLOCK_WHILE_RUNNING, 0) + , MANUAL_LABEL(false, LockMode::UNLOCK_WHILE_RUNNING, "", "Custom Label", true) { ADD_OPTION(X); ADD_OPTION(Y); ADD_OPTION(WIDTH); ADD_OPTION(HEIGHT); + ADD_OPTION(LABEL_TYPE); ADD_OPTION(FORM_LABEL); + ADD_OPTION(CUSTOM_SET_LABEL); + ADD_OPTION(MANUAL_LABEL); + + X.add_listener(*this); + Y.add_listener(*this); + WIDTH.add_listener(*this); + HEIGHT.add_listener(*this); + LABEL_TYPE.add_listener(*this); + FORM_LABEL.add_listener(*this); + CUSTOM_SET_LABEL.add_listener(*this); + MANUAL_LABEL.add_listener(*this); + + // , m_sam_session{RESOURCE_PATH() + "ML/sam_cpu.onnx"} + const std::string sam_model_path = RESOURCE_PATH() + "ML/sam_cpu.onnx"; + if (std::filesystem::exists(sam_model_path)){ + m_sam_session = std::make_unique(sam_model_path); + } else{ + std::cerr << "Error: no such SAM model path " << sam_model_path << "." << std::endl; + QMessageBox box; + box.critical(nullptr, "SAM Model Does Not Exist", + QString::fromStdString("SAM model path" + sam_model_path + " does not exist.")); + } +} +LabelImages::~LabelImages(){ + X.remove_listener(*this); + Y.remove_listener(*this); + WIDTH.remove_listener(*this); + HEIGHT.remove_listener(*this); + LABEL_TYPE.remove_listener(*this); + FORM_LABEL.remove_listener(*this); } + void LabelImages::from_json(const JsonValue& json){ const JsonObject* obj = json.to_object(); if (obj == nullptr){ @@ -223,10 +200,15 @@ void LabelImages::from_json(const JsonValue& json){ m_display_option.load_json(*value); } m_options.load_json(json); + const std::string* file_path = obj->get_string("CUSTOM_LABEL_SET_FILE_PATH"); + if (file_path){ + load_custom_label_set(*file_path); + } } JsonValue LabelImages::to_json() const{ JsonObject obj = std::move(*m_options.to_json().to_object()); obj["ImageSetup"] = m_display_option.to_json(); + obj["CUSTOM_LABEL_SET_FILE_PATH"] = m_custom_label_set_file_path; save_annotation_to_file(); return obj; @@ -245,12 +227,13 @@ void LabelImages::save_annotation_to_file() const{ } void LabelImages::clear_for_new_image(){ + m_overlay_set.clear(); source_image_width = source_image_height = 0; m_image_embedding.clear(); m_output_boolean_mask.clear(); m_mask_image = ImageRGB32(); m_annotations.clear(); - m_last_object_idx = 0; + m_selected_obj_idx = 0; m_annotation_file_path = ""; m_fail_to_load_annotation_file = false; } @@ -259,6 +242,7 @@ QWidget* LabelImages::make_widget(QWidget& parent, PanelHolder& holder){ return new LabelImages_Widget(parent, *this, holder); } +// assuming clear_for_new_image() is already called void LabelImages::load_image_related_data(const std::string& image_path, size_t source_image_width, size_t source_image_height){ this->source_image_height = source_image_height; this->source_image_width = source_image_width; @@ -316,13 +300,14 @@ void LabelImages::load_image_related_data(const std::string& image_path, size_t ); } } - m_last_object_idx = m_annotations.size(); + m_selected_obj_idx = m_annotations.size(); + update_rendered_objects(); cout << "Loaded existing annotation file " << m_annotation_file_path << endl; } -void LabelImages::update_rendered_objects(VideoOverlaySet& overlay_set){ - overlay_set.clear(); - overlay_set.add(COLOR_RED, {X, Y, WIDTH, HEIGHT}); +void LabelImages::update_rendered_objects(){ + m_overlay_set.clear(); + m_overlay_set.add(COLOR_RED, {X, Y, WIDTH, HEIGHT}); for(size_t i_obj = 0; i_obj < m_annotations.size(); i_obj++){ const auto& obj = m_annotations[i_obj]; @@ -333,8 +318,8 @@ void LabelImages::update_rendered_objects(VideoOverlaySet& overlay_set){ if (form != nullptr){ label = form->display_name(); } - Color mask_box_color = (i_obj == m_last_object_idx) ? COLOR_BLACK : COLOR_BLUE; - overlay_set.add(mask_box_color, mask_float_box, label); + Color mask_box_color = (i_obj == m_selected_obj_idx) ? COLOR_BLACK : COLOR_BLUE; + m_overlay_set.add(mask_box_color, mask_float_box, label); size_t mask_width = obj.mask_box.width(); size_t mask_height = obj.mask_box.height(); ImageRGB32 mask_image(mask_width, mask_height); @@ -354,11 +339,11 @@ void LabelImages::update_rendered_objects(VideoOverlaySet& overlay_set){ } } // cout << " count " << count << endl; - overlay_set.add(std::move(mask_image), mask_float_box); + m_overlay_set.add(std::move(mask_image), mask_float_box); } } -void LabelImages::compute_mask(VideoOverlaySet& overlay_set){ +void LabelImages::compute_mask(){ const size_t source_width = source_image_width; const size_t source_height = source_image_height; @@ -370,11 +355,11 @@ void LabelImages::compute_mask(VideoOverlaySet& overlay_set){ return; } - if (m_image_embedding.size() == 0){ + if (!m_sam_session || m_image_embedding.size() == 0){ // no embedding file loaded return; } - m_sam_session.run( + m_sam_session->run( m_image_embedding, (int)source_height, (int)source_width, {}, {}, {box_x, box_y, box_x + box_width, box_y + box_height}, @@ -406,7 +391,7 @@ void LabelImages::compute_mask(VideoOverlaySet& overlay_set){ ImageFloatBox mask_box( min_mask_x/double(source_width), min_mask_y/double(source_height), mask_width/double(source_width), mask_height/double(source_height)); - const std::string label = FORM_LABEL.slug(); + const std::string label = this->selected_label(); ObjectAnnotation annotation; @@ -420,10 +405,10 @@ void LabelImages::compute_mask(VideoOverlaySet& overlay_set){ } annotation.label = label; - m_last_object_idx = m_annotations.size(); + m_selected_obj_idx = m_annotations.size(); m_annotations.emplace_back(std::move(annotation)); - update_rendered_objects(overlay_set); + update_rendered_objects(); } } @@ -433,10 +418,210 @@ void LabelImages::compute_embeddings_for_folder(const std::string& image_folder_ ML::compute_embeddings_for_folder(embedding_model_path, image_folder_path); } +void LabelImages::delete_selected_annotation(){ + if (m_annotations.size() == 0 || m_selected_obj_idx >= m_annotations.size()){ + return; + } + + m_annotations.erase(m_annotations.begin() + m_selected_obj_idx); + + if (m_annotations.size() == 0){ + m_selected_obj_idx = 0; + update_rendered_objects(); + return; + } + + if (m_selected_obj_idx >= m_annotations.size()){ + m_selected_obj_idx = m_annotations.size() - 1; + } else{ + // no change to the currently selected index + } + + std::string& cur_label = m_annotations[m_selected_obj_idx].label; + set_selected_label(cur_label); + update_rendered_objects(); +} + +void LabelImages::change_annotation_selection_by_mouse(double x, double y){ + // no image or no annotation + if (source_image_width == 0 || source_image_height == 0 || m_annotations.size() == 0){ + return; + } + + const size_t px = (size_t)std::max(source_image_width * x + 0.5, 0); + const size_t py = (size_t)std::max(source_image_height * y + 0.5, 0); + + double closest_distance = DBL_MAX; + std::vector zero_distance_annotations; + for(size_t i = 0; i < m_annotations.size(); i++){ + const size_t dx = m_annotations[i].mask_box.distance_to_point_x(px); + const size_t dy = m_annotations[i].mask_box.distance_to_point_y(py); + const size_t d2 = dx*dx + dy*dy; + if (d2 == 0){ + zero_distance_annotations.push_back(i); + } + if (d2 < closest_distance){ + closest_distance = d2; + m_selected_obj_idx = i; + } + } + + if (zero_distance_annotations.size() > 1){ + // this point is inside multiple boxes, we then use the closest to the box center to determine + closest_distance = DBL_MAX; + for(size_t i : zero_distance_annotations){ + const size_t dx = m_annotations[i].mask_box.center_distance_to_point_x(px); + const size_t dy = m_annotations[i].mask_box.center_distance_to_point_y(py); + const size_t d2 = dx*dx + dy*dy; + if (d2 < closest_distance){ + closest_distance = d2; + m_selected_obj_idx = i; + } + } + } + + auto new_label = m_annotations[m_selected_obj_idx].label; + set_selected_label(new_label); +} + +void LabelImages::select_prev_annotation(){ + // no image or no annotation + if (source_image_width == 0 || source_image_height == 0 || m_annotations.size() == 0){ + return; + } + + if (m_selected_obj_idx >= m_annotations.size()){ + m_selected_obj_idx = m_annotations.size() - 1; + } else if (m_selected_obj_idx == 0){ + m_selected_obj_idx = m_annotations.size() - 1; + } else { + m_selected_obj_idx--; + } + + auto new_label = m_annotations[m_selected_obj_idx].label; + set_selected_label(new_label); + update_rendered_objects(); +} +void LabelImages::select_next_annotation(){ + // no image or no annotation + if (source_image_width == 0 || source_image_height == 0 || m_annotations.size() == 0){ + return; + } + + if (m_selected_obj_idx >= m_annotations.size()){ + m_selected_obj_idx = 0; + } else if (m_selected_obj_idx + 1 == m_annotations.size()){ + m_selected_obj_idx = 0; + } else { + m_selected_obj_idx++; + } + + auto new_label = m_annotations[m_selected_obj_idx].label; + set_selected_label(new_label); + update_rendered_objects(); +} + +void LabelImages::on_config_value_changed(void* object){ + // cout << "LabelImages::on_config_value_changed" << endl; + if (object == &LABEL_TYPE){ + const size_t value = LABEL_TYPE.current_value(); + // cout << "LABEL_TYPE value changed to " << value << endl; + // label type changed + if (value == 0){ + FORM_LABEL.set_visibility(ConfigOptionState::ENABLED); + CUSTOM_SET_LABEL.set_visibility(ConfigOptionState::HIDDEN); + MANUAL_LABEL.set_visibility(ConfigOptionState::HIDDEN); + } else if (value == 1){ + FORM_LABEL.set_visibility(ConfigOptionState::HIDDEN); + CUSTOM_SET_LABEL.set_visibility(ConfigOptionState::ENABLED); + MANUAL_LABEL.set_visibility(ConfigOptionState::HIDDEN); + } else { // value == 2 + FORM_LABEL.set_visibility(ConfigOptionState::HIDDEN); + CUSTOM_SET_LABEL.set_visibility(ConfigOptionState::HIDDEN); + MANUAL_LABEL.set_visibility(ConfigOptionState::ENABLED); + } + } + + if (object == &LABEL_TYPE || object == &FORM_LABEL || object == &CUSTOM_SET_LABEL || object == &MANUAL_LABEL){ + // label changed + if (m_annotations.size() > 0 && m_selected_obj_idx < m_annotations.size()){ + std::string& cur_label = m_annotations[m_selected_obj_idx].label; + const std::string ui_slug = this->selected_label(); + if (ui_slug != cur_label){ + cur_label = ui_slug; + } + } + update_rendered_objects(); + } +} + +std::string LabelImages::selected_label() const{ + const size_t label_type = LABEL_TYPE.current_value(); + if (label_type == 0){ + return FORM_LABEL.slug(); + } + if (label_type == 1){ + return CUSTOM_SET_LABEL.slug(); + } + return MANUAL_LABEL; +} +void LabelImages::set_selected_label(const std::string& slug){ + size_t index = FORM_LABEL.database().search_index_by_slug(slug); + if (index != SIZE_MAX){ + LABEL_TYPE.set_value(0); + FORM_LABEL.set_by_index(index); + return; + } + index = CUSTOM_SET_LABEL.database().search_index_by_slug(slug); + if (index != SIZE_MAX){ + LABEL_TYPE.set_value(1); + CUSTOM_SET_LABEL.set_by_index(index); + return; + } + LABEL_TYPE.set_value(2); + MANUAL_LABEL.set(slug); +} + +void LabelImages::load_custom_label_set(const std::string& json_path){ + StringSelectDatabase new_database; + try{ + JsonValue value = load_json_file(json_path); + const JsonArray& json_array = value.to_array_throw(); + for(size_t i = 0; i < json_array.size(); i++){ + const std::string& label_slug = json_array[i].to_string_throw(); + new_database.add_entry(StringSelectEntry(label_slug, label_slug)); + } + } catch(FileException& e){ + std::cerr << "Error: File exception " << e.message() << std::endl; + QMessageBox box; + box.warning(nullptr, "Unable to Load Custom Label Set", + QString::fromStdString("Cannot open JSON file " + json_path + " for the custom label set. Probably wrong permission?")); + return; + } catch(JsonParseException& e){ + std::cerr << "Error: JSON parse exception " << e.message() << std::endl; + QMessageBox box; + box.warning(nullptr, "Unable to Load Custom Label Set", + QString::fromStdString("Cannot parse JSON file " + json_path + " for the custom label set. Probably wrong file content?")); + return; + } + + cout << "Loaded " << new_database.size() << " custom labels from " << json_path << endl; + CUSTOM_LABEL_DATABASE = new_database; + if (&json_path != &m_custom_label_set_file_path){ + m_custom_label_set_file_path = json_path; + } + + // if the current label is set by MANUAL_LABEL but its value appears in the newly loaded custom set, + // the label UI should switch the label to be shown as part of the custom set. + // so call the following line to achieve that + set_selected_label(selected_label()); +} LabelImages_Widget::~LabelImages_Widget(){ - m_program.FORM_LABEL.remove_listener(*this); + m_display_session.overlay().remove_listener(*this); + m_display_session.video_session().remove_state_listener(*this); + delete m_image_display_widget; } LabelImages_Widget::LabelImages_Widget( @@ -447,14 +632,13 @@ LabelImages_Widget::LabelImages_Widget( : PanelWidget(parent, program, holder) , m_program(program) , m_display_session(m_program.m_display_session) - , m_overlay_set(m_display_session.overlay()) - , m_drawn_box(*this, m_display_session.overlay()) { - m_program.FORM_LABEL.add_listener(*this); + m_display_session.overlay().add_listener(*this); m_display_session.video_session().add_state_listener(*this); m_embedding_info_label = new QLabel(this); + // Main layout QVBoxLayout* layout = new QVBoxLayout(this); layout->setContentsMargins(0, 0, 0, 0); layout->addWidget(make_header(*this)); @@ -468,7 +652,7 @@ LabelImages_Widget::LabelImages_Widget( QVBoxLayout* scroll_layout = new QVBoxLayout(scroll_inner); scroll_layout->setAlignment(Qt::AlignTop); - m_image_display_widget = new ImageAnnotationDisplayWidget(*this, m_display_session, 0); + m_image_display_widget = new ImageAnnotationDisplayWidget(*this, m_display_session, this); scroll_layout->addWidget(m_image_display_widget); QHBoxLayout* embedding_info_row = new QHBoxLayout(); @@ -476,26 +660,81 @@ LabelImages_Widget::LabelImages_Widget( embedding_info_row->addWidget(new QLabel("Image Embedding File: ", this)); embedding_info_row->addWidget(m_embedding_info_label); - QPushButton* button = new QPushButton("Delete Last Mask", scroll_inner); - scroll_layout->addWidget(button); - connect(button, &QPushButton::clicked, this, [this](bool){ + // add a row for buttons + + QHBoxLayout* button_row = new QHBoxLayout(); + scroll_layout->addLayout(button_row); + + QPushButton* delete_anno_button = new QPushButton("Delete Selected Annotation", scroll_inner); + button_row->addWidget(delete_anno_button, 1); + + QPushButton* pre_anno_button = new QPushButton("Prev Annotation", scroll_inner); + button_row->addWidget(pre_anno_button, 1); + + QPushButton* next_anno_button = new QPushButton("Next Annotation", scroll_inner); + button_row->addWidget(next_anno_button, 1); + + // add a row for user annotation + // the user can annotate in two modes: + // - set a pokemon form label + // - load a predefined custom string list and select from the list + // The custom list cannot contain pokemon form name. Otherwise it will be set to the pokemon form label + // so underlying data is only a single string. The UI reflects on what dropdown menu is set. + // the UI needs to have a + QHBoxLayout* annotation_row = new QHBoxLayout(); + scroll_layout->addLayout(annotation_row); + + // add a dropdown menu for user to pick whether to choose from pokemon form label or custom label + ConfigWidget* label_type_widget = program.LABEL_TYPE.make_QtWidget(*scroll_inner); + annotation_row->addWidget(&label_type_widget->widget(), 0); + + ConfigWidget* pokemon_label_widget = program.FORM_LABEL.make_QtWidget(*scroll_inner); + annotation_row->addWidget(&pokemon_label_widget->widget(), 2); + ConfigWidget* custom_label_widget = program.CUSTOM_SET_LABEL.make_QtWidget(*scroll_inner); + annotation_row->addWidget(&custom_label_widget->widget(), 2); + ConfigWidget* manual_input_label_widget = program.MANUAL_LABEL.make_QtWidget(*scroll_inner); + annotation_row->addWidget(&manual_input_label_widget->widget(), 2); + QPushButton* load_custom_set_button = new QPushButton("Load Custom Set", scroll_inner); + annotation_row->addWidget(load_custom_set_button, 2); + annotation_row->addWidget(new QLabel(scroll_inner), 10); // an empty label to push other UIs to the left + + // add compute embedding button + + QPushButton* compute_embedding_button = new QPushButton("Compute Image Embeddings (SLOW!)", scroll_inner); + scroll_layout->addWidget(compute_embedding_button); + + // connect button signals to define button actions + + connect(delete_anno_button, &QPushButton::clicked, this, [this](bool){ + auto& program = this->m_program; + program.delete_selected_annotation(); + }); + + connect(pre_anno_button, &QPushButton::clicked, this, [this](bool){ + auto& program = this->m_program; + program.select_prev_annotation(); + }); + connect(next_anno_button, &QPushButton::clicked, this, [this](bool){ auto& program = this->m_program; - if (program.m_annotations.size() > 0){ - program.m_annotations.pop_back(); + program.select_next_annotation(); + }); + + connect(load_custom_set_button, &QPushButton::clicked, this, [this](bool){ + const std::string& last_loaded_file_path = m_program.m_custom_label_set_file_path; + std::string starting_dir = "."; + if (last_loaded_file_path.size() > 0){ + starting_dir = std::filesystem::path(last_loaded_file_path).parent_path().string(); } - if (program.m_annotations.size() > 0){ - program.m_last_object_idx = program.m_annotations.size() - 1; + const std::string path = QFileDialog::getOpenFileName( + nullptr, "Open JSON file", QString::fromStdString(starting_dir), "*.json" + ).toStdString(); + if (path.size() > 0){ + cout << "File dialog returns JSON path " << path << endl; + m_program.load_custom_label_set(path); } - program.update_rendered_objects(this->m_overlay_set); }); - // Add all option UI elements defined by LabelImage program. - m_option_widget = program.m_options.make_QtWidget(*scroll_inner); - scroll_layout->addWidget(&m_option_widget->widget()); - - button = new QPushButton("Compute Image Embeddings (SLOW!)", scroll_inner); - scroll_layout->addWidget(button); - connect(button, &QPushButton::clicked, this, [this](bool){ + connect(compute_embedding_button, &QPushButton::clicked, this, [this](bool){ std::string folder_path = QFileDialog::getExistingDirectory( nullptr, "Open image folder", ".").toStdString(); @@ -507,17 +746,8 @@ LabelImages_Widget::LabelImages_Widget( cout << "LabelImages_Widget built" << endl; } -void LabelImages_Widget::clear_for_new_image(){ - m_overlay_set.clear(); - m_program.clear_for_new_image(); -} void LabelImages_Widget::on_config_value_changed(void* object){ - if (m_program.m_annotations.size() > 0 && m_program.m_last_object_idx < m_program.m_annotations.size()){ - std::string& cur_label = m_program.m_annotations[m_program.m_last_object_idx].label; - cur_label = m_program.FORM_LABEL.slug(); - m_program.update_rendered_objects(m_overlay_set); - } } // This callback function will be called whenever the display source (the image source) is loaded or reloaded: @@ -525,7 +755,7 @@ void LabelImages_Widget::post_startup(VideoSource* source){ const std::string& image_path = m_display_session.option().m_image_path; m_program.save_annotation_to_file(); // save the current annotation file - clear_for_new_image(); + m_program.clear_for_new_image(); if (image_path.size() == 0){ m_embedding_info_label->setText(""); return; @@ -540,7 +770,7 @@ void LabelImages_Widget::post_startup(VideoSource* source){ } m_embedding_info_label->setText(QString::fromStdString(embedding_path_display)); - m_embedding_info_label->setStyleSheet("color: blue"); + m_embedding_info_label->setStyleSheet("color: green"); const auto cur_res = m_display_session.video_session().current_resolution(); if (cur_res.width == 0 || cur_res.height == 0){ @@ -551,10 +781,118 @@ void LabelImages_Widget::post_startup(VideoSource* source){ } m_program.load_image_related_data(image_path, cur_res.width, cur_res.height); - m_program.update_rendered_objects(m_overlay_set); } +void LabelImages_Widget::key_press(QKeyEvent* event){ + const auto key = Qt::Key(event->key()); + switch(key){ + case Qt::Key::Key_Shift: + m_shift_pressed = true; + break; + case Qt::Key::Key_Control: + #ifndef __APPLE__ + m_control_pressed = true; + #endif + break; + case Qt::Key::Key_Meta: + #if defined(__APPLE__) + m_control_pressed = true; + #endif + break; + default:; + } +} +void LabelImages_Widget::key_release(QKeyEvent* event){ + const auto key = Qt::Key(event->key()); + switch(key){ + case Qt::Key::Key_Shift: + m_shift_pressed = false; + break; + case Qt::Key::Key_Control: + #ifndef __APPLE__ + m_control_pressed = false; + #endif + break; + case Qt::Key::Key_Meta: + #if defined(__APPLE__) + m_control_pressed = false; + #endif + break; + case Qt::Key::Key_Delete: + case Qt::Key::Key_Backspace: + m_program.delete_selected_annotation(); + break; + default:; + } +} + +void LabelImages_Widget::on_mouse_press(double x, double y){ + m_program.WIDTH.set(0); + m_program.HEIGHT.set(0); + m_program.X.set(x); + m_program.Y.set(y); + m_mouse_start.emplace(); + m_mouse_end.emplace(); + m_mouse_start->first = m_mouse_end->first = x; + m_mouse_start->second = m_mouse_end->second = y; + m_mouse_start_time = std::chrono::high_resolution_clock::now(); +} + +void LabelImages_Widget::on_mouse_release(double x, double y){ + const std::chrono::duration duration = std::chrono::high_resolution_clock::now() - m_mouse_start_time; + const double rel_x = std::fabs(m_mouse_start->first - m_mouse_end->first); + const double rel_y = std::fabs(m_mouse_start->second - m_mouse_end->second); + + m_mouse_start.reset(); + m_mouse_end.reset(); + + // cout << "Mouse release " << (rel_x) << " " << (rel_y) << " duration " << duration << " " << + // (duration < std::chrono::milliseconds(150)) << endl; + + // user may have very small movement while doing quick clicking. To register this as a simple click, use relative + // screen distance threshold 0.0015 and click duration threshold 0.15 second: + if ((rel_x == 0 && rel_y == 0) || (rel_x < 0.0015 && rel_y < 0.0015 && duration < std::chrono::milliseconds(150))){ + if (m_shift_pressed){ + cout << "shift pressed while at " << x << " " << y << endl; + } + // process mouse clicking + // change currently selected annotation + // also change the option values in the UI + m_program.change_annotation_selection_by_mouse(x, y); + return; + } + + m_program.compute_mask(); +} + +void LabelImages_Widget::on_mouse_move(double x, double y){ + if (!m_mouse_start){ + return; + } + + m_mouse_end->first = x; + m_mouse_end->second = y; + + double xl = m_mouse_start->first; + double yl = m_mouse_start->second; + double xh = x; + double yh = y; + + if (xl > xh){ + std::swap(xl, xh); + } + if (yl > yh){ + std::swap(yl, yh); + } + + m_program.X.set(xl); + m_program.Y.set(yl); + m_program.WIDTH.set(xh - xl); + m_program.HEIGHT.set(yh - yl); + + m_program.update_rendered_objects(); +} } } diff --git a/SerialPrograms/Source/ML/Programs/ML_LabelImages.h b/SerialPrograms/Source/ML/Programs/ML_LabelImages.h index 0197937387..49400419ec 100644 --- a/SerialPrograms/Source/ML/Programs/ML_LabelImages.h +++ b/SerialPrograms/Source/ML/Programs/ML_LabelImages.h @@ -7,9 +7,14 @@ #ifndef PokemonAutomation_ML_LabelImages_H #define PokemonAutomation_ML_LabelImages_H +#include +#include #include #include "Common/Cpp/Options/BatchOption.h" #include "Common/Cpp/Options/FloatingPointOption.h" +#include "Common/Cpp/Options/EnumDropdownOption.h" +#include "Common/Cpp/Options/EnumDropdownDatabase.h" +#include "Common/Cpp/Options/StringOption.h" #include "CommonFramework/Panels/PanelInstance.h" #include "CommonFramework/Panels/UI/PanelWidget.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" @@ -19,6 +24,7 @@ #include "NintendoSwitch/Framework/NintendoSwitch_SwitchSystemOption.h" #include "NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonFramework/VideoPipeline/UI/VideoDisplayWidget.h" #include "ML/DataLabeling/ML_SegmentAnythingModel.h" #include "ML/UI/ML_ImageAnnotationDisplayOption.h" @@ -50,7 +56,7 @@ class LabelImages_Widget; struct ObjectAnnotation{ ImagePixelBox user_box; // user drawn loose bounding box ImagePixelBox mask_box; - std::vector mask; + std::vector mask; // size() equals the total pixels in mask_box std::string label = "unknown"; ObjectAnnotation(); @@ -64,10 +70,11 @@ class LabelImages_Descriptor : public PanelDescriptor{ // Program to annoatation images for training ML models -class LabelImages : public PanelInstance{ +class LabelImages : public PanelInstance, public ConfigOption::Listener { public: LabelImages(const LabelImages_Descriptor& descriptor); virtual QWidget* make_widget(QWidget& parent, PanelHolder& holder) override; + ~LabelImages(); public: // Serialization @@ -86,23 +93,38 @@ class LabelImages : public PanelInstance{ void load_image_related_data(const std::string& image_path, const size_t source_image_width, const size_t source_image_height); // Update rendering data reflect the current annotation - void update_rendered_objects(VideoOverlaySet& overlayset); + void update_rendered_objects(); // Use user currently drawn box to compute per-pixel masks on the image using SAM model - void compute_mask(VideoOverlaySet& overlay_set); + void compute_mask(); // Compute embeddings for all images in a folder. // This can be very slow! void compute_embeddings_for_folder(const std::string& image_folder); + // Delete the currently selected object annotation. + void delete_selected_annotation(); + + void change_annotation_selection_by_mouse(double x, double y); + void select_prev_annotation(); + void select_next_annotation(); + + // return the label selected on UI + std::string selected_label() const; + void set_selected_label(const std::string& label); + + void load_custom_label_set(const std::string& json_path); + private: + void on_config_value_changed(void* object) override; + friend class LabelImages_Widget; - friend class DrawnBoundingBox; // image display options like what image file is loaded ImageAnnotationDisplayOption m_display_option; // handles image display session, holding a reference to m_display_option ImageAnnotationDisplaySession m_display_session; + VideoOverlaySet m_overlay_set; // the group option that holds rest of the options defined below: BatchOption m_options; @@ -110,7 +132,19 @@ class LabelImages : public PanelInstance{ FloatingPointOption Y; FloatingPointOption WIDTH; FloatingPointOption HEIGHT; + + // the database to initialize LABEL_TYPE + IntegerEnumDropdownDatabase LABEL_TYPE_DATABASE; + // a dropdown menu to choose which source below to set label from + IntegerEnumDropdownOption LABEL_TYPE; + // source 1: a dropdown menu for all pokemon forms Pokemon::HomeSpriteSelectCell FORM_LABEL; + // the database to initialize CUSTOM_SET_LABEL + StringSelectDatabase CUSTOM_LABEL_DATABASE; + // source 2: a dropdown menu for custom labels + StringSelectCell CUSTOM_SET_LABEL; + // source 3: editable text input + StringCell MANUAL_LABEL; size_t source_image_height = 0; size_t source_image_width = 0; @@ -120,39 +154,27 @@ class LabelImages : public PanelInstance{ // buffer to compute SAM mask on ImageRGB32 m_mask_image; - SAMSession m_sam_session; + std::unique_ptr m_sam_session; std::vector m_annotations; - size_t m_last_object_idx = 0; + + // currently selected annotated object's index + // if this value == m_annotations.size(), it means the user is not selecting anything + size_t m_selected_obj_idx = 0; std::string m_annotation_file_path; // if we find an annotation file that is supposed to be created by user in a previous session, but // we fail to load it, then we shouldn't overwrite this file to possibly erase the previous work. // so this flag is used to denote if we fail to load an annotation file bool m_fail_to_load_annotation_file = false; -}; - - -class DrawnBoundingBox : public ConfigOption::Listener, public VideoOverlay::MouseListener{ -public: - ~DrawnBoundingBox(); - DrawnBoundingBox(LabelImages_Widget& widget, VideoOverlay& overlay); - virtual void on_config_value_changed(void* object) override; - virtual void on_mouse_press(double x, double y) override; - virtual void on_mouse_release(double x, double y) override; - virtual void on_mouse_move(double x, double y) override; - -private: - void detach(); - -private: - LabelImages_Widget& m_widget; - VideoOverlay& m_overlay; - std::mutex m_lock; - std::optional> m_mouse_start; + std::string m_custom_label_set_file_path; }; -class LabelImages_Widget : public PanelWidget, public ConfigOption::Listener, public VideoSession::StateListener{ +class LabelImages_Widget : public PanelWidget, + public ConfigOption::Listener, + public VideoSession::StateListener, + public CommandReceiver, + public VideoOverlay::MouseListener{ public: ~LabelImages_Widget(); LabelImages_Widget( @@ -161,27 +183,47 @@ class LabelImages_Widget : public PanelWidget, public ConfigOption::Listener, pu PanelHolder& holder ); - // called after loading a new image, clean up all internal data - void clear_for_new_image(); - + // Overwrites ConfigOption::Listener::on_config_value_changed(). virtual void on_config_value_changed(void* object) override; // Overwrites VideoSession::StateListener::post_startup(). virtual void post_startup(VideoSource* source) override; + // Overwrites CommandReceiver::key_press(). + virtual void key_press(QKeyEvent* event) override; + // Overwrites CommandReceiver::key_release(). + virtual void key_release(QKeyEvent* event) override; + // Overwrites CommandReceiver::focus_in(). + virtual void focus_in(QFocusEvent* event) override {} + // Overwrites CommandReceiver::focus_out(). + virtual void focus_out(QFocusEvent* event) override {} + + // Overwrites VideoOverlay::MouseListener::on_mouse_press(). + virtual void on_mouse_press(double x, double y) override; + // Overwrites VideoOverlay::MouseListener::on_mouse_release(). + virtual void on_mouse_release(double x, double y) override; + // Overwrites VideoOverlay::MouseListener::on_mouse_move(). + virtual void on_mouse_move(double x, double y) override; + private: LabelImages& m_program; ImageAnnotationDisplaySession& m_display_session; ImageAnnotationDisplayWidget* m_image_display_widget; - - VideoOverlaySet m_overlay_set; - DrawnBoundingBox m_drawn_box; + // show the info about the loaded image embedding data corresponding to the currently + // displayed image QLabel* m_embedding_info_label = nullptr; - ConfigWidget* m_option_widget; - friend class DrawnBoundingBox; + // TODO: see if we can use + // Common/Cpp/Options/BoxFloatOption.h and Common/Qt/Options/BoxFloatWidget.h as UI options + + std::optional> m_mouse_start; + std::optional> m_mouse_end; + std::chrono::time_point m_mouse_start_time; + + bool m_shift_pressed = false; + bool m_control_pressed = false; }; diff --git a/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplaySession.cpp b/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplaySession.cpp index efa3e911d6..15c1a93cf4 100644 --- a/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplaySession.cpp +++ b/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplaySession.cpp @@ -36,7 +36,7 @@ ImageAnnotationDisplaySession::ImageAnnotationDisplaySession(ImageAnnotationDisp , m_display_option(option) , m_still_image_option(create_still_image_video_source_option()) , m_video_session(m_logger, m_still_image_option) - , m_overlay(option.m_overlay) + , m_overlay(m_logger, option.m_overlay) , m_cpu_utilization(new CpuUtilizationStat()) , m_main_thread_utilization(new ThreadUtilizationStat(current_thread_handle(), "Main Qt Thread:")) { diff --git a/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplayWidget.cpp b/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplayWidget.cpp index f370055618..a3a3883768 100644 --- a/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplayWidget.cpp +++ b/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplayWidget.cpp @@ -40,10 +40,11 @@ ImageAnnotationDisplayWidget::~ImageAnnotationDisplayWidget(){ ImageAnnotationDisplayWidget::ImageAnnotationDisplayWidget( QWidget& parent, ImageAnnotationDisplaySession& session, - uint64_t program_id + CommandReceiver* command_receiver ) : QWidget(&parent) , m_session(session) + , m_command_receiver(command_receiver) { QVBoxLayout* layout = new QVBoxLayout(this); layout->setContentsMargins(0, 0, 0, 0); @@ -94,19 +95,31 @@ void ImageAnnotationDisplayWidget::update_ui(ProgramState state){ void ImageAnnotationDisplayWidget::key_press(QKeyEvent* event){ // cout << "press: " << event->nativeVirtualKey() << endl; m_command->on_key_press(*event); + if (m_command_receiver){ + m_command_receiver->key_press(event); + } } void ImageAnnotationDisplayWidget::key_release(QKeyEvent* event){ // cout << "release: " << event->nativeVirtualKey() << endl; m_command->on_key_release(*event); + if (m_command_receiver){ + m_command_receiver->key_release(event); + } } void ImageAnnotationDisplayWidget::focus_in(QFocusEvent* event){ m_command->set_focus(true); + if (m_command_receiver){ + m_command_receiver->focus_in(event); + } } void ImageAnnotationDisplayWidget::focus_out(QFocusEvent* event){ m_command->set_focus(false); + if (m_command_receiver){ + m_command_receiver->focus_out(event); + } } void ImageAnnotationDisplayWidget::keyPressEvent(QKeyEvent* event){ diff --git a/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplayWidget.h b/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplayWidget.h index 4fd14bb9ba..4e7c3b0fa2 100644 --- a/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplayWidget.h +++ b/SerialPrograms/Source/ML/UI/ML_ImageAnnotationDisplayWidget.h @@ -35,10 +35,12 @@ class ImageAnnotationCommandRow; class ImageAnnotationDisplayWidget final : public QWidget, public CommandReceiver{ public: virtual ~ImageAnnotationDisplayWidget(); + // command_receiver: if not nullptr, ImageAnnotationDisplayWidget will forward key and + // focus events happened on the image display window to this command_receiver. ImageAnnotationDisplayWidget( QWidget& parent, ImageAnnotationDisplaySession& session, - uint64_t program_id + CommandReceiver* command_receiver = nullptr ); public: @@ -47,16 +49,32 @@ class ImageAnnotationDisplayWidget final : public QWidget, public CommandReceive // The public versions of the private QWidget key and focus event handling functions. // They are needed to accept key and focus passed from CommonFramework/VideoPipeline/UI:VideoDisplayWindow. + // Overwrites CommandReceiver::key_press(). + // The public versions of the private QWidget key event handling functions. + // They are needed to accept key passed from CommonFramework/VideoPipeline/UI:VideoDisplayWindow. virtual void key_press(QKeyEvent* event) override; + // Overwrites CommandReceiver::key_release(). + // The public versions of the private QWidget key event handling functions. + // They are needed to accept key passed from CommonFramework/VideoPipeline/UI:VideoDisplayWindow. virtual void key_release(QKeyEvent* event) override; + // Overwrites CommandReceiver::focus_in(). + // The public versions of the private QWidget focus event handling functions. + // They are needed to accept focus passed from CommonFramework/VideoPipeline/UI:VideoDisplayWindow. virtual void focus_in(QFocusEvent* event) override; + // Overwrites CommandReceiver::focus_out(). + // The public versions of the private QWidget focus event handling functions. + // They are needed to accept focus passed from CommonFramework/VideoPipeline/UI:VideoDisplayWindow. virtual void focus_out(QFocusEvent* event) override; private: + // Overwrites QWidget::keyPressEvent(). virtual void keyPressEvent(QKeyEvent* event) override; + // Overwrites QWidget::keyReleaseEvent(). virtual void keyReleaseEvent(QKeyEvent* event) override; + // Overwrites QWidget::focusInEvent(). virtual void focusInEvent(QFocusEvent* event) override; + // Overwrites QWidget::focusOutEvent(). virtual void focusOutEvent(QFocusEvent* event) override; private: @@ -69,6 +87,8 @@ class ImageAnnotationDisplayWidget final : public QWidget, public CommandReceive ImageAnnotationCommandRow* m_command; ImageAnnotationSourceSelectorWidget* m_selector_widget; + + CommandReceiver* m_command_receiver; }; diff --git a/SerialPrograms/Source/ML/UI/ML_ImageAnnotationSourceSelectorWidget.cpp b/SerialPrograms/Source/ML/UI/ML_ImageAnnotationSourceSelectorWidget.cpp index 754beb606c..4257765c27 100644 --- a/SerialPrograms/Source/ML/UI/ML_ImageAnnotationSourceSelectorWidget.cpp +++ b/SerialPrograms/Source/ML/UI/ML_ImageAnnotationSourceSelectorWidget.cpp @@ -39,7 +39,7 @@ ImageAnnotationSourceSelectorWidget::ImageAnnotationSourceSelectorWidget(ImageAn QVBoxLayout* layout = new QVBoxLayout(this); layout->setContentsMargins(0, 0, 0, 0); - QHBoxLayout* image_path_row = new QHBoxLayout(this); + QHBoxLayout* image_path_row = new QHBoxLayout(); layout->addLayout(image_path_row); image_path_row->setContentsMargins(0, 0, 0, 0); @@ -60,12 +60,12 @@ ImageAnnotationSourceSelectorWidget::ImageAnnotationSourceSelectorWidget(ImageAn QPushButton* reload_image_button = new QPushButton("Reload Image", this); image_path_row->addWidget(reload_image_button, 0); - QHBoxLayout* folder_info_row = new QHBoxLayout(this); + // add folder info row to show the index of the current image in the folder and buttons to move between images in this folder + QHBoxLayout* folder_info_row = new QHBoxLayout(); layout->addLayout(folder_info_row); m_folder_info_label = new QLabel(this); - folder_info_row->addWidget(m_folder_info_label, 3); - folder_info_row->addSpacing(3); + folder_info_row->addWidget(m_folder_info_label, 1); QPushButton* prev_image_button = new QPushButton("Prev Image in Folder", this); QPushButton* next_image_button = new QPushButton("Next Image in Folder", this); @@ -73,7 +73,7 @@ ImageAnnotationSourceSelectorWidget::ImageAnnotationSourceSelectorWidget(ImageAn folder_info_row->addSpacing(2); folder_info_row->addWidget(next_image_button, 2); folder_info_row->addSpacing(10); - folder_info_row->addWidget(new QLabel(" ", this), 10); + folder_info_row->addWidget(new QLabel(" ", this), 10); // empty label to push the buttons above to the left // Set the action for the video reset button diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index 398af312e8..556cc0fdbb 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -195,7 +195,7 @@ TestProgram::TestProgram() ) , IMAGE_PATH(false, "Path to image for testing", LockMode::UNLOCK_WHILE_RUNNING, "default.png", "default.png") , STATIC_TEXT("Test text...") -// , PLAYER_LIST("Test Table", LockMode::UNLOCK_WHILE_RUNNING, "Notes") + , BOX("Box", LockMode::UNLOCK_WHILE_RUNNING, 0, 0, 1, 1) , NOTIFICATION_TEST("Test", true, true, ImageAttachmentMode::JPG) , NOTIFICATIONS({ &NOTIFICATION_TEST, @@ -209,7 +209,7 @@ TestProgram::TestProgram() // PA_ADD_OPTION(CONSOLE_MODEL); PA_ADD_OPTION(IMAGE_PATH); PA_ADD_OPTION(STATIC_TEXT); -// PA_ADD_OPTION(PLAYER_LIST); + PA_ADD_OPTION(BOX); // PA_ADD_OPTION(battle_AI); PA_ADD_OPTION(NOTIFICATIONS); BUTTON0.add_listener(*this); @@ -254,10 +254,31 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& ProControllerContext context(scope, console.pro_controller()); VideoOverlaySet overlays(overlay); + auto screenshot = feed.snapshot(); - PokemonSwSh::MaxLairInternal::PathSelectDetector detector; - detector.detect(screenshot); + + DateReader reader(console); + reader.make_overlays(overlays); + reader.read_date(logger, screenshot); + + +#if 0 + BinarySliderDetector detector(COLOR_BLUE, {0.836431, 0.097521, 0.069703, 0.796694}); + auto result = detector.detect(screenshot); + + for (auto& item : result){ + cout << item.first << " : " << item.second.center_y() << endl; + } +#endif + +// LetsGoKillWatcher menu(logger, COLOR_RED, true, {0.23, 0.23, 0.04, 0.20}); +// cout << menu.detect(screenshot) << endl; + + + +// PokemonSwSh::MaxLairInternal::PathSelectDetector detector; +// detector.detect(screenshot); // int ret = PokemonSwSh::MaxLairInternal::read_side(screenshot); // cout << "ret = " << ret << endl; diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.h b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.h index 01e73ed009..a7245f622d 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.h +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.h @@ -13,6 +13,7 @@ #include "Common/Cpp/Options/ButtonOption.h" #include "Common/Cpp/Options/DateOption.h" #include "Common/Cpp/Options/TimeDurationOption.h" +#include "Common/Cpp/Options/BoxFloatOption.h" #include "CommonFramework/Notifications/EventNotificationsTable.h" #include "CommonFramework/ProgramStats/StatsTracking.h" #include "CommonTools/Options/LanguageOCROption.h" @@ -72,6 +73,7 @@ class TestProgram : public MultiSwitchProgramInstance, public ButtonListener{ StringOption IMAGE_PATH; StaticTextOption STATIC_TEXT; + BoxFloatOption BOX; // PokemonSV::SinglesAIOption battle_AI; diff --git a/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_MultiSwitchProgramSession.cpp b/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_MultiSwitchProgramSession.cpp index 0b0b9d3932..161d18bcf7 100644 --- a/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_MultiSwitchProgramSession.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_MultiSwitchProgramSession.cpp @@ -203,9 +203,11 @@ void MultiSwitchProgramSession::internal_run_program(){ // m_setup->wait_for_all_requests(); env.add_overlay_log_to_all_consoles("- Program Finished -"); logger().log("Program finished normally!", COLOR_BLUE); - }catch (OperationCancelledException&){ + }catch (OperationCancelledException& e){ + logger().log("Program Stopped (OperationCancelledException): " + e.message(), COLOR_RED); env.add_overlay_log_to_all_consoles("- Program Stopped -"); - }catch (ProgramCancelledException&){ + }catch (ProgramCancelledException& e){ + logger().log("Program Stopped (ProgramCancelledException): " + e.message(), COLOR_BLUE); env.add_overlay_log_to_all_consoles("- Program Stopped -"); }catch (ProgramFinishedException& e){ logger().log("Program finished early!", COLOR_BLUE); diff --git a/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.cpp b/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.cpp index 3d0cabeadf..d6f506449e 100644 --- a/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Framework/NintendoSwitch_SwitchSystemSession.cpp @@ -44,7 +44,7 @@ SwitchSystemSession::SwitchSystemSession( , m_controller(m_logger, option.m_controller, option.m_required_features) , m_video(m_logger, option.m_video) , m_audio(m_logger, option.m_audio) - , m_overlay(option.m_overlay) + , m_overlay(m_logger, option.m_overlay) , m_history(m_logger) , m_cpu_utilization(new CpuUtilizationStat()) , m_main_thread_utilization(new ThreadUtilizationStat(current_thread_handle(), "Main Qt Thread:")) diff --git a/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.cpp b/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.cpp index cdf2e4966d..18e8178377 100644 --- a/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.cpp @@ -74,10 +74,11 @@ CommandRow::CommandRow( m_overlay_text->setChecked(session.enabled_text()); row->addWidget(m_overlay_text); - if (PreloadSettings::instance().DEVELOPER_MODE){ - m_overlay_images = new QCheckBox("Masks", this); - m_overlay_images->setChecked(session.enabled_images()); - row->addWidget(m_overlay_images); + m_overlay_images = new QCheckBox("Masks", this); + m_overlay_images->setChecked(session.enabled_images()); + row->addWidget(m_overlay_images); + if (!PreloadSettings::instance().DEVELOPER_MODE){ + m_overlay_images->setVisible(false); } m_overlay_log = new QCheckBox("Log", this); diff --git a/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.h b/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.h index 13fe20145d..6076324b84 100644 --- a/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.h +++ b/SerialPrograms/Source/NintendoSwitch/Framework/UI/NintendoSwitch_CommandRow.h @@ -65,19 +65,19 @@ class CommandRow : ControllerSession& m_controller; VideoOverlaySession& m_session; bool m_allow_commands_while_running; - QComboBox* m_command_box; - QLabel* m_status; + QComboBox* m_command_box = nullptr; + QLabel* m_status = nullptr; - QCheckBox* m_overlay_log; - QCheckBox* m_overlay_text; - QCheckBox* m_overlay_images; - QCheckBox* m_overlay_boxes; - QCheckBox* m_overlay_stats; + QCheckBox* m_overlay_log = nullptr; + QCheckBox* m_overlay_text = nullptr; + QCheckBox* m_overlay_images = nullptr; + QCheckBox* m_overlay_boxes = nullptr; + QCheckBox* m_overlay_stats = nullptr; - QPushButton* m_load_profile_button; - QPushButton* m_save_profile_button; - QPushButton* m_screenshot_button; - QPushButton* m_video_button; + QPushButton* m_load_profile_button = nullptr; + QPushButton* m_save_profile_button = nullptr; + QPushButton* m_screenshot_button = nullptr; + QPushButton* m_video_button = nullptr; bool m_last_known_focus; ProgramState m_last_known_state; }; diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch2_BinarySliderDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch2_BinarySliderDetector.cpp index eafcdcbbdf..b85155a895 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch2_BinarySliderDetector.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch2_BinarySliderDetector.cpp @@ -28,7 +28,7 @@ void BinarySliderDetector::make_overlays(VideoOverlaySet& items) const{ items.add(m_color, m_box); } -std::vector> BinarySliderDetector::detect(const ImageViewRGB32& image) const{ +std::vector> BinarySliderDetector::detect(const ImageViewRGB32& screen) const{ using namespace Kernels::Waterfill; static ImageMatch::ExactImageMatcher LIGHT_OFF_CURSOR (RESOURCE_PATH() + "NintendoSwitch2/BinarySlider-Light-Off-Cursor.png"); @@ -40,7 +40,7 @@ std::vector> BinarySliderDetector::detect(const I static ImageMatch::ExactImageMatcher DARK_ON_CURSOR (RESOURCE_PATH() + "NintendoSwitch2/BinarySlider-Dark-On-Cursor.png"); static ImageMatch::ExactImageMatcher DARK_ON_NOCURSOR (RESOURCE_PATH() + "NintendoSwitch2/BinarySlider-Dark-On-NoCursor.png"); - ImageViewRGB32 region = extract_box_reference(image, m_box); + ImageViewRGB32 region = extract_box_reference(screen, m_box); std::vector matrices = compress_rgb32_to_binary_range( region, @@ -56,6 +56,7 @@ std::vector> BinarySliderDetector::detect(const I std::vector> best; +// int c = 0; for (PackedBinaryMatrix& matrix : matrices){ // cout << "-----------------" << endl; session->set_source(matrix); @@ -69,6 +70,12 @@ std::vector> BinarySliderDetector::detect(const I if (aspect_ratio < 0.9 || aspect_ratio > 1.1){ continue; } + + double min_area = screen.total_pixels() * (3000. / (3840*2160)); + if ((double)object.area < min_area){ + continue; + } + ImageViewRGB32 cropped = extract_box_reference(region, object); double best_off_rmsd = 9999; @@ -90,6 +97,8 @@ std::vector> BinarySliderDetector::detect(const I double best_rmsd = on ? best_on_rmsd : best_off_rmsd; if (best_rmsd < 60){ +// cout << "rmsd = " << best_rmsd << endl; +// cropped.save("slider-" + std::to_string(c) + "-" + std::to_string(sliders.size()) + ".png"); sliders.emplace_back(on, object); } @@ -102,6 +111,8 @@ std::vector> BinarySliderDetector::detect(const I if (best.size() < sliders.size()){ best = std::move(sliders); } + +// c++; } return best; diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch2_BinarySliderDetector.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch2_BinarySliderDetector.h index 0c45c26087..7ad1ca55ef 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch2_BinarySliderDetector.h +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch2_BinarySliderDetector.h @@ -22,7 +22,7 @@ class BinarySliderDetector{ BinarySliderDetector(Color color, const ImageFloatBox& box); void make_overlays(VideoOverlaySet& items) const; - std::vector> detect(const ImageViewRGB32& image) const; + std::vector> detect(const ImageViewRGB32& screen) const; private: Color m_color; diff --git a/SerialPrograms/Source/NintendoSwitch/NintendoSwitch_Settings.cpp b/SerialPrograms/Source/NintendoSwitch/NintendoSwitch_Settings.cpp index c68a0f4fd2..2959565bdb 100644 --- a/SerialPrograms/Source/NintendoSwitch/NintendoSwitch_Settings.cpp +++ b/SerialPrograms/Source/NintendoSwitch/NintendoSwitch_Settings.cpp @@ -116,10 +116,10 @@ ConsoleSettings::ConsoleSettings() LockMode::UNLOCK_WHILE_RUNNING, false ) - , SWITCH1_DIGIT_ENTRY(false) - , SWITCH1_KEYBOARD_ENTRY(false) - , SWITCH2_DIGIT_ENTRY(true) - , SWITCH2_KEYBOARD_ENTRY(true) + , SWITCH1_DIGIT_ENTRY0(false) + , SWITCH1_KEYBOARD_ENTRY0(false) + , SWITCH2_DIGIT_ENTRY0(true) + , SWITCH2_KEYBOARD_ENTRY0(true) , KEYBOARD_SECTION("Keyboard to Controller Mappings:") { PA_ADD_OPTION(CONTROLLER_SETTINGS); @@ -133,10 +133,10 @@ ConsoleSettings::ConsoleSettings() PA_ADD_OPTION(ENABLE_SBB3_LOGGING); PA_ADD_OPTION(TIMING_OPTIONS); if (PreloadSettings::instance().DEVELOPER_MODE){ - PA_ADD_OPTION(SWITCH1_DIGIT_ENTRY); - PA_ADD_OPTION(SWITCH1_KEYBOARD_ENTRY); - PA_ADD_OPTION(SWITCH2_DIGIT_ENTRY); - PA_ADD_OPTION(SWITCH2_KEYBOARD_ENTRY); + PA_ADD_OPTION(SWITCH1_DIGIT_ENTRY0); + PA_ADD_OPTION(SWITCH1_KEYBOARD_ENTRY0); + PA_ADD_OPTION(SWITCH2_DIGIT_ENTRY0); + PA_ADD_OPTION(SWITCH2_KEYBOARD_ENTRY0); } PA_ADD_STATIC(KEYBOARD_SECTION); PA_ADD_OPTION(KEYBOARD_MAPPINGS); diff --git a/SerialPrograms/Source/NintendoSwitch/NintendoSwitch_Settings.h b/SerialPrograms/Source/NintendoSwitch/NintendoSwitch_Settings.h index 51cacd4f9d..6f5a28926e 100644 --- a/SerialPrograms/Source/NintendoSwitch/NintendoSwitch_Settings.h +++ b/SerialPrograms/Source/NintendoSwitch/NintendoSwitch_Settings.h @@ -54,10 +54,10 @@ class ConsoleSettings : public BatchOption{ TimingOptions TIMING_OPTIONS; - DigitEntryTimingsOption SWITCH1_DIGIT_ENTRY; - KeyboardEntryTimingsOption SWITCH1_KEYBOARD_ENTRY; - DigitEntryTimingsOption SWITCH2_DIGIT_ENTRY; - KeyboardEntryTimingsOption SWITCH2_KEYBOARD_ENTRY; + DigitEntryTimingsOption SWITCH1_DIGIT_ENTRY0; + KeyboardEntryTimingsOption SWITCH1_KEYBOARD_ENTRY0; + DigitEntryTimingsOption SWITCH2_DIGIT_ENTRY0; + KeyboardEntryTimingsOption SWITCH2_KEYBOARD_ENTRY0; SectionDividerOption KEYBOARD_SECTION; KeyboardMappingOption KEYBOARD_MAPPINGS; diff --git a/SerialPrograms/Source/NintendoSwitch/Options/NintendoSwitch_CodeEntrySettingsOption.cpp b/SerialPrograms/Source/NintendoSwitch/Options/NintendoSwitch_CodeEntrySettingsOption.cpp index 0586f9655d..591164679b 100644 --- a/SerialPrograms/Source/NintendoSwitch/Options/NintendoSwitch_CodeEntrySettingsOption.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Options/NintendoSwitch_CodeEntrySettingsOption.cpp @@ -65,20 +65,20 @@ DigitEntryTimingsOption::DigitEntryTimingsOption(bool switch2) "Controller timing variation will be added to this number.", LockMode::UNLOCK_WHILE_RUNNING, switch2 - ? "48ms" - : PreloadSettings::instance().DEVELOPER_MODE ? "24 ms" : "40ms" + ? PreloadSettings::instance().DEVELOPER_MODE ? "48 ms" : "64 ms" + : PreloadSettings::instance().DEVELOPER_MODE ? "24 ms" : "40 ms" ) , HOLD( "Hold:
Duration to hold each button press down.
" "Controller timing variation will be added to this number.", LockMode::UNLOCK_WHILE_RUNNING, - PreloadSettings::instance().DEVELOPER_MODE ? "48 ms" : "80ms" + "48 ms" ) , COOLDOWN( "Hold:
Do not reuse a button until this long after it is reused.
" "Controller timing variation will be added to this number.", LockMode::UNLOCK_WHILE_RUNNING, - PreloadSettings::instance().DEVELOPER_MODE ? "24 ms" : "40ms" + "24 ms" ) { PA_ADD_OPTION(REORDERING); @@ -100,27 +100,27 @@ KeyboardEntryTimingsOption::KeyboardEntryTimingsOption(bool switch2) , REORDERING( "Character Reordering:
Allow characters to be entered out of order.", LockMode::UNLOCK_WHILE_RUNNING, - PreloadSettings::instance().DEVELOPER_MODE + true ) , TIME_UNIT( "Time Unit:
Timesteps should increment in multiples of this unit.
" "Controller timing variation will be added to this number.", LockMode::UNLOCK_WHILE_RUNNING, switch2 - ? "48ms" - : PreloadSettings::instance().DEVELOPER_MODE ? "24 ms" : "40ms" + ? PreloadSettings::instance().DEVELOPER_MODE ? "48 ms" : "64 ms" + : PreloadSettings::instance().DEVELOPER_MODE ? "24 ms" : "40 ms" ) , HOLD( "Hold:
Duration to hold each button press down.
" "Controller timing variation will be added to this number.", LockMode::UNLOCK_WHILE_RUNNING, - PreloadSettings::instance().DEVELOPER_MODE ? "48 ms" : "80ms" + "48 ms" ) , COOLDOWN( "Hold:
Do not reuse a button until this long after it is reused.
" "Controller timing variation will be added to this number.", LockMode::UNLOCK_WHILE_RUNNING, - PreloadSettings::instance().DEVELOPER_MODE ? "24 ms" : "40ms" + "24 ms" ) { PA_ADD_OPTION(REORDERING); diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/DateManip/NintendoSwitch_DateManipTools.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/DateManip/NintendoSwitch_DateManipTools.cpp index 7f8232ace3..e6f7706cbd 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/DateManip/NintendoSwitch_DateManipTools.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/DateManip/NintendoSwitch_DateManipTools.cpp @@ -24,7 +24,7 @@ namespace DateReaderTools{ ImageRGB32 filter_image(const ImageViewRGB32& image){ double brightness = image_stats(image).average.sum(); - bool white_theme = brightness > 600; + bool white_theme = brightness > 500; ImageRGB32 filtered = to_blackwhite_rgb32_range( image, @@ -39,8 +39,12 @@ int read_box( const ImageViewRGB32& screen, const ImageFloatBox& box ){ ImageViewRGB32 cropped = extract_box_reference(screen, box); + +// static int c = 0; +// cropped.save("image-" + std::to_string(c++) + ".png"); + double brightness = image_stats(cropped).average.sum(); - bool white_theme = brightness > 600; + bool white_theme = brightness > 500; int value; if (white_theme){ diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/DateManip/NintendoSwitch_DateManip_24h.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/DateManip/NintendoSwitch_DateManip_24h.cpp index ef5e670db8..053c75b7c3 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/DateManip/NintendoSwitch_DateManip_24h.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/DateManip/NintendoSwitch_DateManip_24h.cpp @@ -5,6 +5,7 @@ */ #include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/ErrorReports/ErrorReports.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" @@ -38,6 +39,20 @@ DateTime DateReader_24h::read_date(Logger& logger, std::shared_ptr= 12) c -= 12; @@ -122,7 +151,11 @@ void DateReader_US::set_date( } move_horizontal(context, cursor_position, 4); - adjust_wrap(context, 0, 59, current.minute, date.minute); + if (current.minute < 0){ + stream.log("Failed to read minutes. Will not adjust.", COLOR_RED); + }else{ + adjust_wrap(context, 0, 59, current.minute, date.minute); + } move_horizontal(context, cursor_position, 5); if ((date.hour < 12) != (current.hour < 12)){ diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch1_HomeToDateTime.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch1_HomeToDateTime.cpp index 4db8d50f1d..2722d37433 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch1_HomeToDateTime.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch1_HomeToDateTime.cpp @@ -4,6 +4,7 @@ * */ +#include "Common/Cpp/RecursiveThrottler.h" #include "CommonFramework/Exceptions/OperationFailedException.h" #include "CommonFramework/ImageTools/ImageBoxes.h" //#include "CommonFramework/VideoPipeline/VideoFeed.h" @@ -23,7 +24,10 @@ namespace NintendoSwitch{ void home_to_date_time_Switch1_wired_blind( Logger& logger, ProControllerContext& context, bool to_date_change ){ - logger.log("home_to_date_time_Switch1_wired_blind()"); + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("NintendoSwitch::home_to_date_time_Switch1_wired_blind()"); + } ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 32ms); ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 32ms); @@ -93,7 +97,10 @@ void home_to_date_time_Switch1_wired_blind( void home_to_date_time_Switch1_wireless_esp32_blind( Logger& logger, ProControllerContext& context, bool to_date_change ){ - logger.log("home_to_date_time_Switch1_wireless_esp32_blind()"); + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("NintendoSwitch::home_to_date_time_Switch1_wireless_esp32_blind()"); + } Milliseconds tv = context->timing_variation(); Milliseconds unit = 24ms + tv; @@ -167,7 +174,10 @@ void home_to_date_time_Switch1_wireless_esp32_blind( void home_to_date_time_Switch1_sbb_blind( Logger& logger, ProControllerContext& context, bool to_date_change ){ - logger.log("home_to_date_time_Switch1_sbb_blind()"); + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("NintendoSwitch::home_to_date_time_Switch1_sbb_blind()"); + } Milliseconds tv = context->timing_variation(); // ssf_do_nothing(context, 1500ms); @@ -213,34 +223,41 @@ void home_to_date_time_Switch1_wired_feedback( size_t max_attempts = 5; for (size_t i = 0; i < max_attempts; i++){ - ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 32ms); - ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 32ms); - ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 32ms); + { + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("NintendoSwitch::home_to_date_time_Switch1_wired_feedback() - Part 1/3"); + } - // Down twice in case we drop one. - ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); - ssf_issue_scroll(context, SSF_SCROLL_DOWN, 32ms); + ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 32ms); + ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 32ms); + ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 32ms); - // if (i > 0){ // intentionally create a failure, for testing - ssf_issue_scroll(context, SSF_SCROLL_LEFT, 0ms, 40ms, 24ms); - // } + // Down twice in case we drop one. + ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); + ssf_issue_scroll(context, SSF_SCROLL_DOWN, 32ms); + // if (i > 0){ // intentionally create a failure, for testing + ssf_issue_scroll(context, SSF_SCROLL_LEFT, 0ms, 40ms, 24ms); + // } - // ImageFloatBox system_icon(0.685, 0.69, 0.05, 0.03); - // ImageFloatBox other_setting1(0.615, 0.69, 0.05, 0.03); - // ImageFloatBox other_setting2(0.545, 0.69, 0.05, 0.03); - // Two A presses in case we drop the 1st one. - // the program can self recover even if the second button press is registered. - ssf_press_button(context, BUTTON_A, 24ms, 40ms, 24ms); - ssf_press_button(context, BUTTON_A, 24ms, 48ms, 24ms); + // ImageFloatBox system_icon(0.685, 0.69, 0.05, 0.03); + // ImageFloatBox other_setting1(0.615, 0.69, 0.05, 0.03); + // ImageFloatBox other_setting2(0.545, 0.69, 0.05, 0.03); - // Just button mash it. lol - { - auto iterations = Milliseconds(1200) / 24ms + 1; - do{ - ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); - }while (--iterations); + // Two A presses in case we drop the 1st one. + // the program can self recover even if the second button press is registered. + ssf_press_button(context, BUTTON_A, 24ms, 40ms, 24ms); + ssf_press_button(context, BUTTON_A, 24ms, 48ms, 24ms); + + // Just button mash it. lol + { + auto iterations = Milliseconds(1200) / 24ms + 1; + do{ + ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); + }while (--iterations); + } } context.wait_for_all_requests(); @@ -263,26 +280,33 @@ void home_to_date_time_Switch1_wired_feedback( continue; } - { - auto iterations = Milliseconds(312) / 24ms + 1; - do{ - ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 24ms); - }while (--iterations); + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("NintendoSwitch::home_to_date_time_Switch1_wired_feedback() - Part 2/3"); + } + + { + auto iterations = Milliseconds(312) / 24ms + 1; + do{ + ssf_issue_scroll(context, SSF_SCROLL_RIGHT, 24ms); + }while (--iterations); + } + + ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); + ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); + ssf_issue_scroll(context, SSF_SCROLL_DOWN, 80ms, 40ms, 24ms); + ssf_press_dpad(context, DPAD_DOWN, 360ms, 320ms); + ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); + // if (i > 1){ // intentionally create a failure, for testing + ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); + // } + + // only one ButtonA press since the program can self-recover if the button is dropped. + // furthermore, the program can't self-recover if a second button press is registered. + ssf_press_button(context, BUTTON_A, 24ms, 48ms, 24ms); } - ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); - ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); - ssf_issue_scroll(context, SSF_SCROLL_DOWN, 80ms, 40ms, 24ms); - ssf_press_dpad(context, DPAD_DOWN, 360ms, 320ms); - ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); - // if (i > 1){ // intentionally create a failure, for testing - ssf_issue_scroll(context, SSF_SCROLL_DOWN, 24ms); - // } - - // only one ButtonA press since the program can self-recover if the button is dropped. - // furthermore, the program can't self-recover if a second button press is registered. - ssf_press_button(context, BUTTON_A, 24ms, 48ms, 24ms); context.wait_for_all_requests(); context.wait_for(Milliseconds(300)); @@ -306,6 +330,12 @@ void home_to_date_time_Switch1_wired_feedback( return; } + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("NintendoSwitch::home_to_date_time_Switch1_wired_feedback() - Part 3/3"); + } + + { auto iterations = Milliseconds(250) / 24ms + 1; do{ @@ -334,6 +364,11 @@ void home_to_date_time_Switch1_wired_feedback( void home_to_date_time_Switch1_joycon_blind(JoyconContext& context, bool to_date_change){ + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("NintendoSwitch::home_to_date_time_Switch1_joycon_blind()"); + } + Milliseconds tv = context->timing_variation(); Milliseconds unit = 100ms + tv; diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch2_HomeToDateTime.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch2_HomeToDateTime.cpp index 5cc1fd17d2..10dfa54019 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch2_HomeToDateTime.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/DateSpam/NintendoSwitch2_HomeToDateTime.cpp @@ -4,6 +4,7 @@ * */ +#include "Common/Cpp/RecursiveThrottler.h" #include "CommonFramework/Exceptions/OperationFailedException.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" @@ -81,6 +82,11 @@ ConsoleType settings_detect_console_type( void home_to_settings_Switch2_procon_blind( ProControllerContext& context ){ + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("NintendoSwitch::home_to_settings_Switch2_procon_blind()"); + } + Milliseconds tv = context->timing_variation(); Milliseconds unit = 24ms + tv; @@ -115,6 +121,11 @@ void home_to_settings_Switch2_procon_blind( void home_to_settings_Switch2_joycon_blind( JoyconContext& context ){ + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("NintendoSwitch::home_to_settings_Switch2_joycon_blind()"); + } + Milliseconds tv = context->timing_variation(); Milliseconds unit = 24ms + tv; @@ -151,6 +162,11 @@ void settings_to_date_time_Switch2_all_blind( Logger& logger, ControllerContext& context, ConsoleType console_type, bool to_date_change ){ + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("NintendoSwitch::settings_to_date_time_Switch2_all_blind()"); + } + Milliseconds tv = context->timing_variation(); Milliseconds unit = 24ms + tv; @@ -158,7 +174,6 @@ void settings_to_date_time_Switch2_all_blind( ssf_issue_scroll(context, SSF_SCROLL_DOWN, unit); ssf_issue_scroll(context, SSF_SCROLL_DOWN, 192ms, 2*unit, unit); - switch (console_type){ case ConsoleType::Switch2_FW19_International: ssf_issue_scroll(context, SSF_SCROLL_DOWN, 128ms, 2*unit, unit); @@ -185,7 +200,6 @@ void settings_to_date_time_Switch2_all_blind( ); } - if (!to_date_change){ // Triple up this A press to make sure it gets through. ssf_press_button(context, BUTTON_A, unit); diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.cpp index 447307b9c8..ed6430df72 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.cpp @@ -305,15 +305,15 @@ void keyboard_enter_code( Milliseconds cool; bool reordering; if (switch2){ - unit = ConsoleSettings::instance().SWITCH2_KEYBOARD_ENTRY.TIME_UNIT; - hold = ConsoleSettings::instance().SWITCH2_KEYBOARD_ENTRY.HOLD; - cool = ConsoleSettings::instance().SWITCH2_KEYBOARD_ENTRY.COOLDOWN; - reordering = ConsoleSettings::instance().SWITCH2_KEYBOARD_ENTRY.REORDERING; + unit = ConsoleSettings::instance().SWITCH2_KEYBOARD_ENTRY0.TIME_UNIT; + hold = ConsoleSettings::instance().SWITCH2_KEYBOARD_ENTRY0.HOLD; + cool = ConsoleSettings::instance().SWITCH2_KEYBOARD_ENTRY0.COOLDOWN; + reordering = ConsoleSettings::instance().SWITCH2_KEYBOARD_ENTRY0.REORDERING; }else{ - unit = ConsoleSettings::instance().SWITCH1_KEYBOARD_ENTRY.TIME_UNIT; - hold = ConsoleSettings::instance().SWITCH1_KEYBOARD_ENTRY.HOLD; - cool = ConsoleSettings::instance().SWITCH1_KEYBOARD_ENTRY.COOLDOWN; - reordering = ConsoleSettings::instance().SWITCH1_KEYBOARD_ENTRY.REORDERING; + unit = ConsoleSettings::instance().SWITCH1_KEYBOARD_ENTRY0.TIME_UNIT; + hold = ConsoleSettings::instance().SWITCH1_KEYBOARD_ENTRY0.HOLD; + cool = ConsoleSettings::instance().SWITCH1_KEYBOARD_ENTRY0.COOLDOWN; + reordering = ConsoleSettings::instance().SWITCH1_KEYBOARD_ENTRY0.REORDERING; } Milliseconds tv = context->timing_variation(); diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_NumberCodeEntry.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_NumberCodeEntry.cpp index adf50b0fc0..d9d242187b 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_NumberCodeEntry.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_NumberCodeEntry.cpp @@ -213,15 +213,15 @@ void numberpad_enter_code( Milliseconds cool; bool reordering; if (switch2){ - unit = ConsoleSettings::instance().SWITCH2_DIGIT_ENTRY.TIME_UNIT; - hold = ConsoleSettings::instance().SWITCH2_DIGIT_ENTRY.HOLD; - cool = ConsoleSettings::instance().SWITCH2_DIGIT_ENTRY.COOLDOWN; - reordering = ConsoleSettings::instance().SWITCH2_DIGIT_ENTRY.REORDERING; + unit = ConsoleSettings::instance().SWITCH2_DIGIT_ENTRY0.TIME_UNIT; + hold = ConsoleSettings::instance().SWITCH2_DIGIT_ENTRY0.HOLD; + cool = ConsoleSettings::instance().SWITCH2_DIGIT_ENTRY0.COOLDOWN; + reordering = ConsoleSettings::instance().SWITCH2_DIGIT_ENTRY0.REORDERING; }else{ - unit = ConsoleSettings::instance().SWITCH1_DIGIT_ENTRY.TIME_UNIT; - hold = ConsoleSettings::instance().SWITCH1_DIGIT_ENTRY.HOLD; - cool = ConsoleSettings::instance().SWITCH1_DIGIT_ENTRY.COOLDOWN; - reordering = ConsoleSettings::instance().SWITCH1_DIGIT_ENTRY.REORDERING; + unit = ConsoleSettings::instance().SWITCH1_DIGIT_ENTRY0.TIME_UNIT; + hold = ConsoleSettings::instance().SWITCH1_DIGIT_ENTRY0.HOLD; + cool = ConsoleSettings::instance().SWITCH1_DIGIT_ENTRY0.COOLDOWN; + reordering = ConsoleSettings::instance().SWITCH1_DIGIT_ENTRY0.REORDERING; } Milliseconds tv = context->timing_variation(); diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/NintendoSwitch_GameEntry.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/NintendoSwitch_GameEntry.cpp index 13b2658aff..5856e177cf 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/NintendoSwitch_GameEntry.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Programs/NintendoSwitch_GameEntry.cpp @@ -53,6 +53,7 @@ void ensure_at_home(ConsoleHandle& console, ControllerContext& context){ for (size_t attempts = 0; attempts < 10; attempts++){ HomeMenuWatcher home_menu(console, 100ms); + context.wait_for_all_requests(); int ret = wait_until( console, context, 5000ms, {home_menu} @@ -61,7 +62,7 @@ void ensure_at_home(ConsoleHandle& console, ControllerContext& context){ return; } console.log("Unable to detect Home. Pressing Home button...", COLOR_RED); - pbf_press_button(context, BUTTON_HOME, 160ms, 0ms); + pbf_press_button(context, BUTTON_HOME, 160ms, 160ms); } OperationFailedException::fire( ErrorReport::SEND_ERROR_REPORT, diff --git a/SerialPrograms/Source/PokemonSV/Inference/Dialogs/PokemonSV_GradientArrowDetector.cpp b/SerialPrograms/Source/PokemonSV/Inference/Dialogs/PokemonSV_GradientArrowDetector.cpp index ecf0155abe..d32d30dcbe 100644 --- a/SerialPrograms/Source/PokemonSV/Inference/Dialogs/PokemonSV_GradientArrowDetector.cpp +++ b/SerialPrograms/Source/PokemonSV/Inference/Dialogs/PokemonSV_GradientArrowDetector.cpp @@ -37,6 +37,7 @@ const ImageMatch::ExactImageMatcher& GRADIENT_ARROW_VERTICAL(){ bool is_gradient_arrow( GradientArrowType type, + const ImageViewRGB32& original_screen, const ImageViewRGB32& image, WaterfillObject& object, const WaterfillObject& yellow, const WaterfillObject& blue @@ -44,7 +45,9 @@ bool is_gradient_arrow( object = yellow; object.merge_assume_no_overlap(blue); - if (object.width() < 20){ + size_t object_box_area = object.width() * object.height(); + double min_area = original_screen.total_pixels() * (2000. / (1920*1080)); + if (object_box_area < min_area){ return false; } @@ -66,6 +69,9 @@ bool is_gradient_arrow( } double rmsd = GRADIENT_ARROW_HORIZONTAL().rmsd(cropped); // cout << "rmsd = " << rmsd << endl; +// if (rmsd <= THRESHOLD){ +// cout << "object_box_area = " << object_box_area << endl; +// } return rmsd <= THRESHOLD; } case GradientArrowType::DOWN:{ @@ -223,7 +229,7 @@ bool GradientArrowDetector::detect(ImageFloatBox& box, const ImageViewRGB32& scr for (WaterfillObject& yellow : yellows){ for (WaterfillObject& blue : blues){ WaterfillObject object; - if (is_gradient_arrow(m_type, region, object, yellow, blue)){ + if (is_gradient_arrow(m_type, screen, region, object, yellow, blue)){ // hits.emplace_back(translate_to_parent(screen, m_box, object)); // extract_box_reference(region, object).save("object.png"); box = translate_to_parent(screen, m_box, object); diff --git a/SerialPrograms/Source/PokemonSV/Inference/Overworld/PokemonSV_LetsGoKillDetector.cpp b/SerialPrograms/Source/PokemonSV/Inference/Overworld/PokemonSV_LetsGoKillDetector.cpp index 489e0d7244..f47ef2871d 100644 --- a/SerialPrograms/Source/PokemonSV/Inference/Overworld/PokemonSV_LetsGoKillDetector.cpp +++ b/SerialPrograms/Source/PokemonSV/Inference/Overworld/PokemonSV_LetsGoKillDetector.cpp @@ -37,9 +37,16 @@ bool is_kill_icon( WaterfillObject& object, const WaterfillObject& red, const WaterfillObject& white ){ +// cout << "is_kill_icon" << endl; // cout << red.min_x << "-" << red.max_x << ", " << white.min_y << "-" << white.max_y << endl; -// cout << "is_kill_icon" << endl; +#if 0 + static int c = 0; + object = red; + object.merge_assume_no_overlap(white); + extract_box_reference(image, object).save("test-" + std::to_string(c++) + ".png"); +#endif + if (red.max_y >= white.min_y){ // cout << "bad y" << endl; return false; @@ -56,8 +63,6 @@ bool is_kill_icon( object = red; object.merge_assume_no_overlap(white); -// extract_box_reference(image, object).save("test.png"); - if (object.width() < 18 || object.height() < 18){ // cout << "too small: " << object.width() << endl; return false; @@ -91,6 +96,8 @@ void LetsGoKillDetector::make_overlays(VideoOverlaySet& items) const{ bool LetsGoKillDetector::detect(const ImageViewRGB32& screen){ using namespace Kernels::Waterfill; + size_t size_threshold = (size_t)(10. * screen.total_pixels() / 2073600); + ImageViewRGB32 region = extract_box_reference(screen, m_box); std::vector reds; std::vector whites; @@ -99,14 +106,24 @@ bool LetsGoKillDetector::detect(const ImageViewRGB32& screen){ std::vector matrices = compress_rgb32_to_binary_range( region, { + {0xff801e1e, 0xffff5f5f}, + {0xff901e1e, 0xffff5f5f}, + {0xffa01e1e, 0xffff5f5f}, + {0xffb01e1e, 0xffff5f5f}, + {0xffc01e1e, 0xffff5f5f}, + {0xffd01e1e, 0xffff5f5f}, {0xff801e1e, 0xffff7f7f}, - {0xffb31e1e, 0xffff7f7f}, + {0xff901e1e, 0xffff7f7f}, + {0xffa01e1e, 0xffff7f7f}, + {0xffb01e1e, 0xffff7f7f}, + {0xffc01e1e, 0xffff7f7f}, + {0xffd01e1e, 0xffff7f7f}, } ); // size_t c = 0; for (PackedBinaryMatrix& matrix : matrices){ session->set_source(matrix); - auto iter = session->make_iterator(10); + auto iter = session->make_iterator(size_threshold); WaterfillObject object; while (iter->find_next(object, false)){ double aspect_ratio = object.aspect_ratio(); @@ -123,13 +140,16 @@ bool LetsGoKillDetector::detect(const ImageViewRGB32& screen){ std::vector matrices = compress_rgb32_to_binary_range( region, { + {0xffb0b0b0, 0xffffffff}, {0xffc0c0c0, 0xffffffff}, + {0xffd0d0d0, 0xffffffff}, + {0xffe0e0e0, 0xffffffff}, } ); // size_t c = 0; for (PackedBinaryMatrix& matrix : matrices){ session->set_source(matrix); - auto iter = session->make_iterator(10); + auto iter = session->make_iterator(size_threshold); WaterfillObject object; while (iter->find_next(object, false)){ double aspect_ratio = object.aspect_ratio(); diff --git a/SerialPrograms/Source/PokemonSV/Inference/Tera/PokemonSV_TeraRaidSearchDetector.cpp b/SerialPrograms/Source/PokemonSV/Inference/Tera/PokemonSV_TeraRaidSearchDetector.cpp index 1c26a6a06d..33c224fb4f 100644 --- a/SerialPrograms/Source/PokemonSV/Inference/Tera/PokemonSV_TeraRaidSearchDetector.cpp +++ b/SerialPrograms/Source/PokemonSV/Inference/Tera/PokemonSV_TeraRaidSearchDetector.cpp @@ -58,7 +58,7 @@ bool TeraRaidSearchDetector::detect_search_location(ImageFloatBox& box, const Im auto iter = session->make_iterator(100); WaterfillObject object; -// size_t c = 0; +// static size_t c = 0; const TeraSearchGlassMatcher& MATCHER = TeraSearchGlassMatcher::instance(); while (iter->find_next(object, false)){ // cout << "yellow = " << object.area << endl; @@ -67,6 +67,10 @@ bool TeraRaidSearchDetector::detect_search_location(ImageFloatBox& box, const Im double rmsd = MATCHER.rmsd(extract_box_reference(screen, object)); // cout << "rmsd = " << rmsd << endl; if (rmsd < 100){ + +// cout << "rmsd = " << rmsd << endl; +// extract_box_reference(screen, object).save("object-" + std::to_string(c++) + ".png"); + box = translate_to_parent(screen, {0, 0, 1, 1}, object); return true; } @@ -97,8 +101,10 @@ bool TeraRaidSearchDetector::move_cursor_to_search( ok &= arrow.detect(arrow_location, screen); if (!ok){ consecutive_detection_fails++; - if (consecutive_detection_fails > 10){ - dump_image_and_throw_recoverable_exception(info, stream, "UnableToDetectTeraSearch", "Unable to detect Tera Raid Search menu."); + if (consecutive_detection_fails > 3){ +// dump_image_and_throw_recoverable_exception(info, stream, "UnableToDetectTeraSearch", "Unable to detect Tera Raid Search menu."); + stream.log("Unable to detect Tera Raid Search menu.", COLOR_RED); + return false; } context.wait_for(std::chrono::milliseconds(1000)); continue; diff --git a/SerialPrograms/Source/PokemonSV/Programs/PokemonSV_MenuNavigation.cpp b/SerialPrograms/Source/PokemonSV/Programs/PokemonSV_MenuNavigation.cpp index 2adfb6c03d..aa480f05fd 100644 --- a/SerialPrograms/Source/PokemonSV/Programs/PokemonSV_MenuNavigation.cpp +++ b/SerialPrograms/Source/PokemonSV/Programs/PokemonSV_MenuNavigation.cpp @@ -2,6 +2,7 @@ * */ +#include "Common/Cpp/RecursiveThrottler.h" #include "CommonFramework/Exceptions/OperationFailedException.h" #include "CommonFramework/Exceptions/UnexpectedBattleException.h" #include "CommonTools/Async/InferenceRoutines.h" @@ -10,6 +11,8 @@ #include "NintendoSwitch/Commands/NintendoSwitch_Commands_Superscalar.h" #include "NintendoSwitch/Programs/NintendoSwitch_GameEntry.h" #include "NintendoSwitch/Programs/DateSpam/NintendoSwitch_HomeToDateTime.h" +//#include "NintendoSwitch/Programs/DateSpam/NintendoSwitch_RollDateForward1.h" +//#include "NintendoSwitch/Programs/DateSpam/NintendoSwitch_NeutralDateSkip.h" #include "NintendoSwitch/Programs/DateManip/NintendoSwitch_DateManip.h" #include "PokemonSV/PokemonSV_Settings.h" #include "PokemonSV/Inference/Dialogs/PokemonSV_DialogDetector.h" @@ -50,7 +53,10 @@ void set_time_to_12am_from_home(const ProgramInfo& info, ConsoleHandle& console, } void neutral_day_skip_switch1(ConsoleHandle& console, ProControllerContext& context){ - console.log("PokemonSV::neutral_day_skip_switch1()"); + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("PokemonSV::neutral_day_skip_switch1()"); + } Milliseconds tv = context->timing_variation(); if (tv == 0ms){ @@ -88,7 +94,10 @@ void neutral_day_skip_switch1(ConsoleHandle& console, ProControllerContext& cont } } void neutral_day_skip_switch2(ConsoleHandle& console, ProControllerContext& context){ - console.log("PokemonSV::neutral_day_skip_switch2()"); + ThrottleScope scope(context->logging_throttler()); + if (scope){ + context->logger().log("PokemonSV::neutral_day_skip_switch2()"); + } ssf_press_button(context, BUTTON_A, 216ms, 80ms); ssf_issue_scroll_ptv(context, SSF_SCROLL_RIGHT); @@ -109,6 +118,13 @@ void day_skip_from_overworld(ConsoleHandle& console, ProControllerContext& conte neutral_day_skip_switch1(console, context); }else if (is_switch2(console_type)){ neutral_day_skip_switch2(console, context); + +#if 0 + for (int c = 0; c < 10; c++){ + NintendoSwitch::roll_date_forward_1(console, context, true); +// NintendoSwitch::neutral_date_skip(console, context); + } +#endif }else{ throw UserSetupError( console, diff --git a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_AutoHostTools.cpp b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_AutoHostTools.cpp index 20c1ffdc64..1b8fded4fc 100644 --- a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_AutoHostTools.cpp +++ b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_AutoHostTools.cpp @@ -70,6 +70,7 @@ void TeraFailTracker::on_raid_start(){ } m_current_raid_error.store(false, std::memory_order_relaxed); +#if 1 if (m_consecutive_failures > 0 && !m_completed_one){ throw_and_log( m_env.logger(), @@ -77,6 +78,8 @@ void TeraFailTracker::on_raid_start(){ "Failed 1st raid attempt. Will not retry due to risk of ban." ); } +#endif + size_t fail_threshold = m_consecutive_failure_pause; if (m_consecutive_failures >= fail_threshold){ uint16_t minutes = m_failure_pause_minutes; diff --git a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraMultiFarmer.cpp b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraMultiFarmer.cpp index 13cb121762..61ae7d8d8e 100644 --- a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraMultiFarmer.cpp +++ b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraMultiFarmer.cpp @@ -16,7 +16,7 @@ #include "Pokemon/Pokemon_Strings.h" #include "Pokemon/Inference/Pokemon_NameReader.h" #include "PokemonSV/PokemonSV_Settings.h" -#include "PokemonSV/Inference/Dialogs/PokemonSV_DialogDetector.h" +//#include "PokemonSV/Inference/Dialogs/PokemonSV_DialogDetector.h" #include "PokemonSV/Inference/Tera/PokemonSV_TeraCardDetector.h" #include "PokemonSV/Inference/Tera/PokemonSV_TeraRaidSearchDetector.h" #include "PokemonSV/Programs/PokemonSV_GameEntry.h" @@ -285,15 +285,15 @@ bool TeraMultiFarmer::run_raid_host(ProgramEnvironment& env, ConsoleHandle& cons exit_tera_win_without_catching(env.program_info(), console, context, 0); } reset_host(env.program_info(), console, context); - if (HOSTING_MODE == Mode::HOST_ONLINE){ - connect_to_internet_from_overworld(env.program_info(), console, context); - } +// if (HOSTING_MODE == Mode::HOST_ONLINE){ +// connect_to_internet_from_overworld(env.program_info(), console, context); +// } }else{ stats.m_losses++; env.update_stats(); } - open_raid(console, context); +// open_raid(console, context); return win; } @@ -315,135 +315,25 @@ void TeraMultiFarmer::run_raid_joiner(ProgramEnvironment& env, ConsoleHandle& co save_game_from_menu(env.program_info(), console, context); } - enter_tera_search(env.program_info(), console, context, HOSTING_MODE == Mode::HOST_ONLINE); +// enter_tera_search(env.program_info(), console, context, HOSTING_MODE == Mode::HOST_ONLINE); } -void TeraMultiFarmer::join_lobby( - ProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, - size_t host_index, const std::string& normalized_code -){ - if (console.index() == host_index){ - return; - } - TeraMultiFarmer_Descriptor::Stats& stats = env.current_stats(); -// cout << "Joining Lobby" << endl; - - bool seen_code_entry = false; -// bool seen_dialog = false; - size_t attempts = 0; - while (true){ -// cout << "Looping..." << endl; - - if (attempts >= 3){ - OperationFailedException::fire( - ErrorReport::SEND_ERROR_REPORT, - "Failed to join lobby 3 times.", - console - ); - } - - CodeEntryWatcher code_entry(COLOR_GREEN); - TeraLobbyWatcher lobby(console.logger(), COLOR_RED); - AdvanceDialogWatcher dialog(COLOR_YELLOW); - TeraRaidSearchWatcher raid_search(COLOR_CYAN, std::chrono::seconds(5)); - context.wait_for_all_requests(); - context.wait_for(std::chrono::seconds(3)); - int ret = wait_until( - console, context, std::chrono::seconds(60), - { - code_entry, - lobby, - dialog, - raid_search, - } - ); - switch (ret){ - case 0: - console.log("Detected code entry.", COLOR_RED); - if (seen_code_entry){ - console.log("Failed to enter code! Backing out and trying again...", COLOR_RED); - stats.m_errors++; - attempts++; - pbf_press_button(context, BUTTON_X, 20, 480); - enter_tera_search(env.program_info(), console, context, HOSTING_MODE == Mode::HOST_ONLINE); - seen_code_entry = false; - continue; - } - seen_code_entry = true; - enter_code( - console, context, - PLAYERS[console.index()]->keyboard_layout, - normalized_code, - false, - true, - false - ); - context.wait_for(std::chrono::seconds(1)); - continue; - case 1: - console.log("Entered raid lobby!"); - pbf_mash_button(context, BUTTON_A, 5 * TICKS_PER_SECOND); - break; - case 2: - console.log("Detected dialog...", COLOR_ORANGE); -// seen_dialog = true; - pbf_press_button(context, BUTTON_B, 20, 230); - continue; - case 3: -#if 0 - if (!seen_dialog){ - context.wait_for(std::chrono::seconds(1)); - continue; - } -#endif - console.log("Wrong code! Backing out and trying again...", COLOR_RED); - stats.m_errors++; - attempts++; - pbf_press_button(context, BUTTON_B, 20, 230); - enter_tera_search(env.program_info(), console, context, HOSTING_MODE == Mode::HOST_ONLINE); - seen_code_entry = false; - continue; - default: - OperationFailedException::fire( - ErrorReport::SEND_ERROR_REPORT, - "Unable to join lobby.", - console - ); - } - break; - } -} -bool TeraMultiFarmer::run_raid( - MultiSwitchProgramEnvironment& env, CancellableScope& scope, +bool TeraMultiFarmer::start_sequence_host( + MultiSwitchProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, + RaidWaiter& raid_waiter, CancellableScope& joiner_scope, std::string& lobby_code, std::array, 4>& player_names ){ TeraMultiFarmer_Descriptor::Stats& stats = env.current_stats(); - size_t host_index = HOSTING_SWITCH.current_value(); - ConsoleHandle& host_console = env.consoles[host_index]; - ProControllerContext host_context(scope, host_console.pro_controller()); - // Get everyone ready. - env.run_in_parallel(scope, [&](ConsoleHandle& console, ProControllerContext& context){ - try{ - if (console.index() == host_index){ - TeraCardReader card_detector(COLOR_RED); - if (!card_detector.detect(console.video().snapshot())){ - if (HOSTING_MODE == Mode::HOST_ONLINE){ - connect_to_internet_from_overworld(env.program_info(), console, context); - } - open_raid(console, context); - } - }else{ - enter_tera_search(env.program_info(), console, context, HOSTING_MODE == Mode::HOST_ONLINE); - } - }catch (OperationFailedException&){ - stats.m_errors++; - m_reset_required[console.index()] = true; - throw; + TeraCardReader card_detector(COLOR_RED); + if (!card_detector.detect(console.video().snapshot())){ + if (HOSTING_MODE == Mode::HOST_ONLINE){ + connect_to_internet_from_overworld(env.program_info(), console, context); } - }); + open_raid(console, context); + } if (HOSTING_MODE != Mode::FARM_ALONE){ BAN_LIST.refresh_online_table(env.logger()); @@ -452,9 +342,9 @@ bool TeraMultiFarmer::run_raid( // Open lobby and read code. WallClock lobby_start_time; try{ - TeraLobbyReader lobby_reader(host_console.logger()); + TeraLobbyReader lobby_reader(console.logger()); open_hosting_lobby( - env, host_console, host_context, + env, console, context, HOSTING_MODE == Mode::HOST_ONLINE ? HostingMode::ONLINE_CODED : HostingMode::LOCAL @@ -462,7 +352,7 @@ bool TeraMultiFarmer::run_raid( lobby_start_time = current_time(); std::string code = lobby_reader.raid_code( env.logger(), - host_console.video().snapshot() + console.video().snapshot() ); // code.back() = '0'; @@ -472,36 +362,23 @@ bool TeraMultiFarmer::run_raid( OperationFailedException::fire( ErrorReport::SEND_ERROR_REPORT, "Unable to read raid code.", - host_console + console ); } }catch (OperationFailedException&){ stats.m_errors++; - m_reset_required[host_index] = true; + m_reset_required[console.index()].store(true, std::memory_order_relaxed); throw; } -// normalized_code[0] = '0'; - - // Join the lobby with local joiners. If anything throws, we need to reset everyone. - try{ - env.run_in_parallel(scope, [&](ConsoleHandle& console, ProControllerContext& context){ - join_lobby(env, console, context, host_index, lobby_code); - }); - }catch (OperationFailedException&){ - stats.m_errors++; - m_reset_required[0] = true; - m_reset_required[1] = true; - m_reset_required[2] = true; - m_reset_required[3] = true; - throw; - } + raid_waiter.signal_raid_code_is_ready(lobby_code); + raid_waiter.wait_for_joiners(env.consoles.size() - 1); #if 1 // Put it up on auto-host. if (HOSTING_MODE != Mode::FARM_ALONE){ send_host_announcement( - env, host_console, + env, console, lobby_code, HOSTING_OPTIONS.SHOW_RAID_CODE, HOSTING_OPTIONS.DESCRIPTION, @@ -509,7 +386,7 @@ bool TeraMultiFarmer::run_raid( ); TeraLobbyWaiter waiter( - env, host_console, host_context, + env, console, context, (uint8_t)env.consoles.size(), lobby_code, lobby_start_time, HOSTING_OPTIONS.LOBBY_WAIT_DELAY, @@ -524,10 +401,10 @@ bool TeraMultiFarmer::run_raid( if (result == TeraLobbyWaiter::LobbyResult::BANNED_PLAYER){ stats.m_banned++; - m_reset_required[0] = true; - m_reset_required[1] = true; - m_reset_required[2] = true; - m_reset_required[3] = true; + m_reset_required[0].store(true, std::memory_order_relaxed); + m_reset_required[1].store(true, std::memory_order_relaxed); + m_reset_required[2].store(true, std::memory_order_relaxed); + m_reset_required[3].store(true, std::memory_order_relaxed); } if (result != TeraLobbyWaiter::LobbyResult::RAID_STARTED){ return false; @@ -535,19 +412,98 @@ bool TeraMultiFarmer::run_raid( uint8_t hosts = (uint8_t)env.consoles.size(); uint8_t players = waiter.last_known_players(); - stats.m_joiners += players - hosts; + if (players > hosts){ + stats.m_joiners += players - hosts; + } if (players == 4){ stats.m_full++; }else if (players == hosts){ stats.m_empty++; } + + } #endif - bool win = false; - // Start the raid. - pbf_mash_button(host_context, BUTTON_A, 10 * TICKS_PER_SECOND); + pbf_mash_button(context, BUTTON_A, 10 * TICKS_PER_SECOND); + + return true; +} +void TeraMultiFarmer::start_sequence_joiner( + ProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, + RaidWaiter& raid_waiter +){ + join_raid( + env.program_info(), console, context, + HOSTING_MODE == Mode::HOST_ONLINE, + PLAYERS[console.index()]->keyboard_layout, + raid_waiter + ); + + raid_waiter.signal_joiner_is_ready(); + + // Mash A to ready up. Even though this runs for 3 minutes, it will + // automatically be cancelled by the host when the host starts the raid. + pbf_mash_button(context, BUTTON_A, std::chrono::minutes(3)); +} + +bool TeraMultiFarmer::run_raid( + MultiSwitchProgramEnvironment& env, CancellableScope& scope, + std::string& lobby_code, + std::array, 4>& player_names +){ + TeraMultiFarmer_Descriptor::Stats& stats = env.current_stats(); + size_t host_index = HOSTING_SWITCH.current_value(); + + CancellableHolder raid_waiter(scope); + CancellableHolder joiner_scope((CancellableScope&)raid_waiter); + + env.run_in_parallel(scope, [&](ConsoleHandle& console, ProControllerContext& context){ + try{ + if (console.index() == host_index){ + start_sequence_host( + env, console, context, + raid_waiter, + joiner_scope, + lobby_code, + player_names + ); + joiner_scope.cancel(); + return; + } + + ProControllerContext sub_context(joiner_scope, context.controller()); + start_sequence_joiner( + env, console, sub_context, + raid_waiter + ); + sub_context.wait_for_all_requests(); + + }catch (OperationCancelledException&){ + }catch (OperationFailedException&){ + stats.m_errors++; +#if 0 + m_reset_required[console.index()].store(true, std::memory_order_relaxed); +#else + m_reset_required[0].store(true, std::memory_order_relaxed); + m_reset_required[1].store(true, std::memory_order_relaxed); + m_reset_required[2].store(true, std::memory_order_relaxed); + m_reset_required[3].store(true, std::memory_order_relaxed); +#endif + raid_waiter.cancel(); +// cout << "OperationFailedException: " << console.index() << endl; + throw; + }catch (...){ + raid_waiter.cancel(); +// cout << "General Exception: " << console.index() << endl; + throw; + } +// cout << "Finishing: " << console.index() << endl; + }); + + + bool win = false; // Run the raid. env.run_in_parallel(scope, [&](ConsoleHandle& console, ProControllerContext& context){ @@ -561,7 +517,7 @@ bool TeraMultiFarmer::run_raid( // Host throws. Reset the host and keep going. stats.m_errors++; env.update_stats(); - m_reset_required[console.index()] = true; + m_reset_required[console.index()].store(true, std::memory_order_relaxed); throw; } }); @@ -584,10 +540,10 @@ void TeraMultiFarmer::program(MultiSwitchProgramEnvironment& env, CancellableSco std::string report_name = "PokemonSV-AutoHost-JoinReport-" + now_to_filestring() + ".txt"; MultiLanguageJoinTracker join_tracker((uint8_t)env.consoles.size()); - m_reset_required[0] = false; - m_reset_required[1] = false; - m_reset_required[2] = false; - m_reset_required[3] = false; + m_reset_required[0].store(false, std::memory_order_relaxed); + m_reset_required[1].store(false, std::memory_order_relaxed); + m_reset_required[2].store(false, std::memory_order_relaxed); + m_reset_required[3].store(false, std::memory_order_relaxed); if (RECOVERY_MODE == RecoveryMode::SAVE_AND_RESET){ env.run_in_parallel(scope, [&](ConsoleHandle& console, ProControllerContext& context){ @@ -623,7 +579,7 @@ void TeraMultiFarmer::program(MultiSwitchProgramEnvironment& env, CancellableSco // Reset all errored Switches. env.run_in_parallel(scope, [&](ConsoleHandle& console, ProControllerContext& context){ size_t index = console.index(); - if (!m_reset_required[index]){ + if (!m_reset_required[index].load(std::memory_order_relaxed)){ return; } if (index == host_index){ @@ -631,7 +587,7 @@ void TeraMultiFarmer::program(MultiSwitchProgramEnvironment& env, CancellableSco }else{ reset_joiner(env.program_info(), console, context); } - m_reset_required[index] = false; + m_reset_required[index].store(false, std::memory_order_relaxed); }); // Check kill-switch now before we go online. @@ -660,12 +616,14 @@ void TeraMultiFarmer::program(MultiSwitchProgramEnvironment& env, CancellableSco } fail_tracker.report_successful_raid(); }catch (OperationFailedException& e){ +// cout << "caught: TeraMultiFarmer::program" << endl; + e.send_notification(env, NOTIFICATION_ERROR_RECOVERABLE); if (RECOVERY_MODE != RecoveryMode::SAVE_AND_RESET){ // Iterate the errored Switches. If a non-host has errored, // rethrow the exception to stop the program. for (size_t c = 0; c < 4; c++){ - if (m_reset_required[c] && c != host_index){ + if (m_reset_required[c].load(std::memory_order_relaxed) && c != host_index){ throw; } } diff --git a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraMultiFarmer.h b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraMultiFarmer.h index 97bf13aa2b..1ddc703a9f 100644 --- a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraMultiFarmer.h +++ b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraMultiFarmer.h @@ -23,6 +23,7 @@ #include "PokemonSV/Options/PokemonSV_PlayerList.h" #include "PokemonSV_AutoHostTools.h" #include "PokemonSV_JoinTracker.h" +#include "PokemonSV_TeraRoutines.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -96,6 +97,17 @@ class TeraMultiFarmer : public MultiSwitchProgramInstance, private ConfigOption: size_t host_index, const std::string& normalized_code ); + bool start_sequence_host( + MultiSwitchProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, + RaidWaiter& raid_waiter, CancellableScope& joiner_scope, + std::string& lobby_code, + std::array, 4>& player_names + ); + void start_sequence_joiner( + ProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, + RaidWaiter& raid_waiter + ); + bool run_raid( MultiSwitchProgramEnvironment& env, CancellableScope& scope, std::string& lobby_code, @@ -137,7 +149,7 @@ class TeraMultiFarmer : public MultiSwitchProgramInstance, private ConfigOption: WallClock m_last_time_fix; // std::atomic m_raid_error; - bool m_reset_required[4]; + std::atomic m_reset_required[4]; }; diff --git a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoller.cpp b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoller.cpp index e30a416516..96010ecea9 100644 --- a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoller.cpp +++ b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoller.cpp @@ -122,12 +122,23 @@ void TeraRoller::program(SingleSwitchProgramEnvironment& env, ProControllerConte bool first = true; uint32_t skip_counter = 0; + // Keep track of when we last reset. + // Day skips too soon out of a reset may fail. + WallClock last_reset = WallClock::min(); + while (true){ env.update_stats(); send_program_status_notification(env, NOTIFICATION_STATUS_UPDATE); if (!first){ day_skip_from_overworld(env.console, context); + + // Do it again if we're fresh out of a reset. + while (last_reset + std::chrono::seconds(20) > current_time()){ + env.log("Fresh out of a reset. Skipping again."); + day_skip_from_overworld(env.console, context); + } + pbf_wait(context, GameSettings::instance().RAID_SPAWN_DELAY0); context.wait_for_all_requests(); stats.m_skips++; @@ -141,6 +152,7 @@ void TeraRoller::program(SingleSwitchProgramEnvironment& env, ProControllerConte env.log("Resetting game to clear framerate."); save_game_from_overworld(env.program_info(), env.console, context); reset_game(env.program_info(), env.console, context); + last_reset = current_time(); skip_counter = 0; stats.m_resets++; } @@ -172,7 +184,7 @@ void TeraRoller::program(SingleSwitchProgramEnvironment& env, ProControllerConte env.console.overlay().add_log("Entering tera raid...", COLOR_WHITE); // Run away from the tera raid battle - run_from_tera_battle(env.program_info(), env.console, context); + run_from_tera_battle(env, env.console, context, &stats.m_errors); context.wait_for_all_requests(); env.console.log("Checking if tera raid is shiny..."); diff --git a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoutines.cpp b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoutines.cpp index 6831d98dd4..4d7a31ce1f 100644 --- a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoutines.cpp +++ b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoutines.cpp @@ -9,6 +9,7 @@ #include "CommonFramework/Exceptions/ProgramFinishedException.h" #include "CommonFramework/Exceptions/OperationFailedException.h" #include "CommonFramework/Exceptions/FatalProgramException.h" +#include "CommonFramework/ErrorReports/ErrorReports.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" #include "CommonFramework/Tools/ProgramEnvironment.h" #include "CommonFramework/Tools/ErrorDumper.h" @@ -27,6 +28,7 @@ #include "PokemonSV/Inference/Tera/PokemonSV_TeraRaidSearchDetector.h" #include "PokemonSV/Inference/Tera/PokemonSV_TeraRewardsReader.h" #include "PokemonSV/Programs/PokemonSV_ConnectToInternet.h" +#include "PokemonSV/Programs/FastCodeEntry/PokemonSV_CodeEntry.h" #include "PokemonSV/Programs/Battles/PokemonSV_BasicCatcher.h" #include "PokemonSV_TeraRoutines.h" @@ -218,7 +220,7 @@ void enter_tera_search( OverworldWatcher overworld(stream.logger(), COLOR_RED); MainMenuWatcher main_menu(COLOR_YELLOW); PokePortalWatcher poke_portal(COLOR_GREEN); - TeraRaidSearchWatcher raid_search(COLOR_CYAN); + TeraRaidSearchWatcher raid_search(COLOR_CYAN, std::chrono::milliseconds(500)); CodeEntryWatcher code_entry(COLOR_PURPLE); AdvanceDialogWatcher dialog(COLOR_BLUE); context.wait_for_all_requests(); @@ -279,6 +281,129 @@ void enter_tera_search( } } +void join_raid( + const ProgramInfo& info, ConsoleHandle& console, ProControllerContext& context, + bool connect_to_internet, + KeyboardLayout keyboard_layout, + RaidWaiter& raid_waiter +){ + WallClock start = current_time(); + bool connected = false; + while (true){ + if (current_time() - start > std::chrono::minutes(5)){ + dump_image_and_throw_recoverable_exception( + info, console, "JoinRaidFailed", + "join_raid(): Failed to enter Tera raid." + ); + } + + OverworldWatcher overworld(console.logger(), COLOR_RED); + MainMenuWatcher main_menu(COLOR_YELLOW); + PokePortalWatcher poke_portal(COLOR_GREEN); + TeraRaidSearchWatcher raid_search(COLOR_CYAN); + AdvanceDialogWatcher dialog(COLOR_BLUE); + CodeEntryWatcher code_entry(COLOR_PURPLE); + TeraLobbyWatcher lobby(console.logger(), COLOR_ORANGE); + context.wait_for_all_requests(); + int ret = wait_until( + console, context, + std::chrono::seconds(30), + { + overworld, + main_menu, + poke_portal, + raid_search, + dialog, + code_entry, + lobby, + } + ); + context.wait_for(std::chrono::milliseconds(100)); + switch (ret){ + case 0: + console.log("Detected overworld."); + pbf_press_button(context, BUTTON_X, 20, 105); + continue; + + case 1: + console.log("Detected main menu."); + if (connect_to_internet && !connected){ + connect_to_internet_from_menu(info, console, context); + connected = true; + continue; + } + if (main_menu.move_cursor(info, console, context, MenuSide::RIGHT, 3)){ + pbf_press_button(context, BUTTON_A, 20, 230); + } + continue; + + case 2: + console.log("Detected Poke Portal."); + if (poke_portal.move_cursor(info, console, context, 1)){ + pbf_press_button(context, BUTTON_A, 20, 230); + } + continue; + + case 3: + console.log("Detected Tera Raid Search."); + if (raid_search.move_cursor_to_search(info, console, context)){ + pbf_press_button(context, BUTTON_A, 20, 105); + } + continue; + + case 4: + console.log("Detected Dialog."); + pbf_press_button(context, BUTTON_B, 20, 105); + continue; + + case 5:{ + console.log("Detected Code Entry."); + std::string code = raid_waiter.wait_for_raid_code(); + enter_code( + console, context, + keyboard_layout, + code, + false, true, false + ); + +#if 0 + if (console.index() == 1){ + dump_image_and_throw_recoverable_exception( + info, console, "InjectedError", + "join_raid(): Injected Error" + ); + } +#endif + + break; + } + case 6: + console.log("Detected Raid Lobby."); + return; + + default: + auto screen = console.video().snapshot(); + report_error( + &console.logger(), + info, + "join_raid()", + {{"Message", "No recognized state after 30 seconds."}}, + screen, + &console.history() + ); + + pbf_press_button(context, BUTTON_B, 160ms, 840ms); +// OperationFailedException::fire( +// ErrorReport::SEND_ERROR_REPORT, +// "join_raid(): No recognized state after 30 seconds.", +// console, +// std::move(screen) +// ); + } + } +} + + @@ -670,7 +795,12 @@ TeraResult run_tera_summary( } -void run_from_tera_battle(const ProgramInfo& info, VideoStream& stream, ProControllerContext& context){ +void run_from_tera_battle( + ProgramEnvironment& env, + VideoStream& stream, + ProControllerContext& context, + std::atomic* stat_errors +){ stream.log("Running away from tera raid battle..."); WallClock start = current_time(); @@ -688,9 +818,10 @@ void run_from_tera_battle(const ProgramInfo& info, VideoStream& stream, ProContr GradientArrowWatcher leave_confirm( COLOR_RED, GradientArrowType::RIGHT, - {0.557621, 0.471074, 0.388476, 0.247934} + {0.557621, 0.471074, 0.25, 0.247934} ); OverworldWatcher overworld(stream.logger(), COLOR_CYAN); + TeraCardWatcher tera_card(COLOR_BLUE); context.wait_for_all_requests(); int ret = wait_until( @@ -700,6 +831,7 @@ void run_from_tera_battle(const ProgramInfo& info, VideoStream& stream, ProContr battle_menu, leave_confirm, overworld, + tera_card, } ); @@ -718,7 +850,19 @@ void run_from_tera_battle(const ProgramInfo& info, VideoStream& stream, ProContr case 2: stream.log("Detected overworld."); return; + case 3: + stream.log("Detected a raid. (unexpected)", COLOR_RED); + if (stat_errors){ + (*stat_errors)++; + env.update_stats(); + } + pbf_press_button(context, BUTTON_B, 160ms, 80ms); + continue; default: + if (stat_errors){ + (*stat_errors)++; + env.update_stats(); + } OperationFailedException::fire( ErrorReport::SEND_ERROR_REPORT, "run_from_tera_battle(): No recognized state after 1 minutes.", diff --git a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoutines.h b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoutines.h index 35d0b90b0b..a1cbf4252f 100644 --- a/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoutines.h +++ b/SerialPrograms/Source/PokemonSV/Programs/TeraRaids/PokemonSV_TeraRoutines.h @@ -9,15 +9,19 @@ #include #include +#include +#include #include "CommonFramework/Language.h" #include "CommonFramework/Tools/VideoStream.h" #include "NintendoSwitch/Controllers/NintendoSwitch_ProController.h" +#include "NintendoSwitch/Options/NintendoSwitch_CodeEntrySettingsOption.h" namespace PokemonAutomation{ class ProgramEnvironment; class EventNotificationOption; class ImageViewRGB32; struct ProgramInfo; + class ConsoleHandle; namespace NintendoSwitch{ namespace PokemonSV{ @@ -51,6 +55,66 @@ void enter_tera_search( ); +class RaidWaiter : public CancellableScope{ +public: + void signal_joiner_is_ready(){ + { + std::lock_guard lg(m_lock); + m_joiners_ready++; + } + m_cv.notify_all(); + } + void signal_raid_code_is_ready(std::string raid_code){ + { + std::lock_guard lg(m_lock); + m_raid_code = std::move(raid_code); + } + m_cv.notify_all(); + } + + std::string wait_for_raid_code(){ + std::unique_lock lg(m_lock); + m_cv.wait( + lg, + [&]{ + return !m_raid_code.empty() || cancelled(); + } + ); + throw_if_cancelled(); + return m_raid_code; + } + void wait_for_joiners(size_t joiners){ + std::unique_lock lg(m_lock); + m_cv.wait( + lg, + [&]{ + return m_joiners_ready >= joiners || cancelled(); + } + ); + throw_if_cancelled(); + } + virtual bool cancel(std::exception_ptr exception = nullptr) noexcept override{ + bool ret = CancellableScope::cancel(exception); + { + std::unique_lock lg(m_lock); + } + m_cv.notify_all(); + return ret; + } + +private: + size_t m_joiners_ready = 0; + std::string m_raid_code; + std::mutex m_lock; + std::condition_variable m_cv; +}; + +void join_raid( + const ProgramInfo& info, ConsoleHandle& console, ProControllerContext& context, + bool connect_to_internet, + KeyboardLayout keyboard_layout, + RaidWaiter& raid_waiter +); @@ -112,7 +176,12 @@ TeraResult run_tera_summary( ); // Run away from tera battle. -void run_from_tera_battle(const ProgramInfo& info, VideoStream& stream, ProControllerContext& context); +void run_from_tera_battle( + ProgramEnvironment& env, + VideoStream& stream, + ProControllerContext& context, + std::atomic* stat_errors +); bool is_sparkling_raid(VideoStream& stream, ProControllerContext& context); diff --git a/SerialPrograms/Source/PokemonSwSh/MaxLair/Options/PokemonSwSh_MaxLair_Options.h b/SerialPrograms/Source/PokemonSwSh/MaxLair/Options/PokemonSwSh_MaxLair_Options.h index fcb190a986..6b4e8335cf 100644 --- a/SerialPrograms/Source/PokemonSwSh/MaxLair/Options/PokemonSwSh_MaxLair_Options.h +++ b/SerialPrograms/Source/PokemonSwSh/MaxLair/Options/PokemonSwSh_MaxLair_Options.h @@ -27,6 +27,7 @@ extern const std::chrono::milliseconds INFERENCE_RATE; class HostingSwitch : public IntegerEnumDropdownOption{ public: HostingSwitch(); + using ConfigOption::check_validity; std::string check_validity(size_t consoles) const; };