Skip to content

Fix SV Auction Farmer reading currencies and not being able to check all offers #572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mysticial merged 4 commits into from
Apr 11, 2025
Merged

Fix SV Auction Farmer reading currencies and not being able to check all offers #572

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5118981
Fix trying to read currencies as numbers
fyex Apr 4, 2025
ea3ec1c
read both lines in auction offers
fyex Apr 5, 2025
996abec
clamp vertical padding and cutoffs
fyex Apr 9, 2025
a4ef715
only check bottom line for languages that need it
fyex Apr 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 117 additions & 43 deletions SerialPrograms/Source/PokemonSV/Programs/Farming/PokemonSV_AuctionFarmer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "CommonFramework/ProgramStats/StatsTracking.h"
#include "CommonFramework/VideoPipeline/VideoFeed.h"
#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
#include "CommonTools/ImageMatch/ImageCropper.h"
#include "CommonTools/Images/BinaryImage_FilterRgb32.h"
#include "CommonTools/OCR/OCR_NumberReader.h"
#include "CommonTools/Async/InferenceRoutines.h"
Expand Down Expand Up @@ -208,7 +209,7 @@ std::vector<std::pair<AuctionOffer, ImageFloatBox>> AuctionFarmer::check_offers(
VideoSnapshot screen = env.console.video().snapshot();
std::vector<ImagePixelBox> dialog_boxes = detect_dialog_boxes(screen);
std::deque<OverlayBoxScope> bubbles_boxes;
std::deque<OverlayBoxScope> offer_boxes;
std::deque<OverlayBoxScope> offer_overlay_boxes;
std::vector<std::pair<AuctionOffer, ImageFloatBox>> offers;

if (dialog_boxes.empty()){
Expand All @@ -217,57 +218,65 @@ std::vector<std::pair<AuctionOffer, ImageFloatBox>> AuctionFarmer::check_offers(
}

// read dialog bubble
ImageFloatBox top_offer_box(0.05, 0.02, 0.90, 0.49);
ImageFloatBox bottom_offer_box(0.05, 0.49, 0.90, 0.49);
std::vector<ImageFloatBox> offer_boxes = {top_offer_box};
if (LANGUAGE == Language::Spanish || LANGUAGE == Language::ChineseTraditional) {
offer_boxes.emplace_back(bottom_offer_box);
}

for (ImagePixelBox dialog_box : dialog_boxes){
// std::cout << "dialog_box: ["
// << dialog_box.min_x << "," << dialog_box.min_y << "] - ["
// << dialog_box.max_x << "," << dialog_box.max_y << "]" << std::endl;
for (ImageFloatBox offer_box : offer_boxes) {
// std::cout << "dialog_box: ["
// << dialog_box.min_x << "," << dialog_box.min_y << "] - ["
// << dialog_box.max_x << "," << dialog_box.max_y << "]" << std::endl;

ImageFloatBox dialog_float_box = pixelbox_to_floatbox(screen, dialog_box);
bubbles_boxes.emplace_back(env.console, dialog_float_box, COLOR_GREEN);
ImageFloatBox dialog_float_box = pixelbox_to_floatbox(screen, dialog_box);
bubbles_boxes.emplace_back(env.console, dialog_float_box, COLOR_GREEN);


// OverlayBoxScope dialog_overlay(env.console, dialog_box, COLOR_DARK_BLUE);
ImageFloatBox offer_box(0.05, 0.02, 0.90, 0.49);
ImageFloatBox translated_offer_box = translate_to_parent(
screen,
dialog_float_box,
floatbox_to_pixelbox(dialog_box.width(), dialog_box.height(), offer_box)
);
// std::cout << "translated_offer_box: ["
// << translated_offer_box.x << "," << translated_offer_box.y << "] - ["
// << translated_offer_box.width << "," << translated_offer_box.height << "]" << std::endl;
// OverlayBoxScope dialog_overlay(env.console, dialog_box, COLOR_DARK_BLUE);
ImageFloatBox translated_offer_box = translate_to_parent(
screen,
dialog_float_box,
floatbox_to_pixelbox(dialog_box.width(), dialog_box.height(), offer_box)
);
// std::cout << "translated_offer_box: ["
// << translated_offer_box.x << "," << translated_offer_box.y << "] - ["
// << translated_offer_box.width << "," << translated_offer_box.height << "]" << std::endl;

offer_boxes.emplace_back(env.console, translated_offer_box, COLOR_BLUE);
offer_overlay_boxes.emplace_back(env.console, translated_offer_box, COLOR_BLUE);

// OverlayBoxScope offer_overlay(env.console, translated_offer_box, COLOR_BLUE);

ImageViewRGB32 dialog = extract_box_reference(screen, dialog_box);
ImageViewRGB32 offer_image = extract_box_reference(dialog, offer_box);
// OverlayBoxScope offer_overlay(env.console, translated_offer_box, COLOR_BLUE);

// std::cout << offer_image.width() << " x " << offer_image.height() << std::endl;
ImageViewRGB32 dialog = extract_box_reference(screen, dialog_box);
ImageViewRGB32 offer_image = extract_box_reference(dialog, offer_box);

// std::cout << offer_image.width() << " x " << offer_image.height() << std::endl;

const double LOG10P_THRESHOLD = -1.5;
std::string best_item;
OCR::StringMatchResult result = AuctionItemNameReader::instance().read_substring(
env.console, LANGUAGE,
offer_image,
OCR::BLACK_TEXT_FILTERS()
);

result.clear_beyond_log10p(LOG10P_THRESHOLD);
if (best_item.empty() && !result.results.empty()){
auto iter = result.results.begin();
if (iter->first < LOG10P_THRESHOLD){
best_item = iter->second.token;
const double LOG10P_THRESHOLD = -1.5;
std::string best_item;
OCR::StringMatchResult result = AuctionItemNameReader::instance().read_substring(
env.console, LANGUAGE,
offer_image,
OCR::BLACK_TEXT_FILTERS()
);

result.clear_beyond_log10p(LOG10P_THRESHOLD);
if (best_item.empty() && !result.results.empty()) {
auto iter = result.results.begin();
if (iter->first < LOG10P_THRESHOLD) {
best_item = iter->second.token;

AuctionOffer offer{ best_item };
std::pair<AuctionOffer, ImageFloatBox> pair(offer, dialog_float_box);
offers.emplace_back(pair);
AuctionOffer offer{ best_item };
std::pair<AuctionOffer, ImageFloatBox> pair(offer, dialog_float_box);
offers.emplace_back(pair);
}
}
}
}
// context.wait_for(std::chrono::seconds(100));
// context.wait_for(std::chrono::seconds(100));
return offers;
}

Expand Down Expand Up @@ -366,17 +375,82 @@ void AuctionFarmer::reset_position(SingleSwitchProgramEnvironment& env, ProContr
}


uint64_t read_next_bid(VideoStream& stream, ProControllerContext& context, bool high){
uint64_t read_next_bid(VideoStream& stream, ProControllerContext& context, Language language, bool high){
// How much to cut off from the bid text in order to not read currencies and exclamation marks as numbers.
// Values are pixels for a 1920x1080 screen, negative values are padding
static const std::map<Language, std::pair<int8_t, int8_t>> cutoffs = {
{ Language::English, {22, 6} },
{ Language::Japanese, {-5, 54} },
{ Language::Spanish, {6, 32} },
{ Language::French, {-5, 45} },
{ Language::German, {-5, 22} },
{ Language::Italian, {-5, 35} },
{ Language::Korean, {-5, 42} },
{ Language::ChineseSimplified, {22, 7} },
{ Language::ChineseTraditional, {22,7} }
};


static const std::map<Language, float> high_x = {
{ Language::English, 0.75f },
{ Language::Japanese, 0.75f },
{ Language::Spanish, 0.73f },
{ Language::French, 0.68f },
{ Language::German, 0.73f },
{ Language::Italian, 0.75f },
{ Language::Korean, 0.75f },
{ Language::ChineseSimplified, 0.75f },
{ Language::ChineseTraditional, 0.75f }
};

static const std::map<Language, float> low_x = {
{ Language::English, 0.75f },
{ Language::Japanese, 0.75f },
{ Language::Spanish, 0.75f },
{ Language::French, 0.75f },
{ Language::German, 0.73f },
{ Language::Italian, 0.75f },
{ Language::Korean, 0.75f },
{ Language::ChineseSimplified, 0.75f },
{ Language::ChineseTraditional, 0.75f }
};

float box_y = high ? 0.42f : 0.493f;
OverlayBoxScope box(stream.overlay(), { 0.73, box_y, 0.17, 0.048 });
float box_x = high ? high_x.at(language) : low_x.at(language);
float width = 0.9f - box_x; // max_x is always the same for all languages
OverlayBoxScope box(stream.overlay(), { box_x, box_y, width, 0.048 });

std::unordered_map<uint64_t, size_t> read_bids;
size_t highest_read = 0;
uint64_t read_value = 0;

// read next bid multiple times since the selection arrow sometimes blocks the first digit
for (size_t i = 0; i < 10; i++){
VideoSnapshot screen = stream.video().snapshot();
uint64_t read_bid = OCR::read_number(stream.logger(), extract_box_reference(screen, box));
double screen_scale = (double)screen->width() / 1920.0;
double vertical_padding = 5.0; // small amount of pixels so numbers do not touch the edge of the view when reading them

ImageViewRGB32 raw_bid_image = extract_box_reference(screen, box);
ImagePixelBox bid_bounding_box = ImageMatch::enclosing_rectangle_with_pixel_filter(
raw_bid_image,
[](Color pixel) {
return (uint32_t)pixel.red() + pixel.green() + pixel.blue() < 250;
});

int32_t max_width = static_cast<int32_t>(raw_bid_image.width() - 1);
int32_t max_height = static_cast<int32_t>(raw_bid_image.height() - 1);
int32_t scaled_vertical_padding = static_cast<int32_t>(vertical_padding * screen_scale);
int32_t left_cutoff = static_cast<int32_t>(cutoffs.at(language).first * screen_scale);
int32_t right_cutoff = static_cast<int32_t>(cutoffs.at(language).second * screen_scale);

ImagePixelBox cut_bid_bounding_box(
std::max(0, std::min(max_width, static_cast<int32_t>(bid_bounding_box.min_x) + left_cutoff)),
std::max(0, std::min(max_height, static_cast<int32_t>(bid_bounding_box.min_y) - scaled_vertical_padding)),
std::max(0, std::min(max_width, static_cast<int32_t>(bid_bounding_box.max_x) - right_cutoff)),
std::max(0, std::min(max_height, static_cast<int32_t>(bid_bounding_box.max_y) + scaled_vertical_padding))
);

uint64_t read_bid = OCR::read_number(stream.logger(), extract_box_reference(raw_bid_image, cut_bid_bounding_box));

if (read_bids.find(read_bid) == read_bids.end()){
read_bids[read_bid] = 0;
Expand Down Expand Up @@ -418,11 +492,11 @@ void AuctionFarmer::bid_on_item(SingleSwitchProgramEnvironment& env, ProControll
pbf_press_button(context, BUTTON_A, 20, TICKS_PER_SECOND);
break;
case 1:
current_bid = read_next_bid(env.console, context, true);
current_bid = read_next_bid(env.console, context, LANGUAGE, true);
pbf_press_button(context, BUTTON_A, 20, TICKS_PER_SECOND);
break;
case 2:
current_bid = read_next_bid(env.console, context, false);
current_bid = read_next_bid(env.console, context, LANGUAGE, false);
pbf_press_button(context, BUTTON_A, 20, TICKS_PER_SECOND);
break;
case 3:
Expand Down