adding memory usage on macOS and fix Model Cache path

Author: Gin

Common/Cpp/MemoryUtilization/MemoryUtilization.cpp

@@ -0,0 +1,76 @@
+#include <iostream>
+
+#if defined(__linux) || defined(__APPLE__)
+// get process memory and total system memory for Linux and macOS
+#include <libproc.h>
+#include <sys/proc_info.h>
+#include <unistd.h> // For getpid()
+#endif
+
+#if defined(__APPLE__)
+// get total used system memory for macOS
+#include <mach/mach_host.h>
+#include <mach/vm_statistics.h>
+#endif
+
+#include "MemoryUtilization.h"
+
+namespace PokemonAutomation{
+
+
+
+void print_macos_total_used_memory() {
+
+}
+
+
+MemoryUsage process_memory_usage(){
+    MemoryUsage usage;
+
+#if defined(__linux) || defined(__APPLE__)
+    pid_t pid = getpid();
+    struct proc_taskinfo task_info;
+    int ret = proc_pidinfo(pid, PROC_PIDTASKINFO, 0, &task_info, sizeof(task_info));
+
+    if (ret <= 0) {
+        std::cerr << "Error getting process info for PID " << pid << ": " << strerror(errno) << std::endl;
+        return usage;
+    }
+
+    usage.process_physical_memory = task_info.pti_resident_size;
+    usage.process_virtual_memory = task_info.pti_virtual_size;
+    usage.total_system_memory = task_info.pti_total_system;
+
+#if defined(__APPLE__) // compute total used system memory on macOS
+    vm_size_t page_size;
+    vm_statistics_data_t vm_stats;
+    mach_port_t mach_port = mach_host_self();
+    mach_msg_type_number_t count = sizeof(vm_stats) / sizeof(integer_t);
+    
+    // Get the host statistics
+    if (KERN_SUCCESS != host_statistics(mach_port, HOST_VM_INFO, (host_info_t)&vm_stats, &count)) {
+        std::cerr << "Failed to get host statistics." << std::endl;
+        return usage;
+    }
+    // Get the system's page size
+    host_page_size(mach_port, &page_size);
+
+    // Calculate used memory from vm_statistics
+    // Used memory = Wired + Active + Inactive
+    size_t wired_pages = vm_stats.wire_count;
+    size_t active_pages = vm_stats.active_count;
+    size_t inactive_pages = vm_stats.inactive_count;
+
+    usage.total_used_system_memory = (wired_pages + active_pages + inactive_pages) * page_size;
+#else  // compute total used system memory on Linux
+    // TODO
+#endif
+
+#else // compute memory info on Windows
+    // TODO
+#endif
+    return usage;
+}
+
+
+}

Common/Cpp/MemoryUtilization/MemoryUtilization.h

@@ -0,0 +1,29 @@
+/*  Memory Utilization
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#ifndef PokemonAutomation_MemoryUtilization_H
+#define PokemonAutomation_MemoryUtilization_H
+
+#include <cstddef>
+
+namespace PokemonAutomation{
+
+
+struct MemoryUsage{
+    size_t total_system_memory = 0; // in bytes
+    size_t total_used_system_memory = 0;  // in bytes
+    size_t process_physical_memory = 0;  // in bytes
+    size_t process_virtual_memory = 0;  // in bytes
+};
+
+
+MemoryUsage process_memory_usage();
+
+
+}
+
+
+#endif

SerialPrograms/CMakeLists.txt

@@ -146,6 +146,8 @@ file(GLOB MAIN_SOURCES
     ../Common/Cpp/LifetimeSanitizer.cpp
     ../Common/Cpp/LifetimeSanitizer.h
     ../Common/Cpp/ListenerSet.h
+    ../Common/Cpp/MemoryUtilization/MemoryUtilization.cpp
+    ../Common/Cpp/MemoryUtilization/MemoryUtilization.h
     ../Common/Cpp/Options/BatchOption.cpp
     ../Common/Cpp/Options/BatchOption.h
     ../Common/Cpp/Options/BooleanCheckBoxOption.cpp

SerialPrograms/Source/CommonFramework/Globals.cpp

@@ -205,5 +205,10 @@ const std::string& ML_ANNOTATION_PATH(){
     return path;
 }
 
+const std::string& ML_MODEL_CACHE_PATH(){
+    static const std::string path = RUNTIME_BASE_PATH() + "ModelCache/";
+    return path;
+}
+
 }
 

SerialPrograms/Source/CommonFramework/Globals.h

@@ -62,6 +62,10 @@ const std::string& TRAINING_PATH();
 
 // Folder path (end with "/") to hold data annotation for ML labeling programs
 const std::string& ML_ANNOTATION_PATH();
+// Folder path (end with "/") to hold model cache for model inferences. This is only used on macOS
+// for the Apple CoreML model acceleration framework can create model cache for faster model inference
+// sessions.
+const std::string& ML_MODEL_CACHE_PATH();
 
 
 enum class ProgramState{

SerialPrograms/Source/CommonFramework/Main.cpp

@@ -85,7 +85,6 @@ int main(int argc, char *argv[]){
             return 1;
         }
 
-        std::cout << "Loading from program setting JSON: " << PROGRAM_SETTING_JSON_PATH() << std::endl;
         PERSISTENT_SETTINGS().read();
 
         if (!migrate_stats(logger)){

SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp

@@ -14,6 +14,7 @@
 #include <opencv2/imgcodecs.hpp>
 #include <opencv2/imgproc.hpp>
 #include "3rdParty/ONNX/OnnxToolsPA.h"
+#include "CommonFramework/Globals.h"
 #include "ML/Models/ML_ONNXRuntimeHelpers.h"
 #include "ML_SegmentAnythingModelConstants.h"
 #include "ML_SegmentAnythingModel.h"
@@ -24,7 +25,8 @@ namespace ML{
 
 
 SAMEmbedderSession::SAMEmbedderSession(const std::string& model_path)
-    : session{create_session(model_path, "SAMEmbedder")}
+    : m_session_options{create_session_options(ML_MODEL_CACHE_PATH() + "SAMEmbedder/")}
+    , session{create_session(m_env, m_session_options, model_path, ML_MODEL_CACHE_PATH() + "SAMEmbedder/")}
     , memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)}
     , input_names{session.GetInputNames()}
     , output_names{session.GetOutputNames()}
@@ -63,7 +65,8 @@ void SAMEmbedderSession::run(cv::Mat& input_image, std::vector<float>& model_out
 
 
 SAMSession::SAMSession(const std::string& model_path)
-    : session{create_session(model_path, "SAM")}
+    : m_session_options{create_session_options(ML_MODEL_CACHE_PATH() + "SAM/")}
+    , session{create_session(m_env, m_session_options, model_path, ML_MODEL_CACHE_PATH() + "SAM/")}
     , memory_info{Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU)}
     , input_names{session.GetInputNames()}
     , output_names{session.GetOutputNames()}

SerialPrograms/Source/ML/DataLabeling/ML_SegmentAnythingModel.h

@@ -36,6 +36,8 @@ class SAMEmbedderSession{
     void run(cv::Mat& input_image, std::vector<float>& output_image_embedding);
 
 private:
+    Ort::Env m_env;
+    Ort::SessionOptions m_session_options;
     Ort::Session session;
     Ort::MemoryInfo memory_info;
     Ort::RunOptions run_options;
@@ -69,6 +71,8 @@ class SAMSession{
         const std::vector<int>& input_box,
         std::vector<bool>& output_boolean_mask);
 private:
+    Ort::Env m_env;
+    Ort::SessionOptions m_session_options;
     Ort::Session session;
     Ort::MemoryInfo memory_info;
     Ort::RunOptions run_options;

SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp

@@ -24,7 +24,6 @@ namespace fs = std::filesystem;
 namespace PokemonAutomation{
 namespace ML{
 
-const char* MODEL_CACHE_FOLDER = "./ModelCache/";
 
 // Computes the cryptographic hash of a file.
 std::string create_file_hash(const std::string& filepath){
@@ -52,6 +51,7 @@ Ort::SessionOptions create_session_options(const std::string& model_cache_path){
     // "NeuralNetwork" is a faster ModelFormat than "MLProgram".
     provider_options["ModelFormat"] = std::string("NeuralNetwork");
     provider_options["ModelCacheDirectory"] = model_cache_path;
+
     // provider_options["MLComputeUnits"] = "ALL";
     // provider_options["RequireStaticInputShapes"] = "0";
     // provider_options["EnableOnSubgraphs"] = "0";
@@ -113,14 +113,13 @@ void write_cache_flag_file(const std::string& model_cache_path, const std::strin
 }
 
 
-Ort::Session create_session(const std::string& model_path, const std::string& cache_folder_name){
-    const std::string model_cache_path = MODEL_CACHE_FOLDER + cache_folder_name;
-    Ort::SessionOptions so = create_session_options(model_cache_path);
+Ort::Session create_session(const Ort::Env& env, const Ort::SessionOptions& so,
+    const std::string& model_path, const std::string& model_cache_path)
+{
     bool write_flag_file = true;
     std::string file_hash;
     std::tie(write_flag_file, file_hash) = clean_up_old_model_cache(model_cache_path, model_path);
 
-    Ort::Env env;
     Ort::Session session{env, str_to_onnx_str(model_path).c_str(), so};
     // when Ort::Ssssion is created, if possible, it will create a model cache
     if (write_flag_file){

SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.h

@@ -16,14 +16,21 @@
 namespace PokemonAutomation{
 namespace ML{
 
-// Create an ONNX Session.
-// cache_folder_name: the folder name in under ./ModelCache/ to store model caches. This name is better
-//   to be unique for each model for easier file management.
-//
+// Create an ONNX SessionOptions
 // If on macOS, will use CoreML as the backend.
 // Otherwise, use CPU to run the model.
 // TODO: add Cuda backend for Windows machine.
-Ort::Session create_session(const std::string& model_path, const std::string& cache_folder_name);
+//
+// model_cache_path: the path to store model caches. This path is better
+//   to be unique for each model for easier file management.
+Ort::SessionOptions create_session_options(const std::string& model_cache_path);
+
+
+// Create an ONNX Session. It will also update the model cache on macOS if necessary.
+// model_cache_path: the path to store model caches. This path must be the same path
+//   used in `create_session_options()` to construct the passed-in session options so.
+Ort::Session create_session(const Ort::Env& env, const Ort::SessionOptions& so,
+    const std::string& model_path, const std::string& model_cache_path);
 
 // Handy function to create an ONNX Runtime tensor view class from a vector-like `buffer` object holding
 // the tensor data and an array-like `shape` object that represents the dimension of the tensor.