perf: use caching and indexing to increase object relation performance

This commit implements model and query caching/indexing that will help reduce the number of objects that need to be reloaded unnecessarily. This should provide a big performance improvement for instances with large pfSense configurations.

Author: jaredhendrickson13

pfSense-pkg-RESTAPI/files/usr/local/pkg/RESTAPI/Core/Model.inc

@@ -1992,11 +1992,11 @@ class Model {
         }
 
         # Load the ModelSet with all obtained Model objects
-        $modelset = new ModelSet(model_objects: $model_objects);
+        $modelset = new ModelSet(model_objects: $model_objects, signature: $model_name);
 
         # For many models, cache the ModelSet if not exempt
         if ($model->many and !$cache_exempt) {
-            self::get_model_cache()::cache_modelset($model_name, $modelset);
+            self::get_model_cache()::cache_modelset($modelset);
         }
 
         # For many enabled Models return a ModelSet, otherwise return a single Model object

pfSense-pkg-RESTAPI/files/usr/local/pkg/RESTAPI/Core/ModelCache.inc

@@ -20,6 +20,18 @@ class ModelCache {
      */
     public static ?self $instance = null;
 
+    /**
+     * @var array $index An associative array that contains cache Model objects that have been loaded and
+     * indexed by specific fields for quick lookup. Structured as:
+     *
+     * [Model class name] => [
+     *     [index field name] => [
+     *         [index field value] => Model object
+     *     ]
+     * ]
+     */
+    public static array $index = [];
+
     /**
      * @var array $cache An associative array that contains cached ModelSet objects that have been loaded into memory.
      * This includes root ModelSets that contain all Model objects of a given Model class as well as indexed queries
@@ -41,43 +53,185 @@ class ModelCache {
     }
 
     /**
-     * Checks if a given Model class has a cached ModelSet.
-     * @param string $model_class The Model class name to check in the cache.
-     * @return bool True if a cached ModelSet exists for the specified Model class, false otherwise.
+     * Checks if a cached ModelSet exists for a specified signature.
+     * @param string $signature The signature of the ModelSet to check for in the cache.
+     * @return bool True if a cached ModelSet exists with the specified signature, false otherwise.
      */
-    public static function has_modelset(string $model_class): bool {
-        return isset(self::$cache[$model_class]);
+    public static function has_modelset(string $signature): bool {
+        return isset(self::$cache[$signature]);
     }
 
     /**
-     * Caches a ModelSet in the ModelCache.
-     * @param string $model_class The Model class name of the ModelSet to cache.
+     * Caches a ModelSet in the ModelCache by its signature.
      * @param ModelSet $model_set The ModelSet object to cache.
      */
-    public static function cache_modelset(string $model_class, ModelSet $model_set): void {
-        self::$cache[$model_class] = $model_set;
+    public static function cache_modelset(ModelSet $model_set): void {
+        # Do not allow ModelSets to be cached without a valid signature
+        if (!$model_set->signature) {
+            throw new ServerError(
+                message: 'ModelSet cannot be cached without a valid signature.',
+                response_id: 'MODEL_CACHE_MODELSET_MISSING_SIGNATURE',
+            );
+        }
+
+        # Index the ModelSet in the cache by its signature
+        self::$cache[$model_set->signature] = $model_set;
     }
 
     /**
      * Fetches a cached ModelSet by its Model class name.
-     * @param string $model_class The Model class name to load from the cache.
+     * @param string $signature The signature of the ModelSet to fetch from the cache.
      * @return ModelSet The cached ModelSet
-     * @throws NotFoundError If no cached ModelSet exists for the specified Model class.
+     * @throws NotFoundError If no cached ModelSet exists for the specified signature.
      */
-    public static function fetch_modelset(string $model_class): ModelSet {
-        if (!self::has_modelset($model_class)) {
+    public static function fetch_modelset(string $signature): ModelSet {
+        if (!self::has_modelset($signature)) {
             throw new NotFoundError(
-                message: "No cached ModelSet found for Model class '$model_class'.",
+                message: "No cached ModelSet found with signature '$signature'.",
                 response_id: 'MODEL_CACHE_MODELSET_NOT_FOUND',
             );
         }
-        return self::$cache[$model_class];
+        return self::$cache[$signature];
+    }
+
+    /**
+     * Raises an error if the given Model object does not support being indexed by the specified field.
+     * @param Model $model The Model object to check for uniqueness.
+     * @param string $index_field The index field name to check for uniqueness.
+     * @throws ServerError If the Model is attempting to be indexed by a non-unique field.
+     * @throws ServerError If the Model is not many-enabled.
+     */
+    private static function ensure_model_supports_indexing(Model $model, string $index_field): void {
+        # Only many-enabled Models can be indexed
+        if (!$model->many) {
+            throw new ServerError(
+                message: "Cannot index Model class '" . $model->get_class_fqn() . 'because it is not many-enabled.',
+                response_id: 'MODEL_CACHE_INDEX_FIELD_ON_NON_MANY_MODEL',
+            );
+        }
+
+        # Models with parent model classes cannot be indexed
+        if ($model->parent_model_class) {
+            throw new ServerError(
+                message: "Cannot index Model class '" .
+                $model->get_class_fqn() .
+                "' because it has a parent model class '" .
+                $model->parent_model_class .
+                "'.",
+                response_id: 'MODEL_CACHE_INDEX_FIELD_ON_PARENTED_MODEL',
+            );
+        }
+
+        # If indexing by 'id', it's always unique
+        if ($index_field === 'id') {
+            return;
+        }
+
+        # Check if the index field is unique on the Model object
+        if (!$model->$index_field->unique) {
+            throw new ServerError(
+                message: "Cannot index Model class '" .
+                $model->get_class_fqn() .
+                "' by non-unique field " .
+                "'$index_field'.",
+                response_id: 'MODEL_CACHE_INDEX_FIELD_NOT_UNIQUE',
+            );
+        }
+    }
+
+    /**
+     * Indexes the ModelSet cache for a given Model class by a specified field. This method will populate
+     * the $index array for the specified Model class and index field.
+     * @param string $model_class The Model class name whose ModelSet is to be indexed. If no cached ModelSet exists,
+     * one will be created by reading all Model objects from the data source.
+     * @param string $index_field The field name to index the Model objects by.
+     * @return array An associative array of indexed Model objects.
+     */
+    public static function index_modelset_by_field(string $model_class, string $index_field): array {
+        # First, check if this Model class has already been indexed by this field
+        if (isset(self::$index[$model_class][$index_field])) {
+            return self::$index[$model_class][$index_field];
+        }
+
+        # Ensure the Model can be indexed by the specified field
+        $model = new $model_class();
+        self::ensure_model_supports_indexing($model, $index_field);
+
+        # Fetch or create the ModelSet for this Model class
+        $model_set = $model->read_all();
+
+        # Create an associative array to hold the indexed Model objects
+        $indexed_models = [];
+
+        # Index each Model object in the ModelSet by the specified field
+        foreach ($model_set->model_objects as $model) {
+            $index_value = $index_field === 'id' ? $model->id : $model->$index_field->value;
+            $indexed_models[$index_value] = $model;
+        }
+
+        # Store the indexed models in the ModelCache index and return them
+        self::$index[$model_class][$index_field] = $indexed_models;
+        return $indexed_models;
+    }
+
+    /**
+     * Checks if a given Model class has a cached Model object by its index field/value.
+     * @param string $model_class The Model class name to check in the cache.
+     * @param string $index_field The index field name to look up the Model object by
+     * @param mixed $index_value The index field value to look up the Model object by
+     * @return bool True if a cached Model object exists for the specified Model class and index
+     */
+    public static function has_model(string $model_class, string $index_field = 'id', mixed $index_value = null): bool {
+        # Cache is always a miss if the Model class is not indexed
+        if (!self::$index[$model_class]) {
+            return false;
+        }
+
+        # Cache is a hit for non-many enabled models if a single Model object is indexed under the Model class
+        if (self::$index[$model_class] instanceof Model) {
+            return true;
+        }
+
+        # Otherwise, cache is only a hit for many enabled models if the index field/value exists in the index
+        return isset(self::$index[$model_class][$index_field][$index_value]);
+    }
+
+    /**
+     * Fetches a cached Model object by its Model class name and index field/value.
+     * @param string $model_class The Model class name to load from the cache.
+     * @param string $index_field The index field name to look up the Model object by. Only for many enabled Models.
+     * @param mixed $index_value The index field value to look up the Model object by. Only for many enabled Models.
+     * @return Model The cached Model object
+     * @throws NotFoundError If no cached Model object exists for the specified Model class and index.
+     */
+    public static function fetch_model(
+        string $model_class,
+        string $index_field = 'id',
+        mixed $index_value = null,
+    ): Model {
+        if (!self::has_model($model_class, $index_field, $index_value)) {
+            throw new NotFoundError(
+                message: "No cached Model found for Model class '$model_class' with " .
+                "index field '$index_field' and index value '$index_value'.",
+                response_id: 'MODEL_CACHE_MODEL_NOT_FOUND',
+            );
+        }
+
+        # For many enabled models, return the Model object indexed under the specified index field and value
+        if (isset(self::$index[$model_class][$index_field][$index_value])) {
+            return self::$index[$model_class][$index_field][$index_value];
+        }
+
+        # Otherwise, return the Model object indexed under the Model class only
+        return self::$index[$model_class];
     }
 
+
     /**
      * Clears the ModelCache of all cached ModelSets and indexed Model objects.
      */
     public static function clear(): void {
         self::$cache = [];
+        self::$index = [];
     }
 }

pfSense-pkg-RESTAPI/files/usr/local/pkg/RESTAPI/Core/ModelSet.inc

@@ -19,7 +19,7 @@ class ModelSet {
      * a series of query signatures that represent each query that has been applied to this ModelSet to this point
      * (separated by a colon). Example 'RESTAPI\Models\User:queryhash1:queryhash2'
      */
-    public string $signature = '';
+    public string $signature;
 
     /**
      * @var array $model_objects An array of Model objects contained by this ModelSet.
@@ -30,7 +30,10 @@ class ModelSet {
      * Creates a ModelSet object that contains multiple Model objects.
      * @param array $model_objects An array of model objects to include in this model set
      */
-    public function __construct(array $model_objects = []) {
+    public function __construct(array $model_objects = [], string $signature = '') {
+        # Assign the signature if provided
+        $this->signature = $signature;
+
         # Throw an error if any Models are not a Model object
         foreach ($model_objects as $model_object) {
             if (!is_object($model_object) or !in_array('RESTAPI\Core\Model', class_parents($model_object))) {
@@ -180,6 +183,38 @@ class ModelSet {
         return $results;
     }
 
+    /**
+     * Provides access to the ModelCache singleton instance.
+     * @return ModelCache The ModelCache singleton instance.
+     */
+    public static function get_model_cache(): ModelCache {
+        return ModelCache::get_instance();
+    }
+
+    /**
+     * Determines the signature for a given query. This signature is used to index queried ModelSets in the
+     * ModelCache while retaining all query chains.
+     * @param array $query_params An associative array of query parameters to use to generate the signature.
+     * @return string The signature for the given query parameters.
+     */
+    public function get_query_signature(array $query_params = []): string
+    {
+        # ModelSet is cache exempt if there is no existing signature
+        if (!$this->signature) {
+            return '';
+        }
+
+        # Sort the query parameters by key to ensure consistent signatures
+        ksort($query_params);
+
+        # Encode the query parameters as JSON and generate an xxh hash
+        $query_json = json_encode($query_params);
+        $query_hash = hash(algo: 'xxh64', data: $query_json);
+
+        # Append the query hash to the existing signature to retain query chains
+        return "$this->signature:$query_hash";
+    }
+
     /**
      * Filters the ModelSet to only include Model objects that match a specific query.
      * @param array $query_params An associative array of query targets and values. The array key will be the query
@@ -193,9 +228,14 @@ class ModelSet {
     public function query(array $query_params = [], array $excluded = [], ...$vl_query_params): ModelSet {
         # Variables
         $queried_model_set = [];
-
-        # Merge the $query_params and any provided variable-length arguments into a single variable
         $query_params = array_merge($query_params, $vl_query_params);
+        $query_signature = $this->get_query_signature($query_params);
+        $cache_exempt = !$query_signature;
+
+        # Fetch the cached query if it exists and this ModelSet is not cache exempt
+        if (!$cache_exempt and self::get_model_cache()::has_modelset($query_signature)) {
+            return self::get_model_cache()::fetch_modelset($query_signature);
+        }
 
         # Loop through each model object in the provided model set and check it against the query parameters
         foreach ($this->model_objects as $model_object) {
@@ -230,7 +270,16 @@ class ModelSet {
                 $queried_model_set[] = $model_object;
             }
         }
-        return new ModelSet($queried_model_set);
+
+        # Create a new ModelSet with the queried model objects
+        $modelset = new ModelSet($queried_model_set, signature: $query_signature);
+
+        # Cache the queried ModelSet if this ModelSet is not cache exempt
+        if (!$cache_exempt) {
+            self::get_model_cache()::cache_modelset($modelset);
+        }
+
+        return $modelset;
     }
 
     /**