feature #880 [Store] Add metadata filtering support to InMemoryStore and CacheStore (camilleislasse)

chr-hertel · chr-hertel · commit f0fbf2b93361 · 2025-11-20T23:23:41.000+01:00
This PR was merged into the main branch. Discussion ---------- [Store] Add metadata filtering support to `InMemoryStore` and `CacheStore` | Q | A | ------------- | --- | Bug fix? | no | New feature? | yes | Docs? | yes | License | MIT This PR adds metadata filtering support to local stores (InMemoryStore and CacheStore). ### Changes - Add optional filter parameter to query() method accepting a callable - Filter documents by metadata before distance calculation - Add comprehensive test coverage for filtering scenarios - Add documentation for local stores ### Usage ```php $results = $store->query($vector, [ 'filter' => fn(VectorDocument $doc) => $doc->metadata['size'] === 'S', 'maxItems' => 10, ]); ``` Commits ------- 84b3938 Add metadata filtering support to InMemoryStore and CacheStore
diff --git a/docs/components/store/local.rst b/docs/components/store/local.rst
@@ -0,0 +1,108 @@
+Local Stores (InMemory & Cache)
+===============================
+
+The local stores provide in-memory vector storage without external dependencies.
+
+.. note::
+
+    Both ``InMemoryStore`` and ``CacheStore`` load all data into PHP memory during queries.
+    The dataset must fit within PHP's memory limit.
+
+InMemoryStore
+-------------
+
+Stores vectors in a PHP array. Data is not persisted and is lost when the PHP process ends::
+
+    use Symfony\AI\Store\Bridge\Local\InMemoryStore;
+
+    $store = new InMemoryStore();
+    $store->add($document1, $document2);
+    $results = $store->query($vector);
+
+CacheStore
+----------
+
+Stores vectors using a PSR-6 cache implementation. Persistence depends on the cache adapter used::
+
+    use Symfony\AI\Store\Bridge\Local\CacheStore;
+    use Symfony\Component\Cache\Adapter\FilesystemAdapter;
+
+    $cache = new FilesystemAdapter();
+    $store = new CacheStore($cache);
+    $store->add($document1, $document2);
+    $results = $store->query($vector);
+
+Distance Strategies
+-------------------
+
+Both stores support different distance calculation strategies::
+
+    use Symfony\AI\Store\Bridge\Local\DistanceCalculator;
+    use Symfony\AI\Store\Bridge\Local\DistanceStrategy;
+
+    $calculator = new DistanceCalculator(DistanceStrategy::COSINE_DISTANCE);
+    $store = new InMemoryStore($calculator);
+
+Available strategies:
+
+* ``COSINE_DISTANCE`` (default)
+* ``EUCLIDEAN_DISTANCE``
+* ``MANHATTAN_DISTANCE``
+* ``ANGULAR_DISTANCE``
+* ``CHEBYSHEV_DISTANCE``
+
+Metadata Filtering
+------------------
+
+Both stores support filtering search results based on document metadata using a callable::
+
+    use Symfony\AI\Store\Document\VectorDocument;
+
+    $results = $store->query($vector, [
+        'filter' => fn(VectorDocument $doc) => $doc->metadata['category'] === 'products',
+    ]);
+
+You can combine multiple conditions::
+
+    $results = $store->query($vector, [
+        'filter' => fn(VectorDocument $doc) =>
+            $doc->metadata['price'] <= 100
+            && $doc->metadata['stock'] > 0
+            && $doc->metadata['enabled'] === true,
+        'maxItems' => 10,
+    ]);
+
+Filter nested metadata::
+
+    $results = $store->query($vector, [
+        'filter' => fn(VectorDocument $doc) =>
+            $doc->metadata['options']['size'] === 'S'
+            && $doc->metadata['options']['color'] === 'blue',
+    ]);
+
+Use array functions for complex filtering::
+
+    $allowedBrands = ['Nike', 'Adidas', 'Puma'];
+    $results = $store->query($vector, [
+        'filter' => fn(VectorDocument $doc) =>
+            \in_array($doc->metadata['brand'] ?? '', $allowedBrands, true),
+    ]);
+
+.. note::
+
+    Filtering is applied before distance calculation.
+
+Query Options
+-------------
+
+Both stores support the following query options:
+
+* ``maxItems`` (int) - Limit the number of results returned
+* ``filter`` (callable) - Filter documents by metadata before distance calculation
+
+Example combining both options::
+
+    $results = $store->query($vector, [
+        'maxItems' => 5,
+        'filter' => fn(VectorDocument $doc) => $doc->metadata['active'] === true,
+    ]);
diff --git a/src/store/src/Bridge/Local/CacheStore.php b/src/store/src/Bridge/Local/CacheStore.php
@@ -72,8 +72,10 @@ public function add(VectorDocument ...$documents): void
 
     /**
      * @param array{
-     *     maxItems?: positive-int
-     * } $options If maxItems is provided, only the top N results will be returned
+     *     maxItems?: positive-int,
+     *     filter?: callable(VectorDocument): bool
+     * } $options If maxItems is provided, only the top N results will be returned.
+     *            If filter is provided, only documents matching the filter will be considered.
      */
     public function query(Vector $vector, array $options = []): array
     {
@@ -85,6 +87,10 @@ public function query(Vector $vector, array $options = []): array
             metadata: new Metadata($document['metadata']),
         ), $documents);
 
+        if (isset($options['filter'])) {
+            $vectorDocuments = array_values(array_filter($vectorDocuments, $options['filter']));
+        }
+
         return $this->distanceCalculator->calculate($vectorDocuments, $vector, $options['maxItems'] ?? null);
     }
 
diff --git a/src/store/src/Bridge/Local/InMemoryStore.php b/src/store/src/Bridge/Local/InMemoryStore.php
@@ -48,12 +48,20 @@ public function add(VectorDocument ...$documents): void
 
     /**
      * @param array{
-     *     maxItems?: positive-int
-     * } $options If maxItems is provided, only the top N results will be returned
+     *     maxItems?: positive-int,
+     *     filter?: callable(VectorDocument): bool
+     * } $options If maxItems is provided, only the top N results will be returned.
+     *            If filter is provided, only documents matching the filter will be considered.
      */
     public function query(Vector $vector, array $options = []): array
     {
-        return $this->distanceCalculator->calculate($this->documents, $vector, $options['maxItems'] ?? null);
+        $documents = $this->documents;
+
+        if (isset($options['filter'])) {
+            $documents = array_values(array_filter($documents, $options['filter']));
+        }
+
+        return $this->distanceCalculator->calculate($documents, $vector, $options['maxItems'] ?? null);
     }
 
     public function drop(): void
diff --git a/src/store/tests/Bridge/Local/CacheStoreTest.php b/src/store/tests/Bridge/Local/CacheStoreTest.php
@@ -16,6 +16,7 @@
 use Symfony\AI\Store\Bridge\Local\CacheStore;
 use Symfony\AI\Store\Bridge\Local\DistanceCalculator;
 use Symfony\AI\Store\Bridge\Local\DistanceStrategy;
+use Symfony\AI\Store\Document\Metadata;
 use Symfony\AI\Store\Document\VectorDocument;
 use Symfony\Component\Cache\Adapter\ArrayAdapter;
 use Symfony\Component\Uid\Uuid;
@@ -162,4 +163,93 @@ public function testStoreCanSearchUsingChebyshevDistance()
         $this->assertCount(2, $result);
         $this->assertSame([1.0, 2.0, 3.0], $result[0]->vector->getData());
     }
+
+    public function testStoreCanSearchWithFilter()
+    {
+        $store = new CacheStore(new ArrayAdapter());
+        $store->add(
+            new VectorDocument(Uuid::v4(), new Vector([0.1, 0.1, 0.5]), new Metadata(['category' => 'products', 'enabled' => true])),
+            new VectorDocument(Uuid::v4(), new Vector([0.7, -0.3, 0.0]), new Metadata(['category' => 'articles', 'enabled' => true])),
+            new VectorDocument(Uuid::v4(), new Vector([0.3, 0.7, 0.1]), new Metadata(['category' => 'products', 'enabled' => false])),
+        );
+
+        $result = $store->query(new Vector([0.0, 0.1, 0.6]), [
+            'filter' => fn (VectorDocument $doc) => 'products' === $doc->metadata['category'],
+        ]);
+
+        $this->assertCount(2, $result);
+        $this->assertSame('products', $result[0]->metadata['category']);
+        $this->assertSame('products', $result[1]->metadata['category']);
+    }
+
+    public function testStoreCanSearchWithFilterAndMaxItems()
+    {
+        $store = new CacheStore(new ArrayAdapter());
+        $store->add(
+            new VectorDocument(Uuid::v4(), new Vector([0.1, 0.1, 0.5]), new Metadata(['category' => 'products'])),
+            new VectorDocument(Uuid::v4(), new Vector([0.7, -0.3, 0.0]), new Metadata(['category' => 'articles'])),
+            new VectorDocument(Uuid::v4(), new Vector([0.3, 0.7, 0.1]), new Metadata(['category' => 'products'])),
+            new VectorDocument(Uuid::v4(), new Vector([0.0, 0.1, 0.6]), new Metadata(['category' => 'products'])),
+        );
+
+        $result = $store->query(new Vector([0.0, 0.1, 0.6]), [
+            'filter' => fn (VectorDocument $doc) => 'products' === $doc->metadata['category'],
+            'maxItems' => 2,
+        ]);
+
+        $this->assertCount(2, $result);
+        $this->assertSame('products', $result[0]->metadata['category']);
+        $this->assertSame('products', $result[1]->metadata['category']);
+    }
+
+    public function testStoreCanSearchWithComplexFilter()
+    {
+        $store = new CacheStore(new ArrayAdapter());
+        $store->add(
+            new VectorDocument(Uuid::v4(), new Vector([0.1, 0.1, 0.5]), new Metadata(['price' => 100, 'stock' => 5])),
+            new VectorDocument(Uuid::v4(), new Vector([0.7, -0.3, 0.0]), new Metadata(['price' => 200, 'stock' => 0])),
+            new VectorDocument(Uuid::v4(), new Vector([0.3, 0.7, 0.1]), new Metadata(['price' => 50, 'stock' => 10])),
+        );
+
+        $result = $store->query(new Vector([0.0, 0.1, 0.6]), [
+            'filter' => fn (VectorDocument $doc) => $doc->metadata['price'] <= 150 && $doc->metadata['stock'] > 0,
+        ]);
+
+        $this->assertCount(2, $result);
+    }
+
+    public function testStoreCanSearchWithNestedMetadataFilter()
+    {
+        $store = new CacheStore(new ArrayAdapter());
+        $store->add(
+            new VectorDocument(Uuid::v4(), new Vector([0.1, 0.1, 0.5]), new Metadata(['options' => ['size' => 'S', 'color' => 'blue']])),
+            new VectorDocument(Uuid::v4(), new Vector([0.7, -0.3, 0.0]), new Metadata(['options' => ['size' => 'M', 'color' => 'blue']])),
+            new VectorDocument(Uuid::v4(), new Vector([0.3, 0.7, 0.1]), new Metadata(['options' => ['size' => 'S', 'color' => 'red']])),
+        );
+
+        $result = $store->query(new Vector([0.0, 0.1, 0.6]), [
+            'filter' => fn (VectorDocument $doc) => 'S' === $doc->metadata['options']['size'],
+        ]);
+
+        $this->assertCount(2, $result);
+        $this->assertSame('S', $result[0]->metadata['options']['size']);
+        $this->assertSame('S', $result[1]->metadata['options']['size']);
+    }
+
+    public function testStoreCanSearchWithInArrayFilter()
+    {
+        $store = new CacheStore(new ArrayAdapter());
+        $store->add(
+            new VectorDocument(Uuid::v4(), new Vector([0.1, 0.1, 0.5]), new Metadata(['brand' => 'Nike'])),
+            new VectorDocument(Uuid::v4(), new Vector([0.7, -0.3, 0.0]), new Metadata(['brand' => 'Adidas'])),
+            new VectorDocument(Uuid::v4(), new Vector([0.3, 0.7, 0.1]), new Metadata(['brand' => 'Generic'])),
+        );
+
+        $allowedBrands = ['Nike', 'Adidas', 'Puma'];
+        $result = $store->query(new Vector([0.0, 0.1, 0.6]), [
+            'filter' => fn (VectorDocument $doc) => \in_array($doc->metadata['brand'] ?? '', $allowedBrands, true),
+        ]);
+
+        $this->assertCount(2, $result);
+    }
 }
diff --git a/src/store/tests/Bridge/Local/InMemoryStoreTest.php b/src/store/tests/Bridge/Local/InMemoryStoreTest.php
@@ -16,6 +16,7 @@
 use Symfony\AI\Store\Bridge\Local\DistanceCalculator;
 use Symfony\AI\Store\Bridge\Local\DistanceStrategy;
 use Symfony\AI\Store\Bridge\Local\InMemoryStore;
+use Symfony\AI\Store\Document\Metadata;
 use Symfony\AI\Store\Document\VectorDocument;
 use Symfony\Component\Uid\Uuid;
 
@@ -161,4 +162,93 @@ public function testStoreCanSearchUsingChebyshevDistance()
         $this->assertCount(2, $result);
         $this->assertSame([1.0, 2.0, 3.0], $result[0]->vector->getData());
     }
+
+    public function testStoreCanSearchWithFilter()
+    {
+        $store = new InMemoryStore();
+        $store->add(
+            new VectorDocument(Uuid::v4(), new Vector([0.1, 0.1, 0.5]), new Metadata(['category' => 'products', 'enabled' => true])),
+            new VectorDocument(Uuid::v4(), new Vector([0.7, -0.3, 0.0]), new Metadata(['category' => 'articles', 'enabled' => true])),
+            new VectorDocument(Uuid::v4(), new Vector([0.3, 0.7, 0.1]), new Metadata(['category' => 'products', 'enabled' => false])),
+        );
+
+        $result = $store->query(new Vector([0.0, 0.1, 0.6]), [
+            'filter' => fn (VectorDocument $doc) => 'products' === $doc->metadata['category'],
+        ]);
+
+        $this->assertCount(2, $result);
+        $this->assertSame('products', $result[0]->metadata['category']);
+        $this->assertSame('products', $result[1]->metadata['category']);
+    }
+
+    public function testStoreCanSearchWithFilterAndMaxItems()
+    {
+        $store = new InMemoryStore();
+        $store->add(
+            new VectorDocument(Uuid::v4(), new Vector([0.1, 0.1, 0.5]), new Metadata(['category' => 'products'])),
+            new VectorDocument(Uuid::v4(), new Vector([0.7, -0.3, 0.0]), new Metadata(['category' => 'articles'])),
+            new VectorDocument(Uuid::v4(), new Vector([0.3, 0.7, 0.1]), new Metadata(['category' => 'products'])),
+            new VectorDocument(Uuid::v4(), new Vector([0.0, 0.1, 0.6]), new Metadata(['category' => 'products'])),
+        );
+
+        $result = $store->query(new Vector([0.0, 0.1, 0.6]), [
+            'filter' => fn (VectorDocument $doc) => 'products' === $doc->metadata['category'],
+            'maxItems' => 2,
+        ]);
+
+        $this->assertCount(2, $result);
+        $this->assertSame('products', $result[0]->metadata['category']);
+        $this->assertSame('products', $result[1]->metadata['category']);
+    }
+
+    public function testStoreCanSearchWithComplexFilter()
+    {
+        $store = new InMemoryStore();
+        $store->add(
+            new VectorDocument(Uuid::v4(), new Vector([0.1, 0.1, 0.5]), new Metadata(['price' => 100, 'stock' => 5])),
+            new VectorDocument(Uuid::v4(), new Vector([0.7, -0.3, 0.0]), new Metadata(['price' => 200, 'stock' => 0])),
+            new VectorDocument(Uuid::v4(), new Vector([0.3, 0.7, 0.1]), new Metadata(['price' => 50, 'stock' => 10])),
+        );
+
+        $result = $store->query(new Vector([0.0, 0.1, 0.6]), [
+            'filter' => fn (VectorDocument $doc) => $doc->metadata['price'] <= 150 && $doc->metadata['stock'] > 0,
+        ]);
+
+        $this->assertCount(2, $result);
+    }
+
+    public function testStoreCanSearchWithNestedMetadataFilter()
+    {
+        $store = new InMemoryStore();
+        $store->add(
+            new VectorDocument(Uuid::v4(), new Vector([0.1, 0.1, 0.5]), new Metadata(['options' => ['size' => 'S', 'color' => 'blue']])),
+            new VectorDocument(Uuid::v4(), new Vector([0.7, -0.3, 0.0]), new Metadata(['options' => ['size' => 'M', 'color' => 'blue']])),
+            new VectorDocument(Uuid::v4(), new Vector([0.3, 0.7, 0.1]), new Metadata(['options' => ['size' => 'S', 'color' => 'red']])),
+        );
+
+        $result = $store->query(new Vector([0.0, 0.1, 0.6]), [
+            'filter' => fn (VectorDocument $doc) => 'S' === $doc->metadata['options']['size'],
+        ]);
+
+        $this->assertCount(2, $result);
+        $this->assertSame('S', $result[0]->metadata['options']['size']);
+        $this->assertSame('S', $result[1]->metadata['options']['size']);
+    }
+
+    public function testStoreCanSearchWithInArrayFilter()
+    {
+        $store = new InMemoryStore();
+        $store->add(
+            new VectorDocument(Uuid::v4(), new Vector([0.1, 0.1, 0.5]), new Metadata(['brand' => 'Nike'])),
+            new VectorDocument(Uuid::v4(), new Vector([0.7, -0.3, 0.0]), new Metadata(['brand' => 'Adidas'])),
+            new VectorDocument(Uuid::v4(), new Vector([0.3, 0.7, 0.1]), new Metadata(['brand' => 'Generic'])),
+        );
+
+        $allowedBrands = ['Nike', 'Adidas', 'Puma'];
+        $result = $store->query(new Vector([0.0, 0.1, 0.6]), [
+            'filter' => fn (VectorDocument $doc) => \in_array($doc->metadata['brand'] ?? '', $allowedBrands, true),
+        ]);
+
+        $this->assertCount(2, $result);
+    }
 }
