feat: add chunkRows() method in models (#9962)

Author: patel-vansh

system/BaseModel.php

@@ -586,6 +586,22 @@ abstract public function countAllResults(bool $reset = true, bool $test = false)
      */
     abstract public function chunk(int $size, Closure $userFunc);
 
+    /**
+     * Loops over records in batches, allowing you to operate on each chunk at a time.
+     * This method works only with DB calls.
+     *
+     * This method calls the `$userFunc` with the chunk, instead of a single record as in `chunk()`.
+     * This allows you to operate on multiple records at once, which can be more efficient for certain operations.
+     *
+     * @param Closure(list<array<string, string>>|list<object>): mixed $userFunc
+     *
+     * @return void
+     *
+     * @throws DataException
+     * @throws InvalidArgumentException if $size is not a positive integer
+     */
+    abstract public function chunkRows(int $size, Closure $userFunc);
+
     /**
      * Fetches the row of database.
      *

system/Model.php

@@ -26,6 +26,7 @@
 use CodeIgniter\Validation\ValidationInterface;
 use Config\Database;
 use Config\Feature;
+use Generator;
 use stdClass;
 
 /**
@@ -526,16 +527,16 @@ public function countAllResults(bool $reset = true, bool $test = false)
     }
 
     /**
-     * {@inheritDoc}
+     * Iterates over the result set in chunks of the specified size.
      *
-     * Works with `$this->builder` to get the Compiled select to
-     * determine the rows to operate on.
-     * This method works only with dbCalls.
+     * @param int $size The number of records to retrieve in each chunk.
+     *
+     * @return Generator<list<array<string, string>>|list<object>>
      */
-    public function chunk(int $size, Closure $userFunc)
+    private function iterateChunks(int $size): Generator
     {
         if ($size <= 0) {
-            throw new InvalidArgumentException('chunk() requires a positive integer for the $size argument.');
+            throw new InvalidArgumentException('$size must be a positive integer.');
         }
 
         $total  = $this->builder()->countAllResults(false);
@@ -557,6 +558,16 @@ public function chunk(int $size, Closure $userFunc)
                 continue;
             }
 
+            yield $rows;
+        }
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public function chunk(int $size, Closure $userFunc)
+    {
+        foreach ($this->iterateChunks($size) as $rows) {
             foreach ($rows as $row) {
                 if ($userFunc($row) === false) {
                     return;
@@ -565,6 +576,18 @@ public function chunk(int $size, Closure $userFunc)
         }
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    public function chunkRows(int $size, Closure $userFunc): void
+    {
+        foreach ($this->iterateChunks($size) as $rows) {
+            if ($userFunc($rows) === false) {
+                return;
+            }
+        }
+    }
+
     /**
      * Provides a shared instance of the Query Builder.
      *

tests/system/Models/MiscellaneousModelTest.php

@@ -44,15 +44,15 @@ public function testChunk(): void
     public function testChunkThrowsOnZeroSize(): void
     {
         $this->expectException(InvalidArgumentException::class);
-        $this->expectExceptionMessage('chunk() requires a positive integer for the $size argument.');
+        $this->expectExceptionMessage('$size must be a positive integer.');
 
         $this->createModel(UserModel::class)->chunk(0, static function ($row): void {});
     }
 
     public function testChunkThrowsOnNegativeSize(): void
     {
         $this->expectException(InvalidArgumentException::class);
-        $this->expectExceptionMessage('chunk() requires a positive integer for the $size argument.');
+        $this->expectExceptionMessage('$size must be a positive integer.');
 
         $this->createModel(UserModel::class)->chunk(-1, static function ($row): void {});
     }
@@ -97,6 +97,76 @@ public function testChunkEmptyTable(): void
         $this->assertSame(0, $rowCount);
     }
 
+    public function testChunkRows(): void
+    {
+        $chunkCount     = 0;
+        $numRowsInChunk = [];
+
+        $this->createModel(UserModel::class)->chunkRows(2, static function ($rows) use (&$chunkCount, &$numRowsInChunk): void {
+            $chunkCount++;
+            $numRowsInChunk[] = count($rows);
+        });
+
+        $this->assertSame(2, $chunkCount);
+        $this->assertSame([2, 2], $numRowsInChunk);
+    }
+
+    public function testChunkRowsThrowsOnZeroSize(): void
+    {
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('$size must be a positive integer.');
+
+        $this->createModel(UserModel::class)->chunkRows(0, static function ($row): void {});
+    }
+
+    public function testChunkRowsThrowsOnNegativeSize(): void
+    {
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('$size must be a positive integer.');
+
+        $this->createModel(UserModel::class)->chunkRows(-1, static function ($row): void {});
+    }
+
+    public function testChunkRowsEarlyExit(): void
+    {
+        $rowCount = 0;
+
+        $this->createModel(UserModel::class)->chunkRows(2, static function ($rows) use (&$rowCount): bool {
+            $rowCount++;
+
+            return false;
+        });
+
+        $this->assertSame(1, $rowCount);
+    }
+
+    public function testChunkRowsDoesNotRunExtraQuery(): void
+    {
+        $queryCount = 0;
+        $listener   = static function () use (&$queryCount): void {
+            $queryCount++;
+        };
+
+        Events::on('DBQuery', $listener);
+        $this->createModel(UserModel::class)->chunkRows(4, static function ($rows): void {});
+        Events::removeListener('DBQuery', $listener);
+
+        $this->assertSame(2, $queryCount);
+    }
+
+    public function testChunkRowsEmptyTable(): void
+    {
+        $this->db->table('user')->truncate();
+
+        $rowCount = 0;
+
+        $this->createModel(UserModel::class)->chunkRows(2, static function ($row) use (&$rowCount): void {
+            $rowCount++;
+        });
+
+        $this->assertSame(0, $rowCount);
+    }
+
     public function testCanCreateAndSaveEntityClasses(): void
     {
         $entity = $this->createModel(EntityModel::class)->where('name', 'Developer')->first();

user_guide_src/source/changelogs/v4.8.0.rst

@@ -127,6 +127,8 @@ Others
 Model
 =====
 
+- Added new ``chunkRows()`` method to ``CodeIgniter\Model`` for processing large datasets in smaller chunks.
+
 Libraries
 =========
 

user_guide_src/source/models/model.rst

@@ -900,14 +900,27 @@ Processing Large Amounts of Data
 ================================
 
 Sometimes, you need to process large amounts of data and would run the risk of running out of memory.
-To make this simpler, you may use the chunk() method to get smaller chunks of data that you can then
+This is best used during cronjobs, data exports, or other large tasks. To make this simpler, you can
+process the data in smaller, manageable pieces using the methods below.
+
+chunk()
+-------
+
+You may use the ``chunk()`` method to get smaller chunks of data that you can then
 do your work on. The first parameter is the number of rows to retrieve in a single chunk. The second
 parameter is a Closure that will be called for each row of data.
 
-This is best used during cronjobs, data exports, or other large tasks.
-
 .. literalinclude:: model/049.php
 
+chunkRows()
+-----------
+
+.. versionadded:: 4.8.0
+
+On the other hand, if you want the entire chunk to be passed to the Closure at once, you can use the ``chunkRows()`` method.
+
+.. literalinclude:: model/064.php
+
 .. _model-events-callbacks:
 
 Working with Query Builder

user_guide_src/source/models/model/064.php

@@ -0,0 +1,6 @@
+<?php
+
+$userModel->chunkRows(100, static function ($rows) {
+    // do something.
+    // $rows is an array of rows representing chunk of 100 items.
+});