From 30370153cb14067a38b0425f3f95a5ccdff4a132 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:14:32 +1300 Subject: [PATCH 01/17] Add createOrUpdateDocuments --- composer.lock | 56 +++--- src/Database/Adapter.php | 25 +++ src/Database/Adapter/MariaDB.php | 296 ++++++++++++++++++++++++++++-- src/Database/Adapter/Mongo.php | 9 + src/Database/Adapter/Postgres.php | 12 +- src/Database/Adapter/SQL.php | 1 - src/Database/Adapter/SQLite.php | 5 + src/Database/Database.php | 121 +++++++++++- tests/e2e/Adapter/Base.php | 183 ++++++++++++++++++ 9 files changed, 654 insertions(+), 54 deletions(-) diff --git a/composer.lock b/composer.lock index 6c5cb2f70..84049247e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "0ad14b2121f07c2fe936eaaa82186cbc", + "content-hash": "43c3ff88660f90baf3ddf7ec8bc5136d", "packages": [ { "name": "brick/math", @@ -658,16 +658,16 @@ }, { "name": "open-telemetry/gen-otlp-protobuf", - "version": "1.2.1", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/gen-otlp-protobuf.git", - "reference": "66c3b98e998a726691c92e6405a82e6e7b8b169d" + "reference": "585bafddd4ae6565de154610b10a787a455c9ba0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/66c3b98e998a726691c92e6405a82e6e7b8b169d", - "reference": "66c3b98e998a726691c92e6405a82e6e7b8b169d", + "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/585bafddd4ae6565de154610b10a787a455c9ba0", + "reference": "585bafddd4ae6565de154610b10a787a455c9ba0", "shasum": "" }, "require": { @@ -717,7 +717,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-10-30T11:49:49+00:00" + "time": "2025-01-15T23:07:07+00:00" }, { "name": "open-telemetry/sdk", @@ -2052,16 +2052,16 @@ }, { "name": "utopia-php/compression", - "version": "0.1.2", + "version": "0.1.3", "source": { "type": "git", "url": "https://github.com/utopia-php/compression.git", - "reference": "6062f70596415f8d5de40a589367b0eb2a435f98" + "reference": "66f093557ba66d98245e562036182016c7dcfe8a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/compression/zipball/6062f70596415f8d5de40a589367b0eb2a435f98", - "reference": "6062f70596415f8d5de40a589367b0eb2a435f98", + "url": "https://api.github.com/repos/utopia-php/compression/zipball/66f093557ba66d98245e562036182016c7dcfe8a", + "reference": "66f093557ba66d98245e562036182016c7dcfe8a", "shasum": "" }, "require": { @@ -2092,22 +2092,22 @@ ], "support": { "issues": "https://github.com/utopia-php/compression/issues", - "source": "https://github.com/utopia-php/compression/tree/0.1.2" + "source": "https://github.com/utopia-php/compression/tree/0.1.3" }, - "time": "2024-11-08T14:59:54+00:00" + "time": "2025-01-15T15:15:51+00:00" }, { "name": "utopia-php/framework", - "version": "0.33.15", + "version": "0.33.16", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "83b0628900c2c53e8c3efbf069f3e13050295edc" + "reference": "e91d4c560d1b809e25faa63d564fef034363b50f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/83b0628900c2c53e8c3efbf069f3e13050295edc", - "reference": "83b0628900c2c53e8c3efbf069f3e13050295edc", + "url": "https://api.github.com/repos/utopia-php/http/zipball/e91d4c560d1b809e25faa63d564fef034363b50f", + "reference": "e91d4c560d1b809e25faa63d564fef034363b50f", "shasum": "" }, "require": { @@ -2139,9 +2139,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.15" + "source": "https://github.com/utopia-php/http/tree/0.33.16" }, - "time": "2024-12-10T13:07:04+00:00" + "time": "2025-01-16T15:58:50+00:00" }, { "name": "utopia-php/mongo", @@ -2390,16 +2390,16 @@ }, { "name": "laravel/pint", - "version": "v1.19.0", + "version": "v1.20.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0" + "reference": "53072e8ea22213a7ed168a8a15b96fbb8b82d44b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/8169513746e1bac70c85d6ea1524d9225d4886f0", - "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0", + "url": "https://api.github.com/repos/laravel/pint/zipball/53072e8ea22213a7ed168a8a15b96fbb8b82d44b", + "reference": "53072e8ea22213a7ed168a8a15b96fbb8b82d44b", "shasum": "" }, "require": { @@ -2452,7 +2452,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-12-30T16:20:10+00:00" + "time": "2025-01-14T16:20:53+00:00" }, { "name": "myclabs/deep-copy", @@ -2724,16 +2724,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.15", + "version": "1.12.16", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "c91d4e8bc056f46cf653656e6f71004b254574d1" + "reference": "e0bb5cb78545aae631220735aa706eac633a6be9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/c91d4e8bc056f46cf653656e6f71004b254574d1", - "reference": "c91d4e8bc056f46cf653656e6f71004b254574d1", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/e0bb5cb78545aae631220735aa706eac633a6be9", + "reference": "e0bb5cb78545aae631220735aa706eac633a6be9", "shasum": "" }, "require": { @@ -2778,7 +2778,7 @@ "type": "github" } ], - "time": "2025-01-05T16:40:22+00:00" + "time": "2025-01-21T14:50:05+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 73535ea11..04c0ffff8 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -637,6 +637,24 @@ abstract public function updateDocument(string $collection, string $id, Document */ abstract public function updateDocuments(string $collection, Document $updates, array $documents): int; + /** + * Create documents if they do not exist, otherwise increase the value of the attribute by the given value. + * + * @param string $collection + * @param string $attribute + * @param int|float $value + * @param array $documents + * @param int $batchSize + * @return mixed + */ + abstract public function createOrUpdateDocuments( + string $collection, + string $attribute, + int|float $value, + array $documents, + int $batchSize + ); + /** * Delete Document * @@ -875,6 +893,13 @@ abstract public function getSupportForGetConnectionId(): bool; */ abstract public function getSupportForCastIndexArray(): bool; + /** + * Is upserting with an increase supported? + * + * @return bool + */ + abstract public function getSupportForUpsertWithIncrease(): bool; + /** * Get current attribute count from collection document * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index d4c15d032..f7a089a1a 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -330,6 +330,7 @@ public function deleteCollection(string $id): bool * * @param string $collection * @return bool + * @throws DatabaseException */ public function analyzeCollection(string $collection): bool { @@ -980,7 +981,7 @@ public function createDocuments(string $collection, array $documents, int $batch try { $name = $this->filter($collection); $batches = \array_chunk($documents, \max(1, $batchSize)); - $internalIds = []; + $documentIds = \array_map(fn ($document) => $document->getId(), $documents); foreach ($batches as $batch) { $bindIndex = 0; @@ -1075,18 +1076,31 @@ public function createDocuments(string $collection, array $documents, int $batch $stmtPermissions?->execute(); } } - } catch (PDOException $e) { - throw $this->processException($e); - } - foreach ($documents as $document) { - if (!isset($internalIds[$document->getId()])) { - $document['$internalId'] = $this->getDocument( - $collection, - $document->getId(), - [Query::select(['$internalId'])] - )->getInternalId(); + // Get internal IDs + $sql = " + SELECT _id + FROM {$this->getSQLTable($collection)} + WHERE _uid IN (" . implode(',', \array_fill(0, \count($documentIds), '?')) . ") + {$this->getTenantQuery($collection)} + "; + + $stmt = $this->getPDO()->prepare($sql); + foreach ($documentIds as $index => $id) { + $stmt->bindValue($index + 1, $id, PDO::PARAM_STR); + } + + $stmt->execute(); + $internalIds = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] + $stmt->closeCursor(); + + foreach ($documents as $document) { + if (!isset($internalIds[$document->getId()]) && isset($internalIds[$document->getId()])) { + $document['$internalId'] = $internalIds[$document->getId()]; + } } + } catch (PDOException $e) { + throw $this->processException($e); } return $documents; @@ -1381,10 +1395,10 @@ public function updateDocuments(string $collection, Document $updates, array $do } $sql = " - UPDATE {$this->getSQLTable($name)} - SET {$columns} - {$sqlWhere} - "; + UPDATE {$this->getSQLTable($name)} + SET {$columns} + {$sqlWhere} + "; $sql = $this->trigger(Database::EVENT_DOCUMENTS_UPDATE, $sql); $stmt = $this->getPDO()->prepare($sql); @@ -1572,6 +1586,252 @@ public function updateDocuments(string $collection, Document $updates, array $do return $affected; } + /** + * @param string $collection + * @param string $attribute + * @param int|float $value + * @param array $documents + * @param int $batchSize + * @return array + * @throws DatabaseException + */ + public function createOrUpdateDocuments( + string $collection, + string $attribute, + int|float $value, + array $documents, + int $batchSize + ): array { + if (empty($documents)) { + return $documents; + } + + try { + $name = $this->filter($collection); + $attribute = $this->filter($attribute); + $batches = \array_chunk($documents, \max(1, $batchSize)); + $internalIds = []; + + foreach ($batches as $batch) { + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; + + $documentIds = array_map(fn ($doc) => $doc->getId(), $batch); + + foreach ($batch as $index => $document) { + /** + * @var array $attributes + */ + $attributes = $document->getAttributes(); + $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = \json_encode($document->getPermissions()); + + if (!empty($document->getInternalId())) { + $attributes['_id'] = $document->getInternalId(); + } + + if ($this->sharedTables) { + $attributes['_tenant'] = $this->tenant; + } + + $columns = []; + foreach (\array_keys($attributes) as $key => $attr) { + $columns[$key] = "`{$this->filter($attr)}`"; + } + + $columns = '(' . \implode(', ', $columns) . ')'; + + $bindKeys = []; + + foreach ($attributes as $attrValue) { + if (\is_array($attrValue)) { + $attrValue = \json_encode($attrValue); + } + $attrValue = (\is_bool($attrValue)) ? (int)$attrValue : $attrValue; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $attrValue; + $bindIndex++; + } + + $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; + } + + if (empty($attribute) && empty($value)) { + // Update all columns + $updateColumns = []; + foreach (\array_keys($attributes) as $key => $attr) { + $updateColumns[] = "`{$this->filter($attr)}` = VALUES(`{$this->filter($attr)}`)"; + } + } else { + // Increment specific column + $updateColumns = [ + "`{$attribute}` = `{$attribute}` + :_increment" + ]; + } + + $stmt = $this->getPDO()->prepare( + " + INSERT INTO {$this->getSQLTable($name)} {$columns} + VALUES " . \implode(', ', $batchKeys) . " + ON DUPLICATE KEY UPDATE + " . \implode(', ', $updateColumns) + ); + + foreach ($bindValues as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } + + if (!empty($attribute) && !empty($value)) { + $stmt->bindValue(':_increment', $value, $this->getPDOType($value)); + } + + $stmt->execute(); + + // Fetch existing permissions in bulk after data updates + $sql = " + SELECT _document, _type, _permission + FROM {$this->getSQLTable($name . '_perms')} + WHERE _document IN (" . implode(',', \array_fill(0, \count($documentIds), '?')) . ") + {$this->getTenantQuery($collection)} + "; + + $stmt = $this->getPDO()->prepare($sql); + foreach ($documentIds as $index => $id) { + $stmt->bindValue($index + 1, $id, PDO::PARAM_STR); + } + + if ($this->sharedTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + + $stmt->execute(); + $existing = $stmt->fetchAll(); + $stmt->closeCursor(); + + // Group permissions by document + $permissionsByDocument = []; + foreach ($existing as $row) { + $permissionsByDocument[$row['_document']][$row['_type']][] = $row['_permission']; + } + + foreach ($documentIds as $id) { + foreach (Database::PERMISSIONS as $type) { + $permissionsByDocument[$id][$type] = $permissionsByDocument[$id][$type] ?? []; + } + } + + $removeQueries = []; + $removeBindValues = []; + $addQueries = []; + $addBindValues = []; + + foreach ($batch as $index => $document) { + $currentPermissions = $permissionsByDocument[$document->getId()] ?? []; + + // Calculate removals + foreach (Database::PERMISSIONS as $type) { + $toRemove = \array_diff($currentPermissions[$type], $document->getPermissionsByType($type)); + if (!empty($toRemove)) { + $removeQueries[] = "( + _document = :uid_{$index} + {$this->getTenantQuery($collection)} + AND _type = '{$type}' + AND _permission IN (" . \implode(',', \array_map(fn ($i) => ":remove_{$type}_{$index}_{$i}", \array_keys($toRemove))) . ") + )"; + $removeBindValues[":uid_{$index}"] = $document->getId(); + foreach ($toRemove as $i => $perm) { + $removeBindValues[":remove_{$type}_{$index}_{$i}"] = $perm; + } + } + } + + // Calculate additions + foreach (Database::PERMISSIONS as $type) { + $toAdd = \array_diff($document->getPermissionsByType($type), $currentPermissions[$type]); + foreach ($toAdd as $i => $permission) { + $addQuery = "(:uid_{$index}, '{$type}', :add_{$type}_{$index}_{$i}"; + + if ($this->sharedTables) { + $addQuery .= ", :_tenant)"; + } else { + $addQuery .= ")"; + } + + $addQueries[] = $addQuery; + $addBindValues[":uid_{$index}"] = $document->getId(); + $addBindValues[":add_{$type}_{$index}_{$i}"] = $permission; + } + } + } + + // Execute permission removals + if (!empty($removeQueries)) { + $removeQuery = \implode(' OR ', $removeQueries); + $stmtRemovePermissions = $this->getPDO()->prepare("DELETE FROM {$this->getSQLTable($name . '_perms')} WHERE {$removeQuery}"); + foreach ($removeBindValues as $key => $value) { + $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); + } + if ($this->sharedTables) { + $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); + } + $stmtRemovePermissions->execute(); + } + + // Execute permission additions + if (!empty($addQuery)) { + $sqlAddPermissions = "INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission"; + if ($this->sharedTables) { + $sqlAddPermissions .= ", _tenant)"; + } else { + $sqlAddPermissions .= ")"; + } + $addQuery = \implode(', ', $addQueries); + $sqlAddPermissions .= " VALUES {$addQuery}"; + $stmtAddPermissions = $this->getPDO()->prepare($sqlAddPermissions); + foreach ($addBindValues as $key => $value) { + $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); + } + if ($this->sharedTables) { + $stmtAddPermissions->bindValue(':_tenant', $this->tenant); + } + + $stmtAddPermissions->execute(); + } + } + + // Get internal IDs + $sql = " + SELECT _uid, _id + FROM {$this->getSQLTable($collection)} + WHERE _uid IN (" . implode(',', \array_fill(0, count($documentIds), '?')) . ") + {$this->getTenantQuery($collection)} + "; + + $stmt = $this->getPDO()->prepare($sql); + foreach ($documentIds as $index => $id) { + $stmt->bindValue($index + 1, $id, PDO::PARAM_STR); + } + + $stmt->execute(); + $internalIds = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] + $stmt->closeCursor(); + + foreach ($documents as $document) { + if (!isset($internalIds[$document->getId()]) && isset($internalIds[$document->getId()])) { + $document['$internalId'] = $internalIds[$document->getId()]; + } + } + } catch (PDOException $e) { + throw $this->processException($e); + } + + return $documents; + } + /** * Increase or decrease an attribute value * @@ -1602,7 +1862,6 @@ public function increaseDocumentAttribute(string $collection, string $id, string {$this->getTenantQuery($collection)} "; - $sql .= $sqlMax . $sqlMin; $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql); @@ -2346,6 +2605,11 @@ public function getSupportForTimeouts(): bool return true; } + public function getSupportForUpsertWithIncrease(): bool + { + return true; + } + /** * Set max execution time * @param int $milliseconds diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index b67db87ec..dbefb31a6 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -891,6 +891,10 @@ public function updateDocuments(string $collection, Document $updates, array $do return 1; } + public function createOrUpdateDocuments(string $collection, string $attribute, float|int $value, array $documents, int $batchSize) + { + } + /** * Increase or decrease an attribute value * @@ -1803,6 +1807,11 @@ public function getSupportForCastIndexArray(): bool return false; } + public function getSupportForUpsertWithIncrease(): bool + { + return false; + } + /** * Get current attribute count from collection document * diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index d808a82dc..fe03af03e 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1474,6 +1474,7 @@ public function updateDocuments(string $collection, Document $updates, array $do $permissionsStmt->execute(); $permissions = $permissionsStmt->fetchAll(); + $permissionsStmt->closeCursor(); $initial = []; foreach (Database::PERMISSIONS as $type) { @@ -1585,7 +1586,7 @@ public function updateDocuments(string $collection, Document $updates, array $do $sqlAddPermissions .= ')'; } - $sqlAddPermissions .= " VALUES {$addQuery}"; + $sqlAddPermissions .= " VALUES {$addQuery}"; $stmtAddPermissions = $this->getPDO()->prepare($sqlAddPermissions); @@ -1604,6 +1605,10 @@ public function updateDocuments(string $collection, Document $updates, array $do return $affected; } + public function createOrUpdateDocuments(string $collection, string $attribute, float|int $value, array $documents, int $batchSize) + { + } + /** * Increase or decrease an attribute value * @@ -2431,6 +2436,11 @@ public function getSupportForSchemaAttributes(): bool return false; } + public function getSupportForUpsertWithIncrease(): bool + { + return false; + } + /** * @return string */ diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index f731d8bdd..ae05e3d9b 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -232,7 +232,6 @@ public function getDocument(string $collection, string $id, array $queries = [], } $stmt->execute(); - $document = $stmt->fetchAll(); $stmt->closeCursor(); diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 8603a6cbb..bf2371388 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -936,6 +936,11 @@ public function getSupportForSchemaAttributes(): bool return false; } + public function getSupportForUpsertWithIncrease(): bool + { + return false; + } + /** * Get SQL Index Type * diff --git a/src/Database/Database.php b/src/Database/Database.php index 49ecee7e4..c18ac58f5 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3386,15 +3386,13 @@ public function createDocument(string $collection, Document $document): Document * @param string $collection * @param array $documents * @param int $batchSize - * * @return array - * - * @throws AuthorizationException - * @throws StructureException - * @throws Exception */ - public function createDocuments(string $collection, array $documents, int $batchSize = self::INSERT_BATCH_SIZE): array - { + public function createDocuments( + string $collection, + array $documents, + int $batchSize = self::INSERT_BATCH_SIZE, + ): array { if (empty($documents)) { return []; } @@ -3432,7 +3430,11 @@ public function createDocuments(string $collection, array $documents, int $batch } $documents = $this->withTransaction(function () use ($collection, $documents, $batchSize) { - return $this->adapter->createDocuments($collection->getId(), $documents, $batchSize); + return $this->adapter->createDocuments( + $collection->getId(), + $documents, + $batchSize, + ); }); foreach ($documents as $key => $document) { @@ -4532,6 +4534,109 @@ private function getJunctionCollection(Document $collection, Document $relatedCo : '_' . $relatedCollection->getInternalId() . '_' . $collection->getInternalId(); } + /** + * Create or update documents + * + * @param string $collection + * @param array $documents + * @param int $batchSize + * @return array + * @throws StructureException + */ + public function createOrUpdateDocuments( + string $collection, + array $documents, + int $batchSize = self::INSERT_BATCH_SIZE + ): array { + return $this->createOrUpdateDocumentsWithIncrease( + $collection, + '', + 0, + $documents, + $batchSize + ); + } + + /** + * @param string $collection + * @param string $attribute + * @param int|float $value + * @param array $documents + * @param int $batchSize + * @return array + * @throws StructureException + * @throws \Throwable + * @throws Exception + */ + public function createOrUpdateDocumentsWithIncrease( + string $collection, + string $attribute, + int|float $value, + array $documents, + int $batchSize = self::INSERT_BATCH_SIZE + ): array { + if (empty($documents)) { + return []; + } + + $collection = $this->silent(fn () => $this->getCollection($collection)); + + $time = DateTime::now(); + + foreach ($documents as $key => $document) { + $createdAt = $document->getCreatedAt(); + $updatedAt = $document->getUpdatedAt(); + + $document + ->setAttribute('$id', empty($document->getId()) ? ID::unique() : $document->getId()) + ->setAttribute('$collection', $collection->getId()) + ->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt) + ->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt); + + $document = $this->encode($collection, $document); + + $validator = new Structure( + $collection, + $this->adapter->getMinDateTime(), + $this->adapter->getMaxDateTime(), + ); + if (!$validator->isValid($document)) { + throw new StructureException($validator->getDescription()); + } + + if ($this->resolveRelationships) { + $document = $this->silent(fn () => $this->createDocumentRelationships($collection, $document)); + } + + $documents[$key] = $document; + } + + $documents = $this->withTransaction(function () use ($collection, $attribute, $value, $documents, $batchSize) { + return $this->adapter->createOrUpdateDocuments( + $collection->getId(), + $attribute, + $value, + $documents, + $batchSize, + ); + }); + + foreach ($documents as $key => $document) { + if ($this->resolveRelationships) { + $document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document)); + } + + $documents[$key] = $this->decode($collection, $document); + } + + $this->trigger(self::EVENT_DOCUMENTS_CREATE, new Document([ + '$collection' => $collection->getId(), + 'modified' => count($documents) + ])); + + return $documents; + } + /** * Increase a document attribute by a value * diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index ae6ad4653..3db88b40a 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -2230,6 +2230,189 @@ public function testCreateDocuments(): array return $documents; } + public function testCreateOrUpdateDocuments(): void + { + $collection = 'testCreateOrUpdateDocuments'; + + static::getDatabase()->createCollection($collection); + + static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true); + static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true); + static::getDatabase()->createAttribute($collection, 'bigint', Database::VAR_INTEGER, 8, true); + + $documents = [ + new Document([ + '$id' => 'first', + 'string' => 'text📝', + 'integer' => 5, + 'bigint' => Database::BIG_INT_MAX, + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]), + new Document([ + '$id' => 'second', + 'string' => 'text📝', + 'integer' => 5, + 'bigint' => Database::BIG_INT_MAX, + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]), + ]; + + $documents = static::getDatabase()->createOrUpdateDocuments($collection, $documents); + + $this->assertEquals(2, count($documents)); + + foreach ($documents as $document) { + $this->assertNotEmpty(true, $document->getId()); + $this->assertIsString($document->getAttribute('string')); + $this->assertEquals('text📝', $document->getAttribute('string')); // Also makes sure an emoji is working + $this->assertIsInt($document->getAttribute('integer')); + $this->assertEquals(5, $document->getAttribute('integer')); + $this->assertIsInt($document->getAttribute('bigint')); + $this->assertEquals(Database::BIG_INT_MAX, $document->getAttribute('bigint')); + } + + $documents = static::getDatabase()->find($collection); + + $this->assertEquals(2, count($documents)); + + foreach ($documents as $document) { + $this->assertNotEmpty(true, $document->getId()); + $this->assertIsString($document->getAttribute('string')); + $this->assertEquals('text📝', $document->getAttribute('string')); // Also makes sure an emoji is working + $this->assertIsInt($document->getAttribute('integer')); + $this->assertEquals(5, $document->getAttribute('integer')); + $this->assertIsInt($document->getAttribute('bigint')); + $this->assertEquals(Database::BIG_INT_MAX, $document->getAttribute('bigint')); + } + + $documents[0]->setAttribute('string', 'new text📝'); + $documents[0]->setAttribute('integer', 10); + $documents[1]->setAttribute('string', 'new text📝'); + $documents[1]->setAttribute('integer', 10); + + $documents = static::getDatabase()->createOrUpdateDocuments($collection, $documents); + + $this->assertEquals(2, count($documents)); + + foreach ($documents as $document) { + $this->assertNotEmpty(true, $document->getId()); + $this->assertIsString($document->getAttribute('string')); + $this->assertEquals('new text📝', $document->getAttribute('string')); // Also makes sure an emoji is working + $this->assertIsInt($document->getAttribute('integer')); + $this->assertEquals(10, $document->getAttribute('integer')); + $this->assertIsInt($document->getAttribute('bigint')); + $this->assertEquals(Database::BIG_INT_MAX, $document->getAttribute('bigint')); + } + + $documents = static::getDatabase()->find($collection); + + $this->assertEquals(2, count($documents)); + + foreach ($documents as $document) { + $this->assertNotEmpty(true, $document->getId()); + $this->assertIsString($document->getAttribute('string')); + $this->assertEquals('new text📝', $document->getAttribute('string')); // Also makes sure an emoji is working + $this->assertIsInt($document->getAttribute('integer')); + $this->assertEquals(10, $document->getAttribute('integer')); + $this->assertIsInt($document->getAttribute('bigint')); + $this->assertEquals(Database::BIG_INT_MAX, $document->getAttribute('bigint')); + } + } + + public function testCreateOrUpdateDocumentsWithIncrease(): void + { + $collection = 'testCreateOrUpdateWithIncrease'; + + static::getDatabase()->createCollection($collection); + + static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true); + static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true); + + $documents = [ + new Document([ + '$id' => 'first', + 'string' => 'text📝', + 'integer' => 5, + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]), + new Document([ + '$id' => 'second', + 'string' => 'text📝', + 'integer' => 5, + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]), + ]; + + $documents = static::getDatabase()->createOrUpdateDocuments($collection, $documents); + + $this->assertEquals(2, count($documents)); + + foreach ($documents as $document) { + $this->assertNotEmpty(true, $document->getId()); + $this->assertIsString($document->getAttribute('string')); + $this->assertEquals('text📝', $document->getAttribute('string')); // Also makes sure an emoji is working + $this->assertIsInt($document->getAttribute('integer')); + $this->assertEquals(5, $document->getAttribute('integer')); + $this->assertIsInt($document->getAttribute('bigint')); + $this->assertEquals(Database::BIG_INT_MAX, $document->getAttribute('bigint')); + } + + $documents = static::getDatabase()->find($collection); + + $this->assertEquals(2, count($documents)); + + foreach ($documents as $document) { + $this->assertNotEmpty(true, $document->getId()); + $this->assertIsString($document->getAttribute('string')); + $this->assertEquals('text📝', $document->getAttribute('string')); // Also makes sure an emoji is working + $this->assertIsInt($document->getAttribute('integer')); + $this->assertEquals(5, $document->getAttribute('integer')); + $this->assertIsInt($document->getAttribute('bigint')); + $this->assertEquals(Database::BIG_INT_MAX, $document->getAttribute('bigint')); + } + + $documents = static::getDatabase()->createOrUpdateDocumentsWithIncrease( + collection: $collection, + attribute:'integer', + value: 1, + documents: $documents + ); + + $this->assertEquals(2, count($documents)); + + foreach ($documents as $document) { + $this->assertEquals(6, $document->getAttribute('integer')); + } + + $documents = static::getDatabase()->find($collection); + + $this->assertEquals(2, count($documents)); + + foreach ($documents as $document) { + $this->assertEquals(6, $document->getAttribute('integer')); + } + } + public function testRespectNulls(): Document { static::getDatabase()->createCollection('documents_nulls'); From df06456eafa6c1e473d9baefcb8ef9a2923ee60d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:14:54 +1300 Subject: [PATCH 02/17] Fix key pair fetch --- src/Database/Adapter/MariaDB.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index f7a089a1a..52d0cb43f 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1079,7 +1079,7 @@ public function createDocuments(string $collection, array $documents, int $batch // Get internal IDs $sql = " - SELECT _id + SELECT _uid, _id FROM {$this->getSQLTable($collection)} WHERE _uid IN (" . implode(',', \array_fill(0, \count($documentIds), '?')) . ") {$this->getTenantQuery($collection)} @@ -1610,7 +1610,6 @@ public function createOrUpdateDocuments( $name = $this->filter($collection); $attribute = $this->filter($attribute); $batches = \array_chunk($documents, \max(1, $batchSize)); - $internalIds = []; foreach ($batches as $batch) { $bindIndex = 0; From 5c0dd0526de0303052172d1d299c158096262aea Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:15:09 +1300 Subject: [PATCH 03/17] Fix increment not updated on return document --- src/Database/Adapter/MariaDB.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 52d0cb43f..52f56a6e4 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1656,6 +1656,10 @@ public function createOrUpdateDocuments( $bindIndex++; } + if (!empty($attribute) && !empty($value)) { + $document->setAttribute($attribute, $document->getAttribute($attribute) + $value); + } + $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; } From 80c14dc22339edda21f88eb42e2945aa7b5d8769 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:15:27 +1300 Subject: [PATCH 04/17] Fix increment value overwriting --- src/Database/Adapter/MariaDB.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 52f56a6e4..d101815dc 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1684,8 +1684,8 @@ public function createOrUpdateDocuments( " . \implode(', ', $updateColumns) ); - foreach ($bindValues as $key => $value) { - $stmt->bindValue($key, $value, $this->getPDOType($value)); + foreach ($bindValues as $key => $binding) { + $stmt->bindValue($key, $binding, $this->getPDOType($binding)); } if (!empty($attribute) && !empty($value)) { From 93f90810110af8876e49b33123e9c21c219814d0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:16:16 +1300 Subject: [PATCH 05/17] Add permissions check --- src/Database/Database.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Database/Database.php b/src/Database/Database.php index c18ac58f5..beacb73d4 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4584,6 +4584,19 @@ public function createOrUpdateDocumentsWithIncrease( $time = DateTime::now(); foreach ($documents as $key => $document) { + $old = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection->getId(), $document->getId()))); + + if (!$old->isEmpty()) { + $validator = new Authorization(self::PERMISSION_UPDATE); + + if (!$validator->isValid([ + ...$collection->getUpdate(), + ...($collection->getAttribute('documentSecurity') ? $old->getUpdate() : []) + ])) { + throw new AuthorizationException($validator->getDescription()); + } + } + $createdAt = $document->getCreatedAt(); $updatedAt = $document->getUpdatedAt(); @@ -4600,6 +4613,7 @@ public function createOrUpdateDocumentsWithIncrease( $this->adapter->getMinDateTime(), $this->adapter->getMaxDateTime(), ); + if (!$validator->isValid($document)) { throw new StructureException($validator->getDescription()); } From 6b8715e60d797f417bf0e267a716e1f75d18f959 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:16:30 +1300 Subject: [PATCH 06/17] Add document purge --- src/Database/Database.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Database/Database.php b/src/Database/Database.php index beacb73d4..ccc29d0e5 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4641,6 +4641,8 @@ public function createOrUpdateDocumentsWithIncrease( } $documents[$key] = $this->decode($collection, $document); + + $this->purgeCachedDocument($collection->getId(), $document->getId()); } $this->trigger(self::EVENT_DOCUMENTS_CREATE, new Document([ From 56e78f3faff6740abaa9da99b5119264110330b1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:16:59 +1300 Subject: [PATCH 07/17] Add permissions test for createOrUpdate --- tests/e2e/Adapter/Base.php | 104 +++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 28 deletions(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 3db88b40a..b0e7e007a 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -2334,7 +2334,6 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void $collection = 'testCreateOrUpdateWithIncrease'; static::getDatabase()->createCollection($collection); - static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true); static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true); @@ -2363,33 +2362,7 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void ]), ]; - $documents = static::getDatabase()->createOrUpdateDocuments($collection, $documents); - - $this->assertEquals(2, count($documents)); - - foreach ($documents as $document) { - $this->assertNotEmpty(true, $document->getId()); - $this->assertIsString($document->getAttribute('string')); - $this->assertEquals('text📝', $document->getAttribute('string')); // Also makes sure an emoji is working - $this->assertIsInt($document->getAttribute('integer')); - $this->assertEquals(5, $document->getAttribute('integer')); - $this->assertIsInt($document->getAttribute('bigint')); - $this->assertEquals(Database::BIG_INT_MAX, $document->getAttribute('bigint')); - } - - $documents = static::getDatabase()->find($collection); - - $this->assertEquals(2, count($documents)); - - foreach ($documents as $document) { - $this->assertNotEmpty(true, $document->getId()); - $this->assertIsString($document->getAttribute('string')); - $this->assertEquals('text📝', $document->getAttribute('string')); // Also makes sure an emoji is working - $this->assertIsInt($document->getAttribute('integer')); - $this->assertEquals(5, $document->getAttribute('integer')); - $this->assertIsInt($document->getAttribute('bigint')); - $this->assertEquals(Database::BIG_INT_MAX, $document->getAttribute('bigint')); - } + static::getDatabase()->createDocuments($collection, $documents); $documents = static::getDatabase()->createOrUpdateDocumentsWithIncrease( collection: $collection, @@ -2413,6 +2386,81 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void } } + public function testCreateOrUpdateDocumentsPermissions(): void + { + $collection = 'testCreateOrUpdateDocumentPermissions'; + + static::getDatabase()->createCollection($collection); + static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true); + + $document = new Document([ + '$id' => 'first', + 'string' => 'text📝', + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + ], + ]); + + static::getDatabase()->createOrUpdateDocuments($collection, [$document]); + + try { + static::getDatabase()->createOrUpdateDocuments($collection, [$document->setAttribute('string', 'updated')]); + $this->fail('Failed to throw exception'); + } catch (Exception $e) { + $this->assertInstanceOf(AuthorizationException::class, $e); + } + + $document = new Document([ + '$id' => 'second', + 'string' => 'text📝', + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + ], + ]); + + static::getDatabase()->createOrUpdateDocuments($collection, [$document]); + + $documents = static::getDatabase()->createOrUpdateDocuments( + $collection, + [$document->setAttribute('string', 'updated')] + ); + + $this->assertEquals(1, count($documents)); + $this->assertEquals('updated', $documents[0]->getAttribute('string')); + + $document = new Document([ + '$id' => 'third', + 'string' => 'text📝', + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + static::getDatabase()->createOrUpdateDocuments($collection, [$document]); + + $newPermissions = [ + Permission::read(Role::any()), + Permission::update(Role::user('user1')), + Permission::delete(Role::user('user1')), + ]; + + $documents = static::getDatabase()->createOrUpdateDocuments( + $collection, + [$document->setAttribute('$permissions', $newPermissions)] + ); + + $this->assertEquals(1, count($documents)); + $this->assertEquals($newPermissions, $documents[0]->getPermissions()); + + $document = static::getDatabase()->getDocument($collection, 'third'); + + $this->assertEquals($newPermissions, $document->getPermissions()); + } + public function testRespectNulls(): Document { static::getDatabase()->createCollection('documents_nulls'); From 407b801628545dadad9c56ced443706773fdc836 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:19:44 +1300 Subject: [PATCH 08/17] Add decrement test --- tests/e2e/Adapter/Base.php | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index b0e7e007a..f25a4d9b6 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -2379,11 +2379,26 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void $documents = static::getDatabase()->find($collection); - $this->assertEquals(2, count($documents)); - foreach ($documents as $document) { $this->assertEquals(6, $document->getAttribute('integer')); } + + $documents = static::getDatabase()->createOrUpdateDocumentsWithIncrease( + collection: $collection, + attribute:'integer', + value: -1, + documents: $documents + ); + + foreach ($documents as $document) { + $this->assertEquals(5, $document->getAttribute('integer')); + } + + $documents = static::getDatabase()->find($collection); + + foreach ($documents as $document) { + $this->assertEquals(5, $document->getAttribute('integer')); + } } public function testCreateOrUpdateDocumentsPermissions(): void From 390f0d4b8ecb90ab0c56d67dc8aa01999eeac906 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:23:26 +1300 Subject: [PATCH 09/17] Fix isset checks --- src/Database/Adapter/MariaDB.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index d101815dc..87efed163 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1095,7 +1095,7 @@ public function createDocuments(string $collection, array $documents, int $batch $stmt->closeCursor(); foreach ($documents as $document) { - if (!isset($internalIds[$document->getId()]) && isset($internalIds[$document->getId()])) { + if (isset($internalIds[$document->getId()])) { $document['$internalId'] = $internalIds[$document->getId()]; } } @@ -1824,7 +1824,7 @@ public function createOrUpdateDocuments( $stmt->closeCursor(); foreach ($documents as $document) { - if (!isset($internalIds[$document->getId()]) && isset($internalIds[$document->getId()])) { + if (isset($internalIds[$document->getId()])) { $document['$internalId'] = $internalIds[$document->getId()]; } } From 2f6264bbbc970d76b0c117007388aedca1eef0a8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:33:51 +1300 Subject: [PATCH 10/17] Fix flags --- src/Database/Adapter.php | 10 ++++++---- src/Database/Adapter/MariaDB.php | 2 +- src/Database/Adapter/Mongo.php | 5 +++-- src/Database/Adapter/Postgres.php | 5 +++-- src/Database/Adapter/SQLite.php | 2 +- tests/e2e/Adapter/Base.php | 15 +++++++++++++++ 6 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 04c0ffff8..0197c54ef 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -638,7 +638,9 @@ abstract public function updateDocument(string $collection, string $id, Document abstract public function updateDocuments(string $collection, Document $updates, array $documents): int; /** - * Create documents if they do not exist, otherwise increase the value of the attribute by the given value. + * Create documents if they do not exist, otherwise update them. + * + * If both attribute and value are not empty, only the specified attribute will be increased, by the provided value. * * @param string $collection * @param string $attribute @@ -653,7 +655,7 @@ abstract public function createOrUpdateDocuments( int|float $value, array $documents, int $batchSize - ); + ): array; /** * Delete Document @@ -894,11 +896,11 @@ abstract public function getSupportForGetConnectionId(): bool; abstract public function getSupportForCastIndexArray(): bool; /** - * Is upserting with an increase supported? + * Is upserting supported? * * @return bool */ - abstract public function getSupportForUpsertWithIncrease(): bool; + abstract public function getSupportForUpserts(): bool; /** * Get current attribute count from collection document diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 87efed163..cb97b12a3 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -2608,7 +2608,7 @@ public function getSupportForTimeouts(): bool return true; } - public function getSupportForUpsertWithIncrease(): bool + public function getSupportForUpserts(): bool { return true; } diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index dbefb31a6..1a8c36d76 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -891,8 +891,9 @@ public function updateDocuments(string $collection, Document $updates, array $do return 1; } - public function createOrUpdateDocuments(string $collection, string $attribute, float|int $value, array $documents, int $batchSize) + public function createOrUpdateDocuments(string $collection, string $attribute, float|int $value, array $documents, int $batchSize): array { + return $documents; } /** @@ -1807,7 +1808,7 @@ public function getSupportForCastIndexArray(): bool return false; } - public function getSupportForUpsertWithIncrease(): bool + public function getSupportForUpserts(): bool { return false; } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index fe03af03e..70246c5cd 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1605,8 +1605,9 @@ public function updateDocuments(string $collection, Document $updates, array $do return $affected; } - public function createOrUpdateDocuments(string $collection, string $attribute, float|int $value, array $documents, int $batchSize) + public function createOrUpdateDocuments(string $collection, string $attribute, float|int $value, array $documents, int $batchSize): array { + return $documents; } /** @@ -2436,7 +2437,7 @@ public function getSupportForSchemaAttributes(): bool return false; } - public function getSupportForUpsertWithIncrease(): bool + public function getSupportForUpserts(): bool { return false; } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index bf2371388..88b57e590 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -936,7 +936,7 @@ public function getSupportForSchemaAttributes(): bool return false; } - public function getSupportForUpsertWithIncrease(): bool + public function getSupportForUpserts(): bool { return false; } diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index f25a4d9b6..2e0a9c341 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -2232,6 +2232,11 @@ public function testCreateDocuments(): array public function testCreateOrUpdateDocuments(): void { + if (!static::getDatabase()->getAdapter()->getSupportForUpserts()) { + $this->expectNotToPerformAssertions(); + return; + } + $collection = 'testCreateOrUpdateDocuments'; static::getDatabase()->createCollection($collection); @@ -2331,6 +2336,11 @@ public function testCreateOrUpdateDocuments(): void public function testCreateOrUpdateDocumentsWithIncrease(): void { + if (!static::getDatabase()->getAdapter()->getSupportForUpserts()) { + $this->expectNotToPerformAssertions(); + return; + } + $collection = 'testCreateOrUpdateWithIncrease'; static::getDatabase()->createCollection($collection); @@ -2403,6 +2413,11 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void public function testCreateOrUpdateDocumentsPermissions(): void { + if (!static::getDatabase()->getAdapter()->getSupportForUpserts()) { + $this->expectNotToPerformAssertions(); + return; + } + $collection = 'testCreateOrUpdateDocumentPermissions'; static::getDatabase()->createCollection($collection); From e71385134044aa001eb57fa847edae3acc0a50c5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jan 2025 23:48:07 +1300 Subject: [PATCH 11/17] Fix stan --- src/Database/Adapter.php | 2 +- src/Database/Adapter/Mongo.php | 8 ++++++++ src/Database/Adapter/Postgres.php | 8 ++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 0197c54ef..8f831e7d4 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -647,7 +647,7 @@ abstract public function updateDocuments(string $collection, Document $updates, * @param int|float $value * @param array $documents * @param int $batchSize - * @return mixed + * @return array */ abstract public function createOrUpdateDocuments( string $collection, diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 1a8c36d76..27d77ccfe 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -891,6 +891,14 @@ public function updateDocuments(string $collection, Document $updates, array $do return 1; } + /** + * @param string $collection + * @param string $attribute + * @param float|int $value + * @param array $documents + * @param int $batchSize + * @return array + */ public function createOrUpdateDocuments(string $collection, string $attribute, float|int $value, array $documents, int $batchSize): array { return $documents; diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 70246c5cd..127da6bbe 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1605,6 +1605,14 @@ public function updateDocuments(string $collection, Document $updates, array $do return $affected; } + /** + * @param string $collection + * @param string $attribute + * @param float|int $value + * @param array $documents + * @param int $batchSize + * @return array + */ public function createOrUpdateDocuments(string $collection, string $attribute, float|int $value, array $documents, int $batchSize): array { return $documents; From d665a5671109dc2f0a0d3d1e9738e3f3271f22df Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 00:37:28 +1300 Subject: [PATCH 12/17] Remove positional arg usage --- src/Database/Adapter/MariaDB.php | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index cb97b12a3..904b44308 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1081,13 +1081,14 @@ public function createDocuments(string $collection, array $documents, int $batch $sql = " SELECT _uid, _id FROM {$this->getSQLTable($collection)} - WHERE _uid IN (" . implode(',', \array_fill(0, \count($documentIds), '?')) . ") + WHERE _uid IN (" . implode(',', array_map(fn($index) => ":_key_{$index}", array_keys($documentIds))) . ") {$this->getTenantQuery($collection)} "; $stmt = $this->getPDO()->prepare($sql); + foreach ($documentIds as $index => $id) { - $stmt->bindValue($index + 1, $id, PDO::PARAM_STR); + $stmt->bindValue(":_key_{$index}", $id); } $stmt->execute(); @@ -1698,13 +1699,14 @@ public function createOrUpdateDocuments( $sql = " SELECT _document, _type, _permission FROM {$this->getSQLTable($name . '_perms')} - WHERE _document IN (" . implode(',', \array_fill(0, \count($documentIds), '?')) . ") + WHERE _document IN (" . \implode(',', \array_map(fn($index) => ":_key_{$index}", \array_keys($documentIds))) . ") {$this->getTenantQuery($collection)} "; $stmt = $this->getPDO()->prepare($sql); + foreach ($documentIds as $index => $id) { - $stmt->bindValue($index + 1, $id, PDO::PARAM_STR); + $stmt->bindValue(":_key_{$index}", $id); } if ($this->sharedTables) { @@ -1810,13 +1812,14 @@ public function createOrUpdateDocuments( $sql = " SELECT _uid, _id FROM {$this->getSQLTable($collection)} - WHERE _uid IN (" . implode(',', \array_fill(0, count($documentIds), '?')) . ") + WHERE _uid IN (" . \implode(',', \array_map(fn($index) => ":_key_{$index}", \array_keys($documentIds))) . ") {$this->getTenantQuery($collection)} "; $stmt = $this->getPDO()->prepare($sql); + foreach ($documentIds as $index => $id) { - $stmt->bindValue($index + 1, $id, PDO::PARAM_STR); + $stmt->bindValue(":_key_{$index}", $id); } $stmt->execute(); From f086132d5d1e57d2967ea8cd1ee7728414ee0eda Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 00:37:39 +1300 Subject: [PATCH 13/17] Add missing binds --- src/Database/Adapter/MariaDB.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 904b44308..cf7ee77ef 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1091,6 +1091,10 @@ public function createDocuments(string $collection, array $documents, int $batch $stmt->bindValue(":_key_{$index}", $id); } + if ($this->sharedTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + $stmt->execute(); $internalIds = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] $stmt->closeCursor(); @@ -1822,6 +1826,10 @@ public function createOrUpdateDocuments( $stmt->bindValue(":_key_{$index}", $id); } + if ($this->sharedTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + $stmt->execute(); $internalIds = $stmt->fetchAll(PDO::FETCH_KEY_PAIR); // Fetch as [documentId => internalId] $stmt->closeCursor(); From 42584e3ca5104fa04c0a30a89e2bd8a875bf630c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 00:40:27 +1300 Subject: [PATCH 14/17] Lint --- src/Database/Adapter/MariaDB.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index cf7ee77ef..56508c679 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1081,7 +1081,7 @@ public function createDocuments(string $collection, array $documents, int $batch $sql = " SELECT _uid, _id FROM {$this->getSQLTable($collection)} - WHERE _uid IN (" . implode(',', array_map(fn($index) => ":_key_{$index}", array_keys($documentIds))) . ") + WHERE _uid IN (" . implode(',', array_map(fn ($index) => ":_key_{$index}", array_keys($documentIds))) . ") {$this->getTenantQuery($collection)} "; @@ -1703,7 +1703,7 @@ public function createOrUpdateDocuments( $sql = " SELECT _document, _type, _permission FROM {$this->getSQLTable($name . '_perms')} - WHERE _document IN (" . \implode(',', \array_map(fn($index) => ":_key_{$index}", \array_keys($documentIds))) . ") + WHERE _document IN (" . \implode(',', \array_map(fn ($index) => ":_key_{$index}", \array_keys($documentIds))) . ") {$this->getTenantQuery($collection)} "; @@ -1816,7 +1816,7 @@ public function createOrUpdateDocuments( $sql = " SELECT _uid, _id FROM {$this->getSQLTable($collection)} - WHERE _uid IN (" . \implode(',', \array_map(fn($index) => ":_key_{$index}", \array_keys($documentIds))) . ") + WHERE _uid IN (" . \implode(',', \array_map(fn ($index) => ":_key_{$index}", \array_keys($documentIds))) . ") {$this->getTenantQuery($collection)} "; From c9170c85c07997fc3718f35928b252062a02070a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 01:38:29 +1300 Subject: [PATCH 15/17] Add inplace increase --- src/Database/Adapter/MariaDB.php | 21 +++++------ src/Database/Database.php | 26 ++++++++++++++ tests/e2e/Adapter/Base.php | 61 ++++++++++++++++++++++++++++---- 3 files changed, 91 insertions(+), 17 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 56508c679..aaf3b9d7c 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1661,24 +1661,25 @@ public function createOrUpdateDocuments( $bindIndex++; } - if (!empty($attribute) && !empty($value)) { - $document->setAttribute($attribute, $document->getAttribute($attribute) + $value); - } - $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; } - if (empty($attribute) && empty($value)) { + if (!empty($attribute) && !empty($value)) { + // Increment specific column by the specified value + $updateColumns = [ + "`{$attribute}` = `{$attribute}` + :_increment" + ]; + } elseif (!empty($attribute) && empty($value)) { + // Increment specific column by its new value in place + $updateColumns = [ + "`{$attribute}` = `{$attribute}` + VALUES(`{$attribute}`)" + ]; + } else { // Update all columns $updateColumns = []; foreach (\array_keys($attributes) as $key => $attr) { $updateColumns[] = "`{$this->filter($attr)}` = VALUES(`{$this->filter($attr)}`)"; } - } else { - // Increment specific column - $updateColumns = [ - "`{$attribute}` = `{$attribute}` + :_increment" - ]; } $stmt = $this->getPDO()->prepare( diff --git a/src/Database/Database.php b/src/Database/Database.php index a0b0de4af..88958bed8 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4558,6 +4558,32 @@ public function createOrUpdateDocuments( ); } + /** + * Create or update documents + * + * @param string $collection + * @param string $attribute + * @param array $documents + * @param int $batchSize + * @return array + * @throws StructureException + * @throws \Throwable + */ + public function createOrUpdateDocumentsWithInplaceIncrease( + string $collection, + string $attribute, + array $documents, + int $batchSize = self::INSERT_BATCH_SIZE + ): array { + return $this->createOrUpdateDocumentsWithIncrease( + $collection, + $attribute, + 0, + $documents, + $batchSize + ); + } + /** * @param string $collection * @param string $attribute diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index d38675fe5..a35b656d4 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -2374,19 +2374,13 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void static::getDatabase()->createDocuments($collection, $documents); - $documents = static::getDatabase()->createOrUpdateDocumentsWithIncrease( + static::getDatabase()->createOrUpdateDocumentsWithIncrease( collection: $collection, attribute:'integer', value: 1, documents: $documents ); - $this->assertEquals(2, count($documents)); - - foreach ($documents as $document) { - $this->assertEquals(6, $document->getAttribute('integer')); - } - $documents = static::getDatabase()->find($collection); foreach ($documents as $document) { @@ -2411,6 +2405,59 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void } } + public function testCreateOrUpdateDocumentsWithInplaceIncrease(): void + { + if (!static::getDatabase()->getAdapter()->getSupportForUpserts()) { + $this->expectNotToPerformAssertions(); + return; + } + + $collection = 'testCreateOrUpdateInplace'; + + static::getDatabase()->createCollection($collection); + static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true); + static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true); + + $documents = [ + new Document([ + '$id' => 'first', + 'string' => 'text📝', + 'integer' => 5, + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]), + new Document([ + '$id' => 'second', + 'string' => 'text📝', + 'integer' => 5, + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]), + ]; + + static::getDatabase()->createDocuments($collection, $documents); + + static::getDatabase()->createOrUpdateDocumentsWithInplaceIncrease( + collection: $collection, + attribute:'integer', + documents: $documents + ); + + $documents = static::getDatabase()->find($collection); + + foreach ($documents as $document) { + $this->assertEquals(10, $document->getAttribute('integer')); + } + } + public function testCreateOrUpdateDocumentsPermissions(): void { if (!static::getDatabase()->getAdapter()->getSupportForUpserts()) { From fb3fc216f3c8006528f42264216067aceb2f392c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 01:49:21 +1300 Subject: [PATCH 16/17] Fix test --- tests/e2e/Adapter/Base.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index a35b656d4..664d28bfc 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -2387,17 +2387,13 @@ public function testCreateOrUpdateDocumentsWithIncrease(): void $this->assertEquals(6, $document->getAttribute('integer')); } - $documents = static::getDatabase()->createOrUpdateDocumentsWithIncrease( + static::getDatabase()->createOrUpdateDocumentsWithIncrease( collection: $collection, attribute:'integer', value: -1, documents: $documents ); - foreach ($documents as $document) { - $this->assertEquals(5, $document->getAttribute('integer')); - } - $documents = static::getDatabase()->find($collection); foreach ($documents as $document) { From 59288435640262154fbebc62a6cc3d345224962d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Jan 2025 17:09:11 +1300 Subject: [PATCH 17/17] Make it clear that inplace increase uses existing + new value --- tests/e2e/Adapter/Base.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 664d28bfc..815d55b84 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -2441,6 +2441,9 @@ public function testCreateOrUpdateDocumentsWithInplaceIncrease(): void static::getDatabase()->createDocuments($collection, $documents); + $documents[0]->setAttribute('integer', 1); + $documents[1]->setAttribute('integer', 1); + static::getDatabase()->createOrUpdateDocumentsWithInplaceIncrease( collection: $collection, attribute:'integer', @@ -2450,7 +2453,7 @@ public function testCreateOrUpdateDocumentsWithInplaceIncrease(): void $documents = static::getDatabase()->find($collection); foreach ($documents as $document) { - $this->assertEquals(10, $document->getAttribute('integer')); + $this->assertEquals(6, $document->getAttribute('integer')); } }