From c822e8cb4b99eccaeba1294efd483db2981e8b9f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Jan 2026 06:36:10 +0000 Subject: [PATCH 01/29] Add extended attributes and indexes for ClickHouse adapter --- src/Audit/Adapter/ClickHouse.php | 206 +++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index c475edb..4894ec4 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -4,6 +4,7 @@ use Exception; use Utopia\Audit\Log; +use Utopia\Database\Database; use Utopia\Fetch\Client; use Utopia\Validator\Hostname; @@ -248,6 +249,211 @@ public function isSharedTables(): bool return $this->sharedTables; } + /** + * Override getAttributes to provide extended attributes for ClickHouse. + * Includes existing attributes from parent and adds new missing ones. + * + * @return array> + */ + public function getAttributes(): array + { + $parentAttributes = parent::getAttributes(); + + return [ + ...$parentAttributes, + [ + '$id' => 'userType', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => true, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'userInternalId', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'resourceParent', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'resourceType', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => true, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'resourceId', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => true, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'resourceInternalId', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'country', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'projectInternalId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'teamId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'teamInternalId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'hostname', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ]; + } + + /** + * Override getIndexes to provide extended indexes for ClickHouse. + * Includes existing indexes from parent and adds new missing ones. + * + * @return array> + */ + public function getIndexes(): array + { + $parentIndexes = parent::getIndexes(); + + // New indexes to add + return [ + ...$parentIndexes, + [ + '$id' => '_key_user_internal_and_event', + 'type' => Database::INDEX_KEY, + 'attributes' => ['userInternalId', 'event'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => '_key_project_internal_id', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectInternalId'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => '_key_team_internal_id', + 'type' => Database::INDEX_KEY, + 'attributes' => ['teamInternalId'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => '_key_user_internal_id', + 'type' => Database::INDEX_KEY, + 'attributes' => ['userInternalId'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => '_key_user_type', + 'type' => Database::INDEX_KEY, + 'attributes' => ['userType'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => '_key_country', + 'type' => Database::INDEX_KEY, + 'attributes' => ['country'], + 'lengths' => [], + 'orders' => [], + ], + [ + '$id' => '_key_hostname', + 'type' => Database::INDEX_KEY, + 'attributes' => ['hostname'], + 'lengths' => [], + 'orders' => [], + ], + ]; + } + /** * Get the table name with namespace prefix. * Namespace is used to isolate tables for different projects/applications. From d753ebeab0fd13da0d2e70c09729416c606f2d2d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Jan 2026 07:13:26 +0000 Subject: [PATCH 02/29] Add tests for required attributes and indexes in ClickHouse adapter --- tests/Audit/Adapter/ClickHouseTest.php | 60 ++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/Audit/Adapter/ClickHouseTest.php b/tests/Audit/Adapter/ClickHouseTest.php index 2872fbd..c0dfd68 100644 --- a/tests/Audit/Adapter/ClickHouseTest.php +++ b/tests/Audit/Adapter/ClickHouseTest.php @@ -316,4 +316,64 @@ public function testBatchOperationsWithSpecialCharacters(): void $logs = $this->audit->getLogsByUser('user`with`backticks'); $this->assertGreaterThan(0, count($logs)); } + + /** + * Test that ClickHouse adapter has all required attributes + */ + public function testClickHouseAdapterAttributes(): void + { + $adapter = new ClickHouse( + host: 'clickhouse', + username: 'default', + password: 'clickhouse' + ); + + $attributes = $adapter->getAttributes(); + $attributeIds = array_map(fn ($attr) => $attr['$id'], $attributes); + + // Verify all expected attributes exist + $expectedAttributes = [ + 'userType', 'userId', 'userInternalId', 'resourceParent', + 'resourceType', 'resourceId', 'resourceInternalId', 'event', + 'resource', 'userAgent', 'ip', 'country', 'time', 'data', + 'projectId', 'projectInternalId', 'teamId', 'teamInternalId', 'hostname' + ]; + + foreach ($expectedAttributes as $expected) { + $this->assertContains($expected, $attributeIds, "Attribute '{$expected}' not found in ClickHouse adapter"); + } + } + + /** + * Test that ClickHouse adapter has all required indexes + */ + public function testClickHouseAdapterIndexes(): void + { + $adapter = new ClickHouse( + host: 'clickhouse', + username: 'default', + password: 'clickhouse' + ); + + $indexes = $adapter->getIndexes(); + $indexIds = array_map(fn ($idx) => $idx['$id'], $indexes); + + // Verify all expected indexes exist + $expectedIndexes = [ + '_key_event', + '_key_user_internal_and_event', + '_key_resource_and_event', + '_key_time', + '_key_project_internal_id', + '_key_team_internal_id', + '_key_user_internal_id', + '_key_user_type', + '_key_country', + '_key_hostname' + ]; + + foreach ($expectedIndexes as $expected) { + $this->assertContains($expected, $indexIds, "Index '{$expected}' not found in ClickHouse adapter"); + } + } } From aa4f0d43f10fd92533762fb6d71acacb2091d597 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Jan 2026 07:13:42 +0000 Subject: [PATCH 03/29] Foramt --- tests/Audit/Adapter/ClickHouseTest.php | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/Audit/Adapter/ClickHouseTest.php b/tests/Audit/Adapter/ClickHouseTest.php index c0dfd68..5d94839 100644 --- a/tests/Audit/Adapter/ClickHouseTest.php +++ b/tests/Audit/Adapter/ClickHouseTest.php @@ -333,10 +333,25 @@ public function testClickHouseAdapterAttributes(): void // Verify all expected attributes exist $expectedAttributes = [ - 'userType', 'userId', 'userInternalId', 'resourceParent', - 'resourceType', 'resourceId', 'resourceInternalId', 'event', - 'resource', 'userAgent', 'ip', 'country', 'time', 'data', - 'projectId', 'projectInternalId', 'teamId', 'teamInternalId', 'hostname' + 'userType', + 'userId', + 'userInternalId', + 'resourceParent', + 'resourceType', + 'resourceId', + 'resourceInternalId', + 'event', + 'resource', + 'userAgent', + 'ip', + 'country', + 'time', + 'data', + 'projectId', + 'projectInternalId', + 'teamId', + 'teamInternalId', + 'hostname' ]; foreach ($expectedAttributes as $expected) { From 3d467402b09eedb4919960a8163693ba3955c768 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Jan 2026 07:25:12 +0000 Subject: [PATCH 04/29] Refactor ClickHouse index tests to include specific ClickHouse indexes and parent naming conventions --- tests/Audit/Adapter/ClickHouseTest.php | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/Audit/Adapter/ClickHouseTest.php b/tests/Audit/Adapter/ClickHouseTest.php index 5d94839..948ccec 100644 --- a/tests/Audit/Adapter/ClickHouseTest.php +++ b/tests/Audit/Adapter/ClickHouseTest.php @@ -329,7 +329,7 @@ public function testClickHouseAdapterAttributes(): void ); $attributes = $adapter->getAttributes(); - $attributeIds = array_map(fn ($attr) => $attr['$id'], $attributes); + $attributeIds = array_map(fn($attr) => $attr['$id'], $attributes); // Verify all expected attributes exist $expectedAttributes = [ @@ -371,14 +371,11 @@ public function testClickHouseAdapterIndexes(): void ); $indexes = $adapter->getIndexes(); - $indexIds = array_map(fn ($idx) => $idx['$id'], $indexes); + $indexIds = array_map(fn($idx) => $idx['$id'], $indexes); - // Verify all expected indexes exist - $expectedIndexes = [ - '_key_event', + // Verify all ClickHouse-specific indexes exist + $expectedClickHouseIndexes = [ '_key_user_internal_and_event', - '_key_resource_and_event', - '_key_time', '_key_project_internal_id', '_key_team_internal_id', '_key_user_internal_id', @@ -387,8 +384,14 @@ public function testClickHouseAdapterIndexes(): void '_key_hostname' ]; - foreach ($expectedIndexes as $expected) { - $this->assertContains($expected, $indexIds, "Index '{$expected}' not found in ClickHouse adapter"); + foreach ($expectedClickHouseIndexes as $expected) { + $this->assertContains($expected, $indexIds, "ClickHouse index '{$expected}' not found in ClickHouse adapter"); + } + + // Verify parent indexes are also included (with parent naming convention) + $parentExpectedIndexes = ['idx_event', 'idx_userId_event', 'idx_resource_event', 'idx_time_desc']; + foreach ($parentExpectedIndexes as $expected) { + $this->assertContains($expected, $indexIds, "Parent index '{$expected}' not found in ClickHouse adapter"); } } } From fc4a15d7eeb6e169a775d4c05cee66f29f5f45ba Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Jan 2026 07:27:24 +0000 Subject: [PATCH 05/29] format --- tests/Audit/Adapter/ClickHouseTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Audit/Adapter/ClickHouseTest.php b/tests/Audit/Adapter/ClickHouseTest.php index 948ccec..410bc59 100644 --- a/tests/Audit/Adapter/ClickHouseTest.php +++ b/tests/Audit/Adapter/ClickHouseTest.php @@ -329,7 +329,7 @@ public function testClickHouseAdapterAttributes(): void ); $attributes = $adapter->getAttributes(); - $attributeIds = array_map(fn($attr) => $attr['$id'], $attributes); + $attributeIds = array_map(fn ($attr) => $attr['$id'], $attributes); // Verify all expected attributes exist $expectedAttributes = [ @@ -371,7 +371,7 @@ public function testClickHouseAdapterIndexes(): void ); $indexes = $adapter->getIndexes(); - $indexIds = array_map(fn($idx) => $idx['$id'], $indexes); + $indexIds = array_map(fn ($idx) => $idx['$id'], $indexes); // Verify all ClickHouse-specific indexes exist $expectedClickHouseIndexes = [ From 207d1ddbf16db31eec31cdb39e9c3c40f6fad3b5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Jan 2026 08:17:39 +0000 Subject: [PATCH 06/29] Add getById method to Adapter and its implementations for retrieving logs by ID --- src/Audit/Adapter.php | 10 ++++++++++ src/Audit/Adapter/ClickHouse.php | 27 +++++++++++++++++++++++++++ src/Audit/Adapter/Database.php | 24 ++++++++++++++++++++++++ src/Audit/Audit.php | 13 +++++++++++++ tests/Audit/AuditBase.php | 30 ++++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+) diff --git a/src/Audit/Adapter.php b/src/Audit/Adapter.php index 1a39d9c..54e5283 100644 --- a/src/Audit/Adapter.php +++ b/src/Audit/Adapter.php @@ -24,6 +24,16 @@ abstract public function getName(): string; */ abstract public function setup(): void; + /** + * Get a single log by its ID. + * + * @param string $id + * @return Log|null The log entry or null if not found + * + * @throws \Exception + */ + abstract public function getById(string $id): ?Log; + /** * Create an audit log entry. * diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 4894ec4..605f101 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -701,6 +701,33 @@ public function create(array $log): Log return new Log($result); } + /** + * Get a single log by its ID. + * + * @param string $id + * @return Log|null The log entry or null if not found + * @throws Exception + */ + public function getById(string $id): ?Log + { + $tableName = $this->getTableName(); + $tenantFilter = $this->getTenantFilter(); + $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + + $sql = " + SELECT " . $this->getSelectColumns() . " + FROM {$escapedTable} + WHERE id = {id:String}{$tenantFilter} + LIMIT 1 + FORMAT TabSeparated + "; + + $result = $this->query($sql, ['id' => $id]); + $logs = $this->parseResults($result); + + return $logs[0] ?? null; + } + /** * Create multiple audit log entries in batch. * diff --git a/src/Audit/Adapter/Database.php b/src/Audit/Adapter/Database.php index ddc5986..afa6022 100644 --- a/src/Audit/Adapter/Database.php +++ b/src/Audit/Adapter/Database.php @@ -102,6 +102,30 @@ public function createBatch(array $logs): bool return true; } + /** + * Get a single log by its ID. + * + * @param string $id + * @return Log|null The log entry or null if not found + * @throws AuthorizationException|\Exception + */ + public function getById(string $id): ?Log + { + try { + $document = $this->db->getAuthorization()->skip(function () use ($id) { + return $this->db->getDocument($this->getCollectionName(), $id); + }); + + if ($document->isEmpty()) { + return null; + } + + return new Log($document->getArrayCopy()); + } catch (\Exception $e) { + return null; + } + } + /** * Build time-related query conditions. * diff --git a/src/Audit/Audit.php b/src/Audit/Audit.php index f4c0f33..2a2da67 100644 --- a/src/Audit/Audit.php +++ b/src/Audit/Audit.php @@ -84,6 +84,19 @@ public function logBatch(array $events): bool return $this->adapter->createBatch($events); } + /** + * Get a single log by its ID. + * + * @param string $id + * @return Log|null The log entry or null if not found + * + * @throws \Exception + */ + public function getLogById(string $id): ?Log + { + return $this->adapter->getById($id); + } + /** * Get all logs by user ID. * diff --git a/tests/Audit/AuditBase.php b/tests/Audit/AuditBase.php index 25b90cb..f940a72 100644 --- a/tests/Audit/AuditBase.php +++ b/tests/Audit/AuditBase.php @@ -155,6 +155,36 @@ public function testGetLogsByResource(): void $this->assertEquals('127.0.0.1', $logs5[0]['ip']); } + public function testGetLogById(): void + { + // Create a test log + $userId = 'testGetByIdUser'; + $userAgent = 'Mozilla/5.0 Test'; + $ip = '192.168.1.100'; + $location = 'US'; + $data = ['test' => 'getById']; + + $log = $this->audit->log($userId, 'create', 'test/resource/123', $userAgent, $ip, $location, $data); + $logId = $log->getId(); + + // Retrieve the log by ID + $retrievedLog = $this->audit->getLogById($logId); + + $this->assertNotNull($retrievedLog); + $this->assertEquals($logId, $retrievedLog->getId()); + $this->assertEquals($userId, $retrievedLog->getAttribute('userId')); + $this->assertEquals('create', $retrievedLog->getAttribute('event')); + $this->assertEquals('test/resource/123', $retrievedLog->getAttribute('resource')); + $this->assertEquals($userAgent, $retrievedLog->getAttribute('userAgent')); + $this->assertEquals($ip, $retrievedLog->getAttribute('ip')); + $this->assertEquals($location, $retrievedLog->getAttribute('location')); + $this->assertEquals($data, $retrievedLog->getAttribute('data')); + + // Test with non-existent ID + $nonExistentLog = $this->audit->getLogById('non-existent-id-12345'); + $this->assertNull($nonExistentLog); + } + public function testLogByBatch(): void { // First cleanup existing logs From ef6b30f418ea3e604cae42aea658768a893c7a45 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Jan 2026 09:20:20 +0000 Subject: [PATCH 07/29] Query support clickhouse --- src/Audit/Adapter/ClickHouse.php | 152 ++++++++++++++++ src/Audit/Query.php | 288 +++++++++++++++++++++++++++++++ 2 files changed, 440 insertions(+) create mode 100644 src/Audit/Query.php diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 605f101..a1c179c 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -4,6 +4,7 @@ use Exception; use Utopia\Audit\Log; +use Utopia\Audit\Query; use Utopia\Database\Database; use Utopia\Fetch\Client; use Utopia\Validator\Hostname; @@ -728,6 +729,157 @@ public function getById(string $id): ?Log return $logs[0] ?? null; } + /** + * Find logs using Query objects. + * + * @param array $queries + * @return array + * @throws Exception + */ + public function find(array $queries = []): array + { + $tableName = $this->getTableName(); + $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + + // Parse queries + $parsed = $this->parseQueries($queries); + + // Build SELECT clause + $selectColumns = $parsed['select'] ?? $this->getSelectColumns(); + + // Build WHERE clause + $whereClause = ''; + $tenantFilter = $this->getTenantFilter(); + if (!empty($parsed['filters']) || $tenantFilter) { + $conditions = $parsed['filters'] ?? []; + if ($tenantFilter) { + $conditions[] = ltrim($tenantFilter, ' AND'); + } + $whereClause = ' WHERE ' . implode(' AND ', $conditions); + } + + // Build ORDER BY clause + $orderClause = ''; + if (!empty($parsed['orderBy'])) { + $orderClause = ' ORDER BY ' . implode(', ', $parsed['orderBy']); + } + + // Build LIMIT and OFFSET + $limitClause = isset($parsed['limit']) ? ' LIMIT {limit:UInt64}' : ''; + $offsetClause = isset($parsed['offset']) ? ' OFFSET {offset:UInt64}' : ''; + + $sql = " + SELECT {$selectColumns} + FROM {$escapedTable}{$whereClause}{$orderClause}{$limitClause}{$offsetClause} + FORMAT TabSeparated + "; + + $result = $this->query($sql, $parsed['params']); + return $this->parseResults($result); + } + + /** + * Parse Query objects into SQL components. + * + * @param array $queries + * @return array{filters: array, params: array, orderBy?: array, limit?: int, offset?: int} + * @throws Exception + */ + private function parseQueries(array $queries): array + { + $filters = []; + $params = []; + $orderBy = []; + $limit = null; + $offset = null; + $paramCounter = 0; + + foreach ($queries as $query) { + if (!$query instanceof Query) { + continue; + } + + $method = $query->getMethod(); + $attribute = $query->getAttribute(); + $values = $query->getValues(); + + switch ($method) { + case Query::TYPE_EQUAL: + $paramName = 'param_' . $paramCounter++; + $filters[] = "{$attribute} = {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($values[0]); + break; + + case Query::TYPE_LESSER: + $paramName = 'param_' . $paramCounter++; + $filters[] = "{$attribute} < {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($values[0]); + break; + + case Query::TYPE_GREATER: + $paramName = 'param_' . $paramCounter++; + $filters[] = "{$attribute} > {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($values[0]); + break; + + case Query::TYPE_BETWEEN: + $paramName1 = 'param_' . $paramCounter++; + $paramName2 = 'param_' . $paramCounter++; + $filters[] = "{$attribute} BETWEEN {{$paramName1}:String} AND {{$paramName2}:String}"; + $params[$paramName1] = $this->formatParamValue($values[0]); + $params[$paramName2] = $this->formatParamValue($values[1]); + break; + + case Query::TYPE_IN: + $inParams = []; + foreach ($values as $value) { + $paramName = 'param_' . $paramCounter++; + $inParams[] = "{{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($value); + } + $filters[] = "{$attribute} IN (" . implode(', ', $inParams) . ")"; + break; + + case Query::TYPE_ORDER_DESC: + $orderBy[] = "{$attribute} DESC"; + break; + + case Query::TYPE_ORDER_ASC: + $orderBy[] = "{$attribute} ASC"; + break; + + case Query::TYPE_LIMIT: + $limit = (int) $values[0]; + $params['limit'] = $limit; + break; + + case Query::TYPE_OFFSET: + $offset = (int) $values[0]; + $params['offset'] = $offset; + break; + } + } + + $result = [ + 'filters' => $filters, + 'params' => $params, + ]; + + if (!empty($orderBy)) { + $result['orderBy'] = $orderBy; + } + + if ($limit !== null) { + $result['limit'] = $limit; + } + + if ($offset !== null) { + $result['offset'] = $offset; + } + + return $result; + } + /** * Create multiple audit log entries in batch. * diff --git a/src/Audit/Query.php b/src/Audit/Query.php new file mode 100644 index 0000000..3b70b2a --- /dev/null +++ b/src/Audit/Query.php @@ -0,0 +1,288 @@ + + */ + protected array $values = []; + + /** + * Construct a new query object + * + * @param string $method + * @param string $attribute + * @param array $values + */ + public function __construct(string $method, string $attribute = '', array $values = []) + { + $this->method = $method; + $this->attribute = $attribute; + $this->values = $values; + } + + /** + * @return string + */ + public function getMethod(): string + { + return $this->method; + } + + /** + * @return string + */ + public function getAttribute(): string + { + return $this->attribute; + } + + /** + * @return array + */ + public function getValues(): array + { + return $this->values; + } + + /** + * @param mixed $default + * @return mixed + */ + public function getValue(mixed $default = null): mixed + { + return $this->values[0] ?? $default; + } + + /** + * Filter by equal condition + * + * @param string $attribute + * @param mixed $value + * @return self + */ + public static function equal(string $attribute, mixed $value): self + { + return new self(self::TYPE_EQUAL, $attribute, [$value]); + } + + /** + * Filter by less than condition + * + * @param string $attribute + * @param mixed $value + * @return self + */ + public static function lessThan(string $attribute, mixed $value): self + { + return new self(self::TYPE_LESSER, $attribute, [$value]); + } + + /** + * Filter by greater than condition + * + * @param string $attribute + * @param mixed $value + * @return self + */ + public static function greaterThan(string $attribute, mixed $value): self + { + return new self(self::TYPE_GREATER, $attribute, [$value]); + } + + /** + * Filter by BETWEEN condition + * + * @param string $attribute + * @param mixed $start + * @param mixed $end + * @return self + */ + public static function between(string $attribute, mixed $start, mixed $end): self + { + return new self(self::TYPE_BETWEEN, $attribute, [$start, $end]); + } + + /** + * Filter by IN condition + * + * @param string $attribute + * @param array $values + * @return self + */ + public static function in(string $attribute, array $values): self + { + return new self(self::TYPE_IN, $attribute, $values); + } + + /** + * Order by descending + * + * @param string $attribute + * @return self + */ + public static function orderDesc(string $attribute = 'time'): self + { + return new self(self::TYPE_ORDER_DESC, $attribute); + } + + /** + * Order by ascending + * + * @param string $attribute + * @return self + */ + public static function orderAsc(string $attribute = 'time'): self + { + return new self(self::TYPE_ORDER_ASC, $attribute); + } + + /** + * Limit number of results + * + * @param int $limit + * @return self + */ + public static function limit(int $limit): self + { + return new self(self::TYPE_LIMIT, '', [$limit]); + } + + /** + * Offset results + * + * @param int $offset + * @return self + */ + public static function offset(int $offset): self + { + return new self(self::TYPE_OFFSET, '', [$offset]); + } + + /** + * Parse query from JSON string + * + * @param string $query + * @return self + * @throws \Exception + */ + public static function parse(string $query): self + { + try { + $query = \json_decode($query, true, flags: JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + throw new \Exception('Invalid query: ' . $e->getMessage()); + } + + if (!\is_array($query)) { + throw new \Exception('Invalid query. Must be an array, got ' . \gettype($query)); + } + + return self::parseQuery($query); + } + + /** + * Parse an array of queries + * + * @param array $queries + * @return array + * @throws \Exception + */ + public static function parseQueries(array $queries): array + { + $parsed = []; + + foreach ($queries as $query) { + $parsed[] = self::parse($query); + } + + return $parsed; + } + + /** + * Parse query from array + * + * @param array $query + * @return self + * @throws \Exception + */ + protected static function parseQuery(array $query): self + { + $method = $query['method'] ?? ''; + $attribute = $query['attribute'] ?? ''; + $values = $query['values'] ?? []; + + if (!\is_string($method)) { + throw new \Exception('Invalid query method. Must be a string, got ' . \gettype($method)); + } + + if (!\is_string($attribute)) { + throw new \Exception('Invalid query attribute. Must be a string, got ' . \gettype($attribute)); + } + + if (!\is_array($values)) { + throw new \Exception('Invalid query values. Must be an array, got ' . \gettype($values)); + } + + return new self($method, $attribute, $values); + } + + /** + * Convert query to array + * + * @return array + */ + public function toArray(): array + { + $array = ['method' => $this->method]; + + if (!empty($this->attribute)) { + $array['attribute'] = $this->attribute; + } + + $array['values'] = $this->values; + + return $array; + } + + /** + * Convert query to JSON string + * + * @return string + * @throws \Exception + */ + public function toString(): string + { + try { + return \json_encode($this->toArray(), flags: JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + throw new \Exception('Invalid Json: ' . $e->getMessage()); + } + } +} From c83605e87fa988a7cd4cc5761399f688c49151b5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Jan 2026 09:46:44 +0000 Subject: [PATCH 08/29] Add tests for Query class methods and functionality --- tests/Audit/Adapter/ClickHouseTest.php | 273 +++++++++++++++++++++++++ tests/Audit/QueryTest.php | 225 ++++++++++++++++++++ 2 files changed, 498 insertions(+) create mode 100644 tests/Audit/QueryTest.php diff --git a/tests/Audit/Adapter/ClickHouseTest.php b/tests/Audit/Adapter/ClickHouseTest.php index 410bc59..42a7f51 100644 --- a/tests/Audit/Adapter/ClickHouseTest.php +++ b/tests/Audit/Adapter/ClickHouseTest.php @@ -6,6 +6,7 @@ use PHPUnit\Framework\TestCase; use Utopia\Audit\Adapter\ClickHouse; use Utopia\Audit\Audit; +use Utopia\Audit\Query; use Utopia\Tests\Audit\AuditBase; /** @@ -394,4 +395,276 @@ public function testClickHouseAdapterIndexes(): void $this->assertContains($expected, $indexIds, "Parent index '{$expected}' not found in ClickHouse adapter"); } } + + /** + * Test find method with simple query + */ + public function testFindWithSimpleQuery(): void + { + // Create test data + $this->audit->log( + userId: 'testuser1', + event: 'create', + resource: 'document/123', + userAgent: 'Test Agent', + ip: '192.168.1.1', + location: 'US', + data: ['action' => 'test'] + ); + + $this->audit->log( + userId: 'testuser2', + event: 'update', + resource: 'document/456', + userAgent: 'Test Agent', + ip: '192.168.1.2', + location: 'UK', + data: ['action' => 'test'] + ); + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Test equal query + $logs = $adapter->find([ + Query::equal('userId', 'testuser1') + ]); + + $this->assertIsArray($logs); + $this->assertGreaterThan(0, count($logs)); + + foreach ($logs as $log) { + $this->assertEquals('testuser1', $log->getAttribute('userId')); + } + } + + /** + * Test find method with multiple filters + */ + public function testFindWithMultipleFilters(): void + { + // Create test data + $this->audit->log( + userId: 'user123', + event: 'create', + resource: 'collection/test', + userAgent: 'Test Agent', + ip: '10.0.0.1', + location: 'US', + data: [] + ); + + $this->audit->log( + userId: 'user123', + event: 'delete', + resource: 'collection/test2', + userAgent: 'Test Agent', + ip: '10.0.0.1', + location: 'US', + data: [] + ); + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Test with multiple filters + $logs = $adapter->find([ + Query::equal('userId', 'user123'), + Query::equal('event', 'create') + ]); + + $this->assertIsArray($logs); + $this->assertGreaterThan(0, count($logs)); + + foreach ($logs as $log) { + $this->assertEquals('user123', $log->getAttribute('userId')); + $this->assertEquals('create', $log->getAttribute('event')); + } + } + + /** + * Test find method with IN query + */ + public function testFindWithInQuery(): void + { + // Create test data + $events = ['login', 'logout', 'create']; + foreach ($events as $event) { + $this->audit->log( + userId: 'userMulti', + event: $event, + resource: 'test', + userAgent: 'Test Agent', + ip: '127.0.0.1', + location: 'US', + data: [] + ); + } + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Test IN query + $logs = $adapter->find([ + Query::equal('userId', 'userMulti'), + Query::in('event', ['login', 'logout']) + ]); + + $this->assertIsArray($logs); + $this->assertCount(2, $logs); + + foreach ($logs as $log) { + $this->assertContains($log->getAttribute('event'), ['login', 'logout']); + } + } + + /** + * Test find method with ordering + */ + public function testFindWithOrdering(): void + { + // Create test data with different events + $this->audit->log( + userId: 'orderUser', + event: 'zzz_event', + resource: 'test', + userAgent: 'Test Agent', + ip: '127.0.0.1', + location: 'US', + data: [] + ); + + sleep(1); // Ensure different timestamps + + $this->audit->log( + userId: 'orderUser', + event: 'aaa_event', + resource: 'test', + userAgent: 'Test Agent', + ip: '127.0.0.1', + location: 'US', + data: [] + ); + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Test ascending order + $logs = $adapter->find([ + Query::equal('userId', 'orderUser'), + Query::orderAsc('event') + ]); + + $this->assertIsArray($logs); + $this->assertGreaterThanOrEqual(2, count($logs)); + $this->assertEquals('aaa_event', $logs[0]->getAttribute('event')); + + // Test descending order + $logs = $adapter->find([ + Query::equal('userId', 'orderUser'), + Query::orderDesc('event') + ]); + + $this->assertIsArray($logs); + $this->assertGreaterThanOrEqual(2, count($logs)); + $this->assertEquals('zzz_event', $logs[0]->getAttribute('event')); + } + + /** + * Test find method with limit and offset + */ + public function testFindWithLimitAndOffset(): void + { + // Create multiple test logs + for ($i = 1; $i <= 5; $i++) { + $this->audit->log( + userId: 'paginationUser', + event: "event_{$i}", + resource: "resource_{$i}", + userAgent: 'Test Agent', + ip: '127.0.0.1', + location: 'US', + data: ['index' => $i] + ); + } + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Test limit + $logs = $adapter->find([ + Query::equal('userId', 'paginationUser'), + Query::limit(2) + ]); + + $this->assertIsArray($logs); + $this->assertCount(2, $logs); + + // Test offset + $logs = $adapter->find([ + Query::equal('userId', 'paginationUser'), + Query::orderAsc('event'), + Query::limit(2), + Query::offset(2) + ]); + + $this->assertIsArray($logs); + $this->assertLessThanOrEqual(2, count($logs)); + } + + /** + * Test find method with between query + */ + public function testFindWithBetweenQuery(): void + { + $time1 = '2023-01-01 00:00:00+00:00'; + $time2 = '2023-06-01 00:00:00+00:00'; + $time3 = '2023-12-31 23:59:59+00:00'; + + // Create test data with different times using logBatch + $this->audit->logBatch([ + [ + 'userId' => 'betweenUser', + 'event' => 'event1', + 'resource' => 'test', + 'userAgent' => 'Test Agent', + 'ip' => '127.0.0.1', + 'location' => 'US', + 'data' => [], + 'time' => $time1 + ], + [ + 'userId' => 'betweenUser', + 'event' => 'event2', + 'resource' => 'test', + 'userAgent' => 'Test Agent', + 'ip' => '127.0.0.1', + 'location' => 'US', + 'data' => [], + 'time' => $time2 + ], + [ + 'userId' => 'betweenUser', + 'event' => 'event3', + 'resource' => 'test', + 'userAgent' => 'Test Agent', + 'ip' => '127.0.0.1', + 'location' => 'US', + 'data' => [], + 'time' => $time3 + ] + ]); + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Test between query + $logs = $adapter->find([ + Query::equal('userId', 'betweenUser'), + Query::between('time', '2023-05-01 00:00:00+00:00', '2023-12-31 00:00:00+00:00') + ]); + + $this->assertIsArray($logs); + $this->assertGreaterThan(0, count($logs)); + } } diff --git a/tests/Audit/QueryTest.php b/tests/Audit/QueryTest.php new file mode 100644 index 0000000..2dc3fa1 --- /dev/null +++ b/tests/Audit/QueryTest.php @@ -0,0 +1,225 @@ +assertEquals(Query::TYPE_EQUAL, $query->getMethod()); + $this->assertEquals('userId', $query->getAttribute()); + $this->assertEquals(['123'], $query->getValues()); + + // Test lessThan + $query = Query::lessThan('time', '2024-01-01'); + $this->assertEquals(Query::TYPE_LESSER, $query->getMethod()); + $this->assertEquals('time', $query->getAttribute()); + $this->assertEquals(['2024-01-01'], $query->getValues()); + + // Test greaterThan + $query = Query::greaterThan('time', '2023-01-01'); + $this->assertEquals(Query::TYPE_GREATER, $query->getMethod()); + $this->assertEquals('time', $query->getAttribute()); + $this->assertEquals(['2023-01-01'], $query->getValues()); + + // Test between + $query = Query::between('time', '2023-01-01', '2024-01-01'); + $this->assertEquals(Query::TYPE_BETWEEN, $query->getMethod()); + $this->assertEquals('time', $query->getAttribute()); + $this->assertEquals(['2023-01-01', '2024-01-01'], $query->getValues()); + + // Test in + $query = Query::in('event', ['create', 'update', 'delete']); + $this->assertEquals(Query::TYPE_IN, $query->getMethod()); + $this->assertEquals('event', $query->getAttribute()); + $this->assertEquals(['create', 'update', 'delete'], $query->getValues()); + + // Test orderDesc + $query = Query::orderDesc('time'); + $this->assertEquals(Query::TYPE_ORDER_DESC, $query->getMethod()); + $this->assertEquals('time', $query->getAttribute()); + $this->assertEquals([], $query->getValues()); + + // Test orderAsc + $query = Query::orderAsc('userId'); + $this->assertEquals(Query::TYPE_ORDER_ASC, $query->getMethod()); + $this->assertEquals('userId', $query->getAttribute()); + $this->assertEquals([], $query->getValues()); + + // Test limit + $query = Query::limit(10); + $this->assertEquals(Query::TYPE_LIMIT, $query->getMethod()); + $this->assertEquals('', $query->getAttribute()); + $this->assertEquals([10], $query->getValues()); + + // Test offset + $query = Query::offset(5); + $this->assertEquals(Query::TYPE_OFFSET, $query->getMethod()); + $this->assertEquals('', $query->getAttribute()); + $this->assertEquals([5], $query->getValues()); + } + + /** + * Test Query parse and toString methods + */ + public function testQueryParseAndToString(): void + { + // Test parsing equal query + $json = '{"method":"equal","attribute":"userId","values":["123"]}'; + $query = Query::parse($json); + $this->assertEquals(Query::TYPE_EQUAL, $query->getMethod()); + $this->assertEquals('userId', $query->getAttribute()); + $this->assertEquals(['123'], $query->getValues()); + + // Test toString + $query = Query::equal('event', 'create'); + $json = $query->toString(); + $this->assertJson($json); + + $parsed = Query::parse($json); + $this->assertEquals(Query::TYPE_EQUAL, $parsed->getMethod()); + $this->assertEquals('event', $parsed->getAttribute()); + $this->assertEquals(['create'], $parsed->getValues()); + + // Test toArray + $array = $query->toArray(); + $this->assertArrayHasKey('method', $array); + $this->assertArrayHasKey('attribute', $array); + $this->assertArrayHasKey('values', $array); + $this->assertEquals(Query::TYPE_EQUAL, $array['method']); + $this->assertEquals('event', $array['attribute']); + $this->assertEquals(['create'], $array['values']); + } + + /** + * Test Query parseQueries method + */ + public function testQueryParseQueries(): void + { + $queries = [ + '{"method":"equal","attribute":"userId","values":["123"]}', + '{"method":"greaterThan","attribute":"time","values":["2023-01-01"]}', + '{"method":"limit","values":[10]}' + ]; + + $parsed = Query::parseQueries($queries); + + $this->assertCount(3, $parsed); + $this->assertInstanceOf(Query::class, $parsed[0]); + $this->assertInstanceOf(Query::class, $parsed[1]); + $this->assertInstanceOf(Query::class, $parsed[2]); + + $this->assertEquals(Query::TYPE_EQUAL, $parsed[0]->getMethod()); + $this->assertEquals(Query::TYPE_GREATER, $parsed[1]->getMethod()); + $this->assertEquals(Query::TYPE_LIMIT, $parsed[2]->getMethod()); + } + + /** + * Test Query getValue method + */ + public function testGetValue(): void + { + $query = Query::equal('userId', '123'); + $this->assertEquals('123', $query->getValue()); + + $query = Query::limit(10); + $this->assertEquals(10, $query->getValue()); + + // Test with default value + $query = Query::orderAsc('time'); + $this->assertNull($query->getValue()); + $this->assertEquals('default', $query->getValue('default')); + } + + /** + * Test Query with empty attribute + */ + public function testQueryWithEmptyAttribute(): void + { + $query = Query::limit(25); + $this->assertEquals('', $query->getAttribute()); + $this->assertEquals([25], $query->getValues()); + + $query = Query::offset(10); + $this->assertEquals('', $query->getAttribute()); + $this->assertEquals([10], $query->getValues()); + } + + /** + * Test Query parse with invalid JSON + */ + public function testQueryParseInvalidJson(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Invalid query'); + + Query::parse('{"method":"equal","attribute":"userId"'); // Invalid JSON + } + + /** + * Test Query parse with non-array value + */ + public function testQueryParseNonArray(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Invalid query. Must be an array'); + + Query::parse('"string"'); + } + + /** + * Test Query parse with invalid method type + */ + public function testQueryParseInvalidMethodType(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Invalid query method. Must be a string'); + + Query::parse('{"method":["array"],"attribute":"test","values":[]}'); + } + + /** + * Test Query parse with invalid attribute type + */ + public function testQueryParseInvalidAttributeType(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Invalid query attribute. Must be a string'); + + Query::parse('{"method":"equal","attribute":123,"values":[]}'); + } + + /** + * Test Query parse with invalid values type + */ + public function testQueryParseInvalidValuesType(): void + { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Invalid query values. Must be an array'); + + Query::parse('{"method":"equal","attribute":"test","values":"string"}'); + } + + /** + * Test Query toString with complex values + */ + public function testQueryToStringWithComplexValues(): void + { + $query = Query::between('time', '2023-01-01', '2024-12-31'); + $json = $query->toString(); + $this->assertJson($json); + + $parsed = Query::parse($json); + $this->assertEquals(Query::TYPE_BETWEEN, $parsed->getMethod()); + $this->assertEquals('time', $parsed->getAttribute()); + $this->assertEquals(['2023-01-01', '2024-12-31'], $parsed->getValues()); + } +} From d1d0a4331a339bae72c170e1a6acc3bf3c62be07 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 12 Jan 2026 07:21:07 +0000 Subject: [PATCH 09/29] Count method and test --- src/Audit/Adapter/ClickHouse.php | 42 ++++ tests/Audit/Adapter/ClickHouseTest.php | 264 +++++++++++++++++++++++++ 2 files changed, 306 insertions(+) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index a1c179c..abd10e5 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -778,6 +778,48 @@ public function find(array $queries = []): array return $this->parseResults($result); } + /** + * Count logs using Query objects. + * + * @param array $queries + * @return int + * @throws Exception + */ + public function count(array $queries = []): int + { + $tableName = $this->getTableName(); + $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + + // Parse queries - we only need filters and params, not ordering/limit/offset + $parsed = $this->parseQueries($queries); + + // Build WHERE clause + $whereClause = ''; + $tenantFilter = $this->getTenantFilter(); + if (!empty($parsed['filters']) || $tenantFilter) { + $conditions = $parsed['filters'] ?? []; + if ($tenantFilter) { + $conditions[] = ltrim($tenantFilter, ' AND'); + } + $whereClause = ' WHERE ' . implode(' AND ', $conditions); + } + + // Remove limit and offset from params as they don't apply to count + $params = $parsed['params']; + unset($params['limit'], $params['offset']); + + $sql = " + SELECT COUNT(*) as count + FROM {$escapedTable}{$whereClause} + FORMAT TabSeparated + "; + + $result = $this->query($sql, $params); + $trimmed = trim($result); + + return $trimmed !== '' ? (int) $trimmed : 0; + } + /** * Parse Query objects into SQL components. * diff --git a/tests/Audit/Adapter/ClickHouseTest.php b/tests/Audit/Adapter/ClickHouseTest.php index 42a7f51..efc2582 100644 --- a/tests/Audit/Adapter/ClickHouseTest.php +++ b/tests/Audit/Adapter/ClickHouseTest.php @@ -667,4 +667,268 @@ public function testFindWithBetweenQuery(): void $this->assertIsArray($logs); $this->assertGreaterThan(0, count($logs)); } + + /** + * Test count method with no filters + */ + public function testCountWithNoFilters(): void + { + // Create test data + for ($i = 0; $i < 5; $i++) { + $this->audit->log( + userId: 'countUser', + event: "event{$i}", + resource: 'test/count', + userAgent: 'Test Agent', + ip: '127.0.0.1', + location: 'US', + data: [] + ); + } + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Count all logs for this user + $count = $adapter->count([ + Query::equal('userId', 'countUser') + ]); + + $this->assertIsInt($count); + $this->assertEquals(5, $count); + } + + /** + * Test count method with simple query + */ + public function testCountWithSimpleQuery(): void + { + // Create test data + $this->audit->log( + userId: 'countUser1', + event: 'create', + resource: 'document/123', + userAgent: 'Test Agent', + ip: '192.168.1.1', + location: 'US', + data: ['action' => 'test'] + ); + + $this->audit->log( + userId: 'countUser1', + event: 'update', + resource: 'document/456', + userAgent: 'Test Agent', + ip: '192.168.1.2', + location: 'UK', + data: ['action' => 'test'] + ); + + $this->audit->log( + userId: 'countUser2', + event: 'create', + resource: 'document/789', + userAgent: 'Test Agent', + ip: '192.168.1.3', + location: 'FR', + data: ['action' => 'test'] + ); + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Count logs for specific user + $count = $adapter->count([ + Query::equal('userId', 'countUser1') + ]); + + $this->assertIsInt($count); + $this->assertEquals(2, $count); + } + + /** + * Test count method with multiple filters + */ + public function testCountWithMultipleFilters(): void + { + // Create test data + $this->audit->log( + userId: 'multiFilterUser', + event: 'create', + resource: 'collection/test', + userAgent: 'Test Agent', + ip: '10.0.0.1', + location: 'US', + data: [] + ); + + $this->audit->log( + userId: 'multiFilterUser', + event: 'create', + resource: 'collection/test2', + userAgent: 'Test Agent', + ip: '10.0.0.1', + location: 'US', + data: [] + ); + + $this->audit->log( + userId: 'multiFilterUser', + event: 'delete', + resource: 'collection/test3', + userAgent: 'Test Agent', + ip: '10.0.0.1', + location: 'US', + data: [] + ); + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Count with multiple filters + $count = $adapter->count([ + Query::equal('userId', 'multiFilterUser'), + Query::equal('event', 'create') + ]); + + $this->assertIsInt($count); + $this->assertEquals(2, $count); + } + + /** + * Test count method with IN query + */ + public function testCountWithInQuery(): void + { + // Create test data + $events = ['login', 'logout', 'create', 'update', 'delete']; + foreach ($events as $event) { + $this->audit->log( + userId: 'inQueryCountUser', + event: $event, + resource: 'test', + userAgent: 'Test Agent', + ip: '127.0.0.1', + location: 'US', + data: [] + ); + } + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Count with IN query + $count = $adapter->count([ + Query::equal('userId', 'inQueryCountUser'), + Query::in('event', ['login', 'logout', 'create']) + ]); + + $this->assertIsInt($count); + $this->assertEquals(3, $count); + } + + /** + * Test count method ignores limit and offset + */ + public function testCountIgnoresLimitAndOffset(): void + { + // Create test data + for ($i = 0; $i < 10; $i++) { + $this->audit->log( + userId: 'limitOffsetCountUser', + event: "event{$i}", + resource: 'test', + userAgent: 'Test Agent', + ip: '127.0.0.1', + location: 'US', + data: [] + ); + } + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Count should ignore limit and offset and return total count + $count = $adapter->count([ + Query::equal('userId', 'limitOffsetCountUser'), + Query::limit(3), + Query::offset(2) + ]); + + $this->assertIsInt($count); + $this->assertEquals(10, $count); // Should count all 10, not affected by limit/offset + } + + /** + * Test count method with between query + */ + public function testCountWithBetweenQuery(): void + { + $time1 = '2023-01-01 00:00:00+00:00'; + $time2 = '2023-06-01 00:00:00+00:00'; + $time3 = '2023-12-31 23:59:59+00:00'; + + // Create test data with different times using logBatch + $this->audit->logBatch([ + [ + 'userId' => 'betweenCountUser', + 'event' => 'event1', + 'resource' => 'test', + 'userAgent' => 'Test Agent', + 'ip' => '127.0.0.1', + 'location' => 'US', + 'data' => [], + 'time' => $time1 + ], + [ + 'userId' => 'betweenCountUser', + 'event' => 'event2', + 'resource' => 'test', + 'userAgent' => 'Test Agent', + 'ip' => '127.0.0.1', + 'location' => 'US', + 'data' => [], + 'time' => $time2 + ], + [ + 'userId' => 'betweenCountUser', + 'event' => 'event3', + 'resource' => 'test', + 'userAgent' => 'Test Agent', + 'ip' => '127.0.0.1', + 'location' => 'US', + 'data' => [], + 'time' => $time3 + ] + ]); + + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Count with between query + $count = $adapter->count([ + Query::equal('userId', 'betweenCountUser'), + Query::between('time', '2023-05-01 00:00:00+00:00', '2023-12-31 00:00:00+00:00') + ]); + + $this->assertIsInt($count); + $this->assertGreaterThan(0, $count); + } + + /** + * Test count method returns zero for no matches + */ + public function testCountReturnsZeroForNoMatches(): void + { + /** @var ClickHouse $adapter */ + $adapter = $this->audit->getAdapter(); + + // Count with filter that matches nothing + $count = $adapter->count([ + Query::equal('userId', 'nonExistentUserForCountTest12345') + ]); + + $this->assertIsInt($count); + $this->assertEquals(0, $count); + } } From eea7517ec2ec276071bc921e1da8ed90b1b7182f Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 13 Jan 2026 02:10:51 +0000 Subject: [PATCH 10/29] Add find and count methods to Adapter and Database classes with tests --- src/Audit/Adapter.php | 20 + src/Audit/Adapter/Database.php | 71 ++++ src/Audit/Audit.php | 26 ++ tests/Audit/Adapter/ClickHouseTest.php | 537 ------------------------- tests/Audit/AuditBase.php | 149 +++++++ 5 files changed, 266 insertions(+), 537 deletions(-) diff --git a/src/Audit/Adapter.php b/src/Audit/Adapter.php index 54e5283..0b7b484 100644 --- a/src/Audit/Adapter.php +++ b/src/Audit/Adapter.php @@ -212,4 +212,24 @@ abstract public function countByResourceAndEvents( * @throws \Exception */ abstract public function cleanup(\DateTime $datetime): bool; + + /** + * Find logs using custom queries. + * + * @param array<\Utopia\Audit\Query> $queries + * @return array + * + * @throws \Exception + */ + abstract public function find(array $queries = []): array; + + /** + * Count logs using custom queries. + * + * @param array<\Utopia\Audit\Query> $queries + * @return int + * + * @throws \Exception + */ + abstract public function count(array $queries = []): int; } diff --git a/src/Audit/Adapter/Database.php b/src/Audit/Adapter/Database.php index afa6022..ba997fc 100644 --- a/src/Audit/Adapter/Database.php +++ b/src/Audit/Adapter/Database.php @@ -451,4 +451,75 @@ protected function getColumnDefinition(string $id): string return "{$id}: {$type}"; } + + /** + * Find logs using custom queries. + * + * Translates Audit Query objects to Database Query objects. + * + * @param array<\Utopia\Audit\Query> $queries + * @return array<\Utopia\Audit\Log> + * @throws AuthorizationException|\Exception + */ + public function find(array $queries = []): array + { + $dbQueries = []; + + foreach ($queries as $query) { + if (!($query instanceof \Utopia\Audit\Query)) { + throw new \Exception('Invalid query type. Expected Utopia\\Audit\\Query'); + } + + // Convert Audit Query to array and parse as Database Query + // Both use the same structure: method, attribute, values + $dbQueries[] = Query::parseQuery($query->toArray()); + } + + $documents = $this->db->getAuthorization()->skip(function () use ($dbQueries) { + return $this->db->find( + collection: $this->getCollectionName(), + queries: $dbQueries, + ); + }); + + return array_map(fn ($doc) => new Log($doc->getArrayCopy()), $documents); + } + + /** + * Count logs using custom queries. + * + * Translates Audit Query objects to Database Query objects. + * Ignores limit and offset queries as they don't apply to count. + * + * @param array<\Utopia\Audit\Query> $queries + * @return int + * @throws AuthorizationException|\Exception + */ + public function count(array $queries = []): int + { + $dbQueries = []; + + foreach ($queries as $query) { + if (!($query instanceof \Utopia\Audit\Query)) { + throw new \Exception('Invalid query type. Expected Utopia\\Audit\\Query'); + } + + // Skip limit and offset for count queries + $method = $query->getMethod(); + if ($method === \Utopia\Audit\Query::TYPE_LIMIT || $method === \Utopia\Audit\Query::TYPE_OFFSET) { + continue; + } + + // Convert Audit Query to array and parse as Database Query + // Both use the same structure: method, attribute, values + $dbQueries[] = Query::parseQuery($query->toArray()); + } + + return $this->db->getAuthorization()->skip(function () use ($dbQueries) { + return $this->db->count( + collection: $this->getCollectionName(), + queries: $dbQueries, + ); + }); + } } diff --git a/src/Audit/Audit.php b/src/Audit/Audit.php index 2a2da67..dc99f9e 100644 --- a/src/Audit/Audit.php +++ b/src/Audit/Audit.php @@ -256,4 +256,30 @@ public function cleanup(\DateTime $datetime): bool { return $this->adapter->cleanup($datetime); } + + /** + * Find logs using custom queries. + * + * @param array $queries Array of Audit Query objects + * @return array + * + * @throws \Exception + */ + public function find(array $queries = []): array + { + return $this->adapter->find($queries); + } + + /** + * Count logs using custom queries. + * + * @param array $queries Array of Audit Query objects + * @return int + * + * @throws \Exception + */ + public function count(array $queries = []): int + { + return $this->adapter->count($queries); + } } diff --git a/tests/Audit/Adapter/ClickHouseTest.php b/tests/Audit/Adapter/ClickHouseTest.php index efc2582..410bc59 100644 --- a/tests/Audit/Adapter/ClickHouseTest.php +++ b/tests/Audit/Adapter/ClickHouseTest.php @@ -6,7 +6,6 @@ use PHPUnit\Framework\TestCase; use Utopia\Audit\Adapter\ClickHouse; use Utopia\Audit\Audit; -use Utopia\Audit\Query; use Utopia\Tests\Audit\AuditBase; /** @@ -395,540 +394,4 @@ public function testClickHouseAdapterIndexes(): void $this->assertContains($expected, $indexIds, "Parent index '{$expected}' not found in ClickHouse adapter"); } } - - /** - * Test find method with simple query - */ - public function testFindWithSimpleQuery(): void - { - // Create test data - $this->audit->log( - userId: 'testuser1', - event: 'create', - resource: 'document/123', - userAgent: 'Test Agent', - ip: '192.168.1.1', - location: 'US', - data: ['action' => 'test'] - ); - - $this->audit->log( - userId: 'testuser2', - event: 'update', - resource: 'document/456', - userAgent: 'Test Agent', - ip: '192.168.1.2', - location: 'UK', - data: ['action' => 'test'] - ); - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Test equal query - $logs = $adapter->find([ - Query::equal('userId', 'testuser1') - ]); - - $this->assertIsArray($logs); - $this->assertGreaterThan(0, count($logs)); - - foreach ($logs as $log) { - $this->assertEquals('testuser1', $log->getAttribute('userId')); - } - } - - /** - * Test find method with multiple filters - */ - public function testFindWithMultipleFilters(): void - { - // Create test data - $this->audit->log( - userId: 'user123', - event: 'create', - resource: 'collection/test', - userAgent: 'Test Agent', - ip: '10.0.0.1', - location: 'US', - data: [] - ); - - $this->audit->log( - userId: 'user123', - event: 'delete', - resource: 'collection/test2', - userAgent: 'Test Agent', - ip: '10.0.0.1', - location: 'US', - data: [] - ); - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Test with multiple filters - $logs = $adapter->find([ - Query::equal('userId', 'user123'), - Query::equal('event', 'create') - ]); - - $this->assertIsArray($logs); - $this->assertGreaterThan(0, count($logs)); - - foreach ($logs as $log) { - $this->assertEquals('user123', $log->getAttribute('userId')); - $this->assertEquals('create', $log->getAttribute('event')); - } - } - - /** - * Test find method with IN query - */ - public function testFindWithInQuery(): void - { - // Create test data - $events = ['login', 'logout', 'create']; - foreach ($events as $event) { - $this->audit->log( - userId: 'userMulti', - event: $event, - resource: 'test', - userAgent: 'Test Agent', - ip: '127.0.0.1', - location: 'US', - data: [] - ); - } - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Test IN query - $logs = $adapter->find([ - Query::equal('userId', 'userMulti'), - Query::in('event', ['login', 'logout']) - ]); - - $this->assertIsArray($logs); - $this->assertCount(2, $logs); - - foreach ($logs as $log) { - $this->assertContains($log->getAttribute('event'), ['login', 'logout']); - } - } - - /** - * Test find method with ordering - */ - public function testFindWithOrdering(): void - { - // Create test data with different events - $this->audit->log( - userId: 'orderUser', - event: 'zzz_event', - resource: 'test', - userAgent: 'Test Agent', - ip: '127.0.0.1', - location: 'US', - data: [] - ); - - sleep(1); // Ensure different timestamps - - $this->audit->log( - userId: 'orderUser', - event: 'aaa_event', - resource: 'test', - userAgent: 'Test Agent', - ip: '127.0.0.1', - location: 'US', - data: [] - ); - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Test ascending order - $logs = $adapter->find([ - Query::equal('userId', 'orderUser'), - Query::orderAsc('event') - ]); - - $this->assertIsArray($logs); - $this->assertGreaterThanOrEqual(2, count($logs)); - $this->assertEquals('aaa_event', $logs[0]->getAttribute('event')); - - // Test descending order - $logs = $adapter->find([ - Query::equal('userId', 'orderUser'), - Query::orderDesc('event') - ]); - - $this->assertIsArray($logs); - $this->assertGreaterThanOrEqual(2, count($logs)); - $this->assertEquals('zzz_event', $logs[0]->getAttribute('event')); - } - - /** - * Test find method with limit and offset - */ - public function testFindWithLimitAndOffset(): void - { - // Create multiple test logs - for ($i = 1; $i <= 5; $i++) { - $this->audit->log( - userId: 'paginationUser', - event: "event_{$i}", - resource: "resource_{$i}", - userAgent: 'Test Agent', - ip: '127.0.0.1', - location: 'US', - data: ['index' => $i] - ); - } - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Test limit - $logs = $adapter->find([ - Query::equal('userId', 'paginationUser'), - Query::limit(2) - ]); - - $this->assertIsArray($logs); - $this->assertCount(2, $logs); - - // Test offset - $logs = $adapter->find([ - Query::equal('userId', 'paginationUser'), - Query::orderAsc('event'), - Query::limit(2), - Query::offset(2) - ]); - - $this->assertIsArray($logs); - $this->assertLessThanOrEqual(2, count($logs)); - } - - /** - * Test find method with between query - */ - public function testFindWithBetweenQuery(): void - { - $time1 = '2023-01-01 00:00:00+00:00'; - $time2 = '2023-06-01 00:00:00+00:00'; - $time3 = '2023-12-31 23:59:59+00:00'; - - // Create test data with different times using logBatch - $this->audit->logBatch([ - [ - 'userId' => 'betweenUser', - 'event' => 'event1', - 'resource' => 'test', - 'userAgent' => 'Test Agent', - 'ip' => '127.0.0.1', - 'location' => 'US', - 'data' => [], - 'time' => $time1 - ], - [ - 'userId' => 'betweenUser', - 'event' => 'event2', - 'resource' => 'test', - 'userAgent' => 'Test Agent', - 'ip' => '127.0.0.1', - 'location' => 'US', - 'data' => [], - 'time' => $time2 - ], - [ - 'userId' => 'betweenUser', - 'event' => 'event3', - 'resource' => 'test', - 'userAgent' => 'Test Agent', - 'ip' => '127.0.0.1', - 'location' => 'US', - 'data' => [], - 'time' => $time3 - ] - ]); - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Test between query - $logs = $adapter->find([ - Query::equal('userId', 'betweenUser'), - Query::between('time', '2023-05-01 00:00:00+00:00', '2023-12-31 00:00:00+00:00') - ]); - - $this->assertIsArray($logs); - $this->assertGreaterThan(0, count($logs)); - } - - /** - * Test count method with no filters - */ - public function testCountWithNoFilters(): void - { - // Create test data - for ($i = 0; $i < 5; $i++) { - $this->audit->log( - userId: 'countUser', - event: "event{$i}", - resource: 'test/count', - userAgent: 'Test Agent', - ip: '127.0.0.1', - location: 'US', - data: [] - ); - } - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Count all logs for this user - $count = $adapter->count([ - Query::equal('userId', 'countUser') - ]); - - $this->assertIsInt($count); - $this->assertEquals(5, $count); - } - - /** - * Test count method with simple query - */ - public function testCountWithSimpleQuery(): void - { - // Create test data - $this->audit->log( - userId: 'countUser1', - event: 'create', - resource: 'document/123', - userAgent: 'Test Agent', - ip: '192.168.1.1', - location: 'US', - data: ['action' => 'test'] - ); - - $this->audit->log( - userId: 'countUser1', - event: 'update', - resource: 'document/456', - userAgent: 'Test Agent', - ip: '192.168.1.2', - location: 'UK', - data: ['action' => 'test'] - ); - - $this->audit->log( - userId: 'countUser2', - event: 'create', - resource: 'document/789', - userAgent: 'Test Agent', - ip: '192.168.1.3', - location: 'FR', - data: ['action' => 'test'] - ); - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Count logs for specific user - $count = $adapter->count([ - Query::equal('userId', 'countUser1') - ]); - - $this->assertIsInt($count); - $this->assertEquals(2, $count); - } - - /** - * Test count method with multiple filters - */ - public function testCountWithMultipleFilters(): void - { - // Create test data - $this->audit->log( - userId: 'multiFilterUser', - event: 'create', - resource: 'collection/test', - userAgent: 'Test Agent', - ip: '10.0.0.1', - location: 'US', - data: [] - ); - - $this->audit->log( - userId: 'multiFilterUser', - event: 'create', - resource: 'collection/test2', - userAgent: 'Test Agent', - ip: '10.0.0.1', - location: 'US', - data: [] - ); - - $this->audit->log( - userId: 'multiFilterUser', - event: 'delete', - resource: 'collection/test3', - userAgent: 'Test Agent', - ip: '10.0.0.1', - location: 'US', - data: [] - ); - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Count with multiple filters - $count = $adapter->count([ - Query::equal('userId', 'multiFilterUser'), - Query::equal('event', 'create') - ]); - - $this->assertIsInt($count); - $this->assertEquals(2, $count); - } - - /** - * Test count method with IN query - */ - public function testCountWithInQuery(): void - { - // Create test data - $events = ['login', 'logout', 'create', 'update', 'delete']; - foreach ($events as $event) { - $this->audit->log( - userId: 'inQueryCountUser', - event: $event, - resource: 'test', - userAgent: 'Test Agent', - ip: '127.0.0.1', - location: 'US', - data: [] - ); - } - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Count with IN query - $count = $adapter->count([ - Query::equal('userId', 'inQueryCountUser'), - Query::in('event', ['login', 'logout', 'create']) - ]); - - $this->assertIsInt($count); - $this->assertEquals(3, $count); - } - - /** - * Test count method ignores limit and offset - */ - public function testCountIgnoresLimitAndOffset(): void - { - // Create test data - for ($i = 0; $i < 10; $i++) { - $this->audit->log( - userId: 'limitOffsetCountUser', - event: "event{$i}", - resource: 'test', - userAgent: 'Test Agent', - ip: '127.0.0.1', - location: 'US', - data: [] - ); - } - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Count should ignore limit and offset and return total count - $count = $adapter->count([ - Query::equal('userId', 'limitOffsetCountUser'), - Query::limit(3), - Query::offset(2) - ]); - - $this->assertIsInt($count); - $this->assertEquals(10, $count); // Should count all 10, not affected by limit/offset - } - - /** - * Test count method with between query - */ - public function testCountWithBetweenQuery(): void - { - $time1 = '2023-01-01 00:00:00+00:00'; - $time2 = '2023-06-01 00:00:00+00:00'; - $time3 = '2023-12-31 23:59:59+00:00'; - - // Create test data with different times using logBatch - $this->audit->logBatch([ - [ - 'userId' => 'betweenCountUser', - 'event' => 'event1', - 'resource' => 'test', - 'userAgent' => 'Test Agent', - 'ip' => '127.0.0.1', - 'location' => 'US', - 'data' => [], - 'time' => $time1 - ], - [ - 'userId' => 'betweenCountUser', - 'event' => 'event2', - 'resource' => 'test', - 'userAgent' => 'Test Agent', - 'ip' => '127.0.0.1', - 'location' => 'US', - 'data' => [], - 'time' => $time2 - ], - [ - 'userId' => 'betweenCountUser', - 'event' => 'event3', - 'resource' => 'test', - 'userAgent' => 'Test Agent', - 'ip' => '127.0.0.1', - 'location' => 'US', - 'data' => [], - 'time' => $time3 - ] - ]); - - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Count with between query - $count = $adapter->count([ - Query::equal('userId', 'betweenCountUser'), - Query::between('time', '2023-05-01 00:00:00+00:00', '2023-12-31 00:00:00+00:00') - ]); - - $this->assertIsInt($count); - $this->assertGreaterThan(0, $count); - } - - /** - * Test count method returns zero for no matches - */ - public function testCountReturnsZeroForNoMatches(): void - { - /** @var ClickHouse $adapter */ - $adapter = $this->audit->getAdapter(); - - // Count with filter that matches nothing - $count = $adapter->count([ - Query::equal('userId', 'nonExistentUserForCountTest12345') - ]); - - $this->assertIsInt($count); - $this->assertEquals(0, $count); - } } diff --git a/tests/Audit/AuditBase.php b/tests/Audit/AuditBase.php index f940a72..1be57dd 100644 --- a/tests/Audit/AuditBase.php +++ b/tests/Audit/AuditBase.php @@ -586,4 +586,153 @@ public function testRetrievalParameters(): void ); $this->assertGreaterThanOrEqual(0, \count($logsResEvt)); } + + public function testFind(): void + { + $userId = 'userId'; + + // Test 1: Find with equal filter + $logs = $this->audit->find([ + \Utopia\Audit\Query::equal('userId', $userId), + ]); + $this->assertEquals(5, \count($logs)); + + // Test 2: Find with equal and limit + $logs = $this->audit->find([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::limit(2), + ]); + $this->assertEquals(2, \count($logs)); + + // Test 3: Find with equal, limit and offset + $logs = $this->audit->find([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::limit(2), + \Utopia\Audit\Query::offset(1), + ]); + $this->assertEquals(2, \count($logs)); + + // Test 4: Find with multiple filters + $logs = $this->audit->find([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::equal('resource', 'doc/0'), + ]); + $this->assertEquals(1, \count($logs)); + + // Test 5: Find with ordering + $logsDesc = $this->audit->find([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::orderDesc('time'), + ]); + $logsAsc = $this->audit->find([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::orderAsc('time'), + ]); + $this->assertEquals(5, \count($logsDesc)); + $this->assertEquals(5, \count($logsAsc)); + + // Verify order is reversed + if (\count($logsDesc) === \count($logsAsc)) { + for ($i = 0; $i < \count($logsDesc); $i++) { + $this->assertEquals( + $logsDesc[$i]->getId(), + $logsAsc[\count($logsAsc) - 1 - $i]->getId() + ); + } + } + + // Test 6: Find with IN filter + $logs = $this->audit->find([ + \Utopia\Audit\Query::in('event', ['event_0', 'event_1']), + ]); + $this->assertGreaterThanOrEqual(2, \count($logs)); + + // Test 7: Find with between query for time range + $afterTime = new \DateTime('2024-06-15 12:01:00'); + $beforeTime = new \DateTime('2024-06-15 12:04:00'); + $logs = $this->audit->find([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::between('time', DateTime::format($afterTime), DateTime::format($beforeTime)), + ]); + $this->assertGreaterThanOrEqual(0, \count($logs)); + + // Test 8: Find with greater than + $afterTime = new \DateTime('2024-06-15 12:02:00'); + $logs = $this->audit->find([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::greaterThan('time', DateTime::format($afterTime)), + ]); + $this->assertGreaterThanOrEqual(0, \count($logs)); + + // Test 9: Find with less than + $beforeTime = new \DateTime('2024-06-15 12:03:00'); + $logs = $this->audit->find([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::lessThan('time', DateTime::format($beforeTime)), + ]); + $this->assertGreaterThanOrEqual(0, \count($logs)); + } + + public function testCount(): void + { + $userId = 'userId'; + + // Test 1: Count with simple filter + $count = $this->audit->count([ + \Utopia\Audit\Query::equal('userId', $userId), + ]); + $this->assertEquals(5, $count); + + // Test 2: Count with multiple filters + $count = $this->audit->count([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::equal('resource', 'doc/0'), + ]); + $this->assertEquals(1, $count); + + // Test 3: Count with IN filter + $count = $this->audit->count([ + \Utopia\Audit\Query::in('event', ['event_0', 'event_1']), + ]); + $this->assertGreaterThanOrEqual(2, $count); + + // Test 4: Count ignores limit and offset + $count = $this->audit->count([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::limit(2), + \Utopia\Audit\Query::offset(1), + ]); + $this->assertEquals(5, $count); // Should count all 5, not affected by limit/offset + + // Test 5: Count with between query + $afterTime = new \DateTime('2024-06-15 12:01:00'); + $beforeTime = new \DateTime('2024-06-15 12:04:00'); + $count = $this->audit->count([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::between('time', DateTime::format($afterTime), DateTime::format($beforeTime)), + ]); + $this->assertGreaterThanOrEqual(0, $count); + + // Test 6: Count with greater than + $afterTime = new \DateTime('2024-06-15 12:02:00'); + $count = $this->audit->count([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::greaterThan('time', DateTime::format($afterTime)), + ]); + $this->assertGreaterThanOrEqual(0, $count); + + // Test 7: Count with less than + $beforeTime = new \DateTime('2024-06-15 12:03:00'); + $count = $this->audit->count([ + \Utopia\Audit\Query::equal('userId', $userId), + \Utopia\Audit\Query::lessThan('time', DateTime::format($beforeTime)), + ]); + $this->assertGreaterThanOrEqual(0, $count); + + // Test 8: Count returns zero for no matches + $count = $this->audit->count([ + \Utopia\Audit\Query::equal('userId', 'nonExistentUser'), + ]); + $this->assertEquals(0, $count); + } } From 9e2e857a9472d0370cc0e95123831fd22d1b14a4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 03:49:24 +0000 Subject: [PATCH 11/29] more fixes --- tests/Audit/AuditBase.php | 10 +++++----- tests/Audit/QueryTest.php | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/Audit/AuditBase.php b/tests/Audit/AuditBase.php index 1be57dd..23905b7 100644 --- a/tests/Audit/AuditBase.php +++ b/tests/Audit/AuditBase.php @@ -595,7 +595,7 @@ public function testFind(): void $logs = $this->audit->find([ \Utopia\Audit\Query::equal('userId', $userId), ]); - $this->assertEquals(5, \count($logs)); + $this->assertEquals(3, \count($logs)); // Test 2: Find with equal and limit $logs = $this->audit->find([ @@ -628,8 +628,8 @@ public function testFind(): void \Utopia\Audit\Query::equal('userId', $userId), \Utopia\Audit\Query::orderAsc('time'), ]); - $this->assertEquals(5, \count($logsDesc)); - $this->assertEquals(5, \count($logsAsc)); + $this->assertEquals(3, \count($logsDesc)); + $this->assertEquals(3, \count($logsAsc)); // Verify order is reversed if (\count($logsDesc) === \count($logsAsc)) { @@ -681,7 +681,7 @@ public function testCount(): void $count = $this->audit->count([ \Utopia\Audit\Query::equal('userId', $userId), ]); - $this->assertEquals(5, $count); + $this->assertEquals(3, $count); // Test 2: Count with multiple filters $count = $this->audit->count([ @@ -702,7 +702,7 @@ public function testCount(): void \Utopia\Audit\Query::limit(2), \Utopia\Audit\Query::offset(1), ]); - $this->assertEquals(5, $count); // Should count all 5, not affected by limit/offset + $this->assertEquals(3, $count); // Should count all 3, not affected by limit/offset // Test 5: Count with between query $afterTime = new \DateTime('2024-06-15 12:01:00'); diff --git a/tests/Audit/QueryTest.php b/tests/Audit/QueryTest.php index 2dc3fa1..c133bff 100644 --- a/tests/Audit/QueryTest.php +++ b/tests/Audit/QueryTest.php @@ -106,7 +106,7 @@ public function testQueryParseQueries(): void { $queries = [ '{"method":"equal","attribute":"userId","values":["123"]}', - '{"method":"greaterThan","attribute":"time","values":["2023-01-01"]}', + '{"method":"greater","attribute":"time","values":["2023-01-01"]}', '{"method":"limit","values":[10]}' ]; From dfca44dfa33aa633de191c3db13adfdc6da78df4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 04:30:18 +0000 Subject: [PATCH 12/29] Fix codeql --- composer.lock | 88 ++++++++++++++++---------------- src/Audit/Adapter/ClickHouse.php | 16 ++++-- 2 files changed, 56 insertions(+), 48 deletions(-) diff --git a/composer.lock b/composer.lock index 19178c1..7424c17 100644 --- a/composer.lock +++ b/composer.lock @@ -539,16 +539,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.3.3", + "version": "1.3.4", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "07b02bc71838463f6edcc78d3485c04b48fb263d" + "reference": "62e680d587beb42e5247aa6ecd89ad1ca406e8ca" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/07b02bc71838463f6edcc78d3485c04b48fb263d", - "reference": "07b02bc71838463f6edcc78d3485c04b48fb263d", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/62e680d587beb42e5247aa6ecd89ad1ca406e8ca", + "reference": "62e680d587beb42e5247aa6ecd89ad1ca406e8ca", "shasum": "" }, "require": { @@ -599,7 +599,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-11-13T08:04:37+00:00" + "time": "2026-01-15T09:31:34+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -666,16 +666,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.10.0", + "version": "1.11.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99" + "reference": "d91f21addcdb42da9a451c002777f8318432461a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99", - "reference": "3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/d91f21addcdb42da9a451c002777f8318432461a", + "reference": "d91f21addcdb42da9a451c002777f8318432461a", "shasum": "" }, "require": { @@ -759,7 +759,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-11-25T10:59:15+00:00" + "time": "2026-01-15T11:21:03+00:00" }, { "name": "open-telemetry/sem-conv", @@ -2026,16 +2026,16 @@ }, { "name": "utopia-php/cache", - "version": "0.13.2", + "version": "0.13.3", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "5768498c9f451482f0bf3eede4d6452ddcd4a0f6" + "reference": "355707ab2c0090435059216165db86976b68a126" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/5768498c9f451482f0bf3eede4d6452ddcd4a0f6", - "reference": "5768498c9f451482f0bf3eede4d6452ddcd4a0f6", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/355707ab2c0090435059216165db86976b68a126", + "reference": "355707ab2c0090435059216165db86976b68a126", "shasum": "" }, "require": { @@ -2043,7 +2043,7 @@ "ext-memcached": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/pools": "0.8.*", + "utopia-php/pools": "1.*", "utopia-php/telemetry": "*" }, "require-dev": { @@ -2072,9 +2072,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/0.13.2" + "source": "https://github.com/utopia-php/cache/tree/0.13.3" }, - "time": "2025-12-17T08:55:43+00:00" + "time": "2026-01-16T07:54:34+00:00" }, { "name": "utopia-php/compression", @@ -2124,16 +2124,16 @@ }, { "name": "utopia-php/database", - "version": "4.4.0", + "version": "4.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "783193d5cdc723b3784e8fb399068b17d4228d53" + "reference": "b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/783193d5cdc723b3784e8fb399068b17d4228d53", - "reference": "783193d5cdc723b3784e8fb399068b17d4228d53", + "url": "https://api.github.com/repos/utopia-php/database/zipball/b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df", + "reference": "b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df", "shasum": "" }, "require": { @@ -2144,7 +2144,7 @@ "utopia-php/cache": "0.13.*", "utopia-php/framework": "0.33.*", "utopia-php/mongo": "0.11.*", - "utopia-php/pools": "0.8.*" + "utopia-php/pools": "1.*" }, "require-dev": { "fakerphp/faker": "1.23.*", @@ -2176,9 +2176,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/4.4.0" + "source": "https://github.com/utopia-php/database/tree/4.6.0" }, - "time": "2026-01-08T04:54:39+00:00" + "time": "2026-01-16T12:35:16+00:00" }, { "name": "utopia-php/fetch", @@ -2221,28 +2221,29 @@ }, { "name": "utopia-php/framework", - "version": "0.33.30", + "version": "0.33.37", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "07cf699a7c47bd1a03b4da1812f1719a66b3c924" + "reference": "30a119d76531d89da9240496940c84fcd9e1758b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/07cf699a7c47bd1a03b4da1812f1719a66b3c924", - "reference": "07cf699a7c47bd1a03b4da1812f1719a66b3c924", + "url": "https://api.github.com/repos/utopia-php/http/zipball/30a119d76531d89da9240496940c84fcd9e1758b", + "reference": "30a119d76531d89da9240496940c84fcd9e1758b", "shasum": "" }, "require": { - "php": ">=8.1", + "php": ">=8.3", "utopia-php/compression": "0.1.*", - "utopia-php/telemetry": "0.1.*" + "utopia-php/telemetry": "0.1.*", + "utopia-php/validators": "0.2.*" }, "require-dev": { - "laravel/pint": "^1.2", - "phpbench/phpbench": "^1.2", - "phpstan/phpstan": "^1.10", - "phpunit/phpunit": "^9.5.25" + "laravel/pint": "1.*", + "phpbench/phpbench": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "9.*" }, "type": "library", "autoload": { @@ -2262,9 +2263,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.30" + "source": "https://github.com/utopia-php/http/tree/0.33.37" }, - "time": "2025-11-18T12:18:00+00:00" + "time": "2026-01-13T10:10:21+00:00" }, { "name": "utopia-php/mongo", @@ -2329,16 +2330,16 @@ }, { "name": "utopia-php/pools", - "version": "0.8.3", + "version": "1.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "ad7d6ba946376e81c603204285ce9a674b6502b8" + "reference": "74ba7dc985c2f629df8cf08ed95507955e3bcf86" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/ad7d6ba946376e81c603204285ce9a674b6502b8", - "reference": "ad7d6ba946376e81c603204285ce9a674b6502b8", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/74ba7dc985c2f629df8cf08ed95507955e3bcf86", + "reference": "74ba7dc985c2f629df8cf08ed95507955e3bcf86", "shasum": "" }, "require": { @@ -2348,7 +2349,8 @@ "require-dev": { "laravel/pint": "1.*", "phpstan/phpstan": "1.*", - "phpunit/phpunit": "11.*" + "phpunit/phpunit": "11.*", + "swoole/ide-helper": "5.1.2" }, "type": "library", "autoload": { @@ -2375,9 +2377,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.8.3" + "source": "https://github.com/utopia-php/pools/tree/1.0.0" }, - "time": "2025-12-17T09:35:18+00:00" + "time": "2026-01-15T12:34:17+00:00" }, { "name": "utopia-php/telemetry", diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index abd10e5..d7838c5 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -745,13 +745,13 @@ public function find(array $queries = []): array $parsed = $this->parseQueries($queries); // Build SELECT clause - $selectColumns = $parsed['select'] ?? $this->getSelectColumns(); + $selectColumns = $this->getSelectColumns(); // Build WHERE clause $whereClause = ''; $tenantFilter = $this->getTenantFilter(); if (!empty($parsed['filters']) || $tenantFilter) { - $conditions = $parsed['filters'] ?? []; + $conditions = $parsed['filters']; if ($tenantFilter) { $conditions[] = ltrim($tenantFilter, ' AND'); } @@ -797,7 +797,7 @@ public function count(array $queries = []): int $whereClause = ''; $tenantFilter = $this->getTenantFilter(); if (!empty($parsed['filters']) || $tenantFilter) { - $conditions = $parsed['filters'] ?? []; + $conditions = $parsed['filters']; if ($tenantFilter) { $conditions[] = ltrim($tenantFilter, ' AND'); } @@ -891,12 +891,18 @@ private function parseQueries(array $queries): array break; case Query::TYPE_LIMIT: - $limit = (int) $values[0]; + if (!\is_int($values[0])) { + throw new \Exception('Invalid limit value. Expected int'); + } + $limit = $values[0]; $params['limit'] = $limit; break; case Query::TYPE_OFFSET: - $offset = (int) $values[0]; + if (!\is_int($values[0])) { + throw new \Exception('Invalid offset value. Expected int'); + } + $offset = $values[0]; $params['offset'] = $offset; break; } From db20449bc43d685f00d239a7699fad34dc6bd3ea Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 04:47:05 +0000 Subject: [PATCH 13/29] Fix: tests --- src/Audit/Adapter/Database.php | 11 +++---- src/Audit/Query.php | 6 ++-- tests/Audit/AuditBase.php | 54 ++++++++++++++++++++++++++++++++++ tests/Audit/QueryTest.php | 2 +- 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/src/Audit/Adapter/Database.php b/src/Audit/Adapter/Database.php index f9f00e7..fc62c9d 100644 --- a/src/Audit/Adapter/Database.php +++ b/src/Audit/Adapter/Database.php @@ -478,8 +478,8 @@ public function find(array $queries = []): array throw new \Exception('Invalid query type. Expected Utopia\\Audit\\Query'); } - // Convert Audit Query to array and parse as Database Query - // Both use the same structure: method, attribute, values + // Convert Audit Query to Database Query + // Both use the same structure and method names $dbQueries[] = Query::parseQuery($query->toArray()); } @@ -518,9 +518,10 @@ public function count(array $queries = []): int continue; } - // Convert Audit Query to array and parse as Database Query - // Both use the same structure: method, attribute, values - $dbQueries[] = Query::parseQuery($query->toArray()); + // Convert Audit Query to Database Query + // Both use the same structure and method names + $queryArray = $query->toArray(); + $dbQueries[] = Query::parseQuery($queryArray); } return $this->db->getAuthorization()->skip(function () use ($dbQueries) { diff --git a/src/Audit/Query.php b/src/Audit/Query.php index 3b70b2a..ebc7d7a 100644 --- a/src/Audit/Query.php +++ b/src/Audit/Query.php @@ -12,10 +12,10 @@ class Query { // Filter methods public const TYPE_EQUAL = 'equal'; - public const TYPE_GREATER = 'greater'; - public const TYPE_LESSER = 'lesser'; + public const TYPE_GREATER = 'greaterThan'; + public const TYPE_LESSER = 'lessThan'; public const TYPE_BETWEEN = 'between'; - public const TYPE_IN = 'in'; + public const TYPE_IN = 'contains'; // Order methods public const TYPE_ORDER_DESC = 'orderDesc'; diff --git a/tests/Audit/AuditBase.php b/tests/Audit/AuditBase.php index 23905b7..0eb4427 100644 --- a/tests/Audit/AuditBase.php +++ b/tests/Audit/AuditBase.php @@ -589,7 +589,34 @@ public function testRetrievalParameters(): void public function testFind(): void { + // Setup: Create specific test data for find queries + $this->audit->cleanup(new \DateTime()); + $userId = 'userId'; + $userAgent = 'Mozilla/5.0'; + $ip = '192.168.1.1'; + $location = 'US'; + + // Create test logs with specific attributes + $baseTime = new \DateTime('2024-06-15 12:00:00'); + $batchEvents = []; + for ($i = 0; $i < 3; $i++) { + $offset = $i * 60; + $logTime = new \DateTime('2024-06-15 12:00:00'); + $logTime->modify("+{$offset} seconds"); + $timestamp = DateTime::format($logTime); + $batchEvents[] = [ + 'userId' => $userId, + 'event' => 'event_' . $i, + 'resource' => 'doc/' . $i, + 'userAgent' => $userAgent, + 'ip' => $ip, + 'location' => $location, + 'data' => ['sequence' => $i], + 'time' => $timestamp + ]; + } + $this->audit->logBatch($batchEvents); // Test 1: Find with equal filter $logs = $this->audit->find([ @@ -675,7 +702,34 @@ public function testFind(): void public function testCount(): void { + // Setup: Create specific test data for count queries + $this->audit->cleanup(new \DateTime()); + $userId = 'userId'; + $userAgent = 'Mozilla/5.0'; + $ip = '192.168.1.1'; + $location = 'US'; + + // Create test logs with specific attributes + $baseTime = new \DateTime('2024-06-15 12:00:00'); + $batchEvents = []; + for ($i = 0; $i < 3; $i++) { + $offset = $i * 60; + $logTime = new \DateTime('2024-06-15 12:00:00'); + $logTime->modify("+{$offset} seconds"); + $timestamp = DateTime::format($logTime); + $batchEvents[] = [ + 'userId' => $userId, + 'event' => 'event_' . $i, + 'resource' => 'doc/' . $i, + 'userAgent' => $userAgent, + 'ip' => $ip, + 'location' => $location, + 'data' => ['sequence' => $i], + 'time' => $timestamp + ]; + } + $this->audit->logBatch($batchEvents); // Test 1: Count with simple filter $count = $this->audit->count([ diff --git a/tests/Audit/QueryTest.php b/tests/Audit/QueryTest.php index c133bff..2dc3fa1 100644 --- a/tests/Audit/QueryTest.php +++ b/tests/Audit/QueryTest.php @@ -106,7 +106,7 @@ public function testQueryParseQueries(): void { $queries = [ '{"method":"equal","attribute":"userId","values":["123"]}', - '{"method":"greater","attribute":"time","values":["2023-01-01"]}', + '{"method":"greaterThan","attribute":"time","values":["2023-01-01"]}', '{"method":"limit","values":[10]}' ]; From 295b2825e1fe06272187470823180ead42a9e5c6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 10:41:48 +0000 Subject: [PATCH 14/29] feat: add methods for dynamic column handling and datetime formatting in ClickHouse adapter --- src/Audit/Adapter/ClickHouse.php | 241 ++++++++++++++++++++++--------- 1 file changed, 175 insertions(+), 66 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index d7838c5..77d16f8 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -638,6 +638,57 @@ public function setup(): void $this->query($createTableSql); } + /** + * Get column names from attributes. + * Returns an array of column names excluding 'id' and 'tenant' which are handled separately. + * + * @return array Column names + */ + private function getColumnNames(): array + { + $columns = []; + foreach ($this->getAttributes() as $attribute) { + $columnName = $attribute['$id']; + // Exclude id and tenant as they're handled separately + if ($columnName !== 'id' && $columnName !== 'tenant') { + $columns[] = $columnName; + } + } + return $columns; + } + + /** + * Format datetime for ClickHouse compatibility. + * Converts datetime to 'YYYY-MM-DD HH:MM:SS.mmm' format without timezone suffix. + * ClickHouse DateTime64(3) type expects this format as timezone is handled by column metadata. + * + * @param \DateTime|string|null $dateTime The datetime value to format + * @return string The formatted datetime string in ClickHouse compatible format + * @throws Exception If the datetime string cannot be parsed + */ + private function formatDateTimeForClickHouse($dateTime): string + { + if ($dateTime === null) { + return (new \DateTime())->format('Y-m-d H:i:s.v'); + } + + if ($dateTime instanceof \DateTime) { + return $dateTime->format('Y-m-d H:i:s.v'); + } + + if (is_string($dateTime)) { + try { + // Parse the datetime string, handling ISO 8601 format with timezone + $dt = new \DateTime($dateTime); + return $dt->format('Y-m-d H:i:s.v'); + } catch (\Exception $e) { + throw new Exception("Invalid datetime string: {$dateTime}"); + } + } + + throw new Exception('DateTime must be a DateTime object or string'); + } + /** * Create an audit log entry. * @@ -646,25 +697,48 @@ public function setup(): void public function create(array $log): Log { $id = uniqid('', true); - $time = (new \DateTime())->format('Y-m-d H:i:s.v'); + $time = $this->formatDateTimeForClickHouse($log['time'] ?? null); $tableName = $this->getTableName(); - // Build column list and values based on sharedTables setting - $columns = ['id', 'userId', 'event', 'resource', 'userAgent', 'ip', 'location', 'time', 'data']; - $placeholders = ['{id:String}', '{userId:Nullable(String)}', '{event:String}', '{resource:String}', '{userAgent:String}', '{ip:String}', '{location:Nullable(String)}', '{time:String}', '{data:String}']; - - $params = [ - 'id' => $id, - 'userId' => $log['userId'] ?? null, - 'event' => $log['event'], - 'resource' => $log['resource'], - 'userAgent' => $log['userAgent'], - 'ip' => $log['ip'], - 'location' => $log['location'] ?? null, - 'time' => $time, - 'data' => json_encode($log['data'] ?? []), - ]; + // Build column list and placeholders dynamically from attributes + $columns = ['id']; + $placeholders = ['{id:String}']; + $params = ['id' => $id]; + + // Get all column names from attributes + $attributeColumns = $this->getColumnNames(); + + foreach ($attributeColumns as $column) { + if (isset($log[$column])) { + $columns[] = $column; + + // Special handling for time column + if ($column === 'time') { + $params[$column] = $this->formatDateTimeForClickHouse($log[$column]); + $placeholders[] = '{' . $column . ':String}'; + } elseif ($column === 'data') { + $params[$column] = json_encode($log[$column] ?? []); + // data is nullable based on attributes + $placeholders[] = '{' . $column . ':Nullable(String)}'; + } elseif (in_array($column, ['userId', 'location', 'userInternalId', 'resourceParent', 'resourceInternalId', 'country'])) { + // Nullable string fields + $params[$column] = $log[$column]; + $placeholders[] = '{' . $column . ':Nullable(String)}'; + } else { + // Required string fields + $params[$column] = $log[$column]; + $placeholders[] = '{' . $column . ':String}'; + } + } + } + + // Add special handling for time if not provided + if (!isset($log['time'])) { + $columns[] = 'time'; + $params['time'] = $time; + $placeholders[] = '{time:String}'; + } if ($this->sharedTables) { $columns[] = 'tenant'; @@ -683,17 +757,18 @@ public function create(array $log): Log $this->query($insertSql, $params); - $result = [ - '$id' => $id, - 'userId' => $log['userId'] ?? null, - 'event' => $log['event'], - 'resource' => $log['resource'], - 'userAgent' => $log['userAgent'], - 'ip' => $log['ip'], - 'location' => $log['location'] ?? null, - 'time' => $time, - 'data' => $log['data'] ?? [], - ]; + $result = ['$id' => $id]; + + // Add all columns from log to result + foreach ($attributeColumns as $column) { + if ($column === 'time') { + $result[$column] = $time; + } elseif ($column === 'data') { + $result[$column] = $log[$column] ?? []; + } elseif (isset($log[$column])) { + $result[$column] = $log[$column]; + } + } if ($this->sharedTables) { $result['tenant'] = $this->tenant; @@ -942,8 +1017,34 @@ public function createBatch(array $logs): bool $tableName = $this->getTableName(); $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - // Build column list based on sharedTables setting - $columns = ['id', 'userId', 'event', 'resource', 'userAgent', 'ip', 'location', 'time', 'data']; + // Get all attribute column names + $attributeColumns = $this->getColumnNames(); + + // Build column list starting with id + $columns = ['id']; + + // Determine which attribute columns are present in any log + $presentColumns = []; + foreach ($logs as $log) { + foreach ($attributeColumns as $column) { + if (isset($log[$column]) && !in_array($column, $presentColumns, true)) { + $presentColumns[] = $column; + } + } + } + + // Add present columns in the order they're defined in attributes + foreach ($attributeColumns as $column) { + if (in_array($column, $presentColumns, true)) { + $columns[] = $column; + } + } + + // Always include time column + if (!in_array('time', $columns, true)) { + $columns[] = 'time'; + } + if ($this->sharedTables) { $columns[] = 'tenant'; } @@ -959,47 +1060,55 @@ public function createBatch(array $logs): bool // Create parameter placeholders for this row $paramKeys = []; - $paramKeys[] = 'id_' . $paramCounter; - $paramKeys[] = 'userId_' . $paramCounter; - $paramKeys[] = 'event_' . $paramCounter; - $paramKeys[] = 'resource_' . $paramCounter; - $paramKeys[] = 'userAgent_' . $paramCounter; - $paramKeys[] = 'ip_' . $paramCounter; - $paramKeys[] = 'location_' . $paramCounter; - $paramKeys[] = 'time_' . $paramCounter; - $paramKeys[] = 'data_' . $paramCounter; - - // Set parameter values - $params[$paramKeys[0]] = $id; - $params[$paramKeys[1]] = $log['userId'] ?? null; - $params[$paramKeys[2]] = $log['event']; - $params[$paramKeys[3]] = $log['resource']; - $params[$paramKeys[4]] = $log['userAgent']; - $params[$paramKeys[5]] = $log['ip']; - $params[$paramKeys[6]] = $log['location'] ?? null; - - $time = $log['time'] ?? new \DateTime(); - if (is_string($time)) { - $time = new \DateTime($time); - } - $params[$paramKeys[7]] = $time->format('Y-m-d H:i:s.v'); - $params[$paramKeys[8]] = json_encode($log['data'] ?? []); + $paramValues = []; + $placeholders = []; - if ($this->sharedTables) { - $paramKeys[] = 'tenant_' . $paramCounter; - $params[$paramKeys[9]] = $this->tenant; - } + // Add id first + $paramKey = 'id_' . $paramCounter; + $paramKeys[] = $paramKey; + $paramValues[] = $id; + $params[$paramKey] = $id; + $placeholders[] = '{' . $paramKey . ':String}'; + + // Add all present columns in order + foreach ($columns as $column) { + if ($column === 'id') { + continue; // Already added + } - // Build placeholder string for this row - $placeholders = []; - for ($i = 0; $i < count($paramKeys); $i++) { - if ($i === 1 || $i === 6) { // userId and location are nullable - $placeholders[] = '{' . $paramKeys[$i] . ':Nullable(String)}'; - } elseif ($this->sharedTables && $i === 9) { // tenant is nullable UInt64 - $placeholders[] = '{' . $paramKeys[$i] . ':Nullable(UInt64)}'; + if ($column === 'tenant') { + continue; // Handle separately below + } + + $paramKey = $column . '_' . $paramCounter; + $paramKeys[] = $paramKey; + + // Determine value based on column type + if ($column === 'time') { + $value = $this->formatDateTimeForClickHouse($log['time'] ?? null); + $params[$paramKey] = $value; + $placeholders[] = '{' . $paramKey . ':String}'; + } elseif ($column === 'data') { + $value = json_encode($log['data'] ?? []); + $params[$paramKey] = $value; + $placeholders[] = '{' . $paramKey . ':Nullable(String)}'; + } elseif (in_array($column, ['userId', 'location', 'userInternalId', 'resourceParent', 'resourceInternalId', 'country'])) { + $value = $log[$column] ?? null; + $params[$paramKey] = $value; + $placeholders[] = '{' . $paramKey . ':Nullable(String)}'; } else { - $placeholders[] = '{' . $paramKeys[$i] . ':String}'; + $value = $log[$column] ?? null; + $params[$paramKey] = $value; + $placeholders[] = '{' . $paramKey . ':Nullable(String)}'; } + + $paramValues[] = $value; + } + + if ($this->sharedTables) { + $paramKey = 'tenant_' . $paramCounter; + $params[$paramKey] = $this->tenant; + $placeholders[] = '{' . $paramKey . ':Nullable(UInt64)}'; } $valueClauses[] = '(' . implode(', ', $placeholders) . ')'; From 3b5ee79cf47cc62e7217df67df4058d96b39dfdf Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 11:09:57 +0000 Subject: [PATCH 15/29] Fix compatibility and exception swallowing --- src/Audit/Adapter/ClickHouse.php | 151 +++++++++++++++++++++++++------ src/Audit/Adapter/Database.php | 16 ++-- 2 files changed, 131 insertions(+), 36 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 77d16f8..fe719d5 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -616,7 +616,9 @@ public function setup(): void $indexName = $index['$id']; /** @var array $attributes */ $attributes = $index['attributes']; - $attributeList = implode(', ', $attributes); + // Escape each attribute name to prevent SQL injection + $escapedAttributes = array_map(fn ($attr) => $this->escapeIdentifier($attr), $attributes); + $attributeList = implode(', ', $escapedAttributes); $indexes[] = "INDEX {$indexName} ({$attributeList}) TYPE bloom_filter GRANULARITY 1"; } @@ -657,6 +659,51 @@ private function getColumnNames(): array return $columns; } + /** + * Validate that an attribute name exists in the schema. + * Prevents SQL injection by ensuring only valid column names are used. + * + * @param string $attributeName The attribute name to validate + * @return bool True if valid + * @throws Exception If attribute name is invalid + */ + private function validateAttributeName(string $attributeName): bool + { + // Special case: 'id' is always valid + if ($attributeName === 'id') { + return true; + } + + // Check if tenant is valid (only when sharedTables is enabled) + if ($attributeName === 'tenant' && $this->sharedTables) { + return true; + } + + // Check against defined attributes + foreach ($this->getAttributes() as $attribute) { + if ($attribute['$id'] === $attributeName) { + return true; + } + } + + throw new Exception("Invalid attribute name: {$attributeName}"); + } + + /** + * Format datetime values for ClickHouse parameter binding. + * Removes timezone suffixes which are incompatible with DateTime64 type comparisons. + * + * @param mixed $value The value to format + * @return string Formatted string without timezone suffix + */ + private function formatDateTimeParam(mixed $value): string + { + $strValue = $this->formatParamValue($value); + // Remove timezone suffix if present (e.g., +00:00, -05:00) + return preg_replace('/[+\\-]\\d{2}:\\d{2}$/', '', $strValue) ?? $strValue; + } + + /** * Format datetime for ClickHouse compatibility. * Converts datetime to 'YYYY-MM-DD HH:MM:SS.mmm' format without timezone suffix. @@ -789,11 +836,12 @@ public function getById(string $id): ?Log $tableName = $this->getTableName(); $tenantFilter = $this->getTenantFilter(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedId = $this->escapeIdentifier('id'); $sql = " SELECT " . $this->getSelectColumns() . " FROM {$escapedTable} - WHERE id = {id:String}{$tenantFilter} + WHERE {$escapedId} = {id:String}{$tenantFilter} LIMIT 1 FORMAT TabSeparated "; @@ -922,47 +970,70 @@ private function parseQueries(array $queries): array switch ($method) { case Query::TYPE_EQUAL: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); $paramName = 'param_' . $paramCounter++; - $filters[] = "{$attribute} = {{$paramName}:String}"; + $filters[] = "{$escapedAttr} = {{$paramName}:String}"; $params[$paramName] = $this->formatParamValue($values[0]); break; case Query::TYPE_LESSER: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); $paramName = 'param_' . $paramCounter++; - $filters[] = "{$attribute} < {{$paramName}:String}"; + $filters[] = "{$escapedAttr} < {{$paramName}:String}"; $params[$paramName] = $this->formatParamValue($values[0]); break; case Query::TYPE_GREATER: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); $paramName = 'param_' . $paramCounter++; - $filters[] = "{$attribute} > {{$paramName}:String}"; + $filters[] = "{$escapedAttr} > {{$paramName}:String}"; $params[$paramName] = $this->formatParamValue($values[0]); break; case Query::TYPE_BETWEEN: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); $paramName1 = 'param_' . $paramCounter++; $paramName2 = 'param_' . $paramCounter++; - $filters[] = "{$attribute} BETWEEN {{$paramName1}:String} AND {{$paramName2}:String}"; - $params[$paramName1] = $this->formatParamValue($values[0]); - $params[$paramName2] = $this->formatParamValue($values[1]); + // Use DateTime64 type for time column, String for others + // This prevents type mismatch when comparing DateTime64 with timezone-suffixed strings + if ($attribute === 'time') { + $paramType = 'DateTime64(3)'; + $filters[] = "{$escapedAttr} BETWEEN {{$paramName1}:{$paramType}} AND {{$paramName2}:{$paramType}}"; + $params[$paramName1] = $this->formatDateTimeParam($values[0]); + $params[$paramName2] = $this->formatDateTimeParam($values[1]); + } else { + $filters[] = "{$escapedAttr} BETWEEN {{$paramName1}:String} AND {{$paramName2}:String}"; + $params[$paramName1] = $this->formatParamValue($values[0]); + $params[$paramName2] = $this->formatParamValue($values[1]); + } break; case Query::TYPE_IN: + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); $inParams = []; foreach ($values as $value) { $paramName = 'param_' . $paramCounter++; $inParams[] = "{{$paramName}:String}"; $params[$paramName] = $this->formatParamValue($value); } - $filters[] = "{$attribute} IN (" . implode(', ', $inParams) . ")"; + $filters[] = "{$escapedAttr} IN (" . implode(', ', $inParams) . ")"; break; case Query::TYPE_ORDER_DESC: - $orderBy[] = "{$attribute} DESC"; + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $orderBy[] = "{$escapedAttr} DESC"; break; case Query::TYPE_ORDER_ASC: - $orderBy[] = "{$attribute} ASC"; + $this->validateAttributeName($attribute); + $escapedAttr = $this->escapeIdentifier($attribute); + $orderBy[] = "{$escapedAttr} ASC"; break; case Query::TYPE_LIMIT: @@ -1194,20 +1265,34 @@ private function parseResults(string $result): array /** * Get the SELECT column list for queries. - * Returns 9 columns if not using shared tables, 10 if using shared tables. + * Escapes all column names to prevent SQL injection. * * @return string */ private function getSelectColumns(): string { + $columns = [ + $this->escapeIdentifier('id'), + $this->escapeIdentifier('userId'), + $this->escapeIdentifier('event'), + $this->escapeIdentifier('resource'), + $this->escapeIdentifier('userAgent'), + $this->escapeIdentifier('ip'), + $this->escapeIdentifier('location'), + $this->escapeIdentifier('time'), + $this->escapeIdentifier('data'), + ]; + if ($this->sharedTables) { - return 'id, userId, event, resource, userAgent, ip, location, time, data, tenant'; + $columns[] = $this->escapeIdentifier('tenant'); } - return 'id, userId, event, resource, userAgent, ip, location, time, data'; + + return implode(', ', $columns); } /** * Build tenant filter clause based on current tenant context. + * Escapes column name to prevent SQL injection. * * @return string */ @@ -1217,11 +1302,13 @@ private function getTenantFilter(): string return ''; } - return " AND tenant = {$this->tenant}"; + $escapedTenant = $this->escapeIdentifier('tenant'); + return " AND {$escapedTenant} = {$this->tenant}"; } /** * Build time WHERE clause and parameters with safe parameter placeholders. + * Escapes column name to prevent SQL injection. * * @param \DateTime|null $after * @param \DateTime|null $before @@ -1245,8 +1332,10 @@ private function buildTimeClause(?\DateTime $after, ?\DateTime $before): array $beforeStr = \Utopia\Database\DateTime::format($before); } + $escapedTime = $this->escapeIdentifier('time'); + if ($afterStr !== null && $beforeStr !== null) { - $conditions[] = 'time BETWEEN {after:String} AND {before:String}'; + $conditions[] = "{$escapedTime} BETWEEN {after:String} AND {before:String}"; $params['after'] = $afterStr; $params['before'] = $beforeStr; @@ -1254,12 +1343,12 @@ private function buildTimeClause(?\DateTime $after, ?\DateTime $before): array } if ($afterStr !== null) { - $conditions[] = 'time > {after:String}'; + $conditions[] = "{$escapedTime} > {after:String}"; $params['after'] = $afterStr; } if ($beforeStr !== null) { - $conditions[] = 'time < {before:String}'; + $conditions[] = "{$escapedTime} < {before:String}"; $params['before'] = $beforeStr; } @@ -1344,12 +1433,14 @@ public function getByUser( $tableName = $this->getTableName(); $tenantFilter = $this->getTenantFilter(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedUserId = $this->escapeIdentifier('userId'); + $escapedTime = $this->escapeIdentifier('time'); $sql = " SELECT " . $this->getSelectColumns() . " FROM {$escapedTable} - WHERE userId = {userId:String}{$tenantFilter}{$time['clause']} - ORDER BY time {$order} + WHERE {$escapedUserId} = {userId:String}{$tenantFilter}{$time['clause']} + ORDER BY {$escapedTime} {$order} LIMIT {limit:UInt64} OFFSET {offset:UInt64} FORMAT TabSeparated "; @@ -1378,11 +1469,12 @@ public function countByUser( $tableName = $this->getTableName(); $tenantFilter = $this->getTenantFilter(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedUserId = $this->escapeIdentifier('userId'); $sql = " SELECT count() FROM {$escapedTable} - WHERE userId = {userId:String}{$tenantFilter}{$time['clause']} + WHERE {$escapedUserId} = {userId:String}{$tenantFilter}{$time['clause']} FORMAT TabSeparated "; @@ -1412,12 +1504,14 @@ public function getByResource( $tableName = $this->getTableName(); $tenantFilter = $this->getTenantFilter(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedResource = $this->escapeIdentifier('resource'); + $escapedTime = $this->escapeIdentifier('time'); $sql = " SELECT " . $this->getSelectColumns() . " FROM {$escapedTable} - WHERE resource = {resource:String}{$tenantFilter}{$time['clause']} - ORDER BY time {$order} + WHERE {$escapedResource} = {resource:String}{$tenantFilter}{$time['clause']} + ORDER BY {$escapedTime} {$order} LIMIT {limit:UInt64} OFFSET {offset:UInt64} FORMAT TabSeparated "; @@ -1446,11 +1540,12 @@ public function countByResource( $tableName = $this->getTableName(); $tenantFilter = $this->getTenantFilter(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedResource = $this->escapeIdentifier('resource'); $sql = " SELECT count() FROM {$escapedTable} - WHERE resource = {resource:String}{$tenantFilter}{$time['clause']} + WHERE {$escapedResource} = {resource:String}{$tenantFilter}{$time['clause']} FORMAT TabSeparated "; @@ -1516,11 +1611,13 @@ public function countByUserAndEvents( $tableName = $this->getTableName(); $tenantFilter = $this->getTenantFilter(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedUserId = $this->escapeIdentifier('userId'); + $escapedEvent = $this->escapeIdentifier('event'); $sql = " SELECT count() FROM {$escapedTable} - WHERE userId = {userId:String} AND event IN ({$eventList['clause']}){$tenantFilter}{$time['clause']} + WHERE {$escapedUserId} = {userId:String} AND {$escapedEvent} IN ({$eventList['clause']}){$tenantFilter}{$time['clause']} FORMAT TabSeparated "; @@ -1586,11 +1683,13 @@ public function countByResourceAndEvents( $tableName = $this->getTableName(); $tenantFilter = $this->getTenantFilter(); $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $escapedResource = $this->escapeIdentifier('resource'); + $escapedEvent = $this->escapeIdentifier('event'); $sql = " SELECT count() FROM {$escapedTable} - WHERE resource = {resource:String} AND event IN ({$eventList['clause']}){$tenantFilter}{$time['clause']} + WHERE {$escapedResource} = {resource:String} AND {$escapedEvent} IN ({$eventList['clause']}){$tenantFilter}{$time['clause']} FORMAT TabSeparated "; diff --git a/src/Audit/Adapter/Database.php b/src/Audit/Adapter/Database.php index fc62c9d..e9310ee 100644 --- a/src/Audit/Adapter/Database.php +++ b/src/Audit/Adapter/Database.php @@ -111,19 +111,15 @@ public function createBatch(array $logs): bool */ public function getById(string $id): ?Log { - try { - $document = $this->db->getAuthorization()->skip(function () use ($id) { - return $this->db->getDocument($this->getCollectionName(), $id); - }); - - if ($document->isEmpty()) { - return null; - } + $document = $this->db->getAuthorization()->skip(function () use ($id) { + return $this->db->getDocument($this->getCollectionName(), $id); + }); - return new Log($document->getArrayCopy()); - } catch (\Exception $e) { + if ($document->isEmpty()) { return null; } + + return new Log($document->getArrayCopy()); } /** From a40925defe572bd5a37bf355be1f174b18f5bb92 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 11:32:09 +0000 Subject: [PATCH 16/29] feat: enhance ClickHouse adapter to support additional log attributes and improve time handling --- src/Audit/Adapter/ClickHouse.php | 58 ++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index fe719d5..6203b48 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -650,6 +650,7 @@ private function getColumnNames(): array { $columns = []; foreach ($this->getAttributes() as $attribute) { + /** @var string $columnName */ $columnName = $attribute['$id']; // Exclude id and tenant as they're handled separately if ($columnName !== 'id' && $columnName !== 'tenant') { @@ -733,39 +734,52 @@ private function formatDateTimeForClickHouse($dateTime): string } } + // This is unreachable code but kept for completeness - all valid types are handled above + // @phpstan-ignore-next-line throw new Exception('DateTime must be a DateTime object or string'); } /** * Create an audit log entry. * + * @param array $log The log data * @throws Exception */ public function create(array $log): Log { $id = uniqid('', true); - $time = $this->formatDateTimeForClickHouse($log['time'] ?? null); + // Format time - use provided time or current time + /** @var string|\DateTime|null $logTime */ + $logTime = $log['time'] ?? null; + $timeValue = $this->formatDateTimeForClickHouse($logTime); $tableName = $this->getTableName(); // Build column list and placeholders dynamically from attributes - $columns = ['id']; - $placeholders = ['{id:String}']; - $params = ['id' => $id]; + $columns = ['id', 'time']; + $placeholders = ['{id:String}', '{time:String}']; + $params = [ + 'id' => $id, + 'time' => $timeValue, + ]; // Get all column names from attributes $attributeColumns = $this->getColumnNames(); foreach ($attributeColumns as $column) { + if ($column === 'time') { + // Skip time - already handled above + continue; + } + if (isset($log[$column])) { $columns[] = $column; - // Special handling for time column - if ($column === 'time') { - $params[$column] = $this->formatDateTimeForClickHouse($log[$column]); - $placeholders[] = '{' . $column . ':String}'; - } elseif ($column === 'data') { - $params[$column] = json_encode($log[$column] ?? []); + // Special handling for data column + if ($column === 'data') { + /** @var array $dataValue */ + $dataValue = $log['data'] ?? []; + $params[$column] = json_encode($dataValue); // data is nullable based on attributes $placeholders[] = '{' . $column . ':Nullable(String)}'; } elseif (in_array($column, ['userId', 'location', 'userInternalId', 'resourceParent', 'resourceInternalId', 'country'])) { @@ -780,13 +794,6 @@ public function create(array $log): Log } } - // Add special handling for time if not provided - if (!isset($log['time'])) { - $columns[] = 'time'; - $params['time'] = $time; - $placeholders[] = '{time:String}'; - } - if ($this->sharedTables) { $columns[] = 'tenant'; $placeholders[] = '{tenant:Nullable(UInt64)}'; @@ -806,12 +813,17 @@ public function create(array $log): Log $result = ['$id' => $id]; + // Add time + $result['time'] = $timeValue; + // Add all columns from log to result foreach ($attributeColumns as $column) { if ($column === 'time') { - $result[$column] = $time; - } elseif ($column === 'data') { - $result[$column] = $log[$column] ?? []; + continue; // Already added + } + + if ($column === 'data') { + $result[$column] = $log['data'] ?? []; } elseif (isset($log[$column])) { $result[$column] = $log[$column]; } @@ -1077,6 +1089,7 @@ private function parseQueries(array $queries): array /** * Create multiple audit log entries in batch. * + * @param array> $logs The logs to insert * @throws Exception */ public function createBatch(array $logs): bool @@ -1126,6 +1139,7 @@ public function createBatch(array $logs): bool $valueClauses = []; foreach ($logs as $log) { + /** @var array $log */ $id = uniqid('', true); $ids[] = $id; @@ -1156,7 +1170,9 @@ public function createBatch(array $logs): bool // Determine value based on column type if ($column === 'time') { - $value = $this->formatDateTimeForClickHouse($log['time'] ?? null); + /** @var string|\DateTime|null $timeVal */ + $timeVal = $log['time'] ?? null; + $value = $this->formatDateTimeForClickHouse($timeVal); $params[$paramKey] = $value; $placeholders[] = '{' . $paramKey . ':String}'; } elseif ($column === 'data') { From 33d51e88935f533dc7e0135f3fe40c812dd661a0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 12:21:07 +0000 Subject: [PATCH 17/29] feat: add method to retrieve attribute metadata and enforce required fields in ClickHouse log entries --- src/Audit/Adapter/ClickHouse.php | 57 +++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 6203b48..5177b74 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -690,6 +690,23 @@ private function validateAttributeName(string $attributeName): bool throw new Exception("Invalid attribute name: {$attributeName}"); } + /** + * Get attribute metadata by column name. + * Searches through all attributes to find metadata for a specific column. + * + * @param string $columnName The column name to look up + * @return array|null The attribute metadata or null if not found + */ + private function getAttributeMetadata(string $columnName): ?array + { + foreach ($this->getAttributes() as $attribute) { + if ($attribute['$id'] === $columnName) { + return $attribute; + } + } + return null; + } + /** * Format datetime values for ClickHouse parameter binding. * Removes timezone suffixes which are incompatible with DateTime64 type comparisons. @@ -1168,28 +1185,52 @@ public function createBatch(array $logs): bool $paramKey = $column . '_' . $paramCounter; $paramKeys[] = $paramKey; + // Get attribute metadata to determine nullability and requirements + $attributeMeta = $this->getAttributeMetadata($column); + $isRequired = $attributeMeta !== null && isset($attributeMeta['required']) && $attributeMeta['required']; + $value = null; + $placeholder = ''; + // Determine value based on column type if ($column === 'time') { /** @var string|\DateTime|null $timeVal */ $timeVal = $log['time'] ?? null; + + if ($timeVal === null && $isRequired) { + throw new Exception("Required attribute 'time' is missing in batch log entry"); + } + $value = $this->formatDateTimeForClickHouse($timeVal); $params[$paramKey] = $value; - $placeholders[] = '{' . $paramKey . ':String}'; + // time is always non-nullable in ClickHouse + $placeholder = '{' . $paramKey . ':String}'; } elseif ($column === 'data') { - $value = json_encode($log['data'] ?? []); - $params[$paramKey] = $value; - $placeholders[] = '{' . $paramKey . ':Nullable(String)}'; - } elseif (in_array($column, ['userId', 'location', 'userInternalId', 'resourceParent', 'resourceInternalId', 'country'])) { - $value = $log[$column] ?? null; + /** @var array|null $dataVal */ + $dataVal = $log['data'] ?? null; + + if ($dataVal === null && $isRequired) { + throw new Exception("Required attribute 'data' is missing in batch log entry"); + } + + $value = json_encode($dataVal ?? []); $params[$paramKey] = $value; - $placeholders[] = '{' . $paramKey . ':Nullable(String)}'; + // data is nullable in schema + $placeholder = $isRequired ? '{' . $paramKey . ':String}' : '{' . $paramKey . ':Nullable(String)}'; } else { + // Regular attributes $value = $log[$column] ?? null; + + if ($value === null && $isRequired) { + throw new Exception("Required attribute '{$column}' is missing in batch log entry"); + } + $params[$paramKey] = $value; - $placeholders[] = '{' . $paramKey . ':Nullable(String)}'; + // Use metadata to determine if nullable + $placeholder = $isRequired ? '{' . $paramKey . ':String}' : '{' . $paramKey . ':Nullable(String)}'; } $paramValues[] = $value; + $placeholders[] = $placeholder; } if ($this->sharedTables) { From 1788442b7b4369eba62276780cf109aca06ec12e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 12:31:36 +0000 Subject: [PATCH 18/29] fix: improve query validation and formatting in ClickHouse adapter --- src/Audit/Adapter/ClickHouse.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 5177b74..0eae816 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -990,7 +990,9 @@ private function parseQueries(array $queries): array foreach ($queries as $query) { if (!$query instanceof Query) { - continue; + /** @phpstan-ignore-next-line ternary.alwaysTrue - runtime validation despite type hint */ + $type = is_object($query) ? get_class($query) : gettype($query); + throw new \InvalidArgumentException("Invalid query item: expected instance of Query, got {$type}"); } $method = $query->getMethod(); From 667fecf3cac427187f06d3e59b16feec70e38931 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 12:33:39 +0000 Subject: [PATCH 19/29] feat: add validation for required attributes in log entries and improve attribute escaping --- src/Audit/Adapter/ClickHouse.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 0eae816..7ed0f53 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -789,6 +789,15 @@ public function create(array $log): Log continue; } + // Get attribute metadata to check if required + $attributeMeta = $this->getAttributeMetadata($column); + $isRequired = $attributeMeta !== null && isset($attributeMeta['required']) && $attributeMeta['required']; + + // Check if value is missing for required attributes + if ($isRequired && (!isset($log[$column]) || $log[$column] === '')) { + throw new \InvalidArgumentException("Required attribute '{$column}' is missing or empty in log entry"); + } + if (isset($log[$column])) { $columns[] = $column; From 6adc9d185e14b62bd46177d48397f91cfb92375e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 13:30:01 +0000 Subject: [PATCH 20/29] feat: add support for additional attributes in log methods and enhance batch logging functionality --- src/Audit/Audit.php | 16 +++++--- tests/Audit/Adapter/ClickHouseTest.php | 19 +++++++++ tests/Audit/AuditBase.php | 57 ++++++++++++++++++++++---- 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/src/Audit/Audit.php b/src/Audit/Audit.php index dc99f9e..fdd9bc7 100644 --- a/src/Audit/Audit.php +++ b/src/Audit/Audit.php @@ -54,13 +54,14 @@ public function setup(): void * @param string $ip * @param string $location * @param array $data + * @param array $attributes * @return Log * * @throws \Exception */ - public function log(?string $userId, string $event, string $resource, string $userAgent, string $ip, string $location, array $data = []): Log + public function log(?string $userId, string $event, string $resource, string $userAgent, string $ip, string $location, array $data = [], array $attributes = []): Log { - return $this->adapter->create([ + $baseLog = [ 'userId' => $userId, 'event' => $event, 'resource' => $resource, @@ -68,20 +69,25 @@ public function log(?string $userId, string $event, string $resource, string $us 'ip' => $ip, 'location' => $location, 'data' => $data, - ]); + ]; + + return $this->adapter->create(array_merge($baseLog, $attributes)); } /** * Add multiple event logs in batch. * * @param array}> $events + * @param array $defaultAttributes * @return bool * * @throws \Exception */ - public function logBatch(array $events): bool + public function logBatch(array $events, array $defaultAttributes = []): bool { - return $this->adapter->createBatch($events); + $eventsWithDefaults = array_map(static fn (array $event) => array_merge($defaultAttributes, $event), $events); + + return $this->adapter->createBatch($eventsWithDefaults); } /** diff --git a/tests/Audit/Adapter/ClickHouseTest.php b/tests/Audit/Adapter/ClickHouseTest.php index 410bc59..96c40d4 100644 --- a/tests/Audit/Adapter/ClickHouseTest.php +++ b/tests/Audit/Adapter/ClickHouseTest.php @@ -33,6 +33,25 @@ protected function initializeAudit(): void $this->audit->setup(); } + /** + * Provide required attributes for ClickHouse adapter tests. + * + * @return array + */ + protected function getRequiredAttributes(): array + { + return [ + 'userType' => 'member', + 'resourceType' => 'document', + 'resourceId' => 'res-1', + 'projectId' => 'proj-1', + 'projectInternalId' => 'proj-int-1', + 'teamId' => 'team-1', + 'teamInternalId' => 'team-int-1', + 'hostname' => 'example.org', + ]; + } + /** * Test constructor validates host */ diff --git a/tests/Audit/AuditBase.php b/tests/Audit/AuditBase.php index 0eb4427..8ca8432 100644 --- a/tests/Audit/AuditBase.php +++ b/tests/Audit/AuditBase.php @@ -54,10 +54,12 @@ public function createLogs(): void $location = 'US'; $data = ['key1' => 'value1', 'key2' => 'value2']; - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/1', $userAgent, $ip, $location, $data)); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/2', $userAgent, $ip, $location, $data)); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'delete', 'database/document/2', $userAgent, $ip, $location, $data)); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log(null, 'insert', 'user/null', $userAgent, $ip, $location, $data)); + $requiredAttributes = $this->getRequiredAttributes(); + + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/1', $userAgent, $ip, $location, $data, $requiredAttributes)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/2', $userAgent, $ip, $location, $data, $requiredAttributes)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'delete', 'database/document/2', $userAgent, $ip, $location, $data, $requiredAttributes)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log(null, 'insert', 'user/null', $userAgent, $ip, $location, $data, $requiredAttributes)); } public function testGetLogsByUser(): void @@ -164,7 +166,8 @@ public function testGetLogById(): void $location = 'US'; $data = ['test' => 'getById']; - $log = $this->audit->log($userId, 'create', 'test/resource/123', $userAgent, $ip, $location, $data); + $requiredAttributes = $this->getRequiredAttributes(); + $log = $this->audit->log($userId, 'create', 'test/resource/123', $userAgent, $ip, $location, $data, $requiredAttributes); $logId = $log->getId(); // Retrieve the log by ID @@ -243,7 +246,11 @@ public function testLogByBatch(): void ] ]; + $batchEvents = $this->applyRequiredAttributesToBatch($batchEvents); + // Test batch insertion + $batchEvents = $this->applyRequiredAttributesToBatch($batchEvents); + $result = $this->audit->logBatch($batchEvents); $this->assertTrue($result); @@ -324,6 +331,7 @@ public function testLargeBatchInsert(): void } // Insert batch + $batchEvents = $this->applyRequiredAttributesToBatch($batchEvents); $result = $this->audit->logBatch($batchEvents); $this->assertTrue($result); @@ -368,6 +376,7 @@ public function testTimeRangeFilters(): void ] ]; + $batchEvents = $this->applyRequiredAttributesToBatch($batchEvents); $this->audit->logBatch($batchEvents); // Test getting all logs @@ -396,11 +405,13 @@ public function testCleanup(): void $location = 'US'; $data = ['key1' => 'value1', 'key2' => 'value2']; - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/1', $userAgent, $ip, $location, $data)); + $requiredAttributes = $this->getRequiredAttributes(); + + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/1', $userAgent, $ip, $location, $data, $requiredAttributes)); sleep(5); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/2', $userAgent, $ip, $location, $data)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/2', $userAgent, $ip, $location, $data, $requiredAttributes)); sleep(5); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'delete', 'database/document/2', $userAgent, $ip, $location, $data)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'delete', 'database/document/2', $userAgent, $ip, $location, $data, $requiredAttributes)); sleep(5); // DELETE logs older than 11 seconds and check that status is true @@ -447,6 +458,8 @@ public function testRetrievalParameters(): void ]; } + $batchEvents = $this->applyRequiredAttributesToBatch($batchEvents); + $this->audit->logBatch($batchEvents); // Test 1: limit parameter @@ -616,6 +629,8 @@ public function testFind(): void 'time' => $timestamp ]; } + $batchEvents = $this->applyRequiredAttributesToBatch($batchEvents); + $this->audit->logBatch($batchEvents); // Test 1: Find with equal filter @@ -789,4 +804,30 @@ public function testCount(): void ]); $this->assertEquals(0, $count); } + + /** + * Apply adapter-specific required attributes to batch events. + * + * @param array> $batchEvents + * @return array> + */ + protected function applyRequiredAttributesToBatch(array $batchEvents): array + { + $requiredAttributes = $this->getRequiredAttributes(); + if ($requiredAttributes === []) { + return $batchEvents; + } + + return array_map(static fn (array $event) => array_merge($event, $requiredAttributes), $batchEvents); + } + + /** + * Override in adapter-specific tests to provide required attribute defaults. + * + * @return array + */ + protected function getRequiredAttributes(): array + { + return []; + } } From b0a081662aa5cf0d98a692a5507eeeae1facd7ca Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 22:50:31 +0000 Subject: [PATCH 21/29] codeql fix --- src/Audit/Audit.php | 5 ++++- tests/Audit/AuditBase.php | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Audit/Audit.php b/src/Audit/Audit.php index fdd9bc7..fdf3d39 100644 --- a/src/Audit/Audit.php +++ b/src/Audit/Audit.php @@ -71,7 +71,9 @@ public function log(?string $userId, string $event, string $resource, string $us 'data' => $data, ]; - return $this->adapter->create(array_merge($baseLog, $attributes)); + /** @var array{userId?: string|null, event: string, resource: string, userAgent: string, ip: string, location?: string, data?: array} $log */ + $log = array_merge($baseLog, $attributes); + return $this->adapter->create($log); } /** @@ -85,6 +87,7 @@ public function log(?string $userId, string $event, string $resource, string $us */ public function logBatch(array $events, array $defaultAttributes = []): bool { + /** @var array}> $eventsWithDefaults */ $eventsWithDefaults = array_map(static fn (array $event) => array_merge($defaultAttributes, $event), $events); return $this->adapter->createBatch($eventsWithDefaults); diff --git a/tests/Audit/AuditBase.php b/tests/Audit/AuditBase.php index 8ca8432..c79eed9 100644 --- a/tests/Audit/AuditBase.php +++ b/tests/Audit/AuditBase.php @@ -809,15 +809,17 @@ public function testCount(): void * Apply adapter-specific required attributes to batch events. * * @param array> $batchEvents - * @return array> + * @return array}> */ protected function applyRequiredAttributesToBatch(array $batchEvents): array { $requiredAttributes = $this->getRequiredAttributes(); if ($requiredAttributes === []) { + /** @var array}> */ return $batchEvents; } + /** @var array}> */ return array_map(static fn (array $event) => array_merge($event, $requiredAttributes), $batchEvents); } From 5f5cf3d5b48c827e1b5299707cbb08f918df4e0d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 23:07:03 +0000 Subject: [PATCH 22/29] feat: refactor log method to remove attributes parameter and merge required attributes into data --- src/Audit/Adapter/ClickHouse.php | 314 ++++++++++++++++--------------- src/Audit/Audit.php | 8 +- tests/Audit/AuditBase.php | 19 +- 3 files changed, 180 insertions(+), 161 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 7ed0f53..364396e 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -764,102 +764,112 @@ private function formatDateTimeForClickHouse($dateTime): string */ public function create(array $log): Log { - $id = uniqid('', true); + $logId = uniqid('', true); + // Format time - use provided time or current time - /** @var string|\DateTime|null $logTime */ - $logTime = $log['time'] ?? null; - $timeValue = $this->formatDateTimeForClickHouse($logTime); + /** @var string|\DateTime|null $providedTime */ + $providedTime = $log['time'] ?? null; + $formattedTime = $this->formatDateTimeForClickHouse($providedTime); $tableName = $this->getTableName(); + // Extract additional attributes from the data array + /** @var array $logData */ + $logData = $log['data'] ?? []; + // Build column list and placeholders dynamically from attributes - $columns = ['id', 'time']; - $placeholders = ['{id:String}', '{time:String}']; - $params = [ - 'id' => $id, - 'time' => $timeValue, + $insertColumns = ['id', 'time']; + $valuePlaceholders = ['{id:String}', '{time:String}']; + $queryParams = [ + 'id' => $logId, + 'time' => $formattedTime, ]; // Get all column names from attributes - $attributeColumns = $this->getColumnNames(); + $schemaColumns = $this->getColumnNames(); + + // Separate data for the data column (non-schema attributes) + $nonSchemaData = $logData; - foreach ($attributeColumns as $column) { - if ($column === 'time') { + foreach ($schemaColumns as $columnName) { + if ($columnName === 'time') { // Skip time - already handled above continue; } - // Get attribute metadata to check if required - $attributeMeta = $this->getAttributeMetadata($column); - $isRequired = $attributeMeta !== null && isset($attributeMeta['required']) && $attributeMeta['required']; + // Get attribute metadata to determine if required and nullable + $attributeMetadata = $this->getAttributeMetadata($columnName); + $isRequiredAttribute = $attributeMetadata !== null && isset($attributeMetadata['required']) && $attributeMetadata['required']; + $isNullableAttribute = $attributeMetadata !== null && (!isset($attributeMetadata['required']) || !$attributeMetadata['required']); + + // For 'data' column, we'll handle it separately at the end + if ($columnName === 'data') { + continue; + } + + // Check if value exists in main log or in data array + $attributeValue = null; + $hasAttributeValue = false; + + if (isset($log[$columnName]) && $log[$columnName] !== '') { + // Value is in main log (e.g., userId, event, resource, etc.) + $attributeValue = $log[$columnName]; + $hasAttributeValue = true; + } elseif (isset($logData[$columnName]) && $logData[$columnName] !== '') { + // Value is in data array (additional attributes) + $attributeValue = $logData[$columnName]; + $hasAttributeValue = true; + // Remove from non-schema data as it's now a dedicated column + unset($nonSchemaData[$columnName]); + } // Check if value is missing for required attributes - if ($isRequired && (!isset($log[$column]) || $log[$column] === '')) { - throw new \InvalidArgumentException("Required attribute '{$column}' is missing or empty in log entry"); + if ($isRequiredAttribute && !$hasAttributeValue) { + throw new \InvalidArgumentException("Required attribute '{$columnName}' is missing or empty in log entry"); } - if (isset($log[$column])) { - $columns[] = $column; - - // Special handling for data column - if ($column === 'data') { - /** @var array $dataValue */ - $dataValue = $log['data'] ?? []; - $params[$column] = json_encode($dataValue); - // data is nullable based on attributes - $placeholders[] = '{' . $column . ':Nullable(String)}'; - } elseif (in_array($column, ['userId', 'location', 'userInternalId', 'resourceParent', 'resourceInternalId', 'country'])) { - // Nullable string fields - $params[$column] = $log[$column]; - $placeholders[] = '{' . $column . ':Nullable(String)}'; + if ($hasAttributeValue) { + $insertColumns[] = $columnName; + $queryParams[$columnName] = $attributeValue; + + // Determine placeholder type based on attribute metadata + if ($isNullableAttribute) { + $valuePlaceholders[] = '{' . $columnName . ':Nullable(String)}'; } else { - // Required string fields - $params[$column] = $log[$column]; - $placeholders[] = '{' . $column . ':String}'; + $valuePlaceholders[] = '{' . $columnName . ':String}'; } } } + // Add the data column with remaining non-schema attributes + $insertColumns[] = 'data'; + $queryParams['data'] = json_encode($nonSchemaData); + $valuePlaceholders[] = '{data:Nullable(String)}'; + if ($this->sharedTables) { - $columns[] = 'tenant'; - $placeholders[] = '{tenant:Nullable(UInt64)}'; - $params['tenant'] = $this->tenant; + $insertColumns[] = 'tenant'; + $valuePlaceholders[] = '{tenant:Nullable(UInt64)}'; + $queryParams['tenant'] = $this->tenant; } $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); $insertSql = " INSERT INTO {$escapedDatabaseAndTable} - (" . implode(', ', $columns) . ") + (" . implode(', ', $insertColumns) . ") VALUES ( - " . implode(", ", $placeholders) . " + " . implode(", ", $valuePlaceholders) . " ) "; - $this->query($insertSql, $params); - - $result = ['$id' => $id]; - - // Add time - $result['time'] = $timeValue; + $this->query($insertSql, $queryParams); - // Add all columns from log to result - foreach ($attributeColumns as $column) { - if ($column === 'time') { - continue; // Already added - } - - if ($column === 'data') { - $result[$column] = $log['data'] ?? []; - } elseif (isset($log[$column])) { - $result[$column] = $log[$column]; - } - } - - if ($this->sharedTables) { - $result['tenant'] = $this->tenant; + // Retrieve the created log using getById to ensure consistency + $createdLog = $this->getById($logId); + if ($createdLog === null) { + throw new Exception("Failed to retrieve created log with ID: {$logId}"); } - return new Log($result); + return $createdLog; } /** @@ -1130,136 +1140,144 @@ public function createBatch(array $logs): bool $escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); // Get all attribute column names - $attributeColumns = $this->getColumnNames(); + $schemaColumns = $this->getColumnNames(); + + // Process each log to extract additional attributes from data + $processedLogs = []; + foreach ($logs as $log) { + /** @var array $logData */ + $logData = $log['data'] ?? []; + + // Separate data for non-schema attributes + $nonSchemaData = $logData; + $processedLog = $log; + + // Extract schema attributes from data array + foreach ($schemaColumns as $columnName) { + if ($columnName === 'data' || $columnName === 'time') { + continue; + } - // Build column list starting with id - $columns = ['id']; + // If attribute exists in data array and not in main log, move it + if (!isset($processedLog[$columnName]) && isset($logData[$columnName])) { + $processedLog[$columnName] = $logData[$columnName]; + unset($nonSchemaData[$columnName]); + } + } + + // Update data with remaining non-schema attributes + $processedLog['data'] = $nonSchemaData; + $processedLogs[] = $processedLog; + } + + // Build column list starting with id and time + $insertColumns = ['id', 'time']; // Determine which attribute columns are present in any log $presentColumns = []; - foreach ($logs as $log) { - foreach ($attributeColumns as $column) { - if (isset($log[$column]) && !in_array($column, $presentColumns, true)) { - $presentColumns[] = $column; + foreach ($processedLogs as $processedLog) { + foreach ($schemaColumns as $columnName) { + if ($columnName === 'time') { + continue; // Already in insertColumns + } + if (isset($processedLog[$columnName]) && !in_array($columnName, $presentColumns, true)) { + $presentColumns[] = $columnName; } } } // Add present columns in the order they're defined in attributes - foreach ($attributeColumns as $column) { - if (in_array($column, $presentColumns, true)) { - $columns[] = $column; + foreach ($schemaColumns as $columnName) { + if ($columnName === 'time') { + continue; // Already added + } + if (in_array($columnName, $presentColumns, true)) { + $insertColumns[] = $columnName; } - } - - // Always include time column - if (!in_array('time', $columns, true)) { - $columns[] = 'time'; } if ($this->sharedTables) { - $columns[] = 'tenant'; + $insertColumns[] = 'tenant'; } - $ids = []; $paramCounter = 0; - $params = []; + $queryParams = []; $valueClauses = []; - foreach ($logs as $log) { - /** @var array $log */ - $id = uniqid('', true); - $ids[] = $id; - - // Create parameter placeholders for this row - $paramKeys = []; - $paramValues = []; - $placeholders = []; + foreach ($processedLogs as $processedLog) { + $logId = uniqid('', true); + $valuePlaceholders = []; - // Add id first + // Add id $paramKey = 'id_' . $paramCounter; - $paramKeys[] = $paramKey; - $paramValues[] = $id; - $params[$paramKey] = $id; - $placeholders[] = '{' . $paramKey . ':String}'; - - // Add all present columns in order - foreach ($columns as $column) { - if ($column === 'id') { - continue; // Already added - } - - if ($column === 'tenant') { - continue; // Handle separately below + $queryParams[$paramKey] = $logId; + $valuePlaceholders[] = '{' . $paramKey . ':String}'; + + // Add time + /** @var string|\DateTime|null $providedTime */ + $providedTime = $processedLog['time'] ?? null; + $formattedTime = $this->formatDateTimeForClickHouse($providedTime); + $paramKey = 'time_' . $paramCounter; + $queryParams[$paramKey] = $formattedTime; + $valuePlaceholders[] = '{' . $paramKey . ':String}'; + + // Add all other present columns + foreach ($insertColumns as $columnName) { + if ($columnName === 'id' || $columnName === 'time' || $columnName === 'tenant') { + continue; // Already handled } - $paramKey = $column . '_' . $paramCounter; - $paramKeys[] = $paramKey; + $paramKey = $columnName . '_' . $paramCounter; - // Get attribute metadata to determine nullability and requirements - $attributeMeta = $this->getAttributeMetadata($column); - $isRequired = $attributeMeta !== null && isset($attributeMeta['required']) && $attributeMeta['required']; - $value = null; - $placeholder = ''; + // Get attribute metadata to determine if required and nullable + $attributeMetadata = $this->getAttributeMetadata($columnName); + $isRequiredAttribute = $attributeMetadata !== null && isset($attributeMetadata['required']) && $attributeMetadata['required']; + $isNullableAttribute = $attributeMetadata !== null && (!isset($attributeMetadata['required']) || !$attributeMetadata['required']); - // Determine value based on column type - if ($column === 'time') { - /** @var string|\DateTime|null $timeVal */ - $timeVal = $log['time'] ?? null; + $attributeValue = null; + $hasAttributeValue = false; - if ($timeVal === null && $isRequired) { - throw new Exception("Required attribute 'time' is missing in batch log entry"); - } + if ($columnName === 'data') { + // Data column - encode as JSON\n /** @var array $dataValue */ + $dataValue = $processedLog['data']; + $attributeValue = json_encode($dataValue); + $hasAttributeValue = true; + } elseif (isset($processedLog[$columnName]) && $processedLog[$columnName] !== '') { + $attributeValue = $processedLog[$columnName]; + $hasAttributeValue = true; + } - $value = $this->formatDateTimeForClickHouse($timeVal); - $params[$paramKey] = $value; - // time is always non-nullable in ClickHouse - $placeholder = '{' . $paramKey . ':String}'; - } elseif ($column === 'data') { - /** @var array|null $dataVal */ - $dataVal = $log['data'] ?? null; + // Check if value is missing for required attributes + if ($isRequiredAttribute && !$hasAttributeValue) { + throw new \InvalidArgumentException("Required attribute '{$columnName}' is missing or empty in batch log entry"); + } - if ($dataVal === null && $isRequired) { - throw new Exception("Required attribute 'data' is missing in batch log entry"); - } + $queryParams[$paramKey] = $attributeValue; - $value = json_encode($dataVal ?? []); - $params[$paramKey] = $value; - // data is nullable in schema - $placeholder = $isRequired ? '{' . $paramKey . ':String}' : '{' . $paramKey . ':Nullable(String)}'; + // Determine placeholder type based on attribute metadata + if ($isNullableAttribute) { + $valuePlaceholders[] = '{' . $paramKey . ':Nullable(String)}'; } else { - // Regular attributes - $value = $log[$column] ?? null; - - if ($value === null && $isRequired) { - throw new Exception("Required attribute '{$column}' is missing in batch log entry"); - } - - $params[$paramKey] = $value; - // Use metadata to determine if nullable - $placeholder = $isRequired ? '{' . $paramKey . ':String}' : '{' . $paramKey . ':Nullable(String)}'; + $valuePlaceholders[] = '{' . $paramKey . ':String}'; } - - $paramValues[] = $value; - $placeholders[] = $placeholder; } if ($this->sharedTables) { $paramKey = 'tenant_' . $paramCounter; - $params[$paramKey] = $this->tenant; - $placeholders[] = '{' . $paramKey . ':Nullable(UInt64)}'; + $queryParams[$paramKey] = $this->tenant; + $valuePlaceholders[] = '{' . $paramKey . ':Nullable(UInt64)}'; } - $valueClauses[] = '(' . implode(', ', $placeholders) . ')'; + $valueClauses[] = '(' . implode(', ', $valuePlaceholders) . ')'; $paramCounter++; } $insertSql = " INSERT INTO {$escapedDatabaseAndTable} - (" . implode(', ', $columns) . ") + (" . implode(', ', $insertColumns) . ") VALUES " . implode(', ', $valueClauses); - $this->query($insertSql, $params); + $this->query($insertSql, $queryParams); return true; } diff --git a/src/Audit/Audit.php b/src/Audit/Audit.php index fdf3d39..c59ec46 100644 --- a/src/Audit/Audit.php +++ b/src/Audit/Audit.php @@ -54,14 +54,14 @@ public function setup(): void * @param string $ip * @param string $location * @param array $data - * @param array $attributes * @return Log * * @throws \Exception */ - public function log(?string $userId, string $event, string $resource, string $userAgent, string $ip, string $location, array $data = [], array $attributes = []): Log + public function log(?string $userId, string $event, string $resource, string $userAgent, string $ip, string $location, array $data = []): Log { - $baseLog = [ + /** @var array{userId?: string|null, event: string, resource: string, userAgent: string, ip: string, location?: string, data?: array} $log */ + $log = [ 'userId' => $userId, 'event' => $event, 'resource' => $resource, @@ -71,8 +71,6 @@ public function log(?string $userId, string $event, string $resource, string $us 'data' => $data, ]; - /** @var array{userId?: string|null, event: string, resource: string, userAgent: string, ip: string, location?: string, data?: array} $log */ - $log = array_merge($baseLog, $attributes); return $this->adapter->create($log); } diff --git a/tests/Audit/AuditBase.php b/tests/Audit/AuditBase.php index c79eed9..cf6b42b 100644 --- a/tests/Audit/AuditBase.php +++ b/tests/Audit/AuditBase.php @@ -55,11 +55,12 @@ public function createLogs(): void $data = ['key1' => 'value1', 'key2' => 'value2']; $requiredAttributes = $this->getRequiredAttributes(); + $dataWithAttributes = array_merge($data, $requiredAttributes); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/1', $userAgent, $ip, $location, $data, $requiredAttributes)); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/2', $userAgent, $ip, $location, $data, $requiredAttributes)); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'delete', 'database/document/2', $userAgent, $ip, $location, $data, $requiredAttributes)); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log(null, 'insert', 'user/null', $userAgent, $ip, $location, $data, $requiredAttributes)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/1', $userAgent, $ip, $location, $dataWithAttributes)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/2', $userAgent, $ip, $location, $dataWithAttributes)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'delete', 'database/document/2', $userAgent, $ip, $location, $dataWithAttributes)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log(null, 'insert', 'user/null', $userAgent, $ip, $location, $dataWithAttributes)); } public function testGetLogsByUser(): void @@ -167,7 +168,8 @@ public function testGetLogById(): void $data = ['test' => 'getById']; $requiredAttributes = $this->getRequiredAttributes(); - $log = $this->audit->log($userId, 'create', 'test/resource/123', $userAgent, $ip, $location, $data, $requiredAttributes); + $dataWithAttributes = array_merge($data, $requiredAttributes); + $log = $this->audit->log($userId, 'create', 'test/resource/123', $userAgent, $ip, $location, $dataWithAttributes); $logId = $log->getId(); // Retrieve the log by ID @@ -406,12 +408,13 @@ public function testCleanup(): void $data = ['key1' => 'value1', 'key2' => 'value2']; $requiredAttributes = $this->getRequiredAttributes(); + $dataWithAttributes = array_merge($data, $requiredAttributes); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/1', $userAgent, $ip, $location, $data, $requiredAttributes)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/1', $userAgent, $ip, $location, $dataWithAttributes)); sleep(5); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/2', $userAgent, $ip, $location, $data, $requiredAttributes)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'update', 'database/document/2', $userAgent, $ip, $location, $dataWithAttributes)); sleep(5); - $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'delete', 'database/document/2', $userAgent, $ip, $location, $data, $requiredAttributes)); + $this->assertInstanceOf('Utopia\\Audit\\Log', $this->audit->log($userId, 'delete', 'database/document/2', $userAgent, $ip, $location, $dataWithAttributes)); sleep(5); // DELETE logs older than 11 seconds and check that status is true From 47ab63769cce71ec4322004a94bf496aa1144d96 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 23:17:32 +0000 Subject: [PATCH 23/29] feat: enhance attribute validation and extraction logic in ClickHouse adapter --- src/Audit/Adapter/ClickHouse.php | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 364396e..d71ba30 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -807,15 +807,15 @@ public function create(array $log): Log continue; } - // Check if value exists in main log or in data array + // Check if value exists in main log first, then in data array $attributeValue = null; $hasAttributeValue = false; - if (isset($log[$columnName]) && $log[$columnName] !== '') { + if (isset($log[$columnName])) { // Value is in main log (e.g., userId, event, resource, etc.) $attributeValue = $log[$columnName]; $hasAttributeValue = true; - } elseif (isset($logData[$columnName]) && $logData[$columnName] !== '') { + } elseif (isset($logData[$columnName])) { // Value is in data array (additional attributes) $attributeValue = $logData[$columnName]; $hasAttributeValue = true; @@ -823,9 +823,9 @@ public function create(array $log): Log unset($nonSchemaData[$columnName]); } - // Check if value is missing for required attributes + // Validate required attributes if ($isRequiredAttribute && !$hasAttributeValue) { - throw new \InvalidArgumentException("Required attribute '{$columnName}' is missing or empty in log entry"); + throw new \InvalidArgumentException("Required attribute '{$columnName}' is missing in log entry"); } if ($hasAttributeValue) { @@ -1152,16 +1152,19 @@ public function createBatch(array $logs): bool $nonSchemaData = $logData; $processedLog = $log; - // Extract schema attributes from data array + // Extract schema attributes: check main log first, then data array foreach ($schemaColumns as $columnName) { if ($columnName === 'data' || $columnName === 'time') { continue; } - // If attribute exists in data array and not in main log, move it + // If attribute not in main log, check data array if (!isset($processedLog[$columnName]) && isset($logData[$columnName])) { $processedLog[$columnName] = $logData[$columnName]; unset($nonSchemaData[$columnName]); + } elseif (isset($processedLog[$columnName]) && isset($logData[$columnName])) { + // If in both, main log takes precedence, remove from data + unset($nonSchemaData[$columnName]); } } @@ -1238,18 +1241,19 @@ public function createBatch(array $logs): bool $hasAttributeValue = false; if ($columnName === 'data') { - // Data column - encode as JSON\n /** @var array $dataValue */ + // Data column - encode as JSON + /** @var array $dataValue */ $dataValue = $processedLog['data']; $attributeValue = json_encode($dataValue); $hasAttributeValue = true; - } elseif (isset($processedLog[$columnName]) && $processedLog[$columnName] !== '') { + } elseif (isset($processedLog[$columnName])) { $attributeValue = $processedLog[$columnName]; $hasAttributeValue = true; } - // Check if value is missing for required attributes + // Validate required attributes if ($isRequiredAttribute && !$hasAttributeValue) { - throw new \InvalidArgumentException("Required attribute '{$columnName}' is missing or empty in batch log entry"); + throw new \InvalidArgumentException("Required attribute '{$columnName}' is missing in batch log entry"); } $queryParams[$paramKey] = $attributeValue; From a53568820a5711aa496c807f1715e520f90816b3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jan 2026 23:38:03 +0000 Subject: [PATCH 24/29] feat: simplify logBatch method by removing defaultAttributes parameter --- src/Audit/Audit.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Audit/Audit.php b/src/Audit/Audit.php index c59ec46..bb17f83 100644 --- a/src/Audit/Audit.php +++ b/src/Audit/Audit.php @@ -78,17 +78,13 @@ public function log(?string $userId, string $event, string $resource, string $us * Add multiple event logs in batch. * * @param array}> $events - * @param array $defaultAttributes * @return bool * * @throws \Exception */ - public function logBatch(array $events, array $defaultAttributes = []): bool + public function logBatch(array $events): bool { - /** @var array}> $eventsWithDefaults */ - $eventsWithDefaults = array_map(static fn (array $event) => array_merge($defaultAttributes, $event), $events); - - return $this->adapter->createBatch($eventsWithDefaults); + return $this->adapter->createBatch($events); } /** From 3c2b75519a92a47c4e75c7b6464a809eb611d94a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 00:26:24 +0000 Subject: [PATCH 25/29] feat: enhance ClickHouse adapter to dynamically map columns and improve attribute handling --- src/Audit/Adapter/ClickHouse.php | 124 ++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 44 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index d71ba30..6982d25 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -1287,6 +1287,7 @@ public function createBatch(array $logs): bool /** * Parse ClickHouse query result into Log objects. + * Dynamically maps columns based on current attribute definitions. * * @return array */ @@ -1299,28 +1300,32 @@ private function parseResults(string $result): array $lines = explode("\n", trim($result)); $documents = []; + // Build the expected column order dynamically + $selectColumns = []; + foreach ($this->getAttributes() as $attribute) { + $id = $attribute['$id']; + if ($id !== 'data') { + $selectColumns[] = $id; + } + } + $selectColumns[] = 'data'; + + if ($this->sharedTables) { + $selectColumns[] = 'tenant'; + } + + $expectedColumns = count($selectColumns); + foreach ($lines as $line) { if (empty(trim($line))) { continue; } $columns = explode("\t", $line); - // Expect 9 columns without sharedTables, 10 with sharedTables - $expectedColumns = $this->sharedTables ? 10 : 9; if (count($columns) < $expectedColumns) { continue; } - $data = json_decode($columns[8], true) ?? []; - - // Convert ClickHouse timestamp format back to ISO 8601 - // ClickHouse: 2025-12-07 23:33:54.493 - // ISO 8601: 2025-12-07T23:33:54.493+00:00 - $time = $columns[7]; - if (strpos($time, 'T') === false) { - $time = str_replace(' ', 'T', $time) . '+00:00'; - } - // Helper function to parse nullable string fields // ClickHouse TabSeparated format uses \N for NULL, but empty strings are also treated as null for nullable fields $parseNullableString = static function ($value): ?string { @@ -1330,21 +1335,47 @@ private function parseResults(string $result): array return $value; }; - $document = [ - '$id' => $columns[0], - 'userId' => $parseNullableString($columns[1]), - 'event' => $columns[2], - 'resource' => $columns[3], - 'userAgent' => $columns[4], - 'ip' => $columns[5], - 'location' => $parseNullableString($columns[6]), - 'time' => $time, - 'data' => $data, - ]; - - // Add tenant only if sharedTables is enabled - if ($this->sharedTables && isset($columns[9])) { - $document['tenant'] = $columns[9] === '\\N' || $columns[9] === '' ? null : (int) $columns[9]; + // Build document dynamically by mapping columns to values + $document = []; + foreach ($selectColumns as $index => $columnName) { + if (!isset($columns[$index])) { + continue; + } + + $value = $columns[$index]; + + if ($columnName === 'data') { + // Decode JSON data column + $document[$columnName] = json_decode($value, true) ?? []; + } elseif ($columnName === 'tenant') { + // Parse tenant as integer or null + $document[$columnName] = ($value === '\\N' || $value === '') ? null : (int) $value; + } elseif ($columnName === 'time') { + // Convert ClickHouse timestamp format back to ISO 8601 + // ClickHouse: 2025-12-07 23:33:54.493 + // ISO 8601: 2025-12-07T23:33:54.493+00:00 + $parsedTime = $value; + if (strpos($parsedTime, 'T') === false) { + $parsedTime = str_replace(' ', 'T', $parsedTime) . '+00:00'; + } + $document[$columnName] = $parsedTime; + } else { + // Get attribute metadata to check if nullable + $attribute = $this->getAttribute($columnName); + if ($attribute && !$attribute['required']) { + // Nullable field - parse null values + $document[$columnName] = $parseNullableString($value); + } else { + // Required field - use value as-is + $document[$columnName] = $value; + } + } + } + + // Add special $id field if present + if (isset($document['id'])) { + $document['$id'] = $document['id']; + unset($document['id']); } $documents[] = new Log($document); @@ -1355,24 +1386,27 @@ private function parseResults(string $result): array /** * Get the SELECT column list for queries. + * Dynamically builds the column list from attributes, excluding 'data' column. * Escapes all column names to prevent SQL injection. * * @return string */ private function getSelectColumns(): string { - $columns = [ - $this->escapeIdentifier('id'), - $this->escapeIdentifier('userId'), - $this->escapeIdentifier('event'), - $this->escapeIdentifier('resource'), - $this->escapeIdentifier('userAgent'), - $this->escapeIdentifier('ip'), - $this->escapeIdentifier('location'), - $this->escapeIdentifier('time'), - $this->escapeIdentifier('data'), - ]; + $columns = []; + + // Dynamically add all attribute columns except 'data' + foreach ($this->getAttributes() as $attribute) { + $id = $attribute['$id']; + if ($id !== 'data') { + $columns[] = $this->escapeIdentifier($id); + } + } + // Add data column at the end + $columns[] = $this->escapeIdentifier('data'); + + // Add tenant column if shared tables are enabled if ($this->sharedTables) { $columns[] = $this->escapeIdentifier('tenant'); } @@ -1480,6 +1514,9 @@ private function buildEventsList(array $events, int $paramOffset = 0): array /** * Get ClickHouse-specific SQL column definition for a given attribute ID. * + * Dynamically determines the ClickHouse type based on attribute metadata. + * DateTime attributes use DateTime64(3), all others use String. + * * @param string $id Attribute identifier * @return string ClickHouse column definition with appropriate types and nullability * @throws Exception @@ -1492,12 +1529,11 @@ protected function getColumnDefinition(string $id): string throw new Exception("Attribute {$id} not found"); } - // ClickHouse-specific type mapping - $type = match ($id) { - 'userId', 'event', 'resource', 'userAgent', 'ip', 'location', 'data' => 'String', - 'time' => 'DateTime64(3)', - default => 'String', - }; + // Dynamically determine type based on attribute metadata + // DateTime attributes use DateTime64(3), all others use String + $type = (isset($attribute['type']) && $attribute['type'] === Database::VAR_DATETIME) + ? 'DateTime64(3)' + : 'String'; $nullable = !$attribute['required'] ? 'Nullable(' . $type . ')' : $type; From e7c26bb629118254bf35259044ea26be6c80aa48 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 00:36:16 +0000 Subject: [PATCH 26/29] feat: streamline datetime formatting for ClickHouse and enhance column selection logic --- src/Audit/Adapter/ClickHouse.php | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 6982d25..451bbf5 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -707,25 +707,11 @@ private function getAttributeMetadata(string $columnName): ?array return null; } - /** - * Format datetime values for ClickHouse parameter binding. - * Removes timezone suffixes which are incompatible with DateTime64 type comparisons. - * - * @param mixed $value The value to format - * @return string Formatted string without timezone suffix - */ - private function formatDateTimeParam(mixed $value): string - { - $strValue = $this->formatParamValue($value); - // Remove timezone suffix if present (e.g., +00:00, -05:00) - return preg_replace('/[+\\-]\\d{2}:\\d{2}$/', '', $strValue) ?? $strValue; - } - - /** * Format datetime for ClickHouse compatibility. * Converts datetime to 'YYYY-MM-DD HH:MM:SS.mmm' format without timezone suffix. * ClickHouse DateTime64(3) type expects this format as timezone is handled by column metadata. + * Works with DateTime objects, strings, and other datetime representations. * * @param \DateTime|string|null $dateTime The datetime value to format * @return string The formatted datetime string in ClickHouse compatible format @@ -1053,8 +1039,8 @@ private function parseQueries(array $queries): array if ($attribute === 'time') { $paramType = 'DateTime64(3)'; $filters[] = "{$escapedAttr} BETWEEN {{$paramName1}:{$paramType}} AND {{$paramName2}:{$paramType}}"; - $params[$paramName1] = $this->formatDateTimeParam($values[0]); - $params[$paramName2] = $this->formatDateTimeParam($values[1]); + $params[$paramName1] = $this->formatDateTimeForClickHouse($values[0]); + $params[$paramName2] = $this->formatDateTimeForClickHouse($values[1]); } else { $filters[] = "{$escapedAttr} BETWEEN {{$paramName1}:String} AND {{$paramName2}:String}"; $params[$paramName1] = $this->formatParamValue($values[0]); @@ -1300,8 +1286,8 @@ private function parseResults(string $result): array $lines = explode("\n", trim($result)); $documents = []; - // Build the expected column order dynamically - $selectColumns = []; + // Build the expected column order dynamically (matching getSelectColumns order) + $selectColumns = ['id']; foreach ($this->getAttributes() as $attribute) { $id = $attribute['$id']; if ($id !== 'data') { @@ -1395,6 +1381,9 @@ private function getSelectColumns(): string { $columns = []; + // Add id column first (not part of attributes) + $columns[] = $this->escapeIdentifier('id'); + // Dynamically add all attribute columns except 'data' foreach ($this->getAttributes() as $attribute) { $id = $attribute['$id']; From b95f7358a9788fa1a3b49db533f25dcf42d66d16 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 00:37:03 +0000 Subject: [PATCH 27/29] feat: rename formatDateTimeForClickHouse to formatDateTime for clarity --- src/Audit/Adapter/ClickHouse.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 451bbf5..049a652 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -717,7 +717,7 @@ private function getAttributeMetadata(string $columnName): ?array * @return string The formatted datetime string in ClickHouse compatible format * @throws Exception If the datetime string cannot be parsed */ - private function formatDateTimeForClickHouse($dateTime): string + private function formatDateTime($dateTime): string { if ($dateTime === null) { return (new \DateTime())->format('Y-m-d H:i:s.v'); @@ -755,7 +755,7 @@ public function create(array $log): Log // Format time - use provided time or current time /** @var string|\DateTime|null $providedTime */ $providedTime = $log['time'] ?? null; - $formattedTime = $this->formatDateTimeForClickHouse($providedTime); + $formattedTime = $this->formatDateTime($providedTime); $tableName = $this->getTableName(); @@ -1039,8 +1039,8 @@ private function parseQueries(array $queries): array if ($attribute === 'time') { $paramType = 'DateTime64(3)'; $filters[] = "{$escapedAttr} BETWEEN {{$paramName1}:{$paramType}} AND {{$paramName2}:{$paramType}}"; - $params[$paramName1] = $this->formatDateTimeForClickHouse($values[0]); - $params[$paramName2] = $this->formatDateTimeForClickHouse($values[1]); + $params[$paramName1] = $this->formatDateTime($values[0]); + $params[$paramName2] = $this->formatDateTime($values[1]); } else { $filters[] = "{$escapedAttr} BETWEEN {{$paramName1}:String} AND {{$paramName2}:String}"; $params[$paramName1] = $this->formatParamValue($values[0]); @@ -1205,7 +1205,7 @@ public function createBatch(array $logs): bool // Add time /** @var string|\DateTime|null $providedTime */ $providedTime = $processedLog['time'] ?? null; - $formattedTime = $this->formatDateTimeForClickHouse($providedTime); + $formattedTime = $this->formatDateTime($providedTime); $paramKey = 'time_' . $paramCounter; $queryParams[$paramKey] = $formattedTime; $valuePlaceholders[] = '{' . $paramKey . ':String}'; From 210ce93f24b8cb9e6a96e3c1b6985a1ed3b589a8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 00:52:32 +0000 Subject: [PATCH 28/29] feat: remove getAttributeMetadata method and replace its usage with getAttribute for improved clarity --- src/Audit/Adapter/ClickHouse.php | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 049a652..149f623 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -690,23 +690,6 @@ private function validateAttributeName(string $attributeName): bool throw new Exception("Invalid attribute name: {$attributeName}"); } - /** - * Get attribute metadata by column name. - * Searches through all attributes to find metadata for a specific column. - * - * @param string $columnName The column name to look up - * @return array|null The attribute metadata or null if not found - */ - private function getAttributeMetadata(string $columnName): ?array - { - foreach ($this->getAttributes() as $attribute) { - if ($attribute['$id'] === $columnName) { - return $attribute; - } - } - return null; - } - /** * Format datetime for ClickHouse compatibility. * Converts datetime to 'YYYY-MM-DD HH:MM:SS.mmm' format without timezone suffix. @@ -784,7 +767,7 @@ public function create(array $log): Log } // Get attribute metadata to determine if required and nullable - $attributeMetadata = $this->getAttributeMetadata($columnName); + $attributeMetadata = $this->getAttribute($columnName); $isRequiredAttribute = $attributeMetadata !== null && isset($attributeMetadata['required']) && $attributeMetadata['required']; $isNullableAttribute = $attributeMetadata !== null && (!isset($attributeMetadata['required']) || !$attributeMetadata['required']); @@ -1219,7 +1202,7 @@ public function createBatch(array $logs): bool $paramKey = $columnName . '_' . $paramCounter; // Get attribute metadata to determine if required and nullable - $attributeMetadata = $this->getAttributeMetadata($columnName); + $attributeMetadata = $this->getAttribute($columnName); $isRequiredAttribute = $attributeMetadata !== null && isset($attributeMetadata['required']) && $attributeMetadata['required']; $isNullableAttribute = $attributeMetadata !== null && (!isset($attributeMetadata['required']) || !$attributeMetadata['required']); From dcd503b56041cab83a1f95482740c8ed5a3bb964 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 19 Jan 2026 01:05:10 +0000 Subject: [PATCH 29/29] feat: enhance ClickHouse adapter to improve query handling and attribute validation --- src/Audit/Adapter/ClickHouse.php | 276 +++++++++++++------------------ 1 file changed, 118 insertions(+), 158 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 149f623..06a763d 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -1000,16 +1000,26 @@ private function parseQueries(array $queries): array $this->validateAttributeName($attribute); $escapedAttr = $this->escapeIdentifier($attribute); $paramName = 'param_' . $paramCounter++; - $filters[] = "{$escapedAttr} < {{$paramName}:String}"; - $params[$paramName] = $this->formatParamValue($values[0]); + if ($attribute === 'time') { + $filters[] = "{$escapedAttr} < {{$paramName}:DateTime64(3)}"; + $params[$paramName] = $this->formatDateTime($values[0]); + } else { + $filters[] = "{$escapedAttr} < {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($values[0]); + } break; case Query::TYPE_GREATER: $this->validateAttributeName($attribute); $escapedAttr = $this->escapeIdentifier($attribute); $paramName = 'param_' . $paramCounter++; - $filters[] = "{$escapedAttr} > {{$paramName}:String}"; - $params[$paramName] = $this->formatParamValue($values[0]); + if ($attribute === 'time') { + $filters[] = "{$escapedAttr} > {{$paramName}:DateTime64(3)}"; + $params[$paramName] = $this->formatDateTime($values[0]); + } else { + $filters[] = "{$escapedAttr} > {{$paramName}:String}"; + $params[$paramName] = $this->formatParamValue($values[0]); + } break; case Query::TYPE_BETWEEN: @@ -1525,31 +1535,23 @@ public function getByUser( int $offset = 0, bool $ascending = false, ): array { - $time = $this->buildTimeClause($after, $before); - $order = $ascending ? 'ASC' : 'DESC'; - - $tableName = $this->getTableName(); - $tenantFilter = $this->getTenantFilter(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - $escapedUserId = $this->escapeIdentifier('userId'); - $escapedTime = $this->escapeIdentifier('time'); + $queries = [ + Query::equal('userId', $userId), + ]; - $sql = " - SELECT " . $this->getSelectColumns() . " - FROM {$escapedTable} - WHERE {$escapedUserId} = {userId:String}{$tenantFilter}{$time['clause']} - ORDER BY {$escapedTime} {$order} - LIMIT {limit:UInt64} OFFSET {offset:UInt64} - FORMAT TabSeparated - "; + if ($after !== null && $before !== null) { + $queries[] = Query::between('time', $after, $before); + } elseif ($after !== null) { + $queries[] = Query::greaterThan('time', $after); + } elseif ($before !== null) { + $queries[] = Query::lessThan('time', $before); + } - $result = $this->query($sql, array_merge([ - 'userId' => $userId, - 'limit' => $limit, - 'offset' => $offset, - ], $time['params'])); + $queries[] = $ascending ? Query::orderAsc('time') : Query::orderDesc('time'); + $queries[] = Query::limit($limit); + $queries[] = Query::offset($offset); - return $this->parseResults($result); + return $this->find($queries); } /** @@ -1562,25 +1564,19 @@ public function countByUser( ?\DateTime $after = null, ?\DateTime $before = null, ): int { - $time = $this->buildTimeClause($after, $before); - - $tableName = $this->getTableName(); - $tenantFilter = $this->getTenantFilter(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - $escapedUserId = $this->escapeIdentifier('userId'); - - $sql = " - SELECT count() - FROM {$escapedTable} - WHERE {$escapedUserId} = {userId:String}{$tenantFilter}{$time['clause']} - FORMAT TabSeparated - "; + $queries = [ + Query::equal('userId', $userId), + ]; - $result = $this->query($sql, array_merge([ - 'userId' => $userId, - ], $time['params'])); + if ($after !== null && $before !== null) { + $queries[] = Query::between('time', $after, $before); + } elseif ($after !== null) { + $queries[] = Query::greaterThan('time', $after); + } elseif ($before !== null) { + $queries[] = Query::lessThan('time', $before); + } - return (int) trim($result); + return count($this->find($queries)); } /** @@ -1596,31 +1592,23 @@ public function getByResource( int $offset = 0, bool $ascending = false, ): array { - $time = $this->buildTimeClause($after, $before); - $order = $ascending ? 'ASC' : 'DESC'; - - $tableName = $this->getTableName(); - $tenantFilter = $this->getTenantFilter(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - $escapedResource = $this->escapeIdentifier('resource'); - $escapedTime = $this->escapeIdentifier('time'); + $queries = [ + Query::equal('resource', $resource), + ]; - $sql = " - SELECT " . $this->getSelectColumns() . " - FROM {$escapedTable} - WHERE {$escapedResource} = {resource:String}{$tenantFilter}{$time['clause']} - ORDER BY {$escapedTime} {$order} - LIMIT {limit:UInt64} OFFSET {offset:UInt64} - FORMAT TabSeparated - "; + if ($after !== null && $before !== null) { + $queries[] = Query::between('time', $after, $before); + } elseif ($after !== null) { + $queries[] = Query::greaterThan('time', $after); + } elseif ($before !== null) { + $queries[] = Query::lessThan('time', $before); + } - $result = $this->query($sql, array_merge([ - 'resource' => $resource, - 'limit' => $limit, - 'offset' => $offset, - ], $time['params'])); + $queries[] = $ascending ? Query::orderAsc('time') : Query::orderDesc('time'); + $queries[] = Query::limit($limit); + $queries[] = Query::offset($offset); - return $this->parseResults($result); + return $this->find($queries); } /** @@ -1633,25 +1621,19 @@ public function countByResource( ?\DateTime $after = null, ?\DateTime $before = null, ): int { - $time = $this->buildTimeClause($after, $before); - - $tableName = $this->getTableName(); - $tenantFilter = $this->getTenantFilter(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - $escapedResource = $this->escapeIdentifier('resource'); - - $sql = " - SELECT count() - FROM {$escapedTable} - WHERE {$escapedResource} = {resource:String}{$tenantFilter}{$time['clause']} - FORMAT TabSeparated - "; + $queries = [ + Query::equal('resource', $resource), + ]; - $result = $this->query($sql, array_merge([ - 'resource' => $resource, - ], $time['params'])); + if ($after !== null && $before !== null) { + $queries[] = Query::between('time', $after, $before); + } elseif ($after !== null) { + $queries[] = Query::greaterThan('time', $after); + } elseif ($before !== null) { + $queries[] = Query::lessThan('time', $before); + } - return (int) trim($result); + return count($this->find($queries)); } /** @@ -1668,29 +1650,24 @@ public function getByUserAndEvents( int $offset = 0, bool $ascending = false, ): array { - $time = $this->buildTimeClause($after, $before); - $order = $ascending ? 'ASC' : 'DESC'; - $eventList = $this->buildEventsList($events, 0); - $tableName = $this->getTableName(); - $tenantFilter = $this->getTenantFilter(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $queries = [ + Query::equal('userId', $userId), + Query::in('event', $events), + ]; - $sql = " - SELECT " . $this->getSelectColumns() . " - FROM {$escapedTable} - WHERE userId = {userId:String} AND event IN ({$eventList['clause']}){$tenantFilter}{$time['clause']} - ORDER BY time {$order} - LIMIT {limit:UInt64} OFFSET {offset:UInt64} - FORMAT TabSeparated - "; + if ($after !== null && $before !== null) { + $queries[] = Query::between('time', $after, $before); + } elseif ($after !== null) { + $queries[] = Query::greaterThan('time', $after); + } elseif ($before !== null) { + $queries[] = Query::lessThan('time', $before); + } - $result = $this->query($sql, array_merge([ - 'userId' => $userId, - 'limit' => $limit, - 'offset' => $offset, - ], $eventList['params'], $time['params'])); + $queries[] = $ascending ? Query::orderAsc('time') : Query::orderDesc('time'); + $queries[] = Query::limit($limit); + $queries[] = Query::offset($offset); - return $this->parseResults($result); + return $this->find($queries); } /** @@ -1704,26 +1681,20 @@ public function countByUserAndEvents( ?\DateTime $after = null, ?\DateTime $before = null, ): int { - $time = $this->buildTimeClause($after, $before); - $eventList = $this->buildEventsList($events, 0); - $tableName = $this->getTableName(); - $tenantFilter = $this->getTenantFilter(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - $escapedUserId = $this->escapeIdentifier('userId'); - $escapedEvent = $this->escapeIdentifier('event'); - - $sql = " - SELECT count() - FROM {$escapedTable} - WHERE {$escapedUserId} = {userId:String} AND {$escapedEvent} IN ({$eventList['clause']}){$tenantFilter}{$time['clause']} - FORMAT TabSeparated - "; + $queries = [ + Query::equal('userId', $userId), + Query::in('event', $events), + ]; - $result = $this->query($sql, array_merge([ - 'userId' => $userId, - ], $eventList['params'], $time['params'])); + if ($after !== null && $before !== null) { + $queries[] = Query::between('time', $after, $before); + } elseif ($after !== null) { + $queries[] = Query::greaterThan('time', $after); + } elseif ($before !== null) { + $queries[] = Query::lessThan('time', $before); + } - return (int) trim($result); + return count($this->find($queries)); } /** @@ -1740,29 +1711,24 @@ public function getByResourceAndEvents( int $offset = 0, bool $ascending = false, ): array { - $time = $this->buildTimeClause($after, $before); - $order = $ascending ? 'ASC' : 'DESC'; - $eventList = $this->buildEventsList($events, 0); - $tableName = $this->getTableName(); - $tenantFilter = $this->getTenantFilter(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); + $queries = [ + Query::equal('resource', $resource), + Query::in('event', $events), + ]; - $sql = " - SELECT " . $this->getSelectColumns() . " - FROM {$escapedTable} - WHERE resource = {resource:String} AND event IN ({$eventList['clause']}){$tenantFilter}{$time['clause']} - ORDER BY time {$order} - LIMIT {limit:UInt64} OFFSET {offset:UInt64} - FORMAT TabSeparated - "; + if ($after !== null && $before !== null) { + $queries[] = Query::between('time', $after, $before); + } elseif ($after !== null) { + $queries[] = Query::greaterThan('time', $after); + } elseif ($before !== null) { + $queries[] = Query::lessThan('time', $before); + } - $result = $this->query($sql, array_merge([ - 'resource' => $resource, - 'limit' => $limit, - 'offset' => $offset, - ], $eventList['params'], $time['params'])); + $queries[] = $ascending ? Query::orderAsc('time') : Query::orderDesc('time'); + $queries[] = Query::limit($limit); + $queries[] = Query::offset($offset); - return $this->parseResults($result); + return $this->find($queries); } /** @@ -1776,26 +1742,20 @@ public function countByResourceAndEvents( ?\DateTime $after = null, ?\DateTime $before = null, ): int { - $time = $this->buildTimeClause($after, $before); - $eventList = $this->buildEventsList($events, 0); - $tableName = $this->getTableName(); - $tenantFilter = $this->getTenantFilter(); - $escapedTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName); - $escapedResource = $this->escapeIdentifier('resource'); - $escapedEvent = $this->escapeIdentifier('event'); - - $sql = " - SELECT count() - FROM {$escapedTable} - WHERE {$escapedResource} = {resource:String} AND {$escapedEvent} IN ({$eventList['clause']}){$tenantFilter}{$time['clause']} - FORMAT TabSeparated - "; + $queries = [ + Query::equal('resource', $resource), + Query::in('event', $events), + ]; - $result = $this->query($sql, array_merge([ - 'resource' => $resource, - ], $eventList['params'], $time['params'])); + if ($after !== null && $before !== null) { + $queries[] = Query::between('time', $after, $before); + } elseif ($after !== null) { + $queries[] = Query::greaterThan('time', $after); + } elseif ($before !== null) { + $queries[] = Query::lessThan('time', $before); + } - return (int) trim($result); + return count($this->find($queries)); } /**