diff --git a/Dockerfile b/Dockerfile index 405656e47..22ecf2532 100755 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,8 @@ RUN \ git \ brotli-dev \ linux-headers \ + docker-cli \ + docker-cli-compose \ && docker-php-ext-install opcache pgsql pdo_mysql pdo_pgsql \ && apk del postgresql-dev \ && rm -rf /var/cache/apk/* diff --git a/docker-compose.yml b/docker-compose.yml index 953b69838..c49290aaf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,8 @@ services: - ./dev:/usr/src/code/dev - ./phpunit.xml:/usr/src/code/phpunit.xml - ./dev/xdebug.ini:/usr/local/etc/php/conf.d/xdebug.ini + - /var/run/docker.sock:/var/run/docker.sock + - ./docker-compose.yml:/usr/src/code/docker-compose.yml adminer: image: adminer diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index c9cbbdecc..12b74513b 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -900,6 +900,13 @@ abstract public function getSupportForCastIndexArray(): bool; */ abstract public function getSupportForUpserts(): bool; + /** + * Is Cache Fallback supported? + * + * @return bool + */ + abstract public function getSupportForCacheSkipOnFailure(): bool; + /** * Get current attribute count from collection document * diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index c5f0522af..b35add565 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1800,6 +1800,16 @@ public function getSupportForGetConnectionId(): bool return false; } + /** + * Is cache fallback supported? + * + * @return bool + */ + public function getSupportForCacheSkipOnFailure(): bool + { + return false; + } + /** * Is get schema attributes supported? * diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index ae05e3d9b..eb44467e9 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -402,6 +402,16 @@ public function getSupportForGetConnectionId(): bool return true; } + /** + * Is cache fallback supported? + * + * @return bool + */ + public function getSupportForCacheSkipOnFailure(): bool + { + return true; + } + /** * Get current attribute count from collection document * diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 88b57e590..930562c7c 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -220,15 +220,7 @@ public function createCollection(string $name, array $attributes = [], array $in $this->createIndex("{$id}_perms", '_index_2', Database::INDEX_KEY, ['_permission', '_type'], [], []); } catch (PDOException $e) { - $e = $this->processException($e); - - if (!($e instanceof Duplicate)) { - $this->getPDO() - ->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};") - ->execute(); - } - - throw $e; + throw $this->processException($e); } return true; } diff --git a/src/Database/Database.php b/src/Database/Database.php index 1ccea9ec8..872cff833 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4,6 +4,7 @@ use Exception; use Utopia\Cache\Cache; +use Utopia\CLI\Console; use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Conflict as ConflictException; @@ -441,14 +442,20 @@ function (?string $value) { /** * Add listener to events + * Passing a null $callback will remove the listener * * @param string $event * @param string $name - * @param callable $callback + * @param ?callable $callback * @return static */ - public function on(string $event, string $name, callable $callback): static + public function on(string $event, string $name, ?callable $callback): static { + if (empty($callback)) { + unset($this->listeners[$event][$name]); + return $this; + } + if (!isset($this->listeners[$event])) { $this->listeners[$event] = []; } @@ -2992,8 +2999,15 @@ public function getDocument(string $collection, string $id, array $queries = [], $documentCacheHash .= ':' . \md5(\implode($selections)); } - if ($cache = $this->cache->load($documentCacheKey, self::TTL, $documentCacheHash)) { - $document = new Document($cache); + try { + $cached = $this->cache->load($documentCacheKey, self::TTL, $documentCacheHash); + } catch (Exception $e) { + Console::warning('Warning: Failed to get document from cache: ' . $e->getMessage()); + $cached = null; + } + + if ($cached) { + $document = new Document($cached); if ($collection->getId() !== self::METADATA) { if (!$validator->isValid([ @@ -3042,8 +3056,12 @@ public function getDocument(string $collection, string $id, array $queries = [], // Don't save to cache if it's part of a relationship if (empty($relationships)) { - $this->cache->save($documentCacheKey, $document->getArrayCopy(), $documentCacheHash); - $this->cache->save($collectionCacheKey, 'empty', $documentCacheKey); + try { + $this->cache->save($documentCacheKey, $document->getArrayCopy(), $documentCacheHash); + $this->cache->save($collectionCacheKey, 'empty', $documentCacheKey); + } catch (Exception $e) { + Console::warning('Failed to save document to cache: ' . $e->getMessage()); + } } // Remove internal attributes if not queried for select query @@ -3962,6 +3980,7 @@ public function updateDocument(string $collection, string $id, Document $documen } $this->adapter->updateDocument($collection->getId(), $id, $document); + $this->purgeCachedDocument($collection->getId(), $id); return $document; }); @@ -3972,7 +3991,6 @@ public function updateDocument(string $collection, string $id, Document $documen $document = $this->decode($collection, $document); - $this->purgeCachedDocument($collection->getId(), $id); $this->trigger(self::EVENT_DOCUMENT_UPDATE, $document); return $document; @@ -4888,10 +4906,12 @@ public function deleteDocument(string $collection, string $id): bool $document = $this->silent(fn () => $this->deleteDocumentRelationships($collection, $document)); } - return $this->adapter->deleteDocument($collection->getId(), $id); - }); + $result = $this->adapter->deleteDocument($collection->getId(), $id); - $this->purgeCachedDocument($collection->getId(), $id); + $this->purgeCachedDocument($collection->getId(), $id); + + return $result; + }); $this->trigger(self::EVENT_DOCUMENT_DELETE, $document); @@ -5424,6 +5444,7 @@ public function deleteDocuments(string $collection, array $queries = [], int $ba public function purgeCachedCollection(string $collectionId): bool { $collectionKey = $this->cacheName . '-cache-' . $this->getNamespace() . ':' . $this->adapter->getTenant() . ':collection:' . $collectionId; + $documentKeys = $this->cache->list($collectionKey); foreach ($documentKeys as $documentKey) { $this->cache->purge($documentKey); diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index 92cdae23c..6ac448b22 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -157,7 +157,7 @@ public function disableValidation(): static return $this; } - public function on(string $event, string $name, callable $callback): static + public function on(string $event, string $name, ?callable $callback): static { $this->source->on($event, $name, $callback); diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 0191ea853..d35891f0c 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -5,6 +5,7 @@ use Exception; use PHPUnit\Framework\TestCase; use Throwable; +use Utopia\CLI\Console; use Utopia\Database\Adapter\SQL; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -17683,6 +17684,77 @@ public function testEvents(): void $database->deleteAttribute($collectionId, 'attr1'); $database->deleteCollection($collectionId); $database->delete('hellodb'); + + // Remove all listeners + $database->on(Database::EVENT_ALL, 'test', null); + $database->on(Database::EVENT_ALL, 'should-not-execute', null); }); } + + public function testCacheFallback(): void + { + if (!static::getDatabase()->getAdapter()->getSupportForCacheSkipOnFailure()) { + $this->expectNotToPerformAssertions(); + return; + } + + Authorization::cleanRoles(); + Authorization::setRole(Role::any()->toString()); + $database = static::getDatabase(); + + // Write mock data + $database->createCollection('testRedisFallback', attributes: [ + new Document([ + '$id' => ID::custom('string'), + 'type' => Database::VAR_STRING, + 'size' => 767, + 'required' => true, + ]) + ], permissions: [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]); + + $database->createDocument('testRedisFallback', new Document([ + '$id' => 'doc1', + 'string' => 'text📝', + ])); + + $database->createIndex('testRedisFallback', 'index1', Database::INDEX_KEY, ['string']); + $this->assertCount(1, $database->find('testRedisFallback', [Query::equal('string', ['text📝'])])); + + // Bring down Redis + $stdout = ''; + $stderr = ''; + Console::execute('docker ps -a --filter "name=utopia-redis" --format "{{.Names}}" | xargs -r docker stop', "", $stdout, $stderr); + + // Check we can read data still + $this->assertCount(1, $database->find('testRedisFallback', [Query::equal('string', ['text📝'])])); + $this->assertFalse(($database->getDocument('testRedisFallback', 'doc1'))->isEmpty()); + + // Check we cannot modify data + try { + $database->updateDocument('testRedisFallback', 'doc1', new Document([ + 'string' => 'text📝 updated', + ])); + $this->fail('Failed to throw exception'); + } catch (\Throwable $e) { + $this->assertEquals('Redis server redis:6379 went away', $e->getMessage()); + } + + try { + $database->deleteDocument('testRedisFallback', 'doc1'); + $this->fail('Failed to throw exception'); + } catch (\Throwable $e) { + $this->assertEquals('Redis server redis:6379 went away', $e->getMessage()); + } + + // Bring backup Redis + Console::execute('docker ps -a --filter "name=utopia-redis" --format "{{.Names}}" | xargs -r docker start', "", $stdout, $stderr); + sleep(5); + + $this->assertCount(1, $database->find('testRedisFallback', [Query::equal('string', ['text📝'])])); + } } diff --git a/tests/e2e/Adapter/MariaDBTest.php b/tests/e2e/Adapter/MariaDBTest.php index 32d8c0c0c..c60f08c0e 100644 --- a/tests/e2e/Adapter/MariaDBTest.php +++ b/tests/e2e/Adapter/MariaDBTest.php @@ -41,6 +41,7 @@ public static function getDatabase(bool $fresh = false): Database $dbPass = 'password'; $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPDOAttributes()); + $redis = new Redis(); $redis->connect('redis', 6379); $redis->flushAll(); diff --git a/tests/e2e/Adapter/MirrorTest.php b/tests/e2e/Adapter/MirrorTest.php index a3d457624..e0e6fad35 100644 --- a/tests/e2e/Adapter/MirrorTest.php +++ b/tests/e2e/Adapter/MirrorTest.php @@ -45,6 +45,7 @@ protected static function getDatabase(bool $fresh = false): Mirror $dbPass = 'password'; $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPDOAttributes()); + $redis = new Redis(); $redis->connect('redis'); $redis->flushAll(); @@ -59,6 +60,7 @@ protected static function getDatabase(bool $fresh = false): Mirror $mirrorPass = 'password'; $mirrorPdo = new PDO("mysql:host={$mirrorHost};port={$mirrorPort};charset=utf8mb4", $mirrorUser, $mirrorPass, MariaDB::getPDOAttributes()); + $mirrorRedis = new Redis(); $mirrorRedis->connect('redis-mirror'); $mirrorRedis->flushAll(); diff --git a/tests/e2e/Adapter/MySQLTest.php b/tests/e2e/Adapter/MySQLTest.php index 369958f07..0f72d0136 100644 --- a/tests/e2e/Adapter/MySQLTest.php +++ b/tests/e2e/Adapter/MySQLTest.php @@ -45,7 +45,6 @@ public static function getDatabase(): Database $redis = new Redis(); $redis->connect('redis', 6379); $redis->flushAll(); - $cache = new Cache(new RedisAdapter($redis)); $database = new Database(new MySQL($pdo), $cache); diff --git a/tests/e2e/Adapter/SQLiteTest.php b/tests/e2e/Adapter/SQLiteTest.php index 87676b7f5..0dd1b0073 100644 --- a/tests/e2e/Adapter/SQLiteTest.php +++ b/tests/e2e/Adapter/SQLiteTest.php @@ -46,9 +46,8 @@ public static function getDatabase(): Database $pdo = new PDO("sqlite:" . $dsn, null, null, SQLite::getPDOAttributes()); $redis = new Redis(); - $redis->connect('redis'); + $redis->connect('redis', 6379); $redis->flushAll(); - $cache = new Cache(new RedisAdapter($redis)); $database = new Database(new SQLite($pdo), $cache);