Storage adapter

.env

@@ -54,6 +54,9 @@ S3_SECRET_KEY=
 S3_BUCKET=dev-ign-mut-validtri
 S3_REGION=sbg
 
+# Storage type used. Currently supported: local, S3
+STORAGE_TYPE=local
+
 ### validator-worker/validator-cli postgis
 # DB_URL=jdbc:postgresql://${PGHOST}:${PGPORT}/${PGDATABASE}?ssl=true&sslmode=require&sslfactory=org.postgresql.ssl.NonValidatingFactory
 DB_URL=jdbc:postgresql://localhost:5432/validator_api

src/Controller/Api/HealthController.php

@@ -2,6 +2,7 @@
 
 namespace App\Controller\Api;
 
+use App\Storage\ValidationsStorage;
 use Exception;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
 use Symfony\Component\Routing\Annotation\Route;
@@ -17,7 +18,9 @@
  */
 class HealthController extends AbstractController
 {
-    public function __construct(private LoggerInterface $logger){
+    public function __construct(
+        private LoggerInterface $logger,
+        private ValidationsStorage $storage){
 
     }
 
@@ -47,11 +50,11 @@ public function healthDB(EntityManagerInterface $entityManager)
      *
      * @Route("/s3", name="health_s3")
      */
-    public function healthS3(FilesystemOperator $dataStorage)
+    public function healthS3()
     {
         $this->logger->info('list files from S3 bucket...');
         try {
-            $files = $dataStorage->listContents('.', false);
+            $files = $this->storage->getStorage()->listContents('.', false);
             $numFiles = count($files->toArray());
             return new JsonResponse('found '.$numFiles.' files', Response::HTTP_OK);
         } catch (Exception $e) {

src/Controller/Api/ValidationsController.php

@@ -11,7 +11,6 @@
 use App\Storage\ValidationsStorage;
 use JMS\Serializer\Serializer;
 use JMS\Serializer\SerializerInterface;
-use League\Flysystem\FilesystemOperator;
 use Psr\Log\LoggerInterface;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
 use Symfony\Component\Filesystem\Filesystem;
@@ -30,7 +29,6 @@ public function __construct(
         private ValidationRepository $repository,
         private SerializerInterface $serializer,
         private ValidationsStorage $storage,
-        private FilesystemOperator $dataStorage,
         private ValidatorArgumentsService $valArgsService,
         private MimeTypeGuesserService $mimeTypeGuesserService,
         private LoggerInterface $logger
@@ -88,7 +86,7 @@ public function readConsole($uid)
         $outputDirectory = $this->storage->getOutputDirectory($validation);
         $filepath = $outputDirectory . '/validator-debug.log';
 
-        $content = $this->dataStorage->read($filepath);
+        $content = $this->storage->getStorage()->read($filepath);
 
         return new Response(
             $content,
@@ -162,15 +160,15 @@ public function uploadDataset(Request $request)
 
         // Save file to storage
         $uploadDirectory = $this->storage->getUploadDirectory($validation);
-        if (!$this->dataStorage->directoryExists($uploadDirectory)) {
-            $this->dataStorage->createDirectory($uploadDirectory);
+        if (!$this->storage->getStorage()->directoryExists($uploadDirectory)) {
+            $this->storage->getStorage()->createDirectory($uploadDirectory);
         }
         $fileLocation = $uploadDirectory . $validation->getDatasetName() . '.zip';
-        if ($this->dataStorage->fileExists($fileLocation)) {
-            $this->dataStorage->delete($fileLocation);
+        if ($this->storage->getStorage()->fileExists($fileLocation)) {
+            $this->storage->getStorage()->delete($fileLocation);
         }
         $stream = fopen($file->getRealPath(), 'r+');
-        $this->dataStorage->writeStream($fileLocation, $stream);
+        $this->storage->getStorage()->writeStream($fileLocation, $stream);
         fclose($stream);
 
         $fs = new Filesystem;
@@ -259,12 +257,12 @@ public function deleteValidation($uid)
 
         // Delete from storage
         $uploadDirectory = $this->storage->getUploadDirectory($validation);
-        if ($this->dataStorage->directoryExists($uploadDirectory)) {
-            $this->dataStorage->deleteDirectory($uploadDirectory);
+        if ($this->storage->getStorage()->directoryExists($uploadDirectory)) {
+            $this->storage->getStorage()->deleteDirectory($uploadDirectory);
         }
         $outputDirectory = $this->storage->getOutputDirectory($validation);
-        if ($this->dataStorage->directoryExists($outputDirectory)) {
-            $this->dataStorage->deleteDirectory($outputDirectory);
+        if ($this->storage->getStorage()->directoryExists($outputDirectory)) {
+            $this->storage->getStorage()->deleteDirectory($outputDirectory);
         }
 
         return new JsonResponse(null, Response::HTTP_NO_CONTENT);
@@ -333,11 +331,11 @@ public function downloadSourceData($uid)
      */
     private function getDownloadResponse($filepath, $filename)
     {
-        if (!$this->dataStorage->has($filepath)) {
+        if (!$this->storage->getStorage()->has($filepath)) {
             throw new ApiException("Requested files not found for this validation", Response::HTTP_FORBIDDEN);
         }
 
-        $stream = $this->dataStorage->readStream($filepath);
+        $stream = $this->storage->getStorage()->readStream($filepath);
 
         return new StreamedResponse(function () use ($stream) {
             fpassthru($stream);

src/Storage/ValidationsStorage.php

@@ -3,6 +3,7 @@
 namespace App\Storage ;
 
 use App\Entity\Validation;
+use League\Flysystem\FilesystemOperator;
 
 /**
  * Manage validation files
@@ -16,9 +17,24 @@ class ValidationsStorage {
      */
     private $path;
 
-    public function __construct($validationsDir)
+    /**
+     * Flysystem storage
+     *
+     * @var FilesystemOperator
+     */
+    private $storageSystem;
+
+    public function __construct($validationsDir,
+        FilesystemOperator $dataStorage,
+        FilesystemOperator $defaultStorage)
     {
         $this->path = $validationsDir;
+        // Assign storage based on env
+        if (getenv("STORAGE_TYPE") === "S3"){
+            $this->storageSystem = $dataStorage;
+        } else {
+            $this->storageSystem = $defaultStorage;
+        }
     }
 
     /**
@@ -28,6 +44,13 @@ public function getPath(){
         return $this->path;
     }
 
+    /**
+     * @return FilesystemOperator
+     */
+    public function getStorage(){
+        return $this->storageSystem;
+    }
+
     /**
      * @param Validation $validation
      * @return string

src/Validation/ValidationManager.php

@@ -11,7 +11,6 @@
 use Symfony\Component\Filesystem\Filesystem;
 use Symfony\Component\Process\Exception\ProcessFailedException;
 use Symfony\Component\Process\Process;
-use League\Flysystem\FilesystemOperator;
 
 class ValidationManager
 {
@@ -41,11 +40,6 @@ class ValidationManager
      */
     private $zipArchiveValidator;
 
-    /**
-     * @var FilesystemOperator
-     */
-    private $dataStorage;
-
     /**
      * Current validation (in order to handle SIGTERM)
      * @var Validation
@@ -55,14 +49,12 @@ class ValidationManager
     public function __construct(
         EntityManagerInterface $em,
         ValidationsStorage $storage,
-        FilesystemOperator $dataStorage,
         ValidatorCLI $validatorCli,
         ZipArchiveValidator $zipArchiveValidator,
         LoggerInterface $logger
     ) {
         $this->em = $em;
         $this->storage = $storage;
-        $this->dataStorage = $dataStorage;
         $this->validatorCli = $validatorCli;
         $this->zipArchiveValidator = $zipArchiveValidator;
         $this->logger = $logger;
@@ -94,15 +86,15 @@ public function archive(Validation $validation)
             'uid' => $validation->getUid(),
         ]);
         $uploadDirectory = $this->storage->getUploadDirectory($validation);
-        if ($this->dataStorage->directoryExists($uploadDirectory)) {
-            $this->dataStorage->deleteDirectory($uploadDirectory);
+        if ($this->storage->getStorage()->directoryExists($uploadDirectory)) {
+            $this->storage->getStorage()->deleteDirectory($uploadDirectory);
         }
         $this->logger->info('Validation[{uid}] : remove output files', [
             'uid' => $validation->getUid(),
         ]);
         $outputDirectory = $this->storage->getOutputDirectory($validation);
-        if ($this->dataStorage->directoryExists($outputDirectory)) {
-            $this->dataStorage->deleteDirectory($outputDirectory);
+        if ($this->storage->getStorage()->directoryExists($outputDirectory)) {
+            $this->storage->getStorage()->deleteDirectory($outputDirectory);
         }
         $this->logger->info('Validation[{uid}] : archive removing all files : completed', [
             'uid' => $validation->getUid(),
@@ -248,7 +240,7 @@ private function getZip(Validation $validation)
 
         file_put_contents(
             $zipPath,
-            $this->dataStorage->read($uploadFile)
+            $this->storage->getStorage()->read($uploadFile)
         );
     }
 
@@ -343,15 +335,15 @@ private function saveToStorage(Validation $validation)
         $validationDirectory = $this->storage->getDirectory($validation);
         $normDataPath = $validationDirectory . '/validation/' . $validation->getDatasetName() . '.zip';
         $outputDirectory = $this->storage->getOutputDirectory($validation);
-        if (! $this->dataStorage->directoryExists($outputDirectory)){
-            $this->dataStorage->createDirectory($outputDirectory);
+        if (! $this->storage->getStorage()->directoryExists($outputDirectory)){
+            $this->storage->getStorage()->createDirectory($outputDirectory);
         }
         $outputPath = $outputDirectory . $validation->getDatasetName() . '.zip';
-        if ($this->dataStorage->fileExists($outputPath)){
-            $this->dataStorage->delete($outputPath);
+        if ($this->storage->getStorage()->fileExists($outputPath)){
+            $this->storage->getStorage()->delete($outputPath);
         }
         $stream = fopen($normDataPath, 'r+');
-        $this->dataStorage->writeStream($outputPath, $stream);
+        $this->storage->getStorage()->writeStream($outputPath, $stream);
         fclose($stream);
 
         // Saves validator logs to storage
@@ -363,7 +355,7 @@ private function saveToStorage(Validation $validation)
         $outputPath = $outputDirectory . '/validator-debug.log';
 
         $stream = fopen($logPath, 'r+');
-        $this->dataStorage->writeStream($outputPath, $stream);
+        $this->storage->getStorage()->writeStream($outputPath, $stream);
         fclose($stream);
     }
 

tests/Command/Validations/CleanupCommandTest.php

@@ -54,9 +54,6 @@ public function tearDown(): void
      */
     public function testCleanupOneSecond()
     {
-
-        $this->markTestSkipped('TODO : fix test (use local directory for dev and test?)');
-
         static::ensureKernelShutdown();
 
         // wait for 2 seconds

tests/Controller/Api/ValidationControllerTest.php

@@ -126,8 +126,6 @@ public function testGetValidationNotFound()
      */
     public function testUploadDatasetCorrectParams()
     {
-        $this->markTestSkipped('TODO : fix test (use local directory for dev and test?)');
-
         $filename = ValidationsFixtures::FILENAME_SUP_PM3;
         $dataset = $this->createFakeUpload(
             $filename,
@@ -229,8 +227,6 @@ public function testUploadDatasetNoFile()
      */
     public function testDeleteValidation()
     {
-        $this->markTestSkipped('TODO : fix test (use local directory for dev and test?)');
-
         $validation = $this->getValidationFixture(ValidationsFixtures::VALIDATION_ARCHIVED);
 
         $this->client->request(