diff --git a/.env b/.env index 3ab32f0..dea79a9 100755 --- a/.env +++ b/.env @@ -54,6 +54,9 @@ S3_SECRET_KEY= S3_BUCKET=dev-ign-mut-validtri S3_REGION=sbg +# Storage type used. Currently supported: local, S3 +STORAGE_TYPE=local + ### validator-worker/validator-cli postgis # DB_URL=jdbc:postgresql://${PGHOST}:${PGPORT}/${PGDATABASE}?ssl=true&sslmode=require&sslfactory=org.postgresql.ssl.NonValidatingFactory DB_URL=jdbc:postgresql://localhost:5432/validator_api diff --git a/src/Controller/Api/HealthController.php b/src/Controller/Api/HealthController.php index 3683228..bbfc825 100644 --- a/src/Controller/Api/HealthController.php +++ b/src/Controller/Api/HealthController.php @@ -2,6 +2,7 @@ namespace App\Controller\Api; +use App\Storage\ValidationsStorage; use Exception; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; use Symfony\Component\Routing\Annotation\Route; @@ -17,7 +18,9 @@ */ class HealthController extends AbstractController { - public function __construct(private LoggerInterface $logger){ + public function __construct( + private LoggerInterface $logger, + private ValidationsStorage $storage){ } @@ -47,11 +50,11 @@ public function healthDB(EntityManagerInterface $entityManager) * * @Route("/s3", name="health_s3") */ - public function healthS3(FilesystemOperator $dataStorage) + public function healthS3() { $this->logger->info('list files from S3 bucket...'); try { - $files = $dataStorage->listContents('.', false); + $files = $this->storage->getStorage()->listContents('.', false); $numFiles = count($files->toArray()); return new JsonResponse('found '.$numFiles.' files', Response::HTTP_OK); } catch (Exception $e) { diff --git a/src/Controller/Api/ValidationsController.php b/src/Controller/Api/ValidationsController.php index f676aa0..6d5bded 100755 --- a/src/Controller/Api/ValidationsController.php +++ b/src/Controller/Api/ValidationsController.php @@ -11,7 +11,6 @@ use App\Storage\ValidationsStorage; use JMS\Serializer\Serializer; use JMS\Serializer\SerializerInterface; -use League\Flysystem\FilesystemOperator; use Psr\Log\LoggerInterface; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; use Symfony\Component\Filesystem\Filesystem; @@ -30,7 +29,6 @@ public function __construct( private ValidationRepository $repository, private SerializerInterface $serializer, private ValidationsStorage $storage, - private FilesystemOperator $dataStorage, private ValidatorArgumentsService $valArgsService, private MimeTypeGuesserService $mimeTypeGuesserService, private LoggerInterface $logger @@ -88,7 +86,7 @@ public function readConsole($uid) $outputDirectory = $this->storage->getOutputDirectory($validation); $filepath = $outputDirectory . '/validator-debug.log'; - $content = $this->dataStorage->read($filepath); + $content = $this->storage->getStorage()->read($filepath); return new Response( $content, @@ -162,15 +160,15 @@ public function uploadDataset(Request $request) // Save file to storage $uploadDirectory = $this->storage->getUploadDirectory($validation); - if (!$this->dataStorage->directoryExists($uploadDirectory)) { - $this->dataStorage->createDirectory($uploadDirectory); + if (!$this->storage->getStorage()->directoryExists($uploadDirectory)) { + $this->storage->getStorage()->createDirectory($uploadDirectory); } $fileLocation = $uploadDirectory . $validation->getDatasetName() . '.zip'; - if ($this->dataStorage->fileExists($fileLocation)) { - $this->dataStorage->delete($fileLocation); + if ($this->storage->getStorage()->fileExists($fileLocation)) { + $this->storage->getStorage()->delete($fileLocation); } $stream = fopen($file->getRealPath(), 'r+'); - $this->dataStorage->writeStream($fileLocation, $stream); + $this->storage->getStorage()->writeStream($fileLocation, $stream); fclose($stream); $fs = new Filesystem; @@ -259,12 +257,12 @@ public function deleteValidation($uid) // Delete from storage $uploadDirectory = $this->storage->getUploadDirectory($validation); - if ($this->dataStorage->directoryExists($uploadDirectory)) { - $this->dataStorage->deleteDirectory($uploadDirectory); + if ($this->storage->getStorage()->directoryExists($uploadDirectory)) { + $this->storage->getStorage()->deleteDirectory($uploadDirectory); } $outputDirectory = $this->storage->getOutputDirectory($validation); - if ($this->dataStorage->directoryExists($outputDirectory)) { - $this->dataStorage->deleteDirectory($outputDirectory); + if ($this->storage->getStorage()->directoryExists($outputDirectory)) { + $this->storage->getStorage()->deleteDirectory($outputDirectory); } return new JsonResponse(null, Response::HTTP_NO_CONTENT); @@ -333,11 +331,11 @@ public function downloadSourceData($uid) */ private function getDownloadResponse($filepath, $filename) { - if (!$this->dataStorage->has($filepath)) { + if (!$this->storage->getStorage()->has($filepath)) { throw new ApiException("Requested files not found for this validation", Response::HTTP_FORBIDDEN); } - $stream = $this->dataStorage->readStream($filepath); + $stream = $this->storage->getStorage()->readStream($filepath); return new StreamedResponse(function () use ($stream) { fpassthru($stream); diff --git a/src/Storage/ValidationsStorage.php b/src/Storage/ValidationsStorage.php index 2aa6a20..e3daaba 100644 --- a/src/Storage/ValidationsStorage.php +++ b/src/Storage/ValidationsStorage.php @@ -3,6 +3,7 @@ namespace App\Storage ; use App\Entity\Validation; +use League\Flysystem\FilesystemOperator; /** * Manage validation files @@ -16,9 +17,24 @@ class ValidationsStorage { */ private $path; - public function __construct($validationsDir) + /** + * Flysystem storage + * + * @var FilesystemOperator + */ + private $storageSystem; + + public function __construct($validationsDir, + FilesystemOperator $dataStorage, + FilesystemOperator $defaultStorage) { $this->path = $validationsDir; + // Assign storage based on env + if (getenv("STORAGE_TYPE") === "S3"){ + $this->storageSystem = $dataStorage; + } else { + $this->storageSystem = $defaultStorage; + } } /** @@ -28,6 +44,13 @@ public function getPath(){ return $this->path; } + /** + * @return FilesystemOperator + */ + public function getStorage(){ + return $this->storageSystem; + } + /** * @param Validation $validation * @return string diff --git a/src/Validation/ValidationManager.php b/src/Validation/ValidationManager.php index ac207c7..4ac5951 100644 --- a/src/Validation/ValidationManager.php +++ b/src/Validation/ValidationManager.php @@ -11,7 +11,6 @@ use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\Process\Exception\ProcessFailedException; use Symfony\Component\Process\Process; -use League\Flysystem\FilesystemOperator; class ValidationManager { @@ -41,11 +40,6 @@ class ValidationManager */ private $zipArchiveValidator; - /** - * @var FilesystemOperator - */ - private $dataStorage; - /** * Current validation (in order to handle SIGTERM) * @var Validation @@ -55,14 +49,12 @@ class ValidationManager public function __construct( EntityManagerInterface $em, ValidationsStorage $storage, - FilesystemOperator $dataStorage, ValidatorCLI $validatorCli, ZipArchiveValidator $zipArchiveValidator, LoggerInterface $logger ) { $this->em = $em; $this->storage = $storage; - $this->dataStorage = $dataStorage; $this->validatorCli = $validatorCli; $this->zipArchiveValidator = $zipArchiveValidator; $this->logger = $logger; @@ -94,15 +86,15 @@ public function archive(Validation $validation) 'uid' => $validation->getUid(), ]); $uploadDirectory = $this->storage->getUploadDirectory($validation); - if ($this->dataStorage->directoryExists($uploadDirectory)) { - $this->dataStorage->deleteDirectory($uploadDirectory); + if ($this->storage->getStorage()->directoryExists($uploadDirectory)) { + $this->storage->getStorage()->deleteDirectory($uploadDirectory); } $this->logger->info('Validation[{uid}] : remove output files', [ 'uid' => $validation->getUid(), ]); $outputDirectory = $this->storage->getOutputDirectory($validation); - if ($this->dataStorage->directoryExists($outputDirectory)) { - $this->dataStorage->deleteDirectory($outputDirectory); + if ($this->storage->getStorage()->directoryExists($outputDirectory)) { + $this->storage->getStorage()->deleteDirectory($outputDirectory); } $this->logger->info('Validation[{uid}] : archive removing all files : completed', [ 'uid' => $validation->getUid(), @@ -248,7 +240,7 @@ private function getZip(Validation $validation) file_put_contents( $zipPath, - $this->dataStorage->read($uploadFile) + $this->storage->getStorage()->read($uploadFile) ); } @@ -343,15 +335,15 @@ private function saveToStorage(Validation $validation) $validationDirectory = $this->storage->getDirectory($validation); $normDataPath = $validationDirectory . '/validation/' . $validation->getDatasetName() . '.zip'; $outputDirectory = $this->storage->getOutputDirectory($validation); - if (! $this->dataStorage->directoryExists($outputDirectory)){ - $this->dataStorage->createDirectory($outputDirectory); + if (! $this->storage->getStorage()->directoryExists($outputDirectory)){ + $this->storage->getStorage()->createDirectory($outputDirectory); } $outputPath = $outputDirectory . $validation->getDatasetName() . '.zip'; - if ($this->dataStorage->fileExists($outputPath)){ - $this->dataStorage->delete($outputPath); + if ($this->storage->getStorage()->fileExists($outputPath)){ + $this->storage->getStorage()->delete($outputPath); } $stream = fopen($normDataPath, 'r+'); - $this->dataStorage->writeStream($outputPath, $stream); + $this->storage->getStorage()->writeStream($outputPath, $stream); fclose($stream); // Saves validator logs to storage @@ -363,7 +355,7 @@ private function saveToStorage(Validation $validation) $outputPath = $outputDirectory . '/validator-debug.log'; $stream = fopen($logPath, 'r+'); - $this->dataStorage->writeStream($outputPath, $stream); + $this->storage->getStorage()->writeStream($outputPath, $stream); fclose($stream); } diff --git a/tests/Command/Validations/CleanupCommandTest.php b/tests/Command/Validations/CleanupCommandTest.php index 80dacaf..c970cbf 100644 --- a/tests/Command/Validations/CleanupCommandTest.php +++ b/tests/Command/Validations/CleanupCommandTest.php @@ -54,9 +54,6 @@ public function tearDown(): void */ public function testCleanupOneSecond() { - - $this->markTestSkipped('TODO : fix test (use local directory for dev and test?)'); - static::ensureKernelShutdown(); // wait for 2 seconds diff --git a/tests/Controller/Api/ValidationControllerTest.php b/tests/Controller/Api/ValidationControllerTest.php index b1f7d22..a7a28ab 100644 --- a/tests/Controller/Api/ValidationControllerTest.php +++ b/tests/Controller/Api/ValidationControllerTest.php @@ -126,8 +126,6 @@ public function testGetValidationNotFound() */ public function testUploadDatasetCorrectParams() { - $this->markTestSkipped('TODO : fix test (use local directory for dev and test?)'); - $filename = ValidationsFixtures::FILENAME_SUP_PM3; $dataset = $this->createFakeUpload( $filename, @@ -229,8 +227,6 @@ public function testUploadDatasetNoFile() */ public function testDeleteValidation() { - $this->markTestSkipped('TODO : fix test (use local directory for dev and test?)'); - $validation = $this->getValidationFixture(ValidationsFixtures::VALIDATION_ARCHIVED); $this->client->request(