MCLOUD-7476: Add possibility to use released images and cached images (#30)

BaDos · web-flow · commit 449a4c096f1f · 2021-01-13T12:24:29.000-06:00
diff --git a/.travis.yml b/.travis.yml
@@ -89,9 +89,9 @@ before_install:
 install: if [[ $TEST_SUITE != "build-images" ]]; then composer update; fi;
 
 before_script:
-  - if [ $TRAVIS_SECURE_ENV_VARS == "true" ] && [ $TEST_SUITE == "build-images" ]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin; fi;
+  - if [ $TRAVIS_SECURE_ENV_VARS == "true" ] && ([ $TEST_SUITE == "build-images" ] || [ $TEST_SUITE == "functional" ]); then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin; fi;
   - if [ $TEST_SUITE == "functional" ]; then sudo sysctl -w vm.max_map_count=262144; fi;
-  - if [ $TEST_SUITE == "functional" ]; then cp codeception.dist.yml codeception.yml && sed -i "s/use_generated_images:\ false/use_generated_images:\ true/" codeception.yml; fi;
+  - if [ $TEST_SUITE == "functional" ]; then cp codeception.dist.yml codeception.yml && sed -i "s/use_custom_images:\ false/use_custom_images:\ true/" codeception.yml; fi;
 
 script:
   - if [ $TRAVIS_SECURE_ENV_VARS == "true" ] && [ $TEST_SUITE == "functional" ]; then ./tests/travis/functional.sh; fi;
diff --git a/codeception.dist.yml b/codeception.dist.yml
@@ -25,10 +25,10 @@ modules:
       composer_magento_username: "%REPO_USERNAME%"
       composer_magento_password: "%REPO_PASSWORD%"
       composer_github_token: "%GITHUB_TOKEN%"
-      use_generated_images: false
-      generated_images_namespace: "cloudft"
+      use_custom_images: false
+      custom_images_namespace: "cloudft"
+      version_custom_images: "%TRAVIS_BUILD_NUMBER%"
       use_cached_workdir: true
-      version_generated_images: "%TRAVIS_BUILD_NUMBER%"
       printOutput: false
     Magento\CloudDocker\Test\Functional\Codeception\Docker:
       system_magento_dir: "%Magento.docker.settings.system.magento_dir%"
diff --git a/src/Command/BuildCompose.php b/src/Command/BuildCompose.php
@@ -200,6 +200,11 @@ protected function configure(): void
                 InputOption::VALUE_OPTIONAL,
                 'The maximum number of connections that each worker process can handle simultaneously',
                 Source\BaseSource::DEFAULT_NGINX_WORKER_CONNECTIONS
+            )->addOption(
+                Source\CliSource::OPTION_CUSTOM_REGISTRY,
+                null,
+                InputOption::VALUE_OPTIONAL,
+                'The custom registry. It can be your mirror with all needed images. For example: 123.example.com'
             );
 
         $this->addOption(
diff --git a/src/Compose/ProductionBuilder/Service/Blackfire.php b/src/Compose/ProductionBuilder/Service/Blackfire.php
@@ -64,7 +64,8 @@ public function getConfig(Config $config): array
                 ],
                 'ports' => ["8707"]
             ],
-            $config->getServiceImage($this->getServiceName())
+            $config->getServiceImage($this->getServiceName()),
+            $config->getCustomRegistry()
         );
     }
 
diff --git a/src/Compose/ProductionBuilder/Service/Build.php b/src/Compose/ProductionBuilder/Service/Build.php
@@ -73,6 +73,7 @@ public function getConfig(Config $config): array
             $config->getServiceVersion(ServiceInterface::SERVICE_PHP),
             ['volumes' => $this->volume->getBuild($config)],
             $config->getServiceImage(ServiceInterface::SERVICE_PHP),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern(ServiceInterface::SERVICE_PHP_CLI)
         );
     }
diff --git a/src/Compose/ProductionBuilder/Service/Cron.php b/src/Compose/ProductionBuilder/Service/Cron.php
@@ -73,6 +73,7 @@ public function getConfig(Config $config): array
             $config->getServiceVersion(ServiceInterface::SERVICE_PHP),
             ['command' => 'run-cron'],
             $config->getServiceImage(ServiceInterface::SERVICE_PHP),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern(ServiceInterface::SERVICE_PHP_CLI)
         );
         $preparedCronConfig = [];
diff --git a/src/Compose/ProductionBuilder/Service/Database/Db.php b/src/Compose/ProductionBuilder/Service/Database/Db.php
@@ -82,7 +82,8 @@ public function getConfig(Config $config): array
             ServiceInterface::SERVICE_DB,
             $config->getServiceVersion(ServiceInterface::SERVICE_DB),
             $dbConfig,
-            $config->getServiceImage(ServiceInterface::SERVICE_DB)
+            $config->getServiceImage(ServiceInterface::SERVICE_DB),
+            $config->getCustomRegistry()
         );
     }
 
diff --git a/src/Compose/ProductionBuilder/Service/Database/DbQuote.php b/src/Compose/ProductionBuilder/Service/Database/DbQuote.php
@@ -79,7 +79,8 @@ public function getConfig(Config $config): array
             ServiceInterface::SERVICE_DB_QUOTE,
             $config->getServiceVersion(ServiceInterface::SERVICE_DB),
             $dbConfig,
-            $config->getServiceImage(ServiceInterface::SERVICE_DB)
+            $config->getServiceImage(ServiceInterface::SERVICE_DB),
+            $config->getCustomRegistry()
         );
     }
 
diff --git a/src/Compose/ProductionBuilder/Service/Database/DbSales.php b/src/Compose/ProductionBuilder/Service/Database/DbSales.php
@@ -79,7 +79,8 @@ public function getConfig(Config $config): array
             ServiceInterface::SERVICE_DB_SALES,
             $config->getServiceVersion(ServiceInterface::SERVICE_DB),
             $dbConfig,
-            $config->getServiceImage(ServiceInterface::SERVICE_DB)
+            $config->getServiceImage(ServiceInterface::SERVICE_DB),
+            $config->getCustomRegistry()
         );
     }
 
diff --git a/src/Compose/ProductionBuilder/Service/Deploy.php b/src/Compose/ProductionBuilder/Service/Deploy.php
@@ -73,6 +73,7 @@ public function getConfig(Config $config): array
             $config->getServiceVersion(ServiceInterface::SERVICE_PHP),
             ['volumes' => $this->volume->getRo($config)],
             $config->getServiceImage(ServiceInterface::SERVICE_PHP),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern(ServiceInterface::SERVICE_PHP_CLI)
         );
     }
diff --git a/src/Compose/ProductionBuilder/Service/ElasticSearch.php b/src/Compose/ProductionBuilder/Service/ElasticSearch.php
@@ -72,6 +72,7 @@ public function getConfig(Config $config): array
             $config->getServiceVersion($this->getServiceName()),
             !empty($esEnvVars) ? ['environment' => $esEnvVars] : [],
             $config->getServiceImage($this->getServiceName()),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern($this->getServiceName())
         );
     }
diff --git a/src/Compose/ProductionBuilder/Service/Fpm.php b/src/Compose/ProductionBuilder/Service/Fpm.php
@@ -65,6 +65,7 @@ public function getConfig(Config $config): array
             $config->getServiceVersion(ServiceInterface::SERVICE_PHP),
             ['volumes' => $this->volume->getRo($config)],
             $config->getServiceImage(ServiceInterface::SERVICE_PHP),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern($this->getServiceName())
         );
     }
diff --git a/src/Compose/ProductionBuilder/Service/FpmXdebug.php b/src/Compose/ProductionBuilder/Service/FpmXdebug.php
@@ -96,6 +96,7 @@ public function getConfig(Config $config): array
                 'environment' => $this->converter->convert($envVariables)
             ],
             $config->getServiceImage(ServiceInterface::SERVICE_PHP),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern($this->getServiceName())
         );
     }
diff --git a/src/Compose/ProductionBuilder/Service/Generic.php b/src/Compose/ProductionBuilder/Service/Generic.php
@@ -83,6 +83,7 @@ public function getConfig(Config $config): array
                 )
             ],
             $config->getServiceImage(ServiceInterface::SERVICE_PHP),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern(ServiceInterface::SERVICE_PHP_CLI)
         );
     }
diff --git a/src/Compose/ProductionBuilder/Service/Mailhog.php b/src/Compose/ProductionBuilder/Service/Mailhog.php
@@ -61,6 +61,7 @@ public function getConfig(Config $config): array
                 ]
             ],
             $config->getServiceImage($this->getServiceName()),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern($this->getServiceName())
         );
     }
diff --git a/src/Compose/ProductionBuilder/Service/Node.php b/src/Compose/ProductionBuilder/Service/Node.php
@@ -63,6 +63,7 @@ public function getConfig(Config $config): array
             $config->getServiceVersion($this->getServiceName()),
             ['volumes' => $this->volume->getRo($config)],
             $config->getServiceImage($this->getServiceName()),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern($this->getServiceName())
         );
     }
diff --git a/src/Compose/ProductionBuilder/Service/RabbitMq.php b/src/Compose/ProductionBuilder/Service/RabbitMq.php
@@ -55,7 +55,8 @@ public function getConfig(Config $config): array
             $this->getServiceName(),
             $config->getServiceVersion($this->getServiceName()),
             [],
-            $config->getServiceImage($this->getServiceName())
+            $config->getServiceImage($this->getServiceName()),
+            $config->getCustomRegistry()
         );
     }
 
diff --git a/src/Compose/ProductionBuilder/Service/Redis.php b/src/Compose/ProductionBuilder/Service/Redis.php
@@ -62,7 +62,8 @@ public function getConfig(Config $config): array
                     'retries' => 3
                 ]
             ],
-            $config->getServiceImage($this->getServiceName())
+            $config->getServiceImage($this->getServiceName()),
+            $config->getCustomRegistry()
         );
     }
 
diff --git a/src/Compose/ProductionBuilder/Service/Selenium.php b/src/Compose/ProductionBuilder/Service/Selenium.php
@@ -56,7 +56,8 @@ public function getConfig(Config $config): array
             $this->getServiceName(),
             $config->getServiceVersion($this->getServiceName()),
             [],
-            $config->getServiceImage($this->getServiceName())
+            $config->getServiceImage($this->getServiceName()),
+            $config->getCustomRegistry()
         );
     }
 
diff --git a/src/Compose/ProductionBuilder/Service/Test.php b/src/Compose/ProductionBuilder/Service/Test.php
@@ -73,6 +73,7 @@ public function getConfig(Config $config): array
             $config->getServiceVersion(ServiceInterface::SERVICE_PHP),
             ['volumes' => $this->volume->getRw($config)],
             $config->getServiceImage(ServiceInterface::SERVICE_PHP),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern(ServiceInterface::SERVICE_PHP_CLI)
         );
     }
diff --git a/src/Compose/ProductionBuilder/Service/Tls.php b/src/Compose/ProductionBuilder/Service/Tls.php
@@ -69,6 +69,7 @@ public function getConfig(Config $config): array
                 ]
             ],
             $config->getServiceImage($this->getServiceName()),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern($this->getServiceName())
         );
     }
diff --git a/src/Compose/ProductionBuilder/Service/Varnish.php b/src/Compose/ProductionBuilder/Service/Varnish.php
@@ -57,6 +57,7 @@ public function getConfig(Config $config): array
             $config->getServiceVersion($this->getServiceName()),
             [],
             $config->getServiceImage($this->getServiceName()),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern($this->getServiceName())
         );
 
diff --git a/src/Compose/ProductionBuilder/Service/Web.php b/src/Compose/ProductionBuilder/Service/Web.php
@@ -72,6 +72,7 @@ public function getConfig(Config $config): array
                 ],
             ],
             $config->getServiceImage(ServiceInterface::SERVICE_NGINX),
+            $config->getCustomRegistry(),
             $config->getServiceImagePattern(ServiceInterface::SERVICE_NGINX)
         );
 
diff --git a/src/Config/Config.php b/src/Config/Config.php
@@ -215,6 +215,15 @@ public function getServiceImage(string $name): ?string
         return $this->all()->get($this->getKey($name, 'image'));
     }
 
+    /**
+     * @return string|null
+     * @throws ConfigurationMismatchException
+     */
+    public function getCustomRegistry(): ?string
+    {
+        return $this->all()->get(SourceInterface::SYSTEM_CUSTOM_REGISTRY);
+    }
+
     /**
      * @param string $name
      * @return string|null
diff --git a/src/Config/Source/CliSource.php b/src/Config/Source/CliSource.php
@@ -35,6 +35,11 @@ class CliSource implements SourceInterface
     public const OPTION_NO_MAILHOG = 'no-mailhog';
     public const OPTION_NO_TLS = 'no-tls';
 
+    /**
+     * Custom registry
+     */
+    public const OPTION_CUSTOM_REGISTRY = 'custom-registry';
+
     /**
      * MailHog configuration
      */
@@ -145,6 +150,12 @@ public function read(): Repository
     {
         $repository = new Repository();
 
+        if ($customRegistry = $this->input->getOption(self::OPTION_CUSTOM_REGISTRY)) {
+            $repository->set([
+                self::SYSTEM_CUSTOM_REGISTRY => $customRegistry
+            ]);
+        }
+
         if ($mode = $this->input->getOption(self::OPTION_MODE)) {
             $repository->set([
                 self::SYSTEM_MODE => $mode
diff --git a/src/Config/Source/SourceInterface.php b/src/Config/Source/SourceInterface.php
@@ -135,6 +135,7 @@ interface SourceInterface
     /**
      * Config
      */
+    public const SYSTEM_CUSTOM_REGISTRY = 'system.custom_registry';
     public const SYSTEM_SYNC_ENGINE = 'system.sync_engine';
     public const SYSTEM_TMP_MOUNTS = 'system.tmp_mounts';
     public const SYSTEM_MODE = 'system.mode';
diff --git a/src/Service/ServiceFactory.php b/src/Service/ServiceFactory.php
@@ -161,6 +161,7 @@ public function __construct(FileList $fileList)
      * @param string $version
      * @param array $config
      * @param string $image
+     * @param string|null $customRegistry
      * @param string|null $imagePattern
      * @return array
      * @throws ConfigurationMismatchException
@@ -170,6 +171,7 @@ public function create(
         string $version,
         array $config = [],
         string $image = null,
+        string $customRegistry = null,
         string $imagePattern = null
     ): array {
         if (!array_key_exists($name, self::$config)) {
@@ -182,7 +184,7 @@ public function create(
         $metaConfig = self::$config[$name];
         $defaultConfig = $metaConfig['config'] ?? [];
 
-        $image = $image ?: $metaConfig['image'];
+        $image = ($customRegistry ? $customRegistry . '/' : '') . ($image ?: $metaConfig['image']);
         $pattern = $imagePattern ?: $metaConfig['pattern'];
 
         return array_replace(
diff --git a/src/Test/Functional/Acceptance/Acceptance71Cest.php b/src/Test/Functional/Acceptance/Acceptance71Cest.php
diff --git a/src/Test/Functional/Acceptance/AcceptanceCest.php b/src/Test/Functional/Acceptance/AcceptanceCest.php
@@ -20,8 +20,8 @@ class AcceptanceCest extends AbstractCest
      */
     public function testProductionMode(\CliTester $I): void
     {
-        $I->assertTrue($I->runEceDockerCommand('build:compose --mode=production'), 'Command build:compose failed');
-        $I->replaceImagesWithGenerated();
+        $I->assertTrue($I->generateDockerCompose('--mode=production'), 'Command build:compose failed');
+        $I->replaceImagesWithCustom();
         $I->startEnvironment();
         $I->runDockerComposeCommand('run build cloud-build');
         $I->runDockerComposeCommand('run deploy cloud-deploy');
@@ -41,10 +41,10 @@ public function testCustomHost(\CliTester $I): void
     {
         $I->updateBaseUrl('http://magento2.test:8080/');
         $I->assertTrue(
-            $I->runEceDockerCommand('build:compose --mode=production --host=magento2.test --port=8080'),
+            $I->generateDockerCompose('--mode=production --host=magento2.test --port=8080'),
             'Command build:compose failed'
         );
-        $I->replaceImagesWithGenerated();
+        $I->replaceImagesWithCustom();
         $I->startEnvironment();
         $I->assertTrue($I->runDockerComposeCommand('run build cloud-build'), 'Build phase failed');
         $I->assertTrue($I->runDockerComposeCommand('run deploy cloud-deploy'), 'Deploy phase failed');
diff --git a/src/Test/Functional/Acceptance/Elasticsearch72Cest.php b/src/Test/Functional/Acceptance/Elasticsearch72Cest.php
@@ -27,16 +27,6 @@ class Elasticsearch72Cest extends ElasticsearchCest
     protected function dataProvider(): array
     {
         return [
-            [
-                'version' => '1.7',
-                'xms' => '512m',
-                'xmx' => '512m',
-            ],
-            [
-                'version' => '2.4',
-                'xms' => '514m',
-                'xmx' => '514m',
-            ],
             [
                 'version' => '5.2',
                 'xms' => '516m',
diff --git a/src/Test/Functional/Acceptance/ElasticsearchCest.php b/src/Test/Functional/Acceptance/ElasticsearchCest.php
@@ -30,8 +30,8 @@ class ElasticsearchCest extends AbstractCest
      */
     public function testElasticsearch(CliTester $I, Example $data)
     {
-        $I->runEceDockerCommand($this->buildCommand($data));
-        $I->replaceImagesWithGenerated();
+        $I->generateDockerCompose($this->buildCommand($data));
+        $I->replaceImagesWithCustom();
         $I->startEnvironment();
         if (!empty($data['plugins'])) {
             $I->runDockerComposeCommand('logs elasticsearch');
@@ -58,7 +58,7 @@ public function testElasticsearch(CliTester $I, Example $data)
     private function buildCommand(Example $data): string
     {
         $command = sprintf(
-            'build:compose --mode=production --es=%s --es-env-var="ES_JAVA_OPTS=-Xms%s -Xmx%s"',
+            '--mode=production --es=%s --es-env-var="ES_JAVA_OPTS=-Xms%s -Xmx%s"',
             $data['version'],
             $data['xms'],
             $data['xmx']
diff --git a/src/Test/Functional/Acceptance/MailHogCest.php b/src/Test/Functional/Acceptance/MailHogCest.php
@@ -20,7 +20,7 @@ public function testDefaultPorts(\CliTester $I): void
     {
         $I->updateBaseUrl('http://magento2.docker:8025/');
         $I->assertTrue(
-            $I->runEceDockerCommand('build:compose'),
+            $I->generateDockerCompose(),
             'Command build:compose failed'
         );
         $this->runAndAssert($I);
@@ -34,7 +34,7 @@ public function testCustomPorts(\CliTester $I): void
     {
         $I->updateBaseUrl('http://magento2.docker:8026/');
         $I->assertTrue(
-            $I->runEceDockerCommand('build:compose --mailhog-http-port=8026 --mailhog-smtp-port=1026'),
+            $I->generateDockerCompose('--mailhog-http-port=8026 --mailhog-smtp-port=1026'),
             'Command build:compose failed'
         );
         $this->runAndAssert($I);
@@ -46,7 +46,7 @@ public function testCustomPorts(\CliTester $I): void
      */
     private function runAndAssert(\CliTester $I): void
     {
-        $I->replaceImagesWithGenerated();
+        $I->replaceImagesWithCustom();
         $I->startEnvironment();
         $I->amOnPage('/');
         $I->see('MailHog');
diff --git a/src/Test/Functional/Acceptance/Services71Cest.php b/src/Test/Functional/Acceptance/Services71Cest.php
diff --git a/src/Test/Functional/Acceptance/ServicesCest.php b/src/Test/Functional/Acceptance/ServicesCest.php
diff --git a/src/Test/Functional/Acceptance/SplitDb71Cest.php b/src/Test/Functional/Acceptance/SplitDb71Cest.php
diff --git a/src/Test/Functional/Acceptance/SplitDbCest.php b/src/Test/Functional/Acceptance/SplitDbCest.php
diff --git a/src/Test/Integration/BuildComposeTest.php b/src/Test/Integration/BuildComposeTest.php
diff --git a/src/Test/Integration/_files/custom_registry/.docker/.gitignore b/src/Test/Integration/_files/custom_registry/.docker/.gitignore
diff --git a/src/Test/Integration/_files/custom_registry/.docker/config.php.dist b/src/Test/Integration/_files/custom_registry/.docker/config.php.dist
diff --git a/src/Test/Integration/_files/custom_registry/.magento.app.yaml b/src/Test/Integration/_files/custom_registry/.magento.app.yaml
diff --git a/src/Test/Integration/_files/custom_registry/.magento/services.yaml b/src/Test/Integration/_files/custom_registry/.magento/services.yaml
diff --git a/src/Test/Integration/_files/custom_registry/composer.json b/src/Test/Integration/_files/custom_registry/composer.json
diff --git a/src/Test/Integration/_files/custom_registry/docker-compose.exp.yml b/src/Test/Integration/_files/custom_registry/docker-compose.exp.yml
diff --git a/tests/functional/Codeception/BaseModule.php b/tests/functional/Codeception/BaseModule.php
diff --git a/tests/functional/Codeception/TestInfrastructure.php b/tests/functional/Codeception/TestInfrastructure.php
diff --git a/tests/travis/images.sh b/tests/travis/images.sh

Original file line number	Diff line number	Diff line change
`@@ -64,7 +64,8 @@ public function getConfig(Config $config): array`
`64`	`64`	`],`
`65`	`65`	`'ports' => ["8707"]`
`66`	`66`	`],`
`67`		`- $config->getServiceImage($this->getServiceName())`
	`67`	`+ $config->getServiceImage($this->getServiceName()),`
	`68`	`+ $config->getCustomRegistry()`
`68`	`69`	`);`
`69`	`70`	`}`
`70`	`71`
Original file line number	Diff line number	Diff line change
`@@ -73,6 +73,7 @@ public function getConfig(Config $config): array`
`73`	`73`	`$config->getServiceVersion(ServiceInterface::SERVICE_PHP),`
`74`	`74`	`['volumes' => $this->volume->getBuild($config)],`
`75`	`75`	`$config->getServiceImage(ServiceInterface::SERVICE_PHP),`
	`76`	`+ $config->getCustomRegistry(),`
`76`	`77`	`$config->getServiceImagePattern(ServiceInterface::SERVICE_PHP_CLI)`
`77`	`78`	`);`
`78`	`79`	`}`
Original file line number	Diff line number	Diff line change
`@@ -82,7 +82,8 @@ public function getConfig(Config $config): array`
`82`	`82`	`ServiceInterface::SERVICE_DB,`
`83`	`83`	`$config->getServiceVersion(ServiceInterface::SERVICE_DB),`
`84`	`84`	`$dbConfig,`
`85`		`- $config->getServiceImage(ServiceInterface::SERVICE_DB)`
	`85`	`+ $config->getServiceImage(ServiceInterface::SERVICE_DB),`
	`86`	`+ $config->getCustomRegistry()`
`86`	`87`	`);`
`87`	`88`	`}`
`88`	`89`
Original file line number	Diff line number	Diff line change
`@@ -79,7 +79,8 @@ public function getConfig(Config $config): array`
`79`	`79`	`ServiceInterface::SERVICE_DB_QUOTE,`
`80`	`80`	`$config->getServiceVersion(ServiceInterface::SERVICE_DB),`
`81`	`81`	`$dbConfig,`
`82`		`- $config->getServiceImage(ServiceInterface::SERVICE_DB)`
	`82`	`+ $config->getServiceImage(ServiceInterface::SERVICE_DB),`
	`83`	`+ $config->getCustomRegistry()`
`83`	`84`	`);`
`84`	`85`	`}`
`85`	`86`