Merge pull request #62 from QualiSystemsLab/feature/dan-aws_tf_backend

Feature/dan aws tf backend

Author: alexazarh

.gitignore

@@ -61,6 +61,7 @@ docs/_build/
 # PyBuilder
 target/
 cloudshell_config.yml
-shells/generic_terraform_service/tests/.env
-tests/.env
-shells/backends/azure_tf_backend/tests/.env
+
+# env files
+*.env
+!*.template.env

README.md

@@ -28,7 +28,7 @@ Additional workflow recommendation: it is very easy to customize a Blueprint set
 |Github Terraform Module URL|String|Path to target module. Can be provided in three formats: <br/> 1) https://github.com/{ACCOUNT}/{REPO}/tree/{BRANCH}/{PATH_TO_FOLDER} <br/> 2) https://github.com/{ACCOUNT}/{REPO}/blob/{BRANCH}/{PATH_TO_FOLDER}/{FILENAME}.tf <br/> 3) https://raw.githubusercontent.com/{ACCOUNT}/{REPO}/{BRANCH}/{PATH_TO_FOLDER}/{FILENAME}.tf  | Yes |
 |Terraform Version|String|The version of terraform.exe that will be downloaded and used (If not specified latest version will be used)|  No |
 |Github Token|String| Github PAT (Private Access Token) to be used in order to download TF module. The entire repo will be downloaded and then the referenced TF module will be executed |  Yes |
-|Cloud Provider|String| Reference to the CloudProvider resource that should be used to initialize the Terrafom provider. Supported cloud providers: <br> - Azure Shell <br>- Azure Shell 2G| Yes |
+|Cloud Provider|String| Reference to the CloudProvider resource that should be used to initialize the Terrafom provider. Supported cloud providers: <br> - Azure Shell <br>- Azure Shell 2G <br> - AWS Shell <br> - AWS Shell 2G| Yes |
 |Branch|String| In case specified will override the branch in the Github Terraform Module URL |  No |
 |Terraform Outputs|String| Unmapped and *non-sensitive* TF outputs will be stored as a CSV list of key value pairs on this attribute. This attribute is optional. |  No |
 |Terraform Sensitive Outputs|Password| Unmapped and *sensitive* TF outputs will be stored as a CSV list of key value pairs on this attribute. This attribute must be of type "password. The attribute is optional. |  No |
@@ -92,7 +92,17 @@ Cloud Provider|String| Cloud Provider resource name that holds the authenticatio
 Resource Group|String| The resource group of the Storage Account|
 
 ### AWS Remote Provider Shell (backends\aws_tf_backend)
-Coming soon
+The AWS Remote Provider shell is used in order to enable CloudShell access to AWS S3 storage, to then be used to store the remote state file.</br>
+One must create a resource and fill in the attributes - then specify that resource name as the Remote State Provider.
+Only one type of authentication is allowed, either by Access Key+Secret Key or using the Cloud Provider authentication keys. If both options are specified it will throw an error, so please supply only 1 option.
+
+|Attribute|Type|Description|
+|:-----|:-----|:-----|
+Bucket Name|String| The name of the Bucket to be used |
+Region Name|String| The Region to be used with AWS |
+Access Key|String| Access Key of AWS Account|
+Secret Key|String| Secret Key of AWS Account|
+Cloud Provider|String| Cloud Provider resource name that holds the authentication keys in case not filling the Keys|
 
 \* Additional Remote Backend Providers are coming soon
 

deploy.bat

@@ -8,4 +8,7 @@ xcopy /Q /Y . "C:\Program Files (x86)\QualiSystems\CloudShell\Server\Config\Pypi
 cd ..\..
 cd shells\backends\azure_tf_backend
 shellfoundry install
-cd ..\..\..
+cd ..\..
+cd backends\aws_tf_backend
+shellfoundry install
+cd ..\..\..

package/cloudshell/iac/terraform/constants.py

@@ -36,7 +36,13 @@
 TF_WORKING_DIR = "TF_WORKING_DIR"
 
 # CLP models
+AZURE1G_MODEL = "Microsoft Azure"
+AWS1G_MODEL = "AWS EC2"
+
 AZURE2G_MODEL = "Microsoft Azure Cloud Provider 2G"
+AWS2G_MODEL = "Amazon AWS Cloud Provider 2G"
+
+CLP_PROVIDER_MODELS = [AWS1G_MODEL, AWS2G_MODEL, AZURE1G_MODEL, AZURE2G_MODEL]
 
 # Misc
 DIRTY_CHARS = r'''

package/cloudshell/iac/terraform/services/clp_envvar_handler.py

@@ -0,0 +1,67 @@
+import os
+from abc import ABCMeta
+
+from cloudshell.iac.terraform.models.shell_helper import ShellHelperObject
+
+
+class BaseCloudProviderEnvVarHandler(metaclass=ABCMeta):
+    def __init__(self):
+        pass
+
+    def set_env_vars_based_on_clp(self):
+        raise NotImplementedError()
+
+    @staticmethod
+    def does_attribute_match(clp_res_model, clp_attribute, attr_name_to_check) -> bool:
+        if f"{clp_res_model}.{clp_attribute.Name}" == attr_name_to_check or clp_attribute.Name == attr_name_to_check:
+            return True
+        return False
+
+
+class AWSCloudProviderEnvVarHandler(BaseCloudProviderEnvVarHandler):
+    def __init__(self, clp_res_model: str, clp_resource_attributes: list,
+                 shell_helper: ShellHelperObject):
+        BaseCloudProviderEnvVarHandler.__init__(self)
+        self._clp_res_model = clp_res_model
+        self._clp_resource_attributes = clp_resource_attributes
+        self._shell_helper = shell_helper
+
+    def set_env_vars_based_on_clp(self):
+        dec_access_key = ""
+        dec_secret_key = ""
+        region_flag = False
+
+        for attr in self._clp_resource_attributes:
+            if self.does_attribute_match(self._clp_res_model, attr, "AWS Access Key ID"):
+                dec_access_key = self._shell_helper.api.DecryptPassword(attr.Value).Value
+            if self.does_attribute_match(self._clp_res_model, attr, "AWS Secret Access Key"):
+                dec_secret_key = self._shell_helper.api.DecryptPassword(attr.Value).Value
+            if self.does_attribute_match(self._clp_res_model, attr, "Region"):
+                os.environ["AWS_DEFAULT_REGION"] = attr.Value
+                region_flag = True
+        if not region_flag:
+            raise ValueError("Region was not found on AWS Cloud Provider")
+
+        # We must check both keys exist...if not then the EC2 Execution Server profile would be used (Role)
+        if dec_access_key and dec_secret_key:
+            os.environ["AWS_ACCESS_KEY_ID"] = dec_access_key
+            os.environ["AWS_SECRET_ACCESS_KEY"] = dec_secret_key
+
+
+class AzureCloudProviderEnvVarHandler(BaseCloudProviderEnvVarHandler):
+    def __init__(self, clp_res_model, clp_resource_attributes, shell_helper):
+        BaseCloudProviderEnvVarHandler.__init__(self)
+        self._clp_res_model = clp_res_model
+        self._clp_resource_attributes = clp_resource_attributes
+        self._shell_helper = shell_helper
+
+    def set_env_vars_based_on_clp(self):
+        for attr in self._clp_resource_attributes:
+            if self.does_attribute_match(self._clp_res_model, attr, self._shell_helper, "Azure Subscription ID"):
+                os.environ["ARM_SUBSCRIPTION_ID"] = attr.Value
+            if self.does_attribute_match(self._clp_res_model, attr, self._shell_helper, "Azure Tenant ID"):
+                os.environ["Azure Tenant ID"] = attr.Value
+            if self.does_attribute_match(self._clp_res_model, attr, self._shell_helper, "Azure Application ID"):
+                os.environ["ARM_CLIENT_ID"] = attr.Value
+            if self.does_attribute_match(self._clp_res_model, attr, self._shell_helper, "Azure Application Key", True):
+                os.environ["ARM_CLIENT_SECRET"] = self._shell_helper.api.DecryptPassword(attr.Value).Value

package/cloudshell/iac/terraform/services/provider_handler.py

@@ -1,16 +1,19 @@
-import os
 from logging import Logger
 
-from cloudshell.iac.terraform.constants import AZURE2G_MODEL, ATTRIBUTE_NAMES
+from cloudshell.api.cloudshell_api import ResourceInfo
+
+from cloudshell.iac.terraform.constants import AZURE2G_MODEL, ATTRIBUTE_NAMES, AWS2G_MODEL, CLP_PROVIDER_MODELS, \
+    AWS1G_MODEL, AZURE1G_MODEL
 from cloudshell.iac.terraform.models.shell_helper import ShellHelperObject
+from cloudshell.iac.terraform.services.clp_envvar_handler import AWSCloudProviderEnvVarHandler, \
+    AzureCloudProviderEnvVarHandler
 
 
 class ProviderHandler(object):
     def __init__(self, logger: Logger):
         self.logger = logger
 
-    @staticmethod
-    def initialize_provider(shell_helper: ShellHelperObject):
+    def initialize_provider(self, shell_helper: ShellHelperObject):
         clp_resource_name = shell_helper.attr_handler.get_attribute(ATTRIBUTE_NAMES.CLOUD_PROVIDER)
         if not clp_resource_name:
             return
@@ -23,17 +26,8 @@ def initialize_provider(shell_helper: ShellHelperObject):
             raise ValueError(f"{clpr_res_fam} currently not supported")
 
         try:
-            if clp_res_model == 'Microsoft Azure' or clp_res_model == AZURE2G_MODEL:
-                shell_helper.sandbox_messages.write_message("initializing provider...")
-                shell_helper.logger.info("Initializing Environment variables with CloudProvider details")
-                clp_resource_attributes = clp_details.ResourceAttributes
-
-                azure_attr_name_prefix = ""
-                if clp_res_model == AZURE2G_MODEL:
-                    azure_attr_name_prefix = AZURE2G_MODEL + "."
-
-                ProviderHandler._set_azure_env_vars_based_on_clp(azure_attr_name_prefix, clp_resource_attributes,
-                                                                 shell_helper)
+            if clp_res_model in CLP_PROVIDER_MODELS:
+                self._set_cloud_env_vars(clp_details, clp_res_model, shell_helper)
             else:
                 shell_helper.logger.error(f"{clp_res_model} currently not supported")
                 raise ValueError(f"{clp_res_model} currently not supported")
@@ -42,15 +36,25 @@ def initialize_provider(shell_helper: ShellHelperObject):
             shell_helper.logger.error(f"Error Setting environment variables -> {str(e)}")
             raise
 
-    @staticmethod
-    def _set_azure_env_vars_based_on_clp(azure_attr_name_prefix, clp_resource_attributes, shell_helper):
-        for attr in clp_resource_attributes:
-            if attr.Name == azure_attr_name_prefix + "Azure Subscription ID":
-                os.environ["ARM_SUBSCRIPTION_ID"] = attr.Value
-            if attr.Name == azure_attr_name_prefix + "Azure Tenant ID":
-                os.environ["ARM_TENANT_ID"] = attr.Value
-            if attr.Name == azure_attr_name_prefix + "Azure Application ID":
-                os.environ["ARM_CLIENT_ID"] = attr.Value
-            if attr.Name == azure_attr_name_prefix + "Azure Application Key":
-                dec_client_secret = shell_helper.api.DecryptPassword(attr.Value).Value
-                os.environ["ARM_CLIENT_SECRET"] = dec_client_secret
+    def _set_cloud_env_vars(
+            self,
+            clp_details: ResourceInfo,
+            clp_res_model: str,
+            shell_helper: ShellHelperObject,
+    ):
+        shell_helper.sandbox_messages.write_message("initializing provider...")
+        shell_helper.logger.info("Initializing Environment variables with CloudProvider details")
+        clp_resource_attributes = clp_details.ResourceAttributes
+        clp_handler = None
+
+        if clp_res_model in [AWS1G_MODEL, AWS2G_MODEL]:
+            clp_handler = AWSCloudProviderEnvVarHandler(clp_res_model, clp_resource_attributes, shell_helper)
+
+        elif clp_res_model in [AZURE1G_MODEL, AZURE2G_MODEL]:
+            clp_handler = AzureCloudProviderEnvVarHandler(clp_res_model, clp_resource_attributes, shell_helper)
+
+        if clp_handler:
+            clp_handler.set_env_vars_based_on_clp()
+        else:
+            self.logger.error(f"Was not able to initialize provider as {clp_res_model} is not a supported model")
+            raise ValueError(f"Was not able to initialize provider as {clp_res_model} is not a supported model")

package/cloudshell/iac/terraform/terraform_shell.py

@@ -20,6 +20,7 @@ def __init__(self, driver_context: ResourceCommandContext,
         self._tf_service = ObjectFactory.create_tf_service(driver_context)
         self._logger = logger
         self._config = config or TerraformShellConfig()
+        self._provider_handler = ProviderHandler(self._logger)
 
     def execute_terraform(self):
         # initialize a logger if logger wasn't passed during init
@@ -36,7 +37,7 @@ def _execute_procedure(self, sandbox_data_handler: SandboxDataHandler, shell_hel
             tf_proc_executer = ObjectFactory.create_tf_proc_executer(self._config, sandbox_data_handler,
                                                                      shell_helper, tf_working_dir)
             if tf_proc_executer.can_execute_run():
-                ProviderHandler.initialize_provider(shell_helper)
+                self._provider_handler.initialize_provider(shell_helper)
                 tf_proc_executer.init_terraform()
                 tf_proc_executer.tag_terraform()
                 tf_proc_executer.plan_terraform()
@@ -67,7 +68,7 @@ def _destroy_procedure(self, sandbox_data_handler: SandboxDataHandler, shell_hel
             self._handle_error_output(shell_helper, "Destroy failed due to missing local directory")
 
         try:
-            ProviderHandler.initialize_provider(shell_helper)
+            self._provider_handler.initialize_provider(shell_helper)
             tf_proc_executer = ObjectFactory.create_tf_proc_executer(self._config, sandbox_data_handler,
                                                                      shell_helper, tf_working_dir)
             if tf_proc_executer.can_destroy_run():

package/requirements.txt

@@ -2,4 +2,4 @@ cloudshell-shell-core==5.0.5
 requests==2.25.1
 cloudshell-automation-api==2021.1.0.181140
 retry==0.9.2
-python-hcl2==2.0.1
+python-hcl2==2.0.1

package/version.txt

@@ -1 +1 @@
-1.0.0
+1.1.0

shells/backends/aws_tf_backend/.gitignore

@@ -0,0 +1,60 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+cloudshell_config.yml