From 7633034c138fb5d159cf62951a9aea2b82ba0fd7 Mon Sep 17 00:00:00 2001 From: Matthew Casperson Date: Tue, 18 Feb 2025 11:38:45 +1000 Subject: [PATCH 1/3] Add option to exclude library variable sets --- .../octopus-serialize-space-to-terraform.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/step-templates/octopus-serialize-space-to-terraform.json b/step-templates/octopus-serialize-space-to-terraform.json index faeeb2da1..39386991f 100644 --- a/step-templates/octopus-serialize-space-to-terraform.json +++ b/step-templates/octopus-serialize-space-to-terraform.json @@ -3,12 +3,12 @@ "Name": "Octopus - Serialize Space to Terraform", "Description": "Serialize an Octopus space, excluding all projects, as a Terraform module and upload the resulting package to the Octopus built in feed.\n\nThis step is expected to be used in conjunction with the [Octopus - Serialize Project to Terraform](https://library.octopus.com/step-templates/e9526501-09d5-490f-ac3f-5079735fe041/actiontemplate-octopus-serialize-project-to-terraform) step. This step will serialize the global space resources, which typically do not change much, and have those resources recreated in a downstream space. The `Octopus - Serialize Project to Terraform` step then serializes a project, using `data` blocks to reference space level resources by name.", "ActionType": "Octopus.Script", - "Version": 6, + "Version": 7, "CommunityActionTemplateId": null, "Packages": [], "Properties": { "Octopus.Action.RunOnServer": "true", - "Octopus.Action.Script.ScriptBody": "import argparse\nimport os\nimport stat\nimport re\nimport socket\nimport subprocess\nimport sys\nfrom datetime import datetime\nfrom urllib.parse import urlparse\nfrom itertools import chain\nimport platform\nfrom urllib.request import urlretrieve\nimport zipfile\nimport urllib.request\nimport urllib.parse\nimport json\nimport tarfile\nimport random, time\n\n# If this script is not being run as part of an Octopus step, return variables from environment variables.\n# Periods are replaced with underscores, and the variable name is converted to uppercase\nif \"get_octopusvariable\" not in globals():\n def get_octopusvariable(variable):\n return os.environ[re.sub('\\\\.', '_', variable.upper())]\n\n# If this script is not being run as part of an Octopus step, print directly to std out.\nif \"printverbose\" not in globals():\n def printverbose(msg):\n print(msg)\n\n\ndef printverbose_noansi(output):\n \"\"\"\n Strip ANSI color codes and print the output as verbose\n :param output: The output to print\n \"\"\"\n if not output:\n return\n\n # https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python\n output_no_ansi = re.sub(r'\\x1B(?:[@-Z\\\\-_]|\\[[0-?]*[ -/]*[@-~])', '', output)\n printverbose(output_no_ansi)\n\n\ndef get_octopusvariable_quiet(variable):\n \"\"\"\n Gets an octopus variable, or an empty string if it does not exist.\n :param variable: The variable name\n :return: The variable value, or an empty string if the variable does not exist\n \"\"\"\n try:\n return get_octopusvariable(variable)\n except:\n return ''\n\n\ndef retry_with_backoff(fn, retries=5, backoff_in_seconds=1):\n x = 0\n while True:\n try:\n return fn()\n except Exception as e:\n\n print(e)\n\n if x == retries:\n raise\n\n sleep = (backoff_in_seconds * 2 ** x +\n random.uniform(0, 1))\n time.sleep(sleep)\n x += 1\n\n\ndef execute(args, cwd=None, env=None, print_args=None, print_output=printverbose_noansi):\n \"\"\"\n The execute method provides the ability to execute external processes while capturing and returning the\n output to std err and std out and exit code.\n \"\"\"\n process = subprocess.Popen(args,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n text=True,\n cwd=cwd,\n env=env)\n stdout, stderr = process.communicate()\n retcode = process.returncode\n\n if print_args is not None:\n print_output(' '.join(args))\n\n if print_output is not None:\n print_output(stdout)\n print_output(stderr)\n\n return stdout, stderr, retcode\n\n\ndef is_windows():\n return platform.system() == 'Windows'\n\n\ndef init_argparse():\n parser = argparse.ArgumentParser(\n usage='%(prog)s [OPTION] [FILE]...',\n description='Serialize an Octopus project to a Terraform module'\n )\n parser.add_argument('--terraform-backend',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Terraform.Backend') or get_octopusvariable_quiet(\n 'ThisInstance.Terraform.Backend') or 'pg',\n help='Set this to the name of the Terraform backend to be included in the generated module.')\n parser.add_argument('--server-url',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Server.Url') or get_octopusvariable_quiet(\n 'ThisInstance.Server.Url'),\n help='Sets the server URL that holds the project to be serialized.')\n parser.add_argument('--api-key',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Api.Key') or get_octopusvariable_quiet(\n 'ThisInstance.Api.Key'),\n help='Sets the Octopus API key.')\n parser.add_argument('--space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.Id') or get_octopusvariable_quiet(\n 'Exported.Space.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID containing the project to be serialized.')\n parser.add_argument('--upload-space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Octopus.UploadSpace.Id') or get_octopusvariable_quiet(\n 'Octopus.UploadSpace.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID of the Octopus space where ' +\n 'the resulting package will be uploaded to.')\n parser.add_argument('--ignored-library-variable-sets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredLibraryVariableSet') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredLibraryVariableSet'),\n help='A comma separated list of library variable sets to ignore.')\n parser.add_argument('--ignored-tenants',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredTenants') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredTenants'),\n help='A comma separated list of tenants ignore.')\n\n parser.add_argument('--ignored-tenants-with-tag',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredTenantTags') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredTenants'),\n help='A comma separated list of tenant tags that identify tenants to ignore.')\n parser.add_argument('--ignore-all-targets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoreTargets') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoreAllChanges') or 'false',\n help='Set to true to exlude targets from the exported module')\n\n parser.add_argument('--dummy-secret-variables',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.DummySecrets') or get_octopusvariable_quiet(\n 'Exported.Space.DummySecrets') or 'false',\n help='Set to true to set secret values, like account and feed passwords, to a dummy value by default')\n\n parser.add_argument('--default-secret-variables',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.DefaultSecrets') or get_octopusvariable_quiet(\n 'Exported.Space.DefaultSecrets') or 'false',\n help='Set to true to set sensitive variables to the octostache template that represents the variable')\n parser.add_argument('--include-step-templates',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IncludeStepTemplates') or get_octopusvariable_quiet(\n 'Exported.Space.IncludeStepTemplates') or 'false',\n help='Set this to true to include step templates in the exported module. ' +\n 'This disables the default behaviour of detaching step templates.')\n parser.add_argument('--ignored-accounts',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredAccounts') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredAccounts'),\n help='A comma separated list of accounts to ignore.')\n\n return parser.parse_known_args()\n\n\ndef get_latest_github_release(owner, repo, filename):\n url = f\"https://api.github.com/repos/{owner}/{repo}/releases/latest\"\n releases = urllib.request.urlopen(url).read()\n contents = json.loads(releases)\n\n download = [asset for asset in contents.get('assets') if asset.get('name') == filename]\n\n if len(download) != 0:\n return download[0].get('browser_download_url')\n\n return None\n\n\ndef ensure_octo_cli_exists():\n if is_windows():\n print(\"Checking for the Octopus CLI\")\n try:\n stdout, _, exit_code = execute(['octo.exe', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.win-x64.zip',\n 'OctopusTools.zip')\n with zipfile.ZipFile('OctopusTools.zip', 'r') as zip_ref:\n zip_ref.extractall(os.getcwd())\n return os.getcwd()\n else:\n print(\"Checking for the Octopus CLI for Linux\")\n try:\n stdout, _, exit_code = execute(['octo', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI for Linux\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.linux-x64.tar.gz',\n 'OctopusTools.tar.gz')\n with tarfile.open('OctopusTools.tar.gz') as file:\n file.extractall(os.getcwd())\n os.chmod(os.path.join(os.getcwd(), 'octo'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\ndef ensure_octoterra_exists():\n if is_windows():\n print(\"Checking for the Octoterra tool for Windows\")\n try:\n stdout, _, exit_code = execute(['octoterra.exe', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Windows\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_windows_amd64.exe\",\n 'octoterra.exe'), 10, 30)\n return os.getcwd()\n else:\n print(\"Checking for the Octoterra tool for Linux\")\n try:\n stdout, _, exit_code = execute(['octoterra', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Linux\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_linux_amd64\",\n 'octoterra'), 10, 30)\n os.chmod(os.path.join(os.getcwd(), 'octoterra'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\noctocli_path = ensure_octo_cli_exists()\noctoterra_path = ensure_octoterra_exists()\nparser, _ = init_argparse()\n\n# Variable precondition checks\nif len(parser.server_url) == 0:\n print(\"--server-url, ThisInstance.Server.Url, or SerializeSpace.ThisInstance.Server.Url must be defined\")\n sys.exit(1)\n\nif len(parser.api_key) == 0:\n print(\"--api-key, ThisInstance.Api.Key, or SerializeSpace.ThisInstance.Api.Key must be defined\")\n sys.exit(1)\n\n\nprint(\"Octopus URL: \" + parser.server_url)\nprint(\"Octopus Space ID: \" + parser.space_id)\n\n# Build the arguments to ignore library variable sets\nignores_library_variable_sets = parser.ignored_library_variable_sets.split(',')\nignores_library_variable_sets_args = [['-excludeLibraryVariableSetRegex', x] for x in ignores_library_variable_sets if\n x.strip() != '']\n\n# Build the arguments to ignore tenants\nignores_tenants = parser.ignored_tenants.split(',')\nignores_tenants_args = [['-excludeTenants', x] for x in ignores_tenants if x.strip() != '']\n\n# Build the arguments to ignore tenants with tags\nignored_tenants_with_tag = parser.ignored_tenants_with_tag.split(',')\nignored_tenants_with_tag_args = [['-excludeTenantsWithTag', x] for x in ignored_tenants_with_tag if x.strip() != '']\n\n# Build the arguments to ignore accounts\nignored_accounts = parser.ignored_accounts.split(',')\nignored_accounts = [['-excludeAccountsRegex', x] for x in ignored_accounts]\n\nos.mkdir(os.getcwd() + '/export')\n\nexport_args = [os.path.join(octoterra_path, 'octoterra'),\n # the url of the instance\n '-url', parser.server_url,\n # the api key used to access the instance\n '-apiKey', parser.api_key,\n # add a postgres backend to the generated modules\n '-terraformBackend', parser.terraform_backend,\n # dump the generated HCL to the console\n '-console',\n # dump the project from the current space\n '-space', parser.space_id,\n # Use default dummy values for secrets (e.g. a feed password). These values can still be overridden if known,\n # but allows the module to be deployed and have the secrets updated manually later.\n '-dummySecretVariableValues=' + parser.dummy_secret_variables,\n # for any secret variables, add a default value set to the octostache value of the variable\n # e.g. a secret variable called \"database\" has a default value of \"#{database}\"\n '-defaultSecretVariableValues=' + parser.default_secret_variables,\n # Add support for experimental step templates\n '-experimentalEnableStepTemplates=' + parser.include_step_templates,\n # Don't export any projects\n '-excludeAllProjects',\n # Output variables allow the Octopus space and instance to be determined from the Terraform state file.\n '-includeOctopusOutputVars',\n # Provide an option to ignore targets.\n '-excludeAllTargets=' + parser.ignore_all_targets,\n # The directory where the exported files will be saved\n '-dest', os.getcwd() + '/export'] + list(\n chain(*ignores_library_variable_sets_args, *ignores_tenants_args, *ignored_tenants_with_tag_args,\n *ignored_accounts))\n\nprint(\"Exporting Terraform module\")\n_, _, octoterra_exit = execute(export_args)\n\nif not octoterra_exit == 0:\n print(\"Octoterra failed. Please check the verbose logs for more information.\")\n sys.exit(1)\n\ndate = datetime.now().strftime('%Y.%m.%d.%H%M%S')\n\nprint('Looking up space name')\nurl = parser.server_url + '/api/Spaces/' + parser.space_id\nheaders = {\n 'X-Octopus-ApiKey': parser.api_key,\n 'Accept': 'application/json'\n}\nrequest = urllib.request.Request(url, headers=headers)\n\n# Retry the request for up to a minute.\nresponse = None\nfor x in range(12):\n response = urllib.request.urlopen(request)\n if response.getcode() == 200:\n break\n time.sleep(5)\n\nif not response or not response.getcode() == 200:\n print('The API query failed')\n sys.exit(1)\n\ndata = json.loads(response.read().decode(\"utf-8\"))\nprint('Space name is ' + data['Name'])\n\nprint(\"Creating Terraform module package\")\nif is_windows():\n execute([os.path.join(octocli_path, 'octo.exe'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', data['Name']),\n '--version', date,\n '--basePath', os.getcwd() + '\\\\export',\n '--outFolder', os.getcwd()])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', data['Name']),\n '--version', date,\n '--basePath', os.getcwd() + '/export',\n '--outFolder', os.getcwd()])\n\nprint(\"Uploading Terraform module package\")\nif is_windows():\n _, _, _ = execute([os.path.join(octocli_path, 'octo.exe'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"\\\\\" +\n re.sub('[^0-9a-zA-Z]', '_', data['Name']) + '.' + date + '.zip',\n '--replace-existing'])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"/\" +\n re.sub('[^0-9a-zA-Z]', '_', data['Name']) + '.' + date + '.zip',\n '--replace-existing'])\n\nprint(\"##octopus[stdout-default]\")\n\nprint(\"Done\")\n", + "Octopus.Action.Script.ScriptBody": "import argparse\nimport os\nimport stat\nimport re\nimport socket\nimport subprocess\nimport sys\nfrom datetime import datetime\nfrom urllib.parse import urlparse\nfrom itertools import chain\nimport platform\nfrom urllib.request import urlretrieve\nimport zipfile\nimport urllib.request\nimport urllib.parse\nimport json\nimport tarfile\nimport random, time\n\n# If this script is not being run as part of an Octopus step, return variables from environment variables.\n# Periods are replaced with underscores, and the variable name is converted to uppercase\nif \"get_octopusvariable\" not in globals():\n def get_octopusvariable(variable):\n return os.environ[re.sub('\\\\.', '_', variable.upper())]\n\n# If this script is not being run as part of an Octopus step, print directly to std out.\nif \"printverbose\" not in globals():\n def printverbose(msg):\n print(msg)\n\n\ndef printverbose_noansi(output):\n \"\"\"\n Strip ANSI color codes and print the output as verbose\n :param output: The output to print\n \"\"\"\n if not output:\n return\n\n # https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python\n output_no_ansi = re.sub(r'\\x1B(?:[@-Z\\\\-_]|\\[[0-?]*[ -/]*[@-~])', '', output)\n printverbose(output_no_ansi)\n\n\ndef get_octopusvariable_quiet(variable):\n \"\"\"\n Gets an octopus variable, or an empty string if it does not exist.\n :param variable: The variable name\n :return: The variable value, or an empty string if the variable does not exist\n \"\"\"\n try:\n return get_octopusvariable(variable)\n except:\n return ''\n\n\ndef retry_with_backoff(fn, retries=5, backoff_in_seconds=1):\n x = 0\n while True:\n try:\n return fn()\n except Exception as e:\n\n print(e)\n\n if x == retries:\n raise\n\n sleep = (backoff_in_seconds * 2 ** x +\n random.uniform(0, 1))\n time.sleep(sleep)\n x += 1\n\n\ndef execute(args, cwd=None, env=None, print_args=None, print_output=printverbose_noansi):\n \"\"\"\n The execute method provides the ability to execute external processes while capturing and returning the\n output to std err and std out and exit code.\n \"\"\"\n process = subprocess.Popen(args,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n text=True,\n cwd=cwd,\n env=env)\n stdout, stderr = process.communicate()\n retcode = process.returncode\n\n if print_args is not None:\n print_output(' '.join(args))\n\n if print_output is not None:\n print_output(stdout)\n print_output(stderr)\n\n return stdout, stderr, retcode\n\n\ndef is_windows():\n return platform.system() == 'Windows'\n\n\ndef init_argparse():\n parser = argparse.ArgumentParser(\n usage='%(prog)s [OPTION] [FILE]...',\n description='Serialize an Octopus project to a Terraform module'\n )\n parser.add_argument('--terraform-backend',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Terraform.Backend') or get_octopusvariable_quiet(\n 'ThisInstance.Terraform.Backend') or 'pg',\n help='Set this to the name of the Terraform backend to be included in the generated module.')\n parser.add_argument('--server-url',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Server.Url') or get_octopusvariable_quiet(\n 'ThisInstance.Server.Url'),\n help='Sets the server URL that holds the project to be serialized.')\n parser.add_argument('--api-key',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Api.Key') or get_octopusvariable_quiet(\n 'ThisInstance.Api.Key'),\n help='Sets the Octopus API key.')\n parser.add_argument('--space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.Id') or get_octopusvariable_quiet(\n 'Exported.Space.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID containing the project to be serialized.')\n parser.add_argument('--upload-space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Octopus.UploadSpace.Id') or get_octopusvariable_quiet(\n 'Octopus.UploadSpace.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID of the Octopus space where ' +\n 'the resulting package will be uploaded to.')\n parser.add_argument('--ignored-library-variable-sets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredLibraryVariableSet') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredLibraryVariableSet'),\n help='A comma separated list of library variable sets to ignore.')\n\n parser.add_argument('--ignored-all-library-variable-sets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredAllLibraryVariableSet') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredAllLibraryVariableSet') or 'false',\n help='Set to true to exclude library variable sets from the exported module')\n\n parser.add_argument('--ignored-tenants',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredTenants') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredTenants'),\n help='A comma separated list of tenants ignore.')\n\n parser.add_argument('--ignored-tenants-with-tag',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredTenantTags') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredTenants'),\n help='A comma separated list of tenant tags that identify tenants to ignore.')\n parser.add_argument('--ignore-all-targets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoreTargets') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoreTargets') or 'false',\n help='Set to true to exclude targets from the exported module')\n\n parser.add_argument('--dummy-secret-variables',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.DummySecrets') or get_octopusvariable_quiet(\n 'Exported.Space.DummySecrets') or 'false',\n help='Set to true to set secret values, like account and feed passwords, to a dummy value by default')\n\n parser.add_argument('--default-secret-variables',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.DefaultSecrets') or get_octopusvariable_quiet(\n 'Exported.Space.DefaultSecrets') or 'false',\n help='Set to true to set sensitive variables to the octostache template that represents the variable')\n parser.add_argument('--include-step-templates',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IncludeStepTemplates') or get_octopusvariable_quiet(\n 'Exported.Space.IncludeStepTemplates') or 'false',\n help='Set this to true to include step templates in the exported module. ' +\n 'This disables the default behaviour of detaching step templates.')\n parser.add_argument('--ignored-accounts',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredAccounts') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredAccounts'),\n help='A comma separated list of accounts to ignore.')\n\n return parser.parse_known_args()\n\n\ndef get_latest_github_release(owner, repo, filename):\n url = f\"https://api.github.com/repos/{owner}/{repo}/releases/latest\"\n releases = urllib.request.urlopen(url).read()\n contents = json.loads(releases)\n\n download = [asset for asset in contents.get('assets') if asset.get('name') == filename]\n\n if len(download) != 0:\n return download[0].get('browser_download_url')\n\n return None\n\n\ndef ensure_octo_cli_exists():\n if is_windows():\n print(\"Checking for the Octopus CLI\")\n try:\n stdout, _, exit_code = execute(['octo.exe', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.win-x64.zip',\n 'OctopusTools.zip')\n with zipfile.ZipFile('OctopusTools.zip', 'r') as zip_ref:\n zip_ref.extractall(os.getcwd())\n return os.getcwd()\n else:\n print(\"Checking for the Octopus CLI for Linux\")\n try:\n stdout, _, exit_code = execute(['octo', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI for Linux\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.linux-x64.tar.gz',\n 'OctopusTools.tar.gz')\n with tarfile.open('OctopusTools.tar.gz') as file:\n file.extractall(os.getcwd())\n os.chmod(os.path.join(os.getcwd(), 'octo'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\ndef ensure_octoterra_exists():\n if is_windows():\n print(\"Checking for the Octoterra tool for Windows\")\n try:\n stdout, _, exit_code = execute(['octoterra.exe', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Windows\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_windows_amd64.exe\",\n 'octoterra.exe'), 10, 30)\n return os.getcwd()\n else:\n print(\"Checking for the Octoterra tool for Linux\")\n try:\n stdout, _, exit_code = execute(['octoterra', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Linux\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_linux_amd64\",\n 'octoterra'), 10, 30)\n os.chmod(os.path.join(os.getcwd(), 'octoterra'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\noctocli_path = ensure_octo_cli_exists()\noctoterra_path = ensure_octoterra_exists()\nparser, _ = init_argparse()\n\n# Variable precondition checks\nif len(parser.server_url) == 0:\n print(\"--server-url, ThisInstance.Server.Url, or SerializeSpace.ThisInstance.Server.Url must be defined\")\n sys.exit(1)\n\nif len(parser.api_key) == 0:\n print(\"--api-key, ThisInstance.Api.Key, or SerializeSpace.ThisInstance.Api.Key must be defined\")\n sys.exit(1)\n\n\nprint(\"Octopus URL: \" + parser.server_url)\nprint(\"Octopus Space ID: \" + parser.space_id)\n\n# Build the arguments to ignore library variable sets\nignores_library_variable_sets = parser.ignored_library_variable_sets.split(',')\nignores_library_variable_sets_args = [['-excludeLibraryVariableSetRegex', x] for x in ignores_library_variable_sets if\n x.strip() != '']\n\n# Build the arguments to ignore tenants\nignores_tenants = parser.ignored_tenants.split(',')\nignores_tenants_args = [['-excludeTenants', x] for x in ignores_tenants if x.strip() != '']\n\n# Build the arguments to ignore tenants with tags\nignored_tenants_with_tag = parser.ignored_tenants_with_tag.split(',')\nignored_tenants_with_tag_args = [['-excludeTenantsWithTag', x] for x in ignored_tenants_with_tag if x.strip() != '']\n\n# Build the arguments to ignore accounts\nignored_accounts = parser.ignored_accounts.split(',')\nignored_accounts = [['-excludeAccountsRegex', x] for x in ignored_accounts]\n\nos.mkdir(os.getcwd() + '/export')\n\nexport_args = [os.path.join(octoterra_path, 'octoterra'),\n # the url of the instance\n '-url', parser.server_url,\n # the api key used to access the instance\n '-apiKey', parser.api_key,\n # add a postgres backend to the generated modules\n '-terraformBackend', parser.terraform_backend,\n # dump the generated HCL to the console\n '-console',\n # dump the project from the current space\n '-space', parser.space_id,\n # Use default dummy values for secrets (e.g. a feed password). These values can still be overridden if known,\n # but allows the module to be deployed and have the secrets updated manually later.\n '-dummySecretVariableValues=' + parser.dummy_secret_variables,\n # for any secret variables, add a default value set to the octostache value of the variable\n # e.g. a secret variable called \"database\" has a default value of \"#{database}\"\n '-defaultSecretVariableValues=' + parser.default_secret_variables,\n # Add support for experimental step templates\n '-experimentalEnableStepTemplates=' + parser.include_step_templates,\n # Don't export any projects\n '-excludeAllProjects',\n # Output variables allow the Octopus space and instance to be determined from the Terraform state file.\n '-includeOctopusOutputVars',\n # Provide an option to ignore targets.\n '-excludeAllTargets=' + parser.ignore_all_targets,\n # The directory where the exported files will be saved\n '-dest', os.getcwd() + '/export'] + list(\n chain(*ignores_library_variable_sets_args, *ignores_tenants_args, *ignored_tenants_with_tag_args,\n *ignored_accounts))\n\nprint(\"Exporting Terraform module\")\n_, _, octoterra_exit = execute(export_args)\n\nif not octoterra_exit == 0:\n print(\"Octoterra failed. Please check the verbose logs for more information.\")\n sys.exit(1)\n\ndate = datetime.now().strftime('%Y.%m.%d.%H%M%S')\n\nprint('Looking up space name')\nurl = parser.server_url + '/api/Spaces/' + parser.space_id\nheaders = {\n 'X-Octopus-ApiKey': parser.api_key,\n 'Accept': 'application/json'\n}\nrequest = urllib.request.Request(url, headers=headers)\n\n# Retry the request for up to a minute.\nresponse = None\nfor x in range(12):\n response = urllib.request.urlopen(request)\n if response.getcode() == 200:\n break\n time.sleep(5)\n\nif not response or not response.getcode() == 200:\n print('The API query failed')\n sys.exit(1)\n\ndata = json.loads(response.read().decode(\"utf-8\"))\nprint('Space name is ' + data['Name'])\n\nprint(\"Creating Terraform module package\")\nif is_windows():\n execute([os.path.join(octocli_path, 'octo.exe'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', data['Name']),\n '--version', date,\n '--basePath', os.getcwd() + '\\\\export',\n '--outFolder', os.getcwd()])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', data['Name']),\n '--version', date,\n '--basePath', os.getcwd() + '/export',\n '--outFolder', os.getcwd()])\n\nprint(\"Uploading Terraform module package\")\nif is_windows():\n _, _, _ = execute([os.path.join(octocli_path, 'octo.exe'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"\\\\\" +\n re.sub('[^0-9a-zA-Z]', '_', data['Name']) + '.' + date + '.zip',\n '--replace-existing'])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"/\" +\n re.sub('[^0-9a-zA-Z]', '_', data['Name']) + '.' + date + '.zip',\n '--replace-existing'])\n\nprint(\"##octopus[stdout-default]\")\n\nprint(\"Done\")\n", "Octopus.Action.Script.ScriptSource": "Inline", "Octopus.Action.Script.Syntax": "Python" }, @@ -73,6 +73,16 @@ "Octopus.ControlType": "SingleLineText" } }, + { + "Id": "b946a235-a1c1-4b04-83da-ecef622c018a", + "Name": "IgnoredAllLibraryVariableSet", + "Label": "Ignore All Library Variable Sets", + "HelpText": "Check this option to ignore all library variable sets when serializing the Terraform module. This is useful when the downstream space already has library variable sets created.", + "DefaultValue": "True", + "DisplaySettings": { + "Octopus.ControlType": "Checkbox" + } + }, { "Id": "1208c54c-568a-4a48-9340-fbff710079b3", "Name": "SerializeSpace.Exported.Space.IgnoredTenants", From 3ca43f2bffe8d61ca547d726ef41de42ed54039f Mon Sep 17 00:00:00 2001 From: Matthew Casperson Date: Tue, 18 Feb 2025 11:44:39 +1000 Subject: [PATCH 2/3] Fixed parameter --- step-templates/octopus-serialize-space-to-terraform.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/step-templates/octopus-serialize-space-to-terraform.json b/step-templates/octopus-serialize-space-to-terraform.json index 39386991f..d658fb7c1 100644 --- a/step-templates/octopus-serialize-space-to-terraform.json +++ b/step-templates/octopus-serialize-space-to-terraform.json @@ -75,7 +75,7 @@ }, { "Id": "b946a235-a1c1-4b04-83da-ecef622c018a", - "Name": "IgnoredAllLibraryVariableSet", + "Name": "SerializeSpace.Exported.Space.IgnoredAllLibraryVariableSet", "Label": "Ignore All Library Variable Sets", "HelpText": "Check this option to ignore all library variable sets when serializing the Terraform module. This is useful when the downstream space already has library variable sets created.", "DefaultValue": "True", From 9d600c1f70f35da1847b74367711417af67f837e Mon Sep 17 00:00:00 2001 From: Matthew Casperson Date: Tue, 18 Feb 2025 11:59:26 +1000 Subject: [PATCH 3/3] Passed option to octoterra --- step-templates/octopus-serialize-space-to-terraform.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/step-templates/octopus-serialize-space-to-terraform.json b/step-templates/octopus-serialize-space-to-terraform.json index d658fb7c1..f891f001c 100644 --- a/step-templates/octopus-serialize-space-to-terraform.json +++ b/step-templates/octopus-serialize-space-to-terraform.json @@ -8,7 +8,7 @@ "Packages": [], "Properties": { "Octopus.Action.RunOnServer": "true", - "Octopus.Action.Script.ScriptBody": "import argparse\nimport os\nimport stat\nimport re\nimport socket\nimport subprocess\nimport sys\nfrom datetime import datetime\nfrom urllib.parse import urlparse\nfrom itertools import chain\nimport platform\nfrom urllib.request import urlretrieve\nimport zipfile\nimport urllib.request\nimport urllib.parse\nimport json\nimport tarfile\nimport random, time\n\n# If this script is not being run as part of an Octopus step, return variables from environment variables.\n# Periods are replaced with underscores, and the variable name is converted to uppercase\nif \"get_octopusvariable\" not in globals():\n def get_octopusvariable(variable):\n return os.environ[re.sub('\\\\.', '_', variable.upper())]\n\n# If this script is not being run as part of an Octopus step, print directly to std out.\nif \"printverbose\" not in globals():\n def printverbose(msg):\n print(msg)\n\n\ndef printverbose_noansi(output):\n \"\"\"\n Strip ANSI color codes and print the output as verbose\n :param output: The output to print\n \"\"\"\n if not output:\n return\n\n # https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python\n output_no_ansi = re.sub(r'\\x1B(?:[@-Z\\\\-_]|\\[[0-?]*[ -/]*[@-~])', '', output)\n printverbose(output_no_ansi)\n\n\ndef get_octopusvariable_quiet(variable):\n \"\"\"\n Gets an octopus variable, or an empty string if it does not exist.\n :param variable: The variable name\n :return: The variable value, or an empty string if the variable does not exist\n \"\"\"\n try:\n return get_octopusvariable(variable)\n except:\n return ''\n\n\ndef retry_with_backoff(fn, retries=5, backoff_in_seconds=1):\n x = 0\n while True:\n try:\n return fn()\n except Exception as e:\n\n print(e)\n\n if x == retries:\n raise\n\n sleep = (backoff_in_seconds * 2 ** x +\n random.uniform(0, 1))\n time.sleep(sleep)\n x += 1\n\n\ndef execute(args, cwd=None, env=None, print_args=None, print_output=printverbose_noansi):\n \"\"\"\n The execute method provides the ability to execute external processes while capturing and returning the\n output to std err and std out and exit code.\n \"\"\"\n process = subprocess.Popen(args,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n text=True,\n cwd=cwd,\n env=env)\n stdout, stderr = process.communicate()\n retcode = process.returncode\n\n if print_args is not None:\n print_output(' '.join(args))\n\n if print_output is not None:\n print_output(stdout)\n print_output(stderr)\n\n return stdout, stderr, retcode\n\n\ndef is_windows():\n return platform.system() == 'Windows'\n\n\ndef init_argparse():\n parser = argparse.ArgumentParser(\n usage='%(prog)s [OPTION] [FILE]...',\n description='Serialize an Octopus project to a Terraform module'\n )\n parser.add_argument('--terraform-backend',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Terraform.Backend') or get_octopusvariable_quiet(\n 'ThisInstance.Terraform.Backend') or 'pg',\n help='Set this to the name of the Terraform backend to be included in the generated module.')\n parser.add_argument('--server-url',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Server.Url') or get_octopusvariable_quiet(\n 'ThisInstance.Server.Url'),\n help='Sets the server URL that holds the project to be serialized.')\n parser.add_argument('--api-key',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Api.Key') or get_octopusvariable_quiet(\n 'ThisInstance.Api.Key'),\n help='Sets the Octopus API key.')\n parser.add_argument('--space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.Id') or get_octopusvariable_quiet(\n 'Exported.Space.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID containing the project to be serialized.')\n parser.add_argument('--upload-space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Octopus.UploadSpace.Id') or get_octopusvariable_quiet(\n 'Octopus.UploadSpace.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID of the Octopus space where ' +\n 'the resulting package will be uploaded to.')\n parser.add_argument('--ignored-library-variable-sets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredLibraryVariableSet') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredLibraryVariableSet'),\n help='A comma separated list of library variable sets to ignore.')\n\n parser.add_argument('--ignored-all-library-variable-sets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredAllLibraryVariableSet') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredAllLibraryVariableSet') or 'false',\n help='Set to true to exclude library variable sets from the exported module')\n\n parser.add_argument('--ignored-tenants',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredTenants') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredTenants'),\n help='A comma separated list of tenants ignore.')\n\n parser.add_argument('--ignored-tenants-with-tag',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredTenantTags') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredTenants'),\n help='A comma separated list of tenant tags that identify tenants to ignore.')\n parser.add_argument('--ignore-all-targets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoreTargets') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoreTargets') or 'false',\n help='Set to true to exclude targets from the exported module')\n\n parser.add_argument('--dummy-secret-variables',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.DummySecrets') or get_octopusvariable_quiet(\n 'Exported.Space.DummySecrets') or 'false',\n help='Set to true to set secret values, like account and feed passwords, to a dummy value by default')\n\n parser.add_argument('--default-secret-variables',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.DefaultSecrets') or get_octopusvariable_quiet(\n 'Exported.Space.DefaultSecrets') or 'false',\n help='Set to true to set sensitive variables to the octostache template that represents the variable')\n parser.add_argument('--include-step-templates',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IncludeStepTemplates') or get_octopusvariable_quiet(\n 'Exported.Space.IncludeStepTemplates') or 'false',\n help='Set this to true to include step templates in the exported module. ' +\n 'This disables the default behaviour of detaching step templates.')\n parser.add_argument('--ignored-accounts',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredAccounts') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredAccounts'),\n help='A comma separated list of accounts to ignore.')\n\n return parser.parse_known_args()\n\n\ndef get_latest_github_release(owner, repo, filename):\n url = f\"https://api.github.com/repos/{owner}/{repo}/releases/latest\"\n releases = urllib.request.urlopen(url).read()\n contents = json.loads(releases)\n\n download = [asset for asset in contents.get('assets') if asset.get('name') == filename]\n\n if len(download) != 0:\n return download[0].get('browser_download_url')\n\n return None\n\n\ndef ensure_octo_cli_exists():\n if is_windows():\n print(\"Checking for the Octopus CLI\")\n try:\n stdout, _, exit_code = execute(['octo.exe', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.win-x64.zip',\n 'OctopusTools.zip')\n with zipfile.ZipFile('OctopusTools.zip', 'r') as zip_ref:\n zip_ref.extractall(os.getcwd())\n return os.getcwd()\n else:\n print(\"Checking for the Octopus CLI for Linux\")\n try:\n stdout, _, exit_code = execute(['octo', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI for Linux\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.linux-x64.tar.gz',\n 'OctopusTools.tar.gz')\n with tarfile.open('OctopusTools.tar.gz') as file:\n file.extractall(os.getcwd())\n os.chmod(os.path.join(os.getcwd(), 'octo'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\ndef ensure_octoterra_exists():\n if is_windows():\n print(\"Checking for the Octoterra tool for Windows\")\n try:\n stdout, _, exit_code = execute(['octoterra.exe', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Windows\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_windows_amd64.exe\",\n 'octoterra.exe'), 10, 30)\n return os.getcwd()\n else:\n print(\"Checking for the Octoterra tool for Linux\")\n try:\n stdout, _, exit_code = execute(['octoterra', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Linux\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_linux_amd64\",\n 'octoterra'), 10, 30)\n os.chmod(os.path.join(os.getcwd(), 'octoterra'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\noctocli_path = ensure_octo_cli_exists()\noctoterra_path = ensure_octoterra_exists()\nparser, _ = init_argparse()\n\n# Variable precondition checks\nif len(parser.server_url) == 0:\n print(\"--server-url, ThisInstance.Server.Url, or SerializeSpace.ThisInstance.Server.Url must be defined\")\n sys.exit(1)\n\nif len(parser.api_key) == 0:\n print(\"--api-key, ThisInstance.Api.Key, or SerializeSpace.ThisInstance.Api.Key must be defined\")\n sys.exit(1)\n\n\nprint(\"Octopus URL: \" + parser.server_url)\nprint(\"Octopus Space ID: \" + parser.space_id)\n\n# Build the arguments to ignore library variable sets\nignores_library_variable_sets = parser.ignored_library_variable_sets.split(',')\nignores_library_variable_sets_args = [['-excludeLibraryVariableSetRegex', x] for x in ignores_library_variable_sets if\n x.strip() != '']\n\n# Build the arguments to ignore tenants\nignores_tenants = parser.ignored_tenants.split(',')\nignores_tenants_args = [['-excludeTenants', x] for x in ignores_tenants if x.strip() != '']\n\n# Build the arguments to ignore tenants with tags\nignored_tenants_with_tag = parser.ignored_tenants_with_tag.split(',')\nignored_tenants_with_tag_args = [['-excludeTenantsWithTag', x] for x in ignored_tenants_with_tag if x.strip() != '']\n\n# Build the arguments to ignore accounts\nignored_accounts = parser.ignored_accounts.split(',')\nignored_accounts = [['-excludeAccountsRegex', x] for x in ignored_accounts]\n\nos.mkdir(os.getcwd() + '/export')\n\nexport_args = [os.path.join(octoterra_path, 'octoterra'),\n # the url of the instance\n '-url', parser.server_url,\n # the api key used to access the instance\n '-apiKey', parser.api_key,\n # add a postgres backend to the generated modules\n '-terraformBackend', parser.terraform_backend,\n # dump the generated HCL to the console\n '-console',\n # dump the project from the current space\n '-space', parser.space_id,\n # Use default dummy values for secrets (e.g. a feed password). These values can still be overridden if known,\n # but allows the module to be deployed and have the secrets updated manually later.\n '-dummySecretVariableValues=' + parser.dummy_secret_variables,\n # for any secret variables, add a default value set to the octostache value of the variable\n # e.g. a secret variable called \"database\" has a default value of \"#{database}\"\n '-defaultSecretVariableValues=' + parser.default_secret_variables,\n # Add support for experimental step templates\n '-experimentalEnableStepTemplates=' + parser.include_step_templates,\n # Don't export any projects\n '-excludeAllProjects',\n # Output variables allow the Octopus space and instance to be determined from the Terraform state file.\n '-includeOctopusOutputVars',\n # Provide an option to ignore targets.\n '-excludeAllTargets=' + parser.ignore_all_targets,\n # The directory where the exported files will be saved\n '-dest', os.getcwd() + '/export'] + list(\n chain(*ignores_library_variable_sets_args, *ignores_tenants_args, *ignored_tenants_with_tag_args,\n *ignored_accounts))\n\nprint(\"Exporting Terraform module\")\n_, _, octoterra_exit = execute(export_args)\n\nif not octoterra_exit == 0:\n print(\"Octoterra failed. Please check the verbose logs for more information.\")\n sys.exit(1)\n\ndate = datetime.now().strftime('%Y.%m.%d.%H%M%S')\n\nprint('Looking up space name')\nurl = parser.server_url + '/api/Spaces/' + parser.space_id\nheaders = {\n 'X-Octopus-ApiKey': parser.api_key,\n 'Accept': 'application/json'\n}\nrequest = urllib.request.Request(url, headers=headers)\n\n# Retry the request for up to a minute.\nresponse = None\nfor x in range(12):\n response = urllib.request.urlopen(request)\n if response.getcode() == 200:\n break\n time.sleep(5)\n\nif not response or not response.getcode() == 200:\n print('The API query failed')\n sys.exit(1)\n\ndata = json.loads(response.read().decode(\"utf-8\"))\nprint('Space name is ' + data['Name'])\n\nprint(\"Creating Terraform module package\")\nif is_windows():\n execute([os.path.join(octocli_path, 'octo.exe'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', data['Name']),\n '--version', date,\n '--basePath', os.getcwd() + '\\\\export',\n '--outFolder', os.getcwd()])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', data['Name']),\n '--version', date,\n '--basePath', os.getcwd() + '/export',\n '--outFolder', os.getcwd()])\n\nprint(\"Uploading Terraform module package\")\nif is_windows():\n _, _, _ = execute([os.path.join(octocli_path, 'octo.exe'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"\\\\\" +\n re.sub('[^0-9a-zA-Z]', '_', data['Name']) + '.' + date + '.zip',\n '--replace-existing'])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"/\" +\n re.sub('[^0-9a-zA-Z]', '_', data['Name']) + '.' + date + '.zip',\n '--replace-existing'])\n\nprint(\"##octopus[stdout-default]\")\n\nprint(\"Done\")\n", + "Octopus.Action.Script.ScriptBody": "import argparse\nimport os\nimport stat\nimport re\nimport socket\nimport subprocess\nimport sys\nfrom datetime import datetime\nfrom urllib.parse import urlparse\nfrom itertools import chain\nimport platform\nfrom urllib.request import urlretrieve\nimport zipfile\nimport urllib.request\nimport urllib.parse\nimport json\nimport tarfile\nimport random, time\n\n# If this script is not being run as part of an Octopus step, return variables from environment variables.\n# Periods are replaced with underscores, and the variable name is converted to uppercase\nif \"get_octopusvariable\" not in globals():\n def get_octopusvariable(variable):\n return os.environ[re.sub('\\\\.', '_', variable.upper())]\n\n# If this script is not being run as part of an Octopus step, print directly to std out.\nif \"printverbose\" not in globals():\n def printverbose(msg):\n print(msg)\n\n\ndef printverbose_noansi(output):\n \"\"\"\n Strip ANSI color codes and print the output as verbose\n :param output: The output to print\n \"\"\"\n if not output:\n return\n\n # https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python\n output_no_ansi = re.sub(r'\\x1B(?:[@-Z\\\\-_]|\\[[0-?]*[ -/]*[@-~])', '', output)\n printverbose(output_no_ansi)\n\n\ndef get_octopusvariable_quiet(variable):\n \"\"\"\n Gets an octopus variable, or an empty string if it does not exist.\n :param variable: The variable name\n :return: The variable value, or an empty string if the variable does not exist\n \"\"\"\n try:\n return get_octopusvariable(variable)\n except:\n return ''\n\n\ndef retry_with_backoff(fn, retries=5, backoff_in_seconds=1):\n x = 0\n while True:\n try:\n return fn()\n except Exception as e:\n\n print(e)\n\n if x == retries:\n raise\n\n sleep = (backoff_in_seconds * 2 ** x +\n random.uniform(0, 1))\n time.sleep(sleep)\n x += 1\n\n\ndef execute(args, cwd=None, env=None, print_args=None, print_output=printverbose_noansi):\n \"\"\"\n The execute method provides the ability to execute external processes while capturing and returning the\n output to std err and std out and exit code.\n \"\"\"\n process = subprocess.Popen(args,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n text=True,\n cwd=cwd,\n env=env)\n stdout, stderr = process.communicate()\n retcode = process.returncode\n\n if print_args is not None:\n print_output(' '.join(args))\n\n if print_output is not None:\n print_output(stdout)\n print_output(stderr)\n\n return stdout, stderr, retcode\n\n\ndef is_windows():\n return platform.system() == 'Windows'\n\n\ndef init_argparse():\n parser = argparse.ArgumentParser(\n usage='%(prog)s [OPTION] [FILE]...',\n description='Serialize an Octopus project to a Terraform module'\n )\n parser.add_argument('--terraform-backend',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Terraform.Backend') or get_octopusvariable_quiet(\n 'ThisInstance.Terraform.Backend') or 'pg',\n help='Set this to the name of the Terraform backend to be included in the generated module.')\n parser.add_argument('--server-url',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Server.Url') or get_octopusvariable_quiet(\n 'ThisInstance.Server.Url'),\n help='Sets the server URL that holds the project to be serialized.')\n parser.add_argument('--api-key',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.ThisInstance.Api.Key') or get_octopusvariable_quiet(\n 'ThisInstance.Api.Key'),\n help='Sets the Octopus API key.')\n parser.add_argument('--space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.Id') or get_octopusvariable_quiet(\n 'Exported.Space.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID containing the project to be serialized.')\n parser.add_argument('--upload-space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Octopus.UploadSpace.Id') or get_octopusvariable_quiet(\n 'Octopus.UploadSpace.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID of the Octopus space where ' +\n 'the resulting package will be uploaded to.')\n parser.add_argument('--ignored-library-variable-sets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredLibraryVariableSet') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredLibraryVariableSet'),\n help='A comma separated list of library variable sets to ignore.')\n\n parser.add_argument('--ignored-all-library-variable-sets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredAllLibraryVariableSet') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredAllLibraryVariableSet') or 'false',\n help='Set to true to exclude library variable sets from the exported module')\n\n parser.add_argument('--ignored-tenants',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredTenants') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredTenants'),\n help='A comma separated list of tenants ignore.')\n\n parser.add_argument('--ignored-tenants-with-tag',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredTenantTags') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredTenants'),\n help='A comma separated list of tenant tags that identify tenants to ignore.')\n parser.add_argument('--ignore-all-targets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoreTargets') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoreTargets') or 'false',\n help='Set to true to exclude targets from the exported module')\n\n parser.add_argument('--dummy-secret-variables',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.DummySecrets') or get_octopusvariable_quiet(\n 'Exported.Space.DummySecrets') or 'false',\n help='Set to true to set secret values, like account and feed passwords, to a dummy value by default')\n\n parser.add_argument('--default-secret-variables',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.DefaultSecrets') or get_octopusvariable_quiet(\n 'Exported.Space.DefaultSecrets') or 'false',\n help='Set to true to set sensitive variables to the octostache template that represents the variable')\n parser.add_argument('--include-step-templates',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IncludeStepTemplates') or get_octopusvariable_quiet(\n 'Exported.Space.IncludeStepTemplates') or 'false',\n help='Set this to true to include step templates in the exported module. ' +\n 'This disables the default behaviour of detaching step templates.')\n parser.add_argument('--ignored-accounts',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeSpace.Exported.Space.IgnoredAccounts') or get_octopusvariable_quiet(\n 'Exported.Space.IgnoredAccounts'),\n help='A comma separated list of accounts to ignore.')\n\n return parser.parse_known_args()\n\n\ndef get_latest_github_release(owner, repo, filename):\n url = f\"https://api.github.com/repos/{owner}/{repo}/releases/latest\"\n releases = urllib.request.urlopen(url).read()\n contents = json.loads(releases)\n\n download = [asset for asset in contents.get('assets') if asset.get('name') == filename]\n\n if len(download) != 0:\n return download[0].get('browser_download_url')\n\n return None\n\n\ndef ensure_octo_cli_exists():\n if is_windows():\n print(\"Checking for the Octopus CLI\")\n try:\n stdout, _, exit_code = execute(['octo.exe', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.win-x64.zip',\n 'OctopusTools.zip')\n with zipfile.ZipFile('OctopusTools.zip', 'r') as zip_ref:\n zip_ref.extractall(os.getcwd())\n return os.getcwd()\n else:\n print(\"Checking for the Octopus CLI for Linux\")\n try:\n stdout, _, exit_code = execute(['octo', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI for Linux\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.linux-x64.tar.gz',\n 'OctopusTools.tar.gz')\n with tarfile.open('OctopusTools.tar.gz') as file:\n file.extractall(os.getcwd())\n os.chmod(os.path.join(os.getcwd(), 'octo'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\ndef ensure_octoterra_exists():\n if is_windows():\n print(\"Checking for the Octoterra tool for Windows\")\n try:\n stdout, _, exit_code = execute(['octoterra.exe', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Windows\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_windows_amd64.exe\",\n 'octoterra.exe'), 10, 30)\n return os.getcwd()\n else:\n print(\"Checking for the Octoterra tool for Linux\")\n try:\n stdout, _, exit_code = execute(['octoterra', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Linux\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_linux_amd64\",\n 'octoterra'), 10, 30)\n os.chmod(os.path.join(os.getcwd(), 'octoterra'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\noctocli_path = ensure_octo_cli_exists()\noctoterra_path = ensure_octoterra_exists()\nparser, _ = init_argparse()\n\n# Variable precondition checks\nif len(parser.server_url) == 0:\n print(\"--server-url, ThisInstance.Server.Url, or SerializeSpace.ThisInstance.Server.Url must be defined\")\n sys.exit(1)\n\nif len(parser.api_key) == 0:\n print(\"--api-key, ThisInstance.Api.Key, or SerializeSpace.ThisInstance.Api.Key must be defined\")\n sys.exit(1)\n\n\nprint(\"Octopus URL: \" + parser.server_url)\nprint(\"Octopus Space ID: \" + parser.space_id)\n\n# Build the arguments to ignore library variable sets\nignores_library_variable_sets = parser.ignored_library_variable_sets.split(',')\nignores_library_variable_sets_args = [['-excludeLibraryVariableSetRegex', x] for x in ignores_library_variable_sets if\n x.strip() != '']\n\n# Build the arguments to ignore tenants\nignores_tenants = parser.ignored_tenants.split(',')\nignores_tenants_args = [['-excludeTenants', x] for x in ignores_tenants if x.strip() != '']\n\n# Build the arguments to ignore tenants with tags\nignored_tenants_with_tag = parser.ignored_tenants_with_tag.split(',')\nignored_tenants_with_tag_args = [['-excludeTenantsWithTag', x] for x in ignored_tenants_with_tag if x.strip() != '']\n\n# Build the arguments to ignore accounts\nignored_accounts = parser.ignored_accounts.split(',')\nignored_accounts = [['-excludeAccountsRegex', x] for x in ignored_accounts]\n\nos.mkdir(os.getcwd() + '/export')\n\nexport_args = [os.path.join(octoterra_path, 'octoterra'),\n # the url of the instance\n '-url', parser.server_url,\n # the api key used to access the instance\n '-apiKey', parser.api_key,\n # add a postgres backend to the generated modules\n '-terraformBackend', parser.terraform_backend,\n # dump the generated HCL to the console\n '-console',\n # dump the project from the current space\n '-space', parser.space_id,\n # Use default dummy values for secrets (e.g. a feed password). These values can still be overridden if known,\n # but allows the module to be deployed and have the secrets updated manually later.\n '-dummySecretVariableValues=' + parser.dummy_secret_variables,\n # for any secret variables, add a default value set to the octostache value of the variable\n # e.g. a secret variable called \"database\" has a default value of \"#{database}\"\n '-defaultSecretVariableValues=' + parser.default_secret_variables,\n # Add support for experimental step templates\n '-experimentalEnableStepTemplates=' + parser.include_step_templates,\n # Don't export any projects\n '-excludeAllProjects',\n # Output variables allow the Octopus space and instance to be determined from the Terraform state file.\n '-includeOctopusOutputVars',\n # Provide an option to ignore targets.\n '-excludeAllTargets=' + parser.ignore_all_targets,\n # Provide an option to exclude all library variable sets\n '-excludeAllLibraryVariableSets=' + parser.ignored_all_library_variable_sets,\n # The directory where the exported files will be saved\n '-dest', os.getcwd() + '/export'] + list(\n chain(*ignores_library_variable_sets_args, *ignores_tenants_args, *ignored_tenants_with_tag_args,\n *ignored_accounts))\n\nprint(\"Exporting Terraform module\")\n_, _, octoterra_exit = execute(export_args)\n\nif not octoterra_exit == 0:\n print(\"Octoterra failed. Please check the verbose logs for more information.\")\n sys.exit(1)\n\ndate = datetime.now().strftime('%Y.%m.%d.%H%M%S')\n\nprint('Looking up space name')\nurl = parser.server_url + '/api/Spaces/' + parser.space_id\nheaders = {\n 'X-Octopus-ApiKey': parser.api_key,\n 'Accept': 'application/json'\n}\nrequest = urllib.request.Request(url, headers=headers)\n\n# Retry the request for up to a minute.\nresponse = None\nfor x in range(12):\n response = urllib.request.urlopen(request)\n if response.getcode() == 200:\n break\n time.sleep(5)\n\nif not response or not response.getcode() == 200:\n print('The API query failed')\n sys.exit(1)\n\ndata = json.loads(response.read().decode(\"utf-8\"))\nprint('Space name is ' + data['Name'])\n\nprint(\"Creating Terraform module package\")\nif is_windows():\n execute([os.path.join(octocli_path, 'octo.exe'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', data['Name']),\n '--version', date,\n '--basePath', os.getcwd() + '\\\\export',\n '--outFolder', os.getcwd()])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', data['Name']),\n '--version', date,\n '--basePath', os.getcwd() + '/export',\n '--outFolder', os.getcwd()])\n\nprint(\"Uploading Terraform module package\")\nif is_windows():\n _, _, _ = execute([os.path.join(octocli_path, 'octo.exe'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"\\\\\" +\n re.sub('[^0-9a-zA-Z]', '_', data['Name']) + '.' + date + '.zip',\n '--replace-existing'])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"/\" +\n re.sub('[^0-9a-zA-Z]', '_', data['Name']) + '.' + date + '.zip',\n '--replace-existing'])\n\nprint(\"##octopus[stdout-default]\")\n\nprint(\"Done\")\n", "Octopus.Action.Script.ScriptSource": "Inline", "Octopus.Action.Script.Syntax": "Python" },