diff --git a/.asf.yaml b/.asf.yaml index 2b2708bb24..27a374ff3e 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -44,9 +44,13 @@ github: strict: true # contexts are the names of checks that must pass contexts: + - "Unit Tests (Python 3.9)" - "Unit Tests (Python 3.10)" - "Unit Tests (Python 3.11)" - "Unit Tests (Python 3.12)" + - "Unit Tests (Python 3.13)" + - "Run Various Lint and Other Checks (3.9)" + - "Build and upload Documentation (3.9)" - "Dependency Review" notifications: diff --git a/.github/workflows/install_test.yml b/.github/workflows/install_test.yml index 3283111e72..7a2a0dd2d7 100644 --- a/.github/workflows/install_test.yml +++ b/.github/workflows/install_test.yml @@ -28,6 +28,8 @@ jobs: - "3.10" - "3.11" - "pypy-3.7" + - "pypy-3.8" + - "pypy-3.9" include: # python 3.6 is not supported with ubuntu-latest anymore so we need to # use ubuntu 20.04 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 048c691c69..f2140c7a55 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -32,7 +32,7 @@ jobs: strategy: matrix: - python_version: [3.8] + python_version: [3.9] steps: - uses: actions/checkout@master diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3afddcd6ac..b766c6cd9d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,13 +32,14 @@ jobs: fail-fast: false matrix: python_version: - - 3.8 - - 3.9 + - "3.9" - "3.10" - "3.11" - "3.12" + - "3.13" # cryptography is not compatible with older PyPy versions - - "pypy-3.8" + - "pypy-3.9" + - "pypy-3.10" os: - ubuntu-latest @@ -74,8 +75,7 @@ jobs: tox -e py${{ matrix.python_version }} - name: Run dist install checks tox target - # NOTE: 3.12 will be failing until we migrate away from setup.py - if: ${{ matrix.python_version != 'pypy-3.7' && matrix.python_version != 'pypy-3.8' && matrix.python_version != '3.12-dev' }} + if: ${{ matrix.python_version != 'pypy-3.9' && matrix.python_version != 'pypy-3.10' }} run: | tox -e py${{ matrix.python_version }}-dist,py${{ matrix.python_version }}-dist-wheel @@ -85,7 +85,7 @@ jobs: strategy: matrix: - python_version: [3.8] + python_version: [3.9] steps: - uses: actions/checkout@master @@ -133,7 +133,7 @@ jobs: strategy: matrix: - python_version: [3.8] + python_version: [3.9] steps: - uses: actions/checkout@master @@ -176,7 +176,7 @@ jobs: strategy: matrix: - python_version: [3.8] + python_version: [3.9] steps: - uses: actions/checkout@master @@ -200,7 +200,7 @@ jobs: strategy: matrix: - python_version: [3.8] + python_version: [3.9] steps: - uses: actions/checkout@master @@ -268,7 +268,7 @@ jobs: strategy: matrix: - python_version: [3.8] + python_version: [3.9] steps: - uses: actions/checkout@master @@ -307,7 +307,7 @@ jobs: strategy: matrix: - python_version: [3.8] + python_version: [3.9] steps: - name: Print Environment Info diff --git a/.github/workflows/publish_dev_artifact.yml b/.github/workflows/publish_dev_artifact.yml index 5bbf828029..a155a8a428 100644 --- a/.github/workflows/publish_dev_artifact.yml +++ b/.github/workflows/publish_dev_artifact.yml @@ -35,7 +35,7 @@ jobs: - name: Use Python ${{ matrix.python_version }} uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Install Dependencies run: | diff --git a/.github/workflows/publish_pricing_to_s3.yml b/.github/workflows/publish_pricing_to_s3.yml index 393eaa4ba5..681acc8505 100644 --- a/.github/workflows/publish_pricing_to_s3.yml +++ b/.github/workflows/publish_pricing_to_s3.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: - python_version: [3.8] + python_version: [3.9] steps: - name: Print Environment Info diff --git a/CHANGES.rst b/CHANGES.rst index 37d847781f..968b2b3703 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,6 +7,17 @@ Changes in Apache Libcloud 3.9.0 Common ~~~~~~ +- Indicate we also support Python 3.12 (non beta) and Python 3.13. + (#2050) + [Tomaz Muraus - @Kami] + +- Support for Python 3.8 which is EOL has been removed. + + If you still want to use Libcloud with Python 3.8, you should use an older + release which still supports Python 3.8. + (#2050) + [Tomaz Muraus - @Kami] + - Support for Python 3.7 which is EOL has been removed. If you still want to use Libcloud with Python 3.7, you should use an older diff --git a/README.rst b/README.rst index 6152a5365d..73a26e6f8b 100644 --- a/README.rst +++ b/README.rst @@ -59,11 +59,11 @@ through a unified and easy to use API. :Issues: https://issues.apache.org/jira/projects/LIBCLOUD/issues :Website: https://libcloud.apache.org/ :Documentation: https://libcloud.readthedocs.io -:Supported Python Versions: Python >= 3.8, PyPy >= 3.8, Python 3.10 + Pyjion +:Supported Python Versions: Python >= 3.9, PyPy >= 3.9, Python 3.10 + Pyjion (Python 2.7 and Python 3.4 is supported by the v2.8.x release series, last version which supports Python 3.5 is v3.4.0, v3.6.x for Python 3.6, and - v3.8.x for Python 3.7) + v3.8.x for Python 3.7 and 3.8) Resources you can manage with Libcloud are divided into the following categories: @@ -86,10 +86,10 @@ Documentation can be found at . Note on Python Version Compatibility ==================================== -Libcloud supports Python >= 3.8 and PyPy >= 3.8. +Libcloud supports Python >= 3.9 and PyPy >= 3.9. -* Support for Python 3.7 has been dropped in v3.9.0 release. - Last release series which supports Python 3.6 is v3.6.x. +* Support for Python 3.7 and 3.8 has been dropped in v3.9.0 release. + Last release series which supports Python 3.7 and 3.8 is v3.8.x. * Support for Python 3.6 has been dropped in v3.7.0 release. Last release series which supports Python 3.6 is v3.6.x. * Support for Python 3.5 has been dropped in v3.5.0 release. diff --git a/contrib/Dockerfile b/contrib/Dockerfile index 9351f39630..489a707a9e 100644 --- a/contrib/Dockerfile +++ b/contrib/Dockerfile @@ -29,29 +29,27 @@ RUN set -e && \ add-apt-repository ppa:pypy/ppa && \ apt-get update && \ apt-get -y install \ - python3.8 \ python3.9 \ python3.10 \ python3.11 \ + python3.12 \ + python3.13 \ python3-dev \ - python3.8-dev \ python3.9-dev \ python3.10-dev \ python3.11-dev \ - python3.8-distutils \ - python3.9-distutils \ + python3.12-dev \ + python3.13-dev \ + # Uses 3.10 pypy3 \ pypy3-dev \ python3-pip \ python3-distutils \ + python3.9-distutils \ libvirt-dev \ # Needed by libvirt driver - pkg-config \ - # Needed by cryptography library for pypy - libssl-dev + pkg-config -# Workaround for zipp import error issue - https://github.com/pypa/virtualenv/issues/1630 -RUN python3.8 -m pip install --upgrade pip COPY . /libcloud @@ -60,6 +58,6 @@ RUN if [ ! -f "/libcloud/README.rst" ]; then echo "libcloud/README.rst file not WORKDIR /libcloud RUN set -e && \ - python3.8 -m pip install --no-cache-dir -r requirements-ci.txt + python3.9 -m pip install --no-cache-dir -r requirements-ci.txt -CMD ["tox", "-e", "lint,isort-check,black-check,bandit,py3.7,py3.8,py3.9,py3.10,py3.11,pypypy3.8"] +CMD ["tox", "-e", "lint,isort-check,black-check,bandit,py3.9,py3.10,py3.11,py3.12,py3.13,pypypy3.10"] diff --git a/contrib/generate_contributor_list.py b/contrib/generate_contributor_list.py index 201c2263da..4f1dcb5626 100755 --- a/contrib/generate_contributor_list.py +++ b/contrib/generate_contributor_list.py @@ -111,12 +111,14 @@ def convert_to_markdown(contributors_map, include_tickets=False): def compare(item1, item2): lastname1 = item1.split(" ")[-1].lower() lastname2 = item2.split(" ")[-1].lower() + return (lastname1 > lastname2) - (lastname1 < lastname2) names = contributors_map.keys() names = sorted(names, cmp=compare) result = [] + for name in names: tickets = contributors_map[name] @@ -125,6 +127,7 @@ def compare(item1, item2): for ticket in tickets: if "-" not in ticket: # Invalid ticket number + continue number = ticket.split("-")[1] @@ -133,6 +136,8 @@ def compare(item1, item2): url = JIRA_URL % (number) elif ticket.startswith("GITHUB-") or ticket.startswith("GH-"): url = GITHUB_URL % (number) + else: + url = None values = {"ticket": ticket, "url": url} tickets_string.append("[%(ticket)s](%(url)s)" % values) @@ -147,6 +152,7 @@ def compare(item1, item2): result.append(line.strip()) result = "\n".join(result) + return result diff --git a/docs/upgrade_notes.rst b/docs/upgrade_notes.rst index 217d6e878d..488b2eae73 100644 --- a/docs/upgrade_notes.rst +++ b/docs/upgrade_notes.rst @@ -8,11 +8,10 @@ preserve the old behavior when this is possible. Libcloud 3.9.0 -------------- -* Support for Python 3.7 which has been EOL for more than a year now has been - removed. +* Support for Python 3.7 and 3.8 which have been EOL has been removed. - If you still want to use Libcloud with Python 3.7, you should use an older - release which still supports Python 3.7. + If you still want to use Libcloud with Python 3.7 or 3.8, you should use an older + release which still supports Python 3.7 and 3.8. * [AZURE ARM] Added a new argument to destroy_node() to also delete node's managed OS disk as part of the node's deletion. Defaults to true. This can be reverted by diff --git a/integration/storage/test_minio.py b/integration/storage/test_minio.py index ecacb40102..8c2665f116 100644 --- a/integration/storage/test_minio.py +++ b/integration/storage/test_minio.py @@ -33,7 +33,8 @@ class MinioTest(Integration.ContainerTestBase): # Output seemed to have changed recently, see # https://github.com/apache/libcloud/runs/7481114211?check_suite_focus=true # ready_message = b'Console endpoint is listening on a dynamic port' - ready_message = b"1 Online" + # ready_message = b"1 Online" + ready_message = b"MinIO Object Storage Server" def test_cdn_url(self): self.skipTest("Not implemented in driver") diff --git a/libcloud/common/gandi.py b/libcloud/common/gandi.py index f0e5194695..6d76c0ff9f 100644 --- a/libcloud/common/gandi.py +++ b/libcloud/common/gandi.py @@ -36,10 +36,12 @@ class GandiException(Exception): def __str__(self): # pylint: disable=unsubscriptable-object + return "({}) {}".format(self.args[0], self.args[1]) def __repr__(self): # pylint: disable=unsubscriptable-object + return ''.format(self.args[0], self.args[1]) @@ -83,6 +85,7 @@ def __init__( def request(self, method, *args): args = (self.key,) + args + return super().request(method, *args) @@ -106,6 +109,7 @@ def _wait_operation(self, id, timeout=DEFAULT_TIMEOUT, check_interval=DEFAULT_IN if op["step"] == "DONE": return True + if op["step"] in ["ERROR", "CANCEL"]: return False except (KeyError, IndexError): @@ -114,6 +118,7 @@ def _wait_operation(self, id, timeout=DEFAULT_TIMEOUT, check_interval=DEFAULT_IN raise GandiException(1002, e) time.sleep(check_interval) + return False @@ -147,7 +152,8 @@ def get_uuid(self): same UUID! """ hashstring = "{}:{}:{}".format(self.uuid_prefix, self.id, self.driver.type) - return hashlib.sha1(b(hashstring)).hexdigest() + + return hashlib.sha1(b(hashstring)).hexdigest() # nosec class IPAddress(BaseObject): diff --git a/libcloud/common/nfsn.py b/libcloud/common/nfsn.py index 95089562d5..426e843835 100644 --- a/libcloud/common/nfsn.py +++ b/libcloud/common/nfsn.py @@ -45,10 +45,13 @@ def parse_error(self): # If we only have one of "error" or "debug", use the one that we have. # If we have both, use both, with a space character in between them. value = "No message specified" + if error is not None: value = error + if debug is not None: value = debug + if error is not None and value is not None: value = error + " " + value value = value + " (HTTP Code: %d)" % self.status @@ -70,22 +73,25 @@ def _header(self, action, data): salt = self._salt() api_key = self.key data = urlencode(data) - data_hash = hashlib.sha1(data.encode("utf-8")).hexdigest() + data_hash = hashlib.sha1(data.encode("utf-8")).hexdigest() # nosec string = ";".join((login, timestamp, salt, api_key, action, data_hash)) - string_hash = hashlib.sha1(string.encode("utf-8")).hexdigest() + string_hash = hashlib.sha1(string.encode("utf-8")).hexdigest() # nosec return ";".join((login, timestamp, salt, string_hash)) def request(self, action, params=None, data="", headers=None, method="GET"): """Add the X-NFSN-Authentication header to an HTTP request.""" + if not headers: headers = {} + if not params: params = {} header = self._header(action, data) headers["X-NFSN-Authentication"] = header + if method == "POST": headers["Content-Type"] = "application/x-www-form-urlencoded" @@ -94,16 +100,20 @@ def request(self, action, params=None, data="", headers=None, method="GET"): def encode_data(self, data): """NFSN expects the body to be regular key-value pairs that are not JSON-encoded.""" + if data: data = urlencode(data) + return data def _salt(self): """Return a 16-character alphanumeric string.""" r = random.SystemRandom() + return "".join(r.choice(SALT_CHARACTERS) for _ in range(16)) def _timestamp(self): """Return the current number of seconds since the Unix epoch, as a string.""" + return str(int(time.time())) diff --git a/libcloud/common/ovh.py b/libcloud/common/ovh.py index 08d82e30a4..d2233c1e40 100644 --- a/libcloud/common/ovh.py +++ b/libcloud/common/ovh.py @@ -109,11 +109,13 @@ class OvhConnection(ConnectionUserAndKey): def __init__(self, user_id, *args, **kwargs): region = kwargs.pop("region", "") + if region: self.host = ("{}.{}".format(region, API_HOST)).lstrip(".") else: self.host = API_HOST self.consumer_key = kwargs.pop("ex_consumer_key", None) + if self.consumer_key is None: consumer_key_json = self.request_consumer_key(user_id) msg = ( @@ -145,27 +147,32 @@ def request_consumer_key(self, user_id): json_response = response.parse_body() httpcon.close() + return json_response def get_timestamp(self): if not self._timedelta: url = "https://{}{}/auth/time".format(self.host, API_ROOT) response = get_response_object(url=url, method="GET", headers={}) + if not response or not response.body: raise Exception("Failed to get current time from Ovh API") timestamp = int(response.body) self._timedelta = timestamp - int(time.time()) + return int(time.time()) + self._timedelta def make_signature(self, method, action, params, data, timestamp): full_url = "https://{}{}".format(self.host, action) + if params: full_url += "?" + for key, value in params.items(): full_url += "{}={}&".format(key, value) full_url = full_url[:-1] - sha1 = hashlib.sha1() + sha1 = hashlib.sha1() # nosec base_signature = "+".join( [ self.key, @@ -178,6 +185,7 @@ def make_signature(self, method, action, params, data, timestamp): ) sha1.update(base_signature.encode()) signature = "$1$" + sha1.hexdigest() + return signature def add_default_params(self, params): @@ -191,6 +199,7 @@ def add_default_headers(self, headers): "Content-type": "application/json", } ) + return headers def request(self, action, params=None, data=None, headers=None, method="GET", raw=False): diff --git a/libcloud/common/worldwidedns.py b/libcloud/common/worldwidedns.py index cd64b65951..07b3074fdc 100644 --- a/libcloud/common/worldwidedns.py +++ b/libcloud/common/worldwidedns.py @@ -92,6 +92,7 @@ def __init__(self, http_code, driver=None): class ErrorOnReloadInNameServer(WorldWideDNSException): def __init__(self, server, http_code, driver=None): + value, code = "unknown", "unknown" if server == 1: value = "Name server #1 kicked an error on reload, contact support" code = 411 diff --git a/libcloud/compute/base.py b/libcloud/compute/base.py index 7ec7772c17..58f3c0f4c1 100644 --- a/libcloud/compute/base.py +++ b/libcloud/compute/base.py @@ -144,14 +144,18 @@ def get_uuid(self): :rtype: ``str`` """ + if not self._uuid: - self._uuid = hashlib.sha1(b("{}:{}".format(self.id, self.driver.type))).hexdigest() + self._uuid = hashlib.sha1( + b("{}:{}".format(self.id, self.driver.type)) + ).hexdigest() # nosec return self._uuid @property def uuid(self): # type: () -> str + return self.get_uuid() @@ -280,6 +284,7 @@ def reboot(self): >>> node.state == NodeState.REBOOTING True """ + return self.driver.reboot_node(self) def start(self): @@ -289,6 +294,7 @@ def start(self): :return: ``bool`` """ + return self.driver.start_node(self) def stop_node(self): @@ -298,6 +304,7 @@ def stop_node(self): :return: ``bool`` """ + return self.driver.stop_node(self) def destroy(self): @@ -321,6 +328,7 @@ def destroy(self): False """ + return self.driver.destroy_node(self) def __repr__(self): @@ -688,6 +696,7 @@ def list_snapshots(self): """ :rtype: ``list`` of ``VolumeSnapshot`` """ + return self.driver.list_volume_snapshots(volume=self) def attach(self, node, device=None): @@ -705,6 +714,7 @@ def attach(self, node, device=None): :return: ``True`` if attach was successful, ``False`` otherwise. :rtype: ``bool`` """ + return self.driver.attach_volume(node=node, volume=self, device=device) def detach(self): @@ -715,6 +725,7 @@ def detach(self): :return: ``True`` if detach was successful, ``False`` otherwise. :rtype: ``bool`` """ + return self.driver.detach_volume(volume=self) def snapshot(self, name): @@ -725,6 +736,7 @@ def snapshot(self, name): :return: Created snapshot. :rtype: ``VolumeSnapshot`` """ + return self.driver.create_volume_snapshot(volume=self, name=name) def destroy(self): @@ -804,6 +816,7 @@ def destroy(self): :rtype: ``bool`` """ + return self.driver.destroy_volume_snapshot(snapshot=self) def __repr__(self): @@ -1148,6 +1161,7 @@ def deploy_node( :type wait_period: ``int`` """ + if not libcloud.compute.ssh.have_paramiko: raise RuntimeError( "paramiko is not installed. You can install " + "it using pip: pip install paramiko" @@ -1163,6 +1177,7 @@ def deploy_node( pass elif "create_node" in self.features: f = self.features["create_node"] + if "generates_password" not in f and "password" not in f: raise NotImplementedError("deploy_node not implemented for this driver") else: @@ -1181,6 +1196,7 @@ def deploy_node( try: # NOTE: We only pass auth to the method if auth argument is # provided + if auth: node = self.create_node(auth=auth, **create_node_kwargs) else: @@ -1188,6 +1204,7 @@ def deploy_node( except TypeError as e: msg_1_re = r"create_node\(\) missing \d+ required " "positional arguments.*" msg_2_re = r"create_node\(\) takes at least \d+ arguments.*" + if re.match(msg_1_re, str(e)) or re.match(msg_2_re, str(e)): # pylint: disable=unexpected-keyword-arg node = self.create_node( # type: ignore @@ -1211,6 +1228,7 @@ def deploy_node( atexit.register(at_exit_func, driver=self, node=node) password = None + if auth: if isinstance(auth, NodeAuthPassword): password = auth.password @@ -1261,6 +1279,7 @@ def deploy_node( else: # Script successfully executed, don't try alternate username deploy_error = None + break if deploy_error is not None: @@ -1674,8 +1693,10 @@ def is_supported(address): """ Return True for supported address. """ + if force_ipv4 and not is_valid_ip_address(address=address, family=socket.AF_INET): return False + return True def filter_addresses(addresses): @@ -1683,6 +1704,7 @@ def filter_addresses(addresses): """ Return list of supported addresses. """ + return [address for address in addresses if is_supported(address)] if ssh_interface not in ["public_ips", "private_ips"]: @@ -1708,8 +1730,10 @@ def filter_addresses(addresses): running_nodes = [node for node in matching_nodes if node.state == NodeState.RUNNING] addresses = [] + for node in running_nodes: node_addresses = filter_addresses(getattr(node, ssh_interface)) + if len(node_addresses) >= 1: addresses.append(node_addresses) @@ -1717,6 +1741,7 @@ def filter_addresses(addresses): return list(zip(running_nodes, addresses)) else: time.sleep(wait_period) + continue raise LibcloudError(value="Timed out after %s seconds" % (timeout), driver=self) @@ -1757,6 +1782,7 @@ def _get_and_check_auth(self, auth): # Some providers require password to also include uppercase # characters so convert some characters to uppercase password = "" + for char in value: if not char.isdigit() and char.islower(): if random.randint(0, 1) == 1: @@ -1784,6 +1810,7 @@ def _wait_until_running( # type: (Node, float, int, str, bool) -> List[Tuple[Node, List[str]]] # This is here for backward compatibility and will be removed in the # next major release + return self.wait_until_running( nodes=[node], wait_period=wait_period, @@ -1836,6 +1863,7 @@ def _ssh_client_connect(self, ssh_client, wait_period=1.5, timeout=300): pass time.sleep(wait_period) + continue else: return ssh_client @@ -1881,6 +1909,7 @@ def _connect_and_run_deployment_script( node = self._run_deployment_script( task=task, node=node, ssh_client=ssh_client, max_tries=max_tries ) + return node def _run_deployment_script(self, task, node, ssh_client, max_tries=3): @@ -1940,6 +1969,7 @@ def _run_deployment_script(self, task, node, ssh_client, max_tries=3): else: # Deployment succeeded ssh_client.close() + return node return node @@ -1949,6 +1979,7 @@ def _get_size_price(self, size_id): """ Return pricing information for the provided size id. """ + return get_size_price(driver_type="compute", driver_name=self.api_name, size_id=size_id) diff --git a/libcloud/compute/drivers/abiquo.py b/libcloud/compute/drivers/abiquo.py index 24c3ffcdc3..73bd208c0a 100644 --- a/libcloud/compute/drivers/abiquo.py +++ b/libcloud/compute/drivers/abiquo.py @@ -138,6 +138,7 @@ def create_node(self, image, name=None, size=None, location=None, ex_group_name= edit_vm = get_href(vm, "edit") headers = {"Accept": self.NODE_MIME_TYPE} vm = self.connection.request(edit_vm, headers=headers).object + return self._to_node(vm, self) def destroy_node(self, node): @@ -163,6 +164,8 @@ def destroy_node(self, node): if state in ["ALLOCATED", "CONFIGURED", "LOCKED", "UNKNOWN"]: raise LibcloudError("Invalid Node state", self) + res = None + if state != "NOT_ALLOCATED": # prepare the element that forces the undeploy vm_task = ET.Element("virtualmachinetask") @@ -183,9 +186,11 @@ def destroy_node(self, node): ) # pylint: disable=maybe-no-member - if state == "NOT_ALLOCATED" or res.async_success(): + + if state == "NOT_ALLOCATED" or (res and res.async_success()): # pylint: enable=maybe-no-member self.connection.request(action=node.extra["uri_id"], method="DELETE") + return True else: return False @@ -231,6 +236,7 @@ def ex_run_node(self, node): edit_vm = get_href(e_vm, "edit") headers = {"Accept": self.NODE_MIME_TYPE} e_vm = self.connection.request(edit_vm, headers=headers).object + return self._to_node(e_vm, self) def ex_populate_cache(self): @@ -270,14 +276,17 @@ def ex_populate_cache(self): "/admin/datacenters", headers=dcs_headers, params=params ).object dc_dict = {} + for dc in e_dcs.findall("datacenter"): key = get_href(dc, "self") dc_dict[key] = dc # Populate locations name cache self.connection.cache["locations"] = {} + for e_vdc in e_vdcs.findall("virtualDatacenter"): loc = get_href(e_vdc, "location") + if loc is not None: self.connection.cache["locations"][loc] = get_href(e_vdc, "edit") @@ -344,6 +353,8 @@ def ex_destroy_group(self, group): error = "Can not destroy group because of current state" raise LibcloudError(error, self) + res = None + if state == "DEPLOYED": # prepare the element that forces the undeploy vm_task = ET.Element("virtualmachinetask") @@ -366,10 +377,12 @@ def ex_destroy_group(self, group): ) # pylint: disable=maybe-no-member - if state == "NOT_DEPLOYED" or res.async_success(): + + if state == "NOT_DEPLOYED" or (res and res.async_success()): # pylint: enable=maybe-no-member # The node is no longer deployed. Unregister it. self.connection.request(action=group.uri, method="DELETE") + return True else: return False @@ -384,6 +397,7 @@ def ex_list_groups(self, location=None): :return: the list of :class:`NodeGroup` """ groups = [] + for vdc in self._get_locations(location): link_vdc = self.connection.cache["locations"][vdc] hdr_vdc = {"Accept": self.VDC_MIME_TYPE} @@ -391,11 +405,13 @@ def ex_list_groups(self, location=None): apps_link = get_href(e_vdc, "virtualappliances") hdr_vapps = {"Accept": self.VAPPS_MIME_TYPE} vapps = self.connection.request(apps_link, headers=hdr_vapps).object + for vapp in vapps.findall("virtualAppliance"): nodes = [] vms_link = get_href(vapp, "virtualmachines") headers = {"Accept": self.NODES_MIME_TYPE} vms = self.connection.request(vms_link, headers=headers).object + for vm in vms.findall("virtualMachine"): nodes.append(self._to_node(vm, self)) group = NodeGroup(self, vapp.findtext("name"), nodes, get_href(vapp, "edit")) @@ -419,9 +435,11 @@ def list_images(self, location=None): repos = self.connection.request(uri, headers=repos_hdr).object images = [] + for repo in repos.findall("datacenterRepository"): # filter by location. Skips when the name of the location # is different from the 'datacenterRepository' element + for vdc in self._get_locations(location): # Check if the virtual datacenter belongs to this repo link_vdc = self.connection.cache["locations"][vdc] @@ -439,10 +457,12 @@ def list_images(self, location=None): templates = self.connection.request( url_templates, params, headers=headers ).object + for templ in templates.findall("virtualMachineTemplate"): # Avoid duplicated templates id_template = templ.findtext("id") ids = [image.id for image in images] + if id_template not in ids: images.append(self._to_nodeimage(templ, self, get_href(repo, "edit"))) @@ -456,6 +476,7 @@ def list_locations(self): user :rtype: ``list`` of :class:`NodeLocation` """ + return list(self.connection.cache["locations"].keys()) def list_nodes(self, location=None): @@ -491,6 +512,7 @@ def list_sizes(self, location=None): :return: The list of sizes :rtype: ``list`` of :class:`NodeSizes` """ + return [ NodeSize( id=1, @@ -543,6 +565,7 @@ def reboot_node(self, node): reboot_uri = node.extra["uri_id"] + "/action/reset" reboot_hdr = {"Accept": self.AR_MIME_TYPE} res = self.connection.async_request(action=reboot_uri, method="POST", headers=reboot_hdr) + return res.async_success() # pylint: disable=maybe-no-member # ------------------------- @@ -579,6 +602,7 @@ def _deploy_remote(self, e_vm): res = self.connection.async_request( action=link_deploy, method="POST", data=tostring(vm_task), headers=headers ) + if not res.async_success(): # pylint: disable=maybe-no-member raise LibcloudError("Could not run the node", self) @@ -589,6 +613,7 @@ def _to_location(self, vdc, dc, driver): identifier = vdc.findtext("id") name = vdc.findtext("name") country = dc.findtext("name") + return NodeLocation(identifier, name, country, driver) def _to_node(self, vm, driver): @@ -610,10 +635,13 @@ def _to_node(self, vm, driver): public_ips = [] nics_hdr = {"Accept": self.NICS_MIME_TYPE} nics_element = self.connection.request(get_href(vm, "nics"), headers=nics_hdr).object + for nic in nics_element.findall("nic"): ip = nic.findtext("ip") + for link in nic.findall("link"): rel = link.attrib["rel"] + if rel == "privatenetwork": private_ips.append(ip) elif rel in ["publicnetwork", "externalnetwork", "unmanagednetwork"]: @@ -645,12 +673,14 @@ def _to_nodeimage(self, template, driver, repo): url = get_href(template, "edit") hdreqd = template.findtext("hdRequired") extra = {"repo": repo, "url": url, "hdrequired": hdreqd} + return NodeImage(identifier, name, driver, extra) def _get_locations(self, location=None): """ Returns the locations as a generator. """ + if location is not None: yield location else: @@ -660,6 +690,7 @@ def _get_enterprise_id(self): """ Returns the identifier of the logged user's enterprise. """ + return self.connection.cache["enterprise"].findtext("id") def _define_create_node_location(self, image, location): @@ -670,11 +701,13 @@ def _define_create_node_location(self, image, location): location will be created. """ # First, get image location + if not image: error = "'image' parameter is mandatory" raise LibcloudError(error, self) # Get the location argument + if location: if location not in self.list_locations(): raise LibcloudError("Location does not exist") @@ -683,14 +716,17 @@ def _define_create_node_location(self, image, location): # the input location loc = None target_loc = None + for candidate_loc in self._get_locations(location): link_vdc = self.connection.cache["locations"][candidate_loc] hdr_vdc = {"Accept": self.VDC_MIME_TYPE} e_vdc = self.connection.request(link_vdc, headers=hdr_vdc).object + for img in self.list_images(candidate_loc): if img.id == image.id: loc = e_vdc target_loc = candidate_loc + break if loc is None: @@ -705,6 +741,7 @@ def _define_create_node_group(self, xml_loc, loc, group_name=None): If we can not find any group, create it into argument 'location' """ + if not group_name: group_name = NodeGroup.DEFAULT_GROUP_NAME @@ -713,14 +750,17 @@ def _define_create_node_group(self, xml_loc, loc, group_name=None): groups_hdr = {"Accept": self.VAPPS_MIME_TYPE} vapps_element = self.connection.request(groups_link, headers=groups_hdr).object target_group = None + for vapp in vapps_element.findall("virtualAppliance"): if vapp.findtext("name") == group_name: uri_vapp = get_href(vapp, "edit") + return NodeGroup(self, vapp.findtext("name"), uri=uri_vapp) # target group not found: create it. Since it is an extension of # the basic 'libcloud' functionality, we try to be as flexible as # possible. + if target_group is None: return self.ex_create_group(group_name, loc) @@ -732,6 +772,7 @@ def _define_create_node_node(self, group, name=None, size=None, image=None): the API before to create it into the target hypervisor. """ vm = ET.Element("virtualMachine") + if name: vmname = ET.SubElement(vm, "label") vmname.text = name @@ -795,4 +836,5 @@ def destroy(self): Destroys the group delegating the execution to :class:`AbiquoNodeDriver`. """ + return self.driver.ex_destroy_group(self) diff --git a/libcloud/compute/drivers/cloudsigma.py b/libcloud/compute/drivers/cloudsigma.py index 30e6eea340..a012b480bf 100644 --- a/libcloud/compute/drivers/cloudsigma.py +++ b/libcloud/compute/drivers/cloudsigma.py @@ -82,22 +82,26 @@ def __new__( cls = CloudSigma_2_0_NodeDriver else: raise NotImplementedError("Unsupported API version: %s" % (api_version)) + return super().__new__(cls) class CloudSigmaException(Exception): def __str__(self): # pylint: disable=unsubscriptable-object + return self.args[0] def __repr__(self): # pylint: disable=unsubscriptable-object + return "" % (self.args[0]) class CloudSigmaInsufficientFundsException(Exception): def __repr__(self): # pylint: disable=unsubscriptable-object + return "" % (self.args[0]) @@ -157,6 +161,7 @@ def add_default_headers(self, headers): headers["Authorization"] = "Basic %s" % ( base64.b64encode(b("{}:{}".format(self.user_id, self.key))).decode("utf-8") ) + return headers @@ -237,6 +242,7 @@ def destroy_node(self, node): state = node.state # Node cannot be destroyed while running so it must be stopped first + if state == NodeState.RUNNING: stopped = self.ex_stop_node(node) else: @@ -246,6 +252,7 @@ def destroy_node(self, node): raise CloudSigmaException("Could not stop node with id %s" % (node.id)) response = self.connection.request(action="/servers/%s/destroy" % (node.id), method="POST") + return response.status == 204 def list_images(self, location=None): @@ -258,6 +265,7 @@ def list_images(self, location=None): response = self.connection.request(action="/drives/standard/info").object images = [] + for value in response: if value.get("type"): if value["type"] == "disk": @@ -273,6 +281,7 @@ def list_images(self, location=None): def list_sizes(self, location=None): sizes = [] + for value in INSTANCE_TYPES: key = value["id"] size = CloudSigmaNodeSize( @@ -293,10 +302,13 @@ def list_nodes(self): response = self.connection.request(action="/servers/info").object nodes = [] + for data in response: node = self._to_node(data) + if node: nodes.append(node) + return nodes def create_node( @@ -331,6 +343,7 @@ def create_node( :keyword drive_type: Drive type (ssd|hdd). Defaults to hdd. :type drive_type: ``str`` """ + if nic_model not in ["e1000", "rtl8139", "virtio"]: raise CloudSigmaException("Invalid NIC model specified") @@ -355,10 +368,12 @@ def create_node( response = self.connection.request(action="/drives/%s/info" % (drive_uuid)).object imaging_start = time.time() + while "imaging" in response[0]: response = self.connection.request(action="/drives/%s/info" % (drive_uuid)).object elapsed_time = time.time() - imaging_start timed_out = elapsed_time >= self.IMAGING_TIMEOUT + if "imaging" in response[0] and timed_out: raise CloudSigmaException("Drive imaging timed out") time.sleep(1) @@ -387,6 +402,7 @@ def create_node( response = [response] node = self._to_node(response[0]) + if node is None: # Insufficient funds, destroy created drive self.ex_drive_destroy(drive_uuid) @@ -412,6 +428,7 @@ def ex_destroy_node_and_drives(self, node): node = self._get_node_info(node) drive_uuids = [] + for key, value in node.items(): if ( key.startswith("ide:") or key.startswith("scsi") or key.startswith("block") @@ -442,6 +459,7 @@ def ex_static_ip_list(self): raise CloudSigmaException("Could not retrieve IP list") ips = str2list(response.body) + return ips def ex_drives_list(self): @@ -453,6 +471,7 @@ def ex_drives_list(self): response = self.connection.request(action="/drives/info", method="GET") result = str2dicts(response.body) + return result def ex_static_ip_create(self): @@ -464,6 +483,7 @@ def ex_static_ip_create(self): response = self.connection.request(action="/resources/ip/create", method="GET") result = str2dicts(response.body) + return result def ex_static_ip_destroy(self, ip_address): @@ -532,12 +552,16 @@ def ex_set_node_configuration(self, node, **kwargs): invalid_keys = [] keys = list(kwargs.keys()) + for key in keys: matches = False + for regex in valid_keys: if re.match(regex, key): matches = True + break + if not matches: invalid_keys.append(key) @@ -569,6 +593,7 @@ def ex_stop_node(self, node): # NOTE: This method is here for backward compatibility reasons after # this method was promoted to be part of the standard compute API in # Libcloud v2.7.0 + return self.stop_node(node=node) def stop_node(self, node): @@ -581,6 +606,7 @@ def stop_node(self, node): :rtype: ``bool`` """ response = self.connection.request(action="/servers/%s/stop" % (node.id), method="POST") + return response.status == 204 def ex_shutdown_node(self, node): @@ -589,6 +615,7 @@ def ex_shutdown_node(self, node): @inherits: :class:`CloudSigmaBaseNodeDriver.ex_stop_node` """ + return self.ex_stop_node(node) def ex_destroy_drive(self, drive_uuid): @@ -603,6 +630,7 @@ def ex_destroy_drive(self, drive_uuid): response = self.connection.request( action="/drives/%s/destroy" % (drive_uuid), method="POST" ) + return response.status == 204 def _ex_connection_class_kwargs(self): @@ -610,6 +638,7 @@ def _ex_connection_class_kwargs(self): Return the host value based on the user supplied region. """ kwargs = {} + if not self._host_argument_set: kwargs["host"] = API_ENDPOINTS_1_0[self.region]["host"] @@ -625,9 +654,11 @@ def _to_node(self, data): if "server" not in data: # Response does not contain server UUID if the server # creation failed because of insufficient funds. + return None public_ips = [] + if "nic:0:dhcp" in data: if isinstance(data["nic:0:dhcp"], list): public_ips = data["nic:0:dhcp"] @@ -641,6 +672,7 @@ def _to_node(self, data): ("mem", "int"), ("status", "str"), ] + for key, value_type in extra_keys: if key in data: value = data[key] @@ -669,6 +701,7 @@ def _to_node(self, data): ) return node + return None def _get_node(self, node_id): @@ -684,6 +717,7 @@ def _get_node_info(self, node): response = self.connection.request(action="/servers/%s/info" % (node.id)) result = str2dicts(response.body) + return result[0] @@ -1017,6 +1051,7 @@ def _parse_errors_from_body(self, body): for item in body: if "error_type" not in item: # Unrecognized error + continue error = CloudSigmaError( @@ -1043,10 +1078,12 @@ def add_default_headers(self, headers): headers["Authorization"] = "Basic %s" % ( base64.b64encode(b("{}:{}".format(self.user_id, self.key))).decode("utf-8") ) + return headers def encode_data(self, data): data = json.dumps(data) + return data def request(self, action, params=None, data=None, headers=None, method="GET", raw=False): @@ -1130,6 +1167,7 @@ def list_nodes(self, ex_tag=None): provided tag. :type ex_tag: :class:`CloudSigmaTag` """ + if ex_tag: action = "/tags/%s/servers/detail/" % (ex_tag.id) else: @@ -1137,6 +1175,7 @@ def list_nodes(self, ex_tag=None): response = self.connection.request(action=action, method="GET").object nodes = [self._to_node(data=item) for item in response["objects"]] + return nodes def list_sizes(self): @@ -1144,6 +1183,7 @@ def list_sizes(self): List available sizes. """ sizes = [] + for value in INSTANCE_TYPES: key = value["id"] size = CloudSigmaNodeSize( @@ -1175,6 +1215,7 @@ def list_images(self): # they can't be used directly following a default Libcloud server # creation flow. images = [image for image in images if image.extra["image_type"] == "preinst"] + return images def create_node( @@ -1237,6 +1278,7 @@ def create_node( drive_name = "%s-drive" % (name) drive_size = size.disk + if not is_installation_cd: # 1. Clone library drive so we can use it drive = self.ex_clone_drive(drive=image, name=drive_name) @@ -1246,6 +1288,7 @@ def create_node( # 2. Resize drive to the desired disk size if the desired disk size # is larger than the cloned drive size. + if drive_size > drive.size: drive = self.ex_resize_drive(drive=drive, size=drive_size) @@ -1287,6 +1330,7 @@ def create_node( nics.append(nic) # Need to use IDE for installation CDs + if is_installation_cd: device_type = "ide" else: @@ -1321,11 +1365,13 @@ def destroy_node(self, node, ex_delete_drives=False): :rtype: ``bool`` """ action = "/servers/%s/" % (node.id) + if ex_delete_drives is True: params = {"recurse": "all_drives"} else: params = None response = self.connection.request(action=action, method="DELETE", params=params) + return response.status == httplib.NO_CONTENT def reboot_node(self, node): @@ -1349,11 +1395,13 @@ def reboot_node(self, node): raise CloudSigmaException("Could not stop node with id %s" % (node.id)) success = False + for _ in range(5): try: success = self.start_node(node) except CloudSigmaError: time.sleep(1) + continue else: break @@ -1393,6 +1441,7 @@ def ex_edit_node(self, node, params): action = "/servers/%s/" % (node.id) response = self.connection.request(action=action, method="PUT", data=data).object node = self._to_node(data=response) + return node def start_node(self, node, ex_avoid=None): @@ -1416,17 +1465,20 @@ def start_node(self, node, ex_avoid=None): path = "/servers/%s/action/" % (node.id) response = self._perform_action(path=path, action="start", params=params, method="POST") + return response.status == httplib.ACCEPTED def stop_node(self, node): path = "/servers/%s/action/" % (node.id) response = self._perform_action(path=path, action="stop", method="POST") + return response.status == httplib.ACCEPTED def ex_start_node(self, node, ex_avoid=None): # NOTE: This method is here for backward compatibility reasons after # this method was promoted to be part of the standard compute API in # Libcloud v2.7.0 + return self.start_node(node=node, ex_avoid=ex_avoid) def ex_stop_node(self, node): @@ -1436,6 +1488,7 @@ def ex_stop_node(self, node): """ Stop a node. """ + return self.stop_node(node=node) def ex_clone_node(self, node, name=None, random_vnc_password=None): @@ -1461,13 +1514,16 @@ def ex_clone_node(self, node, name=None, random_vnc_password=None): path = "/servers/%s/action/" % (node.id) response = self._perform_action(path=path, action="clone", method="POST", data=data).object node = self._to_node(data=response) + return node def ex_get_node(self, node_id, return_json=False): action = "/servers/%s/" % (node_id) response = self.connection.request(action=action).object + if return_json is True: return response + return self._to_node(response) def ex_open_vnc_tunnel(self, node): @@ -1483,6 +1539,7 @@ def ex_open_vnc_tunnel(self, node): path = "/servers/%s/action/" % (node.id) response = self._perform_action(path=path, action="open_vnc", method="POST").object vnc_url = response["vnc_url"] + return vnc_url def ex_close_vnc_tunnel(self, node): @@ -1497,6 +1554,7 @@ def ex_close_vnc_tunnel(self, node): """ path = "/servers/%s/action/" % (node.id) response = self._perform_action(path=path, action="close_vnc", method="POST") + return response.status == httplib.ACCEPTED # Drive extension methods @@ -1510,6 +1568,7 @@ def ex_list_library_drives(self): """ response = self.connection.request(action="/libdrives/").object drives = [self._to_drive(data=item) for item in response["objects"]] + return drives def ex_list_user_drives(self): @@ -1520,6 +1579,7 @@ def ex_list_user_drives(self): """ response = self.connection.request(action="/drives/detail/").object drives = [self._to_drive(data=item) for item in response["objects"]] + return drives def list_volumes(self): @@ -1559,6 +1619,7 @@ def ex_create_drive(self, name, size, media="disk", ex_avoid=None): action=action, method="POST", params=params, data=data ).object drive = self._to_drive(data=response["objects"][0]) + return drive def create_volume(self, name, size, media="disk", ex_avoid=None): @@ -1599,6 +1660,7 @@ def ex_clone_drive(self, drive, name=None, ex_avoid=None): path=path, action="clone", params=params, data=data, method="POST" ) drive = self._to_drive(data=response.object["objects"][0]) + return drive def ex_resize_drive(self, drive, size): @@ -1618,6 +1680,7 @@ def ex_resize_drive(self, drive, size): response = self._perform_action(path=path, action="resize", method="POST", data=data) drive = self._to_drive(data=response.object["objects"][0]) + return drive def ex_attach_drive(self, node, drive): @@ -1641,11 +1704,14 @@ def ex_attach_drive(self, node, drive): # https://cloudsigma-docs.readthedocs.io/en/2.14.3/servers_kvm.html?highlight=dev%20channel#device-channel dev_channels = [item["dev_channel"] for item in data["drives"]] dev_channel = None + for controller in range(MAX_VIRTIO_CONTROLLERS): for unit in range(MAX_VIRTIO_UNITS): if "{}:{}".format(controller, unit) not in dev_channels: dev_channel = "{}:{}".format(controller, unit) + break + if dev_channel: break else: @@ -1660,6 +1726,7 @@ def ex_attach_drive(self, node, drive): data["drives"].append(item) action = "/servers/%s/" % (node.id) response = self.connection.request(action=action, data=data, method="PUT") + return response.status == 200 def attach_volume(self, node, volume): @@ -1670,6 +1737,7 @@ def ex_detach_drive(self, node, drive): data["drives"] = [item for item in data["drives"] if item["drive"]["uuid"] != drive.id] action = "/servers/%s/" % (node.id) response = self.connection.request(action=action, data=data, method="PUT") + return response.status == 200 def detach_volume(self, node, volume): @@ -1688,11 +1756,13 @@ def ex_get_drive(self, drive_id): action = "/drives/%s/" % (drive_id) response = self.connection.request(action=action).object drive = self._to_drive(data=response) + return drive def ex_destroy_drive(self, drive): action = "/drives/%s/" % (drive.id) response = self.connection.request(action=action, method="DELETE") + return response.status == httplib.NO_CONTENT def destroy_volume(self, drive): @@ -1709,6 +1779,7 @@ def ex_list_firewall_policies(self): action = "/fwpolicies/detail/" response = self.connection.request(action=action, method="GET").object policies = [self._to_firewall_policy(data=item) for item in response["objects"]] + return policies def ex_create_firewall_policy(self, name, rules=None): @@ -1737,6 +1808,7 @@ def ex_create_firewall_policy(self, name, rules=None): action = "/fwpolicies/" response = self.connection.request(action=action, method="POST", data=data).object policy = self._to_firewall_policy(data=response["objects"][0]) + return policy def ex_attach_firewall_policy(self, policy, node, nic_mac=None): @@ -1772,6 +1844,7 @@ def ex_attach_firewall_policy(self, policy, node, nic_mac=None): params = {"nics": nics} node = self.ex_edit_node(node=node, params=params) + return node def ex_delete_firewall_policy(self, policy): @@ -1786,6 +1859,7 @@ def ex_delete_firewall_policy(self, policy): """ action = "/fwpolicies/%s/" % (policy.id) response = self.connection.request(action=action, method="DELETE") + return response.status == httplib.NO_CONTENT # Availability groups extension methods @@ -1801,6 +1875,7 @@ def ex_list_servers_availability_groups(self): """ action = "/servers/availability_groups/" response = self.connection.request(action=action, method="GET") + return response.object def ex_list_drives_availability_groups(self): @@ -1814,6 +1889,7 @@ def ex_list_drives_availability_groups(self): """ action = "/drives/availability_groups/" response = self.connection.request(action=action, method="GET") + return response.object # Tag extension methods @@ -1842,6 +1918,7 @@ def ex_get_tag(self, tag_id): action = "/tags/%s/" % (tag_id) response = self.connection.request(action=action, method="GET").object tag = self._to_tag(data=response) + return tag def ex_create_tag(self, name, resource_uuids=None): @@ -1867,6 +1944,7 @@ def ex_create_tag(self, name, resource_uuids=None): action = "/tags/" response = self.connection.request(action=action, method="POST", data=data).object tag = self._to_tag(data=response["objects"][0]) + return tag def ex_tag_resource(self, resource, tag): @@ -1883,6 +1961,7 @@ def ex_tag_resource(self, resource, tag): :return: Updated tag object. :rtype: :class:`.CloudSigmaTag` """ + if not hasattr(resource, "id"): raise ValueError("Resource doesn't have id attribute") @@ -1918,6 +1997,7 @@ def ex_tag_resources(self, resources, tag): action = "/tags/%s/" % (tag.id) response = self.connection.request(action=action, method="PUT", data=data).object tag = self._to_tag(data=response) + return tag def ex_delete_tag(self, tag): @@ -1932,6 +2012,7 @@ def ex_delete_tag(self, tag): """ action = "/tags/%s/" % (tag.id) response = self.connection.request(action=action, method="DELETE") + return response.status == httplib.NO_CONTENT # Account extension methods @@ -1945,6 +2026,7 @@ def ex_get_balance(self): """ action = "/balance/" response = self.connection.request(action=action, method="GET") + return response.object def ex_get_pricing(self): @@ -1956,6 +2038,7 @@ def ex_get_pricing(self): """ action = "/pricing/" response = self.connection.request(action=action, method="GET") + return response.object def ex_get_usage(self): @@ -1967,6 +2050,7 @@ def ex_get_usage(self): """ action = "/currentusage/" response = self.connection.request(action=action, method="GET") + return response.object def ex_list_subscriptions(self, status="all", resources=None): @@ -1992,6 +2076,7 @@ def ex_list_subscriptions(self, status="all", resources=None): response = self.connection.request(action="/subscriptions/", params=params).object subscriptions = self._to_subscriptions(data=response) + return subscriptions def ex_toggle_subscription_auto_renew(self, subscription): @@ -2006,6 +2091,7 @@ def ex_toggle_subscription_auto_renew(self, subscription): """ path = "/subscriptions/%s/action/" % (subscription.id) response = self._perform_action(path=path, action="auto_renew", method="POST") + return response.status == httplib.OK def ex_create_subscription(self, amount, period, resource, auto_renew=False): @@ -2038,6 +2124,7 @@ def ex_create_subscription(self, amount, period, resource, auto_renew=False): response = self.connection.request(action="/subscriptions/", data=data, method="POST") data = response.object["objects"][0] subscription = self._to_subscription(data=data) + return subscription # Misc extension methods @@ -2051,6 +2138,7 @@ def ex_list_capabilities(self): action = "/capabilities/" response = self.connection.request(action=action, method="GET") capabilities = response.object + return capabilities def list_key_pairs(self): @@ -2091,6 +2179,7 @@ def create_key_pair(self, name): action = "/keypairs/" data = {"objects": [{"name": name}]} response = self.connection.request(action=action, method="POST", data=data).object + return self._to_key_pair(response["objects"][0]) def import_key_pair_from_string(self, name, key_material): @@ -2108,6 +2197,7 @@ def import_key_pair_from_string(self, name, key_material): action = "/keypairs/" data = {"objects": [{"name": name, "public_key": key_material.replace("\n", "")}]} response = self.connection.request(action=action, method="POST", data=data).object + return self._to_key_pair(response["objects"][0]) def delete_key_pair(self, key_pair): @@ -2121,6 +2211,7 @@ def delete_key_pair(self, key_pair): """ action = "/keypairs/%s/" % (key_pair.extra["uuid"]) response = self.connection.request(action=action, method="DELETE") + return response.status == 204 def _parse_ips_from_nic(self, nic): @@ -2196,6 +2287,7 @@ def _to_node(self, data): # find image name and boot drive size image = None drive_size = 0 + for item in extra["drives"]: if item["boot_order"] == 1: drive = self.ex_get_drive(item["drive"]["uuid"]) @@ -2203,6 +2295,7 @@ def _to_node(self, data): image = "{} {}".format( drive.extra.get("distribution", ""), drive.extra.get("version", "") ) + break # try to find if node size is from example sizes given by CloudSigma try: @@ -2210,7 +2303,7 @@ def _to_node(self, data): size = CloudSigmaNodeSize(**kwargs, driver=self) except KeyError: id_to_hash = str(extra["cpus"]) + str(extra["memory"]) + str(drive_size) - size_id = hashlib.md5(id_to_hash.encode("utf-8")).hexdigest() + size_id = hashlib.md5(id_to_hash.encode("utf-8")).hexdigest() # nosec size_name = "custom, {} CPUs, {}MB RAM, {}GB disk".format( extra["cpus"], extra["memory"], drive_size ) # noqa @@ -2242,6 +2335,7 @@ def _to_node(self, data): size=size, extra=extra, ) + return node def _to_image(self, data): @@ -2260,6 +2354,7 @@ def _to_image(self, data): extra = self._extract_values(obj=data, keys=extra_keys) image = NodeImage(id=id, name=name, driver=self, extra=extra) + return image def _to_drive(self, data): @@ -2296,6 +2391,7 @@ def _to_tag(self, data): resources = [resource["uuid"] for resource in resources] tag = CloudSigmaTag(id=data["uuid"], name=data["name"], resources=resources) + return tag def _to_subscriptions(self, data): @@ -2332,6 +2428,7 @@ def _to_subscription(self, data): auto_renew=data["auto_renew"], subscribed_object=obj_uuid, ) + return subscription def _to_firewall_policy(self, data): @@ -2351,6 +2448,7 @@ def _to_firewall_policy(self, data): rules.append(rule) policy = CloudSigmaFirewallPolicy(id=data["uuid"], name=data["name"], rules=rules) + return policy def _to_key_pair(self, data): @@ -2361,6 +2459,7 @@ def _to_key_pair(self, data): "permissions": data["permissions"], "meta": data["meta"], } + return KeyPair( name=data["name"], public_key=data["public_key"], @@ -2374,6 +2473,7 @@ def _perform_action(self, path, action, method="POST", params=None, data=None): """ Perform API action and return response object. """ + if params: params = params.copy() else: @@ -2381,6 +2481,7 @@ def _perform_action(self, path, action, method="POST", params=None, data=None): params["do"] = action response = self.connection.request(action=path, method=method, params=params, data=data) + return response def _is_installation_cd(self, image): @@ -2389,6 +2490,7 @@ def _is_installation_cd(self, image): :rtype: ``bool`` """ + if isinstance(image, CloudSigmaDrive) and image.media == "cdrom": return True diff --git a/libcloud/compute/drivers/equinixmetal.py b/libcloud/compute/drivers/equinixmetal.py index ba2f713b15..1b2dc7d35e 100644 --- a/libcloud/compute/drivers/equinixmetal.py +++ b/libcloud/compute/drivers/equinixmetal.py @@ -292,6 +292,7 @@ def create_node( raise Exception("ex_project_id needs to be specified") location_code = location.extra["code"] + if not self._valid_location: raise ValueError( "Failed to create node: valid parameter metro [code] is required in the input" @@ -464,6 +465,11 @@ def _to_node(self, data): if "ip_addresses" in data and data["ip_addresses"] is not None: ips = self._parse_ips(data["ip_addresses"]) + public_ips = ips["public"] + private_ips = ips["private"] + else: + public_ips = [] + private_ips = [] if "operating_system" in data and data["operating_system"] is not None: image = self._to_image(data["operating_system"]) @@ -479,6 +485,7 @@ def _to_node(self, data): if "facility" in data: extra["facility"] = data["facility"] + if "metro" in data and data["metro"] is not None: extra["metro"] = data["metro"] @@ -490,8 +497,8 @@ def _to_node(self, data): id=data["id"], name=data["hostname"], state=state, - public_ips=ips["public"], - private_ips=ips["private"], + public_ips=public_ips, + private_ips=private_ips, size=size, image=image, extra=extra, @@ -776,9 +783,11 @@ def _valid_location(self, metro_code): if not metro_code: return False metros = self.connection.request("/metal/v1/locations/metros").object["metros"] + for metro in metros: if metro["code"] == metro_code: return True + return False diff --git a/libcloud/compute/drivers/gce.py b/libcloud/compute/drivers/gce.py index 108e0e854e..58a3468d78 100644 --- a/libcloud/compute/drivers/gce.py +++ b/libcloud/compute/drivers/gce.py @@ -69,6 +69,7 @@ def timestamp_to_datetime(timestamp): tz_hours = int(timestamp[-5:-3]) tz_mins = int(timestamp[-2:]) * int(timestamp[-6:-5] + "1") tz_delta = datetime.timedelta(hours=tz_hours, minutes=tz_mins) + return ts + tz_delta @@ -132,8 +133,10 @@ def pre_connect_hook(self, params, headers): @inherits: :class:`GoogleBaseConnection.pre_connect_hook` """ params, headers = super().pre_connect_hook(params, headers) + if self.gce_params: params.update(self.gce_params) + return params, headers def paginated_request(self, *args, **kwargs): @@ -147,6 +150,7 @@ def paginated_request(self, *args, **kwargs): items = [] max_results = kwargs["max_results"] if "max_results" in kwargs else 500 params = {"maxResults": max_results} + while more_results: self.gce_params = params response = self.request(*args, **kwargs) @@ -165,6 +169,7 @@ def request(self, *args, **kwargs): # If gce_params has been set, then update the pageToken with the # nextPageToken so it can be used in the next request. + if self.gce_params: if "nextPageToken" in response.object: self.gce_params["pageToken"] = response.object["nextPageToken"] @@ -194,6 +199,7 @@ def request_aggregated_items(self, api_name, zone=None): ex: { 'items': {'zones/us-central1-a': {disks: []}} } :rtype: ``dict`` """ + if zone: request_path = "/zones/{}/{}".format(zone.name, api_name) else: @@ -203,9 +209,11 @@ def request_aggregated_items(self, api_name, zone=None): params = {"maxResults": 500} more_results = True + while more_results: self.gce_params = params response = self.request(request_path, method="GET").object + if "items" in response: if zone: # Special case when we are handling pagination for a @@ -214,6 +222,7 @@ def request_aggregated_items(self, api_name, zone=None): response["items"] = {"zones/%s" % (zone): {api_name: items}} api_responses.append(response) more_results = "pageToken" in params + return self._merge_response_items(api_name, api_responses) def _merge_response_items(self, list_name, response_list): @@ -248,14 +257,17 @@ def _merge_response_items(self, list_name, response_list): :rtype: ``dict`` """ merged_items = {} + for resp in response_list: # example k would be a zone or region name # example v would be { "disks" : [], "otherkey" : "..." } + for k, v in resp.get("items", {}).items(): if list_name in v: merged_items.setdefault(k, {}).setdefault(list_name, []) # Combine the list with the existing list. merged_items[k][list_name] += v[list_name] + return {"items": merged_items} @@ -294,6 +306,7 @@ def __init__(self, driver, list_fn, **kwargs): def __iter__(self): list_fn = self.list_fn more_results = True + while more_results: self.driver.connection.gce_params = self.params yield list_fn(**self.kwargs) @@ -334,6 +347,7 @@ def filter(self, expression): :rtype: :class:`GCEList` """ self.params["filter"] = expression + return self def page(self, max_results=500): @@ -358,6 +372,7 @@ def page(self, max_results=500): :rtype: :class:`GCEList` """ self.params["maxResults"] = max_results + return self @@ -472,6 +487,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_address(address=self) def __repr__(self): @@ -531,6 +547,7 @@ def _gen_id(self): :rtype: ``str`` """ zone_name = self.instance_group.zone.name + return "{}/instanceGroups/{}".format(zone_name, self.instance_group.name) def to_backend_dict(self): @@ -545,12 +562,16 @@ def to_backend_dict(self): if self.balancing_mode: d["balancingMode"] = self.balancing_mode + if self.max_utilization: d["maxUtilization"] = self.max_utilization + if self.max_rate: d["maxRate"] = self.max_rate + if self.max_rate_per_instance: d["maxRatePerInstance"] = self.max_rate_per_instance + if self.capacity_scaler: d["capacityScaler"] = self.capacity_scaler @@ -601,6 +622,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_backendservice(backendservice=self) @@ -663,6 +685,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_healthcheck(healthcheck=self) def update(self): @@ -672,6 +695,7 @@ def update(self): :return: Updated Healthcheck object :rtype: :class:`GCEHealthcheck` """ + return self.driver.ex_update_healthcheck(healthcheck=self) def __repr__(self): @@ -728,6 +752,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_firewall(firewall=self) def update(self): @@ -737,6 +762,7 @@ def update(self): :return: Updated Firewall object :rtype: :class:`GCEFirewall` """ + return self.driver.ex_update_firewall(firewall=self) def __repr__(self): @@ -768,6 +794,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_forwarding_rule(forwarding_rule=self) def __repr__(self): @@ -791,6 +818,7 @@ def delete(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_delete_image(image=self) def deprecate(self, replacement, state, deprecated=None, obsolete=None, deleted=None): @@ -816,6 +844,7 @@ def deprecate(self, replacement, state, deprecated=None, obsolete=None, deleted= :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_deprecate_image( self, replacement, state, deprecated, obsolete, deleted ) @@ -878,6 +907,7 @@ def destroy(self): :return: Return True if successful. :rtype: ``bool`` """ + return self.driver.ex_destroy_sslcertificate(sslcertificate=self) @@ -901,6 +931,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_subnetwork(self) def __repr__(self): @@ -924,6 +955,7 @@ def __init__(self, id, name, cidr, driver, extra=None): self.extra = extra self.mode = "legacy" self.subnetworks = [] + if "mode" in extra and extra["mode"] != "legacy": self.mode = extra["mode"] self.subnetworks = extra["subnetworks"] @@ -936,6 +968,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_network(network=self) def __repr__(self): @@ -978,10 +1011,12 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_route(route=self) def __repr__(self): network_name = getattr(self.network, "name", self.network) + return ''.format( self.id, self.name, @@ -1034,6 +1069,7 @@ def set_common_instance_metadata(self, metadata=None, force=False): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_set_common_instance_metadata(metadata=metadata, force=force) def set_usage_export_bucket(self, bucket, prefix=None): @@ -1056,6 +1092,7 @@ def set_usage_export_bucket(self, bucket, prefix=None): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_set_usage_export_bucket(bucket=bucket, prefix=prefix) def __repr__(self): @@ -1107,6 +1144,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_targethttpproxy(targethttpproxy=self) @@ -1186,6 +1224,7 @@ def set_sslcertificates(self, sslcertificates): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_targethttpsproxy_set_sslcertificates( targethttpsproxy=self, sslcertificates=sslcertificates ) @@ -1218,6 +1257,7 @@ def destroy(self): :return: Return True if successful. :rtype: ``bool`` """ + return self.driver.ex_destroy_targethttpsproxy(targethttpsproxy=self) @@ -1238,6 +1278,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_targetinstance(targetinstance=self) def __repr__(self): @@ -1269,6 +1310,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_autoscaler(autoscaler=self) def __repr__(self): @@ -1304,6 +1346,7 @@ def destroy(self): :return: Return True if successful. :rtype: ``bool`` """ + return self.driver.ex_destroy_instancetemplate(instancetemplate=self) @@ -1369,6 +1412,7 @@ def destroy(self): :return: Return True if successful. :rtype: ``bool`` """ + return self.driver.ex_destroy_instancegroup(instancegroup=self) def add_instances(self, node_list): @@ -1392,6 +1436,7 @@ def add_instances(self, node_list): :return: Return True if successful. :rtype: ``bool`` """ + return self.driver.ex_instancegroup_add_instances(instancegroup=self, node_list=node_list) def list_instances(self): @@ -1406,6 +1451,7 @@ def list_instances(self): :return: List of :class:`GCENode` objects. :rtype: ``list`` of :class:`GCENode` objects. """ + return self.driver.ex_instancegroup_list_instances(instancegroup=self) def remove_instances(self, node_list): @@ -1428,6 +1474,7 @@ def remove_instances(self, node_list): :return: Return True if successful. :rtype: ``bool`` """ + return self.driver.ex_instancegroup_remove_instances( instancegroup=self, node_list=node_list ) @@ -1453,6 +1500,7 @@ def set_named_ports(self, named_ports): :return: Return True if successful. :rtype: ``bool`` """ + return self.driver.ex_instancegroup_set_named_ports( instancegroup=self, named_ports=named_ports ) @@ -1511,6 +1559,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_instancegroupmanager(manager=self) def list_managed_instances(self): @@ -1528,6 +1577,7 @@ def list_managed_instances(self): more details. :rtype: ``list`` """ + return self.driver.ex_instancegroupmanager_list_managed_instances(manager=self) def set_instancetemplate(self, instancetemplate): @@ -1540,6 +1590,7 @@ def set_instancetemplate(self, instancetemplate): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_instancegroupmanager_set_instancetemplate( manager=self, instancetemplate=instancetemplate ) @@ -1554,6 +1605,7 @@ def recreate_instances(self): more details. :rtype: ``list`` """ + return self.driver.ex_instancegroupmanager_recreate_instances(manager=self) def delete_instances(self, node_list): @@ -1572,6 +1624,7 @@ def delete_instances(self, node_list): :return: Return True if successful. :rtype: ``bool`` """ + return self.driver.ex_instancegroupmanager_delete_instances( manager=self, node_list=node_list ) @@ -1588,6 +1641,7 @@ def resize(self, size): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_instancegroupmanager_resize(manager=self, size=size) def set_named_ports(self, named_ports): @@ -1611,6 +1665,7 @@ def set_named_ports(self, named_ports): :return: Return True if successful. :rtype: ``bool`` """ + return self.driver.ex_instancegroup_set_named_ports( instancegroup=self.instance_group, named_ports=named_ports ) @@ -1631,6 +1686,7 @@ def set_autohealingpolicies(self, healthcheck, initialdelaysec): :return: Return True if successful. :rtype: ``bool`` """ + return self.driver.ex_instancegroupmanager_set_autohealingpolicies( manager=self, healthcheck=healthcheck, initialdelaysec=initialdelaysec ) @@ -1664,6 +1720,7 @@ def add_node(self, node): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_targetpool_add_node(targetpool=self, node=node) def remove_node(self, node): @@ -1676,6 +1733,7 @@ def remove_node(self, node): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_targetpool_remove_node(targetpool=self, node=node) def add_healthcheck(self, healthcheck): @@ -1688,6 +1746,7 @@ def add_healthcheck(self, healthcheck): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_targetpool_add_healthcheck(targetpool=self, healthcheck=healthcheck) def remove_healthcheck(self, healthcheck): @@ -1700,6 +1759,7 @@ def remove_healthcheck(self, healthcheck): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_targetpool_remove_healthcheck( targetpool=self, healthcheck=healthcheck ) @@ -1720,6 +1780,7 @@ def set_backup_targetpool(self, backup_targetpool, failover_ratio=0.1): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_targetpool_set_backup_targetpool( targetpool=self, backup_targetpool=backup_targetpool, @@ -1737,6 +1798,7 @@ def get_health(self, node=None): :return: List of hashes of nodes and their respective health :rtype: ``list`` of ``dict`` """ + return self.driver.ex_targetpool_get_health(targetpool=self, node=node) def destroy(self): @@ -1746,6 +1808,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_targetpool(targetpool=self) def __repr__(self): @@ -1790,6 +1853,7 @@ def destroy(self): :return: True if successful :rtype: ``bool`` """ + return self.driver.ex_destroy_urlmap(urlmap=self) @@ -1810,6 +1874,7 @@ def time_until_mw(self): Returns the time until the next Maintenance Window as a datetime.timedelta object. """ + return self._get_time_until_mw() @property @@ -1818,6 +1883,7 @@ def next_mw_duration(self): Returns the duration of the next Maintenance Window as a datetime.timedelta object. """ + return self._get_next_mw_duration() def _now(self): @@ -1826,6 +1892,7 @@ def _now(self): Can be overridden in unittests. """ + return datetime.datetime.utcnow() def _get_next_maint(self): @@ -1843,15 +1910,20 @@ def _get_next_maint(self): """ begin = None next_window = None + if not self.maintenance_windows: return None + if len(self.maintenance_windows) == 1: return self.maintenance_windows[0] + for mw in self.maintenance_windows: begin_next = timestamp_to_datetime(mw["beginTime"]) + if (not begin) or (begin_next < begin): begin = begin_next next_window = mw + return next_window def _get_time_until_mw(self): @@ -1863,10 +1935,12 @@ def _get_time_until_mw(self): :rtype: :class:`datetime.timedelta` or ``None`` """ next_window = self._get_next_maint() + if not next_window: return None now = self._now() next_begin = timestamp_to_datetime(next_window["beginTime"]) + return next_begin - now def _get_next_mw_duration(self): @@ -1878,10 +1952,12 @@ def _get_next_mw_duration(self): :rtype: :class:`datetime.timedelta` or ``None`` """ next_window = self._get_next_maint() + if not next_window: return None next_begin = timestamp_to_datetime(next_window["beginTime"]) next_end = timestamp_to_datetime(next_window["endTime"]) + return next_end - next_begin def __repr__(self): @@ -2073,6 +2149,7 @@ def __init__( information used by GCEConnection. :type credential_file: ``str`` """ + if not project: raise ValueError("Project name must be specified using " '"project" keyword.') @@ -2114,12 +2191,14 @@ def zone_dict(self): if self._zone_dict is None: zones = self.ex_list_zones() self._zone_dict = {zone.name: zone for zone in zones} + return self._zone_dict @property def zone_list(self): if self._zone_list is None: self._zone_list = list(self.zone_dict.values()) + return self._zone_list @property @@ -2127,12 +2206,14 @@ def region_dict(self): if self._region_dict is None: regions = self.ex_list_regions() self._region_dict = {region.name: region for region in regions} + return self._region_dict @property def region_list(self): if self._region_list is None: self._region_list = list(self.region_dict.values()) + return self._region_list def ex_add_access_config(self, node, name, nic, nat_ip=None, config_type=None): @@ -2158,12 +2239,14 @@ def ex_add_access_config(self, node, name, nic, nat_ip=None, config_type=None): :return: True if successful :rtype: ``bool`` """ + if not isinstance(node, Node): raise ValueError("Must specify a valid libcloud node object.") node_name = node.name zone_name = node.extra["zone"].name config = {"name": name} + if config_type is None: config_type = "ONE_TO_ONE_NAT" config["type"] = config_type @@ -2173,6 +2256,7 @@ def ex_add_access_config(self, node, name, nic, nat_ip=None, config_type=None): params = {"networkInterface": nic} request = "/zones/{}/instances/{}/addAccessConfig".format(zone_name, node_name) self.connection.async_request(request, method="POST", data=config, params=params) + return True def ex_delete_access_config(self, node, name, nic): @@ -2191,6 +2275,7 @@ def ex_delete_access_config(self, node, name, nic): :return: True if successful :rtype: ``bool`` """ + if not isinstance(node, Node): raise ValueError("Must specify a valid libcloud node object.") node_name = node.name @@ -2199,6 +2284,7 @@ def ex_delete_access_config(self, node, name, nic): params = {"accessConfig": name, "networkInterface": nic} request = "/zones/{}/instances/{}/deleteAccessConfig".format(zone_name, node_name) self.connection.async_request(request, method="POST", params=params) + return True def ex_set_node_metadata(self, node, metadata): @@ -2215,10 +2301,12 @@ def ex_set_node_metadata(self, node, metadata): :return: True if successful :rtype: ``bool`` """ + if not isinstance(node, Node): raise ValueError("Must specify a valid libcloud node object.") node_name = node.name zone_name = node.extra["zone"].name + if "metadata" in node.extra and "fingerprint" in node.extra["metadata"]: current_fp = node.extra["metadata"]["fingerprint"] else: @@ -2226,6 +2314,7 @@ def ex_set_node_metadata(self, node, metadata): body = self._format_metadata(current_fp, metadata) request = "/zones/{}/instances/{}/setMetadata".format(zone_name, node_name) self.connection.async_request(request, method="POST", data=body) + return True def ex_set_node_labels(self, node, labels): @@ -2241,6 +2330,7 @@ def ex_set_node_labels(self, node, labels): :return: True if successful :rtype: ``bool`` """ + if not isinstance(node, Node): raise ValueError("Must specify a valid libcloud node object.") node_name = node.name @@ -2249,6 +2339,7 @@ def ex_set_node_labels(self, node, labels): body = {"labels": labels, "labelFingerprint": current_fp} request = "/zones/{}/instances/{}/setLabels".format(zone_name, node_name) self.connection.async_request(request, method="POST", data=body) + return True def ex_set_image_labels(self, image, labels): @@ -2264,12 +2355,14 @@ def ex_set_image_labels(self, image, labels): :return: True if successful :rtype: ``bool`` """ + if not isinstance(image, NodeImage): raise ValueError("Must specify a valid libcloud image object.") current_fp = image.extra["labelFingerprint"] body = {"labels": labels, "labelFingerprint": current_fp} request = "/global/images/%s/setLabels" % (image.name) self.connection.async_request(request, method="POST", data=body) + return True def ex_set_volume_labels(self, volume, labels): @@ -2295,6 +2388,7 @@ def ex_set_volume_labels(self, volume, labels): body = {"labels": labels, "labelFingerprint": current_fp} request = "/zones/{}/disks/{}/setLabels".format(zone_name, volume_name) self.connection.async_request(request, method="POST", data=body) + return True def ex_get_serial_output(self, node): @@ -2307,12 +2401,14 @@ def ex_get_serial_output(self, node): :return: A string containing serial port output of the node. :rtype: ``str`` """ + if not isinstance(node, Node): raise ValueError("Must specify a valid libcloud node object.") node_name = node.name zone_name = node.extra["zone"].name request = "/zones/{}/instances/{}/serialPort".format(zone_name, node_name) response = self.connection.request(request, method="GET").object + return response["contents"] def ex_list(self, list_fn, **kwargs): @@ -2332,6 +2428,7 @@ def ex_list(self, list_fn, **kwargs): :return: An iterator that returns sublists from list_fn. :rtype: :class:`GCEList` """ + return GCEList(driver=self, list_fn=list_fn, **kwargs) def ex_list_disktypes(self, zone=None): @@ -2348,6 +2445,7 @@ def ex_list_disktypes(self, zone=None): """ list_disktypes = [] zone = self._set_zone(zone) + if zone is None: request = "/aggregated/diskTypes" else: @@ -2356,12 +2454,14 @@ def ex_list_disktypes(self, zone=None): if "items" in response: # The aggregated result returns dictionaries for each region + if zone is None: for v in response["items"].values(): zone_disktypes = [self._to_disktype(a) for a in v.get("diskTypes", [])] list_disktypes.extend(zone_disktypes) else: list_disktypes = [self._to_disktype(a) for a in response["items"]] + return list_disktypes def ex_set_usage_export_bucket(self, bucket, prefix=None): @@ -2384,15 +2484,18 @@ def ex_set_usage_export_bucket(self, bucket, prefix=None): :return: True if successful :rtype: ``bool`` """ + if bucket.startswith("https://www.googleapis.com/") or bucket.startswith("gs://"): data = {"bucketName": bucket} else: raise ValueError("Invalid bucket name: %s" % bucket) + if prefix: data["reportNamePrefix"] = prefix request = "/setUsageExportBucket" self.connection.async_request(request, method="POST", data=data) + return True def ex_set_common_instance_metadata(self, metadata=None, force=False): @@ -2419,6 +2522,7 @@ def ex_set_common_instance_metadata(self, metadata=None, force=False): :return: True if successful :rtype: ``bool`` """ + if metadata: metadata = self._format_metadata("na", metadata) @@ -2428,11 +2532,13 @@ def ex_set_common_instance_metadata(self, metadata=None, force=False): current_metadata = project.extra["commonInstanceMetadata"] fingerprint = current_metadata["fingerprint"] md_items = [] + if "items" in current_metadata: md_items = current_metadata["items"] # grab copy of current 'sshKeys' in case we want to retain them current_keys = "" + for md in md_items: if md["key"] == "sshKeys": current_keys = md["value"] @@ -2441,6 +2547,7 @@ def ex_set_common_instance_metadata(self, metadata=None, force=False): md = {"fingerprint": fingerprint, "items": new_md} self.connection.async_request(request, method="POST", data=md) + return True def ex_list_addresses(self, region=None): @@ -2458,8 +2565,10 @@ def ex_list_addresses(self, region=None): :rtype: ``list`` of :class:`GCEAddress` """ list_addresses = [] + if region != "global": region = self._set_region(region) + if region is None: request = "/aggregated/addresses" elif region == "global": @@ -2470,12 +2579,14 @@ def ex_list_addresses(self, region=None): if "items" in response: # The aggregated result returns dictionaries for each region + if region is None: for v in response["items"].values(): region_addresses = [self._to_address(a) for a in v.get("addresses", [])] list_addresses.extend(region_addresses) else: list_addresses = [self._to_address(a) for a in response["items"]] + return list_addresses def ex_list_backendservices(self): @@ -2503,6 +2614,7 @@ def ex_list_healthchecks(self): request = "/global/httpHealthChecks" response = self.connection.request(request, method="GET").object list_healthchecks = [self._to_healthcheck(h) for h in response.get("items", [])] + return list_healthchecks def ex_list_firewalls(self): @@ -2516,6 +2628,7 @@ def ex_list_firewalls(self): request = "/global/firewalls" response = self.connection.request(request, method="GET").object list_firewalls = [self._to_firewall(f) for f in response.get("items", [])] + return list_firewalls def ex_list_forwarding_rules(self, region=None, global_rules=False): @@ -2539,11 +2652,13 @@ def ex_list_forwarding_rules(self, region=None, global_rules=False): :rtype: ``list`` of :class:`GCEForwardingRule` """ list_forwarding_rules = [] + if global_rules: region = None request = "/global/forwardingRules" else: region = self._set_region(region) + if region is None: request = "/aggregated/forwardingRules" else: @@ -2552,6 +2667,7 @@ def ex_list_forwarding_rules(self, region=None, global_rules=False): if "items" in response: # The aggregated result returns dictionaries for each region + if not global_rules and region is None: for v in response["items"].values(): region_forwarding_rules = [ @@ -2560,6 +2676,7 @@ def ex_list_forwarding_rules(self, region=None, global_rules=False): list_forwarding_rules.extend(region_forwarding_rules) else: list_forwarding_rules = [self._to_forwarding_rule(f) for f in response["items"]] + return list_forwarding_rules def list_images(self, ex_project=None, ex_include_deprecated=False): @@ -2579,9 +2696,11 @@ def list_images(self, ex_project=None, ex_include_deprecated=False): :rtype: ``list`` of :class:`GCENodeImage` """ dep = ex_include_deprecated + if ex_project is not None: return self.ex_list_project_images(ex_project=ex_project, ex_include_deprecated=dep) image_list = self.ex_list_project_images(ex_project=None, ex_include_deprecated=dep) + for img_proj in list(self.IMAGE_PROJECTS.keys()): try: image_list.extend( @@ -2590,6 +2709,7 @@ def list_images(self, ex_project=None, ex_include_deprecated=False): except Exception: # do not break if an OS type is invalid pass + return image_list def ex_list_project_images(self, ex_project=None, ex_include_deprecated=False): @@ -2609,8 +2729,10 @@ def ex_list_project_images(self, ex_project=None, ex_include_deprecated=False): """ list_images = [] request = "/global/images" + if ex_project is None: response = self.connection.paginated_request(request, method="GET") + for img in response.get("items", []): if "deprecated" not in img: list_images.append(self._to_node_image(img)) @@ -2621,8 +2743,10 @@ def ex_list_project_images(self, ex_project=None, ex_include_deprecated=False): list_images = [] # Save the connection request_path save_request_path = self.connection.request_path + if isinstance(ex_project, str): ex_project = [ex_project] + for proj in ex_project: # Override the connection request path new_request_path = save_request_path.replace(self.project, proj) @@ -2634,12 +2758,14 @@ def ex_list_project_images(self, ex_project=None, ex_include_deprecated=False): finally: # Restore the connection request_path self.connection.request_path = save_request_path + for img in response.get("items", []): if "deprecated" not in img: list_images.append(self._to_node_image(img)) else: if ex_include_deprecated: list_images.append(self._to_node_image(img)) + return list_images def list_locations(self): @@ -2656,6 +2782,7 @@ def list_locations(self): request = "/zones" response = self.connection.request(request, method="GET").object list_locations = [self._to_node_location(loc) for loc in response["items"]] + return list_locations def ex_list_routes(self): @@ -2669,6 +2796,7 @@ def ex_list_routes(self): request = "/global/routes" response = self.connection.request(request, method="GET").object list_routes = [self._to_route(n) for n in response.get("items", [])] + return list_routes def ex_list_sslcertificates(self): @@ -2688,6 +2816,7 @@ def ex_list_sslcertificates(self): request = "/global/sslCertificates" response = self.connection.request(request, method="GET").object list_data = [self._to_sslcertificate(a) for a in response.get("items", [])] + return list_data def ex_list_subnetworks(self, region=None): @@ -2702,6 +2831,7 @@ def ex_list_subnetworks(self, region=None): :rtype: ``list`` of :class:`GCESubnetwork` """ region = self._set_region(region) + if region is None: request = "/aggregated/subnetworks" else: @@ -2738,6 +2868,7 @@ def ex_list_networks(self): request = "/global/networks" response = self.connection.request(request, method="GET").object list_networks = [self._to_network(n) for n in response.get("items", [])] + return list_networks def list_nodes(self, ex_zone=None, ex_use_disk_cache=True): @@ -2783,12 +2914,14 @@ def list_nodes(self, ex_zone=None, ex_use_disk_cache=True): # # Just ignore that node and return the list of the # other nodes. + continue list_nodes.append(node) # Clear the volume cache as lookups are complete. self._ex_volume_dict = {} + return list_nodes def ex_list_regions(self): @@ -2802,6 +2935,7 @@ def ex_list_regions(self): request = "/regions" response = self.connection.request(request, method="GET").object list_regions = [self._to_region(r) for r in response["items"]] + return list_regions def list_sizes(self, location=None): @@ -2817,6 +2951,7 @@ def list_sizes(self, location=None): """ list_sizes = [] zone = self._set_zone(location) + if zone is None: request = "/aggregated/machineTypes" else: @@ -2825,8 +2960,10 @@ def list_sizes(self, location=None): response = self.connection.request(request, method="GET").object # getting pricing data here so it is done only once instance_prices = get_pricing(driver_type="compute", driver_name="gce_instances") + if "items" in response: # The aggregated response returns a dict for each zone + if zone is None: for v in response["items"].values(): zone_sizes = [ @@ -2835,6 +2972,7 @@ def list_sizes(self, location=None): list_sizes.extend(zone_sizes) else: list_sizes = [self._to_node_size(s, instance_prices) for s in response["items"]] + return list_sizes def ex_list_snapshots(self): @@ -2848,6 +2986,7 @@ def ex_list_snapshots(self): request = "/global/snapshots" response = self.connection.request(request, method="GET").object list_snapshots = [self._to_snapshot(s) for s in response.get("items", [])] + return list_snapshots def ex_list_targethttpproxies(self): @@ -2859,6 +2998,7 @@ def ex_list_targethttpproxies(self): """ request = "/global/targetHttpProxies" response = self.connection.request(request, method="GET").object + return [self._to_targethttpproxy(u) for u in response.get("items", [])] def ex_list_targethttpsproxies(self): @@ -2870,6 +3010,7 @@ def ex_list_targethttpsproxies(self): """ request = "/global/targetHttpsProxies" response = self.connection.request(request, method="GET").object + return [self._to_targethttpsproxy(x) for x in response.get("items", [])] def ex_list_targetinstances(self, zone=None): @@ -2881,6 +3022,7 @@ def ex_list_targetinstances(self, zone=None): """ list_targetinstances = [] zone = self._set_zone(zone) + if zone is None: request = "/aggregated/targetInstances" else: @@ -2889,6 +3031,7 @@ def ex_list_targetinstances(self, zone=None): if "items" in response: # The aggregated result returns dictionaries for each region + if zone is None: for v in response["items"].values(): zone_targetinstances = [ @@ -2897,6 +3040,7 @@ def ex_list_targetinstances(self, zone=None): list_targetinstances.extend(zone_targetinstances) else: list_targetinstances = [self._to_targetinstance(t) for t in response["items"]] + return list_targetinstances def ex_list_targetpools(self, region=None): @@ -2908,6 +3052,7 @@ def ex_list_targetpools(self, region=None): """ list_targetpools = [] region = self._set_region(region) + if region is None: request = "/aggregated/targetPools" else: @@ -2916,12 +3061,14 @@ def ex_list_targetpools(self, region=None): if "items" in response: # The aggregated result returns dictionaries for each region + if region is None: for v in response["items"].values(): region_targetpools = [self._to_targetpool(t) for t in v.get("targetPools", [])] list_targetpools.extend(region_targetpools) else: list_targetpools = [self._to_targetpool(t) for t in response["items"]] + return list_targetpools def ex_list_urlmaps(self): @@ -2933,6 +3080,7 @@ def ex_list_urlmaps(self): """ request = "/global/urlMaps" response = self.connection.request(request, method="GET").object + return [self._to_urlmap(u) for u in response.get("items", [])] def ex_list_instancegroups(self, zone): @@ -2955,19 +3103,23 @@ def ex_list_instancegroups(self, zone): list_data = [] zone = self._set_zone(zone) + if zone is None: request = "/aggregated/instanceGroups" else: request = "/zones/%s/instanceGroups" % (zone.name) response = self.connection.request(request, method="GET").object + if "items" in response: # The aggregated result returns dictionaries for each region + if zone is None: for v in response["items"].values(): zone_data = [self._to_instancegroup(a) for a in v.get("instanceGroups", [])] list_data.extend(zone_data) else: list_data = [self._to_instancegroup(a) for a in response["items"]] + return list_data def ex_list_instancegroupmanagers(self, zone=None): @@ -2985,6 +3137,7 @@ def ex_list_instancegroupmanagers(self, zone=None): """ list_managers = [] zone = self._set_zone(zone) + if zone is None: request = "/aggregated/instanceGroupManagers" else: @@ -2993,6 +3146,7 @@ def ex_list_instancegroupmanagers(self, zone=None): if "items" in response: # The aggregated result returns dictionaries for each region + if zone is None: for v in response["items"].values(): zone_managers = [ @@ -3001,6 +3155,7 @@ def ex_list_instancegroupmanagers(self, zone=None): list_managers.extend(zone_managers) else: list_managers = [self._to_instancegroupmanager(a) for a in response["items"]] + return list_managers def ex_list_instancetemplates(self): @@ -3012,6 +3167,7 @@ def ex_list_instancetemplates(self): """ request = "/global/instanceTemplates" response = self.connection.request(request, method="GET").object + return [self._to_instancetemplate(u) for u in response.get("items", [])] def ex_list_autoscalers(self, zone=None): @@ -3029,20 +3185,24 @@ def ex_list_autoscalers(self, zone=None): """ list_autoscalers = [] zone = self._set_zone(zone) + if zone is None: request = "/aggregated/autoscalers" else: request = "/zones/%s/autoscalers" % (zone.name) response = self.connection.request(request, method="GET").object + if "items" in response: # The aggregated result returns dictionaries for each zone. + if zone is None: for v in response["items"].values(): zone_as = [self._to_autoscaler(a) for a in v.get("autoscalers", [])] list_autoscalers.extend(zone_as) else: list_autoscalers = [self._to_autoscaler(a) for a in response["items"]] + return list_autoscalers def list_volumes(self, ex_zone=None): @@ -3061,20 +3221,24 @@ def list_volumes(self, ex_zone=None): """ list_volumes = [] zone = self._set_zone(ex_zone) + if zone is None: request = "/aggregated/disks" else: request = "/zones/%s/disks" % (zone.name) response = self.connection.request(request, method="GET").object + if "items" in response: # The aggregated response returns a dict for each zone + if zone is None: for v in response["items"].values(): zone_volumes = [self._to_storage_volume(d) for d in v.get("disks", [])] list_volumes.extend(zone_volumes) else: list_volumes = [self._to_storage_volume(d) for d in response["items"]] + return list_volumes def ex_list_zones(self): @@ -3088,6 +3252,7 @@ def ex_list_zones(self): request = "/zones" response = self.connection.request(request, method="GET").object list_zones = [self._to_zone(z) for z in response["items"]] + return list_zones def ex_create_address( @@ -3133,15 +3298,20 @@ def ex_create_address( :rtype: :class:`GCEAddress` """ region = region or self.region + if region is None: raise ValueError("REGION_NOT_SPECIFIED", "Region must be provided for an address") + if region != "global" and not hasattr(region, "name"): region = self.ex_get_region(region) address_data = {"name": name} + if address: address_data["address"] = address + if description: address_data["description"] = description + if address_type: if address_type not in ["EXTERNAL", "INTERNAL"]: raise ValueError( @@ -3151,19 +3321,23 @@ def ex_create_address( ) else: address_data["addressType"] = address_type + if subnetwork and address_type != "INTERNAL": raise ValueError( "INVALID_ARGUMENT_COMBINATION", "Address type must be internal if subnetwork \ provided", ) + if subnetwork and not hasattr(subnetwork, "name"): subnetwork = self.ex_get_subnetwork(subnetwork, region) + if region == "global": request = "/global/addresses" else: request = "/regions/%s/addresses" % (region.name) self.connection.async_request(request, method="POST", data=address_data) + return self.ex_get_address(name, region=region) def ex_create_autoscaler(self, name, zone, instance_group, policy, description=None): @@ -3189,6 +3363,7 @@ def ex_create_autoscaler(self, name, zone, instance_group, policy, description=N zone = zone or self.zone autoscaler_data = {} autoscaler_data = {"name": name} + if not hasattr(zone, "name"): zone = self.ex_get_zone(zone) autoscaler_data["zone"] = zone.extra["selfLink"] @@ -3198,6 +3373,7 @@ def ex_create_autoscaler(self, name, zone, instance_group, policy, description=N request = "/zones/%s/autoscalers" % zone.name autoscaler_data["target"] = instance_group.extra["selfLink"] self.connection.async_request(request, method="POST", data=autoscaler_data) + return self.ex_get_autoscaler(name, zone) def ex_create_backend( @@ -3359,12 +3535,16 @@ def ex_create_backendservice( backendservice_data["backends"].append(be.to_backend_dict()) else: backendservice_data["backends"].append(be) + if port: backendservice_data["port"] = port + if port_name: backendservice_data["portName"] = port_name + if timeout_sec: backendservice_data["timeoutSec"] = timeout_sec + if protocol: if protocol in self.BACKEND_SERVICE_PROTOCOLS: backendservice_data["protocol"] = protocol @@ -3372,11 +3552,13 @@ def ex_create_backendservice( raise ValueError( "Protocol must be one of %s" % ",".join(self.BACKEND_SERVICE_PROTOCOLS) ) + if description: backendservice_data["description"] = description request = "/global/backendServices" self.connection.async_request(request, method="POST", data=backendservice_data) + return self.ex_get_backendservice(name) def ex_create_healthcheck( @@ -3429,8 +3611,10 @@ def ex_create_healthcheck( """ hc_data = {} hc_data["name"] = name + if host: hc_data["host"] = host + if description: hc_data["description"] = description # As of right now, the 'default' values aren't getting set when called @@ -3445,6 +3629,7 @@ def ex_create_healthcheck( request = "/global/httpHealthChecks" self.connection.async_request(request, method="POST", data=hc_data) + return self.ex_get_healthcheck(name) def ex_create_firewall( @@ -3542,6 +3727,7 @@ def ex_create_firewall( :rtype: :class:`GCEFirewall` """ firewall_data = {} + if not hasattr(network, "name"): nw = self.ex_get_network(network) else: @@ -3551,29 +3737,38 @@ def ex_create_firewall( firewall_data["direction"] = direction firewall_data["priority"] = priority firewall_data["description"] = description + if direction == "INGRESS": firewall_data["allowed"] = allowed elif direction == "EGRESS": firewall_data["denied"] = denied firewall_data["network"] = nw.extra["selfLink"] + if source_ranges is None and source_tags is None and source_service_accounts is None: source_ranges = ["0.0.0.0/0"] + if source_ranges is not None: firewall_data["sourceRanges"] = source_ranges + if source_tags is not None: firewall_data["sourceTags"] = source_tags + if source_service_accounts is not None: firewall_data["sourceServiceAccounts"] = source_service_accounts + if target_tags is not None: firewall_data["targetTags"] = target_tags + if target_service_accounts is not None: firewall_data["targetServiceAccounts"] = target_service_accounts + if target_ranges is not None: firewall_data["destinationRanges"] = target_ranges request = "/global/firewalls" self.connection.async_request(request, method="POST", data=firewall_data) + return self.ex_get_firewall(name) def ex_create_forwarding_rule( @@ -3638,28 +3833,34 @@ def ex_create_forwarding_rule( :rtype: :class:`GCEForwardingRule` """ forwarding_rule_data = {"name": name} + if global_rule: if not hasattr(target, "name"): target = self.ex_get_targethttpproxy(target) else: region = region or self.region + if not hasattr(region, "name"): region = self.ex_get_region(region) forwarding_rule_data["region"] = region.extra["selfLink"] if not target: target = targetpool # Backwards compatibility + if not hasattr(target, "name"): target = self.ex_get_targetpool(target, region) forwarding_rule_data["target"] = target.extra["selfLink"] forwarding_rule_data["IPProtocol"] = protocol.upper() + if address: if not hasattr(address, "name"): address = self.ex_get_address(address, "global" if global_rule else region) forwarding_rule_data["IPAddress"] = address.address + if port_range: forwarding_rule_data["portRange"] = port_range + if description: forwarding_rule_data["description"] = description @@ -3739,6 +3940,7 @@ def ex_create_image( image_data["name"] = name image_data["description"] = description image_data["family"] = family + if isinstance(volume, StorageVolume): image_data["sourceDisk"] = volume.extra["selfLink"] image_data["zone"] = volume.extra["zone"].name @@ -3748,6 +3950,7 @@ def ex_create_image( image_data["rawDisk"] = {"source": volume, "containerType": "TAR"} else: raise ValueError("Source must be instance of StorageVolume or URI") + if ex_licenses: if isinstance(ex_licenses, str): ex_licenses = [ex_licenses] @@ -3758,8 +3961,10 @@ def ex_create_image( if guest_os_features: image_data["guestOsFeatures"] = [] + if isinstance(guest_os_features, str): guest_os_features = [guest_os_features] + for feature in guest_os_features: image_data["guestOsFeatures"].append({"type": feature}) request = "/global/images" @@ -3801,6 +4006,7 @@ def ex_copy_image(self, name, url, description=None, family=None, guest_os_featu """ # The URL for an image can start with gs:// + if url.startswith("gs://"): url = url.replace("gs://", "https://storage.googleapis.com/", 1) @@ -3814,13 +4020,16 @@ def ex_copy_image(self, name, url, description=None, family=None, guest_os_featu if guest_os_features: image_data["guestOsFeatures"] = [] + if isinstance(guest_os_features, str): guest_os_features = [guest_os_features] + for feature in guest_os_features: image_data["guestOsFeatures"].append({"type": feature}) request = "/global/images" self.connection.async_request(request, method="POST", data=image_data) + return self.ex_get_image(name) def ex_create_instancegroup( @@ -3875,18 +4084,23 @@ def ex_create_instancegroup( :rtype: :class:`GCEInstanceGroup` """ zone = zone or self.zone + if not hasattr(zone, "name"): zone = self.ex_get_zone(zone) request = "/zones/%s/instanceGroups" % (zone.name) request_data = {} request_data["name"] = name request_data["zone"] = zone.extra["selfLink"] + if description: request_data["description"] = description + if network: request_data["network"] = network.extra["selfLink"] + if subnetwork: request_data["subnetwork"] = subnetwork.extra["selfLink"] + if named_ports: request_data["namedPorts"] = named_ports @@ -3921,6 +4135,7 @@ def ex_create_instancegroupmanager( :rtype: :class:`GCEInstanceGroupManager` """ zone = zone or self.zone + if not hasattr(zone, "name"): zone = self.ex_get_zone(zone) @@ -3929,12 +4144,14 @@ def ex_create_instancegroupmanager( manager_data = {} # If the user gave us a name, we fetch the GCEInstanceTemplate for it. + if not hasattr(template, "name"): template = self.ex_get_instancetemplate(template) manager_data["instanceTemplate"] = template.extra["selfLink"] # If base_instance_name is not set, we use name. manager_data["baseInstanceName"] = name + if base_instance_name is not None: manager_data["baseInstanceName"] = base_instance_name @@ -3992,6 +4209,7 @@ def ex_create_route( route_data["destRange"] = dest_range route_data["priority"] = priority route_data["description"] = description + if isinstance(network, str) and network.startswith("https://"): network_uri = network elif isinstance(network, str): @@ -4001,6 +4219,7 @@ def ex_create_route( network_uri = network.extra["selfLink"] route_data["network"] = network_uri route_data["tags"] = tags + if next_hop is None: url = "https://www.googleapis.com/compute/{}/projects/{}/{}".format( API_VERSION, @@ -4112,6 +4331,7 @@ def ex_create_subnetwork( :return: Subnetwork object :rtype: :class:`GCESubnetwork` """ + if not cidr: raise ValueError("Must provide an IP network in CIDR notation.") @@ -4186,10 +4406,12 @@ def ex_create_network(self, name, cidr, description=None, mode="legacy", routing network_data = {} network_data["name"] = name network_data["description"] = description + if mode.lower() not in ["auto", "custom", "legacy"]: raise ValueError( "Invalid network mode: '%s'. Must be 'auto', " "'custom', or 'legacy'." % mode ) + if cidr and mode in ["auto", "custom"]: raise ValueError("Can only specify IPv4Range with 'legacy' mode.") @@ -4388,6 +4610,7 @@ def create_node( :return: A Node object for the new node. :rtype: :class:`Node` """ + if ex_boot_disk and ex_disks_gce_struct: raise ValueError("Cannot specify both 'ex_boot_disk' and " "'ex_disks_gce_struct'") @@ -4403,24 +4626,33 @@ def create_node( ) location = location or self.zone + if location and not hasattr(location, "name"): location = self.ex_get_zone(location) + if not hasattr(size, "name"): size = self.ex_get_size(size, location) + if not hasattr(ex_network, "name"): ex_network = self.ex_get_network(ex_network) + if ex_subnetwork and not hasattr(ex_subnetwork, "name"): ex_subnetwork = self.ex_get_subnetwork( ex_subnetwork, region=self._get_region_from_zone(location) ) + if ex_image_family: image = self.ex_get_image_from_family(ex_image_family) + if image and not hasattr(image, "name"): image = self.ex_get_image(image) + if ex_disk_type and not hasattr(ex_disk_type, "name"): ex_disk_type = self.ex_get_disktype(ex_disk_type, zone=location) + if ex_boot_disk and not hasattr(ex_boot_disk, "name"): ex_boot_disk = self.ex_get_volume(ex_boot_disk, zone=location) + if ex_accelerator_type and not hasattr(ex_accelerator_type, "name"): if ex_accelerator_count is None: raise ValueError( @@ -4431,6 +4663,7 @@ def create_node( ex_accelerator_type = self.ex_get_accelerator_type(ex_accelerator_type, zone=location) # Use disks[].initializeParams to auto-create the boot disk + if not ex_disks_gce_struct and not ex_boot_disk: ex_disks_gce_struct = [ { @@ -4485,6 +4718,7 @@ def create_node( ex_accelerator_count=ex_accelerator_count, ) self.connection.async_request(request, method="POST", data=node_data) + return self.ex_get_node(name, location.name) def ex_create_instancetemplate( @@ -4827,6 +5061,7 @@ def _create_instance_properties( instance_properties = {} # build disks + if not image and not source and not disks_gce_struct: raise ValueError( "Missing root device or image. Must specify an " @@ -4844,6 +5079,7 @@ def _create_instance_properties( else: disk_name = None device_name = None + if source: disk_name = source.name # TODO(supertom): what about device name? @@ -4867,6 +5103,7 @@ def _create_instance_properties( ] # build network interfaces + if nic_gce_struct is not None: if hasattr(external_ip, "address"): raise ValueError( @@ -4874,6 +5111,7 @@ def _create_instance_properties( "and 'nic_gce_struct'. Use one or the " "other." ) + if hasattr(network, "name"): if network.name == "default": # pylint: disable=no-member # assume this is just the default value from create_node() @@ -4902,6 +5140,7 @@ def _create_instance_properties( scheduling = self._build_scheduling_gce_struct( on_host_maintenance, automatic_restart, preemptible ) + if scheduling: instance_properties["scheduling"] = scheduling @@ -4911,22 +5150,28 @@ def _create_instance_properties( ) # build accelerators + if accelerator_type is not None: instance_properties["guestAccelerators"] = self._format_guest_accelerators( accelerator_type, accelerator_count ) # include general properties + if description: instance_properties["description"] = str(description) + if tags: instance_properties["tags"] = {"items": tags} + if metadata: instance_properties["metadata"] = self._format_metadata( fingerprint="na", metadata=metadata ) + if labels: instance_properties["labels"] = labels + if can_ip_forward: instance_properties["canIpForward"] = True @@ -4999,6 +5244,7 @@ def _build_disk_gce_struct( :rtype: ``dict`` """ # validation + if source is None and image is None: raise ValueError("Either the 'source' or 'image' argument must be specified.") @@ -5009,13 +5255,16 @@ def _build_disk_gce_struct( raise ValueError("disk_size must be a digit, '%s' provided." % str(disk_size)) mount_modes = ["READ_WRITE", "READ_ONLY"] + if mount_mode not in mount_modes: raise ValueError("mount mode must be one of: %s." % (",".join(mount_modes))) usage_types = ["PERSISTENT", "SCRATCH"] + if usage_type not in usage_types: raise ValueError("usage type must be one of: %s." % (",".join(usage_types))) disk = {} + if not disk_name: disk_name = device_name @@ -5037,6 +5286,7 @@ def _build_disk_gce_struct( "diskType": disk_type, "sourceImage": image, } + if disk_size is not None: disk["initializeParams"]["diskSizeGb"] = disk_size @@ -5050,6 +5300,7 @@ def _build_disk_gce_struct( "autoDelete": auto_delete, } ) + return disk def _get_selflink_or_name(self, obj, get_selflinks=True, objname=None): @@ -5073,6 +5324,7 @@ def _get_selflink_or_name(self, obj, get_selflinks=True, objname=None): :return: URL from extra['selfLink'] or name :rtype: ``str`` """ + if get_selflinks: if not hasattr(obj, "name"): if objname: @@ -5080,6 +5332,7 @@ def _get_selflink_or_name(self, obj, get_selflinks=True, objname=None): obj = getobj(obj) else: raise ValueError("objname must be set if selflinks is True.") + return obj.extra["selfLink"] else: if not hasattr(obj, "name"): @@ -5121,6 +5374,7 @@ def _build_network_gce_struct( """ ni = {} ni = {"kind": "compute#instanceNetworkInterface"} + if network is None: network = "default" @@ -5135,6 +5389,7 @@ def _build_network_gce_struct( if external_ip: access_configs = [{"name": "External NAT", "type": "ONE_TO_ONE_NAT"}] + if hasattr(external_ip, "address"): access_configs[0]["natIP"] = external_ip.address ni["accessConfigs"] = access_configs @@ -5170,12 +5425,14 @@ def _build_service_account_gce_struct( :return: dict usable in GCE API call. :rtype: ``dict`` """ + if not isinstance(service_account, dict): raise ValueError( "service_account not in the correct format," "'%s - %s'" % (str(type(service_account)), str(service_account)) ) sa = {} + if "email" not in service_account: sa["email"] = default_email else: @@ -5185,6 +5442,7 @@ def _build_service_account_gce_struct( sa["scopes"] = [self.AUTH_URL + default_scope] else: ps = [] + for scope in service_account["scopes"]: if scope.startswith(self.AUTH_URL): ps.append(scope) @@ -5227,6 +5485,7 @@ def _build_service_accounts_gce_list( :rtype: ``list`` of ``dict`` """ gce_service_accounts = [] + if service_accounts is None: gce_service_accounts = [ {"email": default_email, "scopes": [self.AUTH_URL + default_scope]} @@ -5272,13 +5531,16 @@ def _build_scheduling_gce_struct( :rtype: ``dict`` """ scheduling = {} + if preemptible is not None: if isinstance(preemptible, bool): scheduling["preemptible"] = preemptible else: raise ValueError("boolean expected for preemptible") + if on_host_maintenance is not None: maint_opts = ["MIGRATE", "TERMINATE"] + if isinstance(on_host_maintenance, str) and on_host_maintenance in maint_opts: if preemptible is True and on_host_maintenance == "MIGRATE": raise ValueError( @@ -5287,6 +5549,7 @@ def _build_scheduling_gce_struct( scheduling["onHostMaintenance"] = on_host_maintenance else: raise ValueError("host maintenance must be one of %s" % (",".join(maint_opts))) + if automatic_restart is not None: if isinstance(automatic_restart, bool): if automatic_restart is True and preemptible is True: @@ -5479,6 +5742,7 @@ def ex_create_multiple_nodes( :rtype: ``list`` of :class:`Node` """ + if image and ex_disks_gce_struct: raise ValueError("Cannot specify both 'image' and " "'ex_disks_gce_struct'.") @@ -5486,20 +5750,27 @@ def ex_create_multiple_nodes( raise ValueError("Cannot specify both 'image' and " "'ex_image_family'") location = location or self.zone + if not hasattr(location, "name"): location = self.ex_get_zone(location) + if not hasattr(size, "name"): size = self.ex_get_size(size, location) + if not hasattr(ex_network, "name"): ex_network = self.ex_get_network(ex_network) + if ex_subnetwork and not hasattr(ex_subnetwork, "name"): ex_subnetwork = self.ex_get_subnetwork( ex_subnetwork, region=self._get_region_from_zone(location) ) + if ex_image_family: image = self.ex_get_image_from_family(ex_image_family) + if image and not hasattr(image, "name"): image = self.ex_get_image(image) + if not hasattr(ex_disk_type, "name"): ex_disk_type = self.ex_get_disktype(ex_disk_type, zone=location) @@ -5538,13 +5809,16 @@ def ex_create_multiple_nodes( start_time = time.time() complete = False + while not complete: if time.time() - start_time >= timeout: raise Exception("Timeout (%s sec) while waiting for multiple " "instances") complete = True time.sleep(poll_interval) + for status in status_list: # Create the node or check status if already in progress. + if not status["node"]: if not status["node_response"]: self._multi_create_node(status, node_attrs) @@ -5552,13 +5826,16 @@ def ex_create_multiple_nodes( self._multi_check_node(status, node_attrs) # If any of the nodes have not been created (or failed) we are # not done yet. + if not status["node"]: complete = False # Return list of nodes node_list = [] + for status in status_list: node_list.append(status["node"]) + return node_list def ex_create_targethttpproxy(self, name, urlmap): @@ -5665,19 +5942,23 @@ def ex_create_targetinstance( zone = zone or self.zone targetinstance_data = {} targetinstance_data["name"] = name + if not hasattr(zone, "name"): zone = self.ex_get_zone(zone) targetinstance_data["zone"] = zone.extra["selfLink"] + if node is not None: if not hasattr(node, "name"): node = self.ex_get_node(node, zone) targetinstance_data["instance"] = node.extra["selfLink"] targetinstance_data["natPolicy"] = nat_policy + if description: targetinstance_data["description"] = description request = "/zones/%s/targetInstances" % (zone.name) self.connection.async_request(request, method="POST", data=targetinstance_data) + return self.ex_get_targetinstance(name, zone) def ex_create_targetpool( @@ -5724,14 +6005,17 @@ def ex_create_targetpool( """ targetpool_data = {} region = region or self.region + if backup_pool and not failover_ratio: failover_ratio = 0.1 targetpool_data["failoverRatio"] = failover_ratio targetpool_data["backupPool"] = backup_pool.extra["selfLink"] + if failover_ratio and not backup_pool: e = "Must supply a backup targetPool when setting failover_ratio" raise ValueError(e) targetpool_data["name"] = name + if not hasattr(region, "name"): region = self.ex_get_region(region) targetpool_data["region"] = region.extra["selfLink"] @@ -5742,12 +6026,14 @@ def ex_create_targetpool( else: hc_list = [h.extra["selfLink"] for h in healthchecks] targetpool_data["healthChecks"] = hc_list + if nodes: if not hasattr(nodes[0], "name"): node_list = [self.ex_get_node(n, "all").extra["selfLink"] for n in nodes] else: node_list = [n.extra["selfLink"] for n in nodes] targetpool_data["instances"] = node_list + if session_affinity: targetpool_data["sessionAffinity"] = session_affinity @@ -5773,6 +6059,7 @@ def ex_create_urlmap(self, name, default_service): urlmap_data = {"name": name} # TODO: support hostRules, pathMatchers, tests + if not hasattr(default_service, "name"): default_service = self.ex_get_backendservice(default_service) urlmap_data["defaultService"] = default_service.extra["selfLink"] @@ -5830,6 +6117,7 @@ def create_volume( :return: Storage Volume object :rtype: :class:`StorageVolume` """ + if image and ex_image_family: raise ValueError("Cannot specify both 'image' and " "'ex_image_family'") @@ -5884,9 +6172,11 @@ def list_volume_snapshots(self, volume): volume_snapshots = [] volume_link = volume.extra["selfLink"] all_snapshots = self.ex_list_snapshots() + for snapshot in all_snapshots: if snapshot.extra["sourceDisk"] == volume_link: volume_snapshots.append(snapshot) + return volume_snapshots def ex_update_autoscaler(self, autoscaler): @@ -5933,8 +6223,10 @@ def ex_update_healthcheck(self, healthcheck): hc_data["timeoutSec"] = healthcheck.timeout hc_data["unhealthyThreshold"] = healthcheck.unhealthy_threshold hc_data["healthyThreshold"] = healthcheck.healthy_threshold + if healthcheck.extra["host"]: hc_data["host"] = healthcheck.extra["host"] + if healthcheck.extra["description"]: hc_data["description"] = healthcheck.extra["description"] @@ -5963,18 +6255,25 @@ def ex_update_firewall(self, firewall): firewall_data["denied"] = firewall.denied # Priority updates not yet exposed via API firewall_data["network"] = firewall.network.extra["selfLink"] + if firewall.source_ranges: firewall_data["sourceRanges"] = firewall.source_ranges + if firewall.source_tags: firewall_data["sourceTags"] = firewall.source_tags + if firewall.source_service_accounts: firewall_data["sourceServiceAccounts"] = firewall.source_service_accounts + if firewall.target_tags: firewall_data["targetTags"] = firewall.target_tags + if firewall.target_service_accounts: firewall_data["targetServiceAccounts"] = firewall.target_service_accounts + if firewall.target_ranges: firewall_data["destinationRanges"] = firewall.target_ranges + if firewall.extra["description"]: firewall_data["description"] = firewall.extra["description"] @@ -6061,8 +6360,11 @@ def ex_targetpool_get_health(self, targetpool, node=None): node_name = node.name else: node_name = node + else: + node_name = None nodes = targetpool.nodes + for node_object in nodes: if node: if node_name == node_object.name: @@ -6075,6 +6377,7 @@ def ex_targetpool_get_health(self, targetpool, node=None): resp = self.connection.request(request, method="POST", data=body).object status = resp["healthStatus"][0]["healthState"] health.append({"node": node_object, "health": status}) + return health def ex_targetpool_set_backup_targetpool( @@ -6105,6 +6408,7 @@ def ex_targetpool_set_backup_targetpool( request = "/regions/{}/targetPools/{}/setBackup".format(region, name) self.connection.async_request(request, method="POST", data=req_data, params=params) + return True def ex_targetpool_add_node(self, targetpool, node): @@ -6120,8 +6424,10 @@ def ex_targetpool_add_node(self, targetpool, node): :return: True if successful :rtype: ``bool`` """ + if not hasattr(targetpool, "name"): targetpool = self.ex_get_targetpool(targetpool) + if hasattr(node, "name"): node_uri = node.extra["selfLink"] else: @@ -6138,11 +6444,13 @@ def ex_targetpool_add_node(self, targetpool, node): targetpool.name, ) self.connection.async_request(request, method="POST", data=targetpool_data) + if all( (node_uri != n) and (not hasattr(n, "extra") or n.extra["selfLink"] != node_uri) for n in targetpool.nodes ): targetpool.nodes.append(node) + return True def ex_targetpool_add_healthcheck(self, targetpool, healthcheck): @@ -6158,8 +6466,10 @@ def ex_targetpool_add_healthcheck(self, targetpool, healthcheck): :return: True if successful :rtype: ``bool`` """ + if not hasattr(targetpool, "name"): targetpool = self.ex_get_targetpool(targetpool) + if not hasattr(healthcheck, "name"): healthcheck = self.ex_get_healthcheck(healthcheck) @@ -6171,6 +6481,7 @@ def ex_targetpool_add_healthcheck(self, targetpool, healthcheck): ) self.connection.async_request(request, method="POST", data=targetpool_data) targetpool.healthchecks.append(healthcheck) + return True def ex_targetpool_remove_node(self, targetpool, node): @@ -6186,6 +6497,7 @@ def ex_targetpool_remove_node(self, targetpool, node): :return: True if successful :rtype: ``bool`` """ + if not hasattr(targetpool, "name"): targetpool = self.ex_get_targetpool(targetpool) @@ -6207,12 +6519,16 @@ def ex_targetpool_remove_node(self, targetpool, node): self.connection.async_request(request, method="POST", data=targetpool_data) # Remove node object from node list index = None + for i, nd in enumerate(targetpool.nodes): if nd == node_uri or (hasattr(nd, "extra") and nd.extra["selfLink"] == node_uri): index = i + break + if index is not None: targetpool.nodes.pop(index) + return True def ex_targetpool_remove_healthcheck(self, targetpool, healthcheck): @@ -6228,8 +6544,10 @@ def ex_targetpool_remove_healthcheck(self, targetpool, healthcheck): :return: True if successful :rtype: ``bool`` """ + if not hasattr(targetpool, "name"): targetpool = self.ex_get_targetpool(targetpool) + if not hasattr(healthcheck, "name"): healthcheck = self.ex_get_healthcheck(healthcheck) @@ -6242,11 +6560,14 @@ def ex_targetpool_remove_healthcheck(self, targetpool, healthcheck): self.connection.async_request(request, method="POST", data=targetpool_data) # Remove healthcheck object from healthchecks list index = None + for i, hc in enumerate(targetpool.healthchecks): if hc.name == healthcheck.name: index = i + if index is not None: targetpool.healthchecks.pop(index) + return True def ex_instancegroup_add_instances(self, instancegroup, node_list): @@ -6276,6 +6597,7 @@ def ex_instancegroup_add_instances(self, instancegroup, node_list): ) request_data = {"instances": [{"instance": x.extra["selfLink"]} for x in node_list]} self.connection.async_request(request, method="POST", data=request_data) + return True def ex_instancegroup_remove_instances(self, instancegroup, node_list): @@ -6304,6 +6626,7 @@ def ex_instancegroup_remove_instances(self, instancegroup, node_list): ) request_data = {"instances": [{"instance": x.extra["selfLink"]} for x in node_list]} self.connection.async_request(request, method="POST", data=request_data) + return True def ex_instancegroup_list_instances(self, instancegroup): @@ -6332,10 +6655,12 @@ def ex_instancegroup_list_instances(self, instancegroup): response = self.connection.request(request, method="POST").object list_data = [] + if "items" in response: for v in response["items"]: instance_info = self._get_components_from_path(v["instance"]) list_data.append(self.ex_get_node(instance_info["name"], instance_info["zone"])) + return list_data def ex_instancegroup_set_named_ports(self, instancegroup, named_ports=[]): @@ -6376,6 +6701,7 @@ def ex_instancegroup_set_named_ports(self, instancegroup, named_ports=[]): "fingerprint": instancegroup.extra["fingerprint"], } self.connection.async_request(request, method="POST", data=request_data) + return True def ex_destroy_instancegroup(self, instancegroup): @@ -6441,6 +6767,7 @@ def ex_instancegroupmanager_list_managed_instances(self, manager): response = self.connection.request(request, method="POST").object instance_data = [] + if "managedInstances" in response: for i in response["managedInstances"]: i["name"] = self._get_components_from_path(i["instance"])["name"] @@ -6476,6 +6803,7 @@ def ex_instancegroupmanager_set_autohealingpolicies( manager.name, ) self.connection.async_request(request, method="PATCH", data=request_data) + return True def ex_instancegroupmanager_set_instancetemplate(self, manager, instancetemplate): @@ -6499,6 +6827,7 @@ def ex_instancegroupmanager_set_instancetemplate(self, manager, instancetemplate manager.name, ) self.connection.async_request(request, method="POST", data=req_data) + return True def ex_instancegroupmanager_recreate_instances(self, manager, instances=None): @@ -6538,6 +6867,7 @@ def ex_instancegroupmanager_recreate_instances(self, manager, instances=None): "InstanceGroupManager must be of type str or " "GCEInstanceGroupManager. Type '%s' provided" % (type(manager)) ) + if isinstance(manager, str): manager = self.ex_get_instancegroupmanager(manager) @@ -6616,6 +6946,7 @@ def ex_instancegroupmanager_resize(self, manager, size): manager.name, ) self.connection.async_request(request, method="POST", params=req_params) + return True def reboot_node(self, node): @@ -6630,6 +6961,7 @@ def reboot_node(self, node): """ request = "/zones/{}/instances/{}/reset".format(node.extra["zone"].name, node.name) self.connection.async_request(request, method="POST", data="ignored") + return True def ex_set_node_tags(self, node, tags): @@ -6660,6 +6992,7 @@ def ex_set_node_tags(self, node, tags): new_node = self.ex_get_node(node.name, node.extra["zone"]) node.extra["tags"] = new_node.extra["tags"] node.extra["tags_fingerprint"] = new_node.extra["tags_fingerprint"] + return True def ex_set_node_scheduling(self, node, on_host_maintenance=None, automatic_restart=None): @@ -6689,11 +7022,14 @@ def ex_set_node_scheduling(self, node, on_host_maintenance=None, automatic_resta :return: True if successful. :rtype: ``bool`` """ + if not hasattr(node, "name"): node = self.ex_get_node(node, "all") + if on_host_maintenance is not None: on_host_maintenance = on_host_maintenance.upper() ohm_values = ["MIGRATE", "TERMINATE"] + if on_host_maintenance not in ohm_values: raise ValueError("on_host_maintenance must be one of %s" % ",".join(ohm_values)) @@ -6703,8 +7039,10 @@ def ex_set_node_scheduling(self, node, on_host_maintenance=None, automatic_resta ) scheduling_data = {} + if on_host_maintenance is not None: scheduling_data["onHostMaintenance"] = on_host_maintenance + if automatic_restart is not None: scheduling_data["automaticRestart"] = automatic_restart @@ -6717,8 +7055,10 @@ def ex_set_node_scheduling(self, node, on_host_maintenance=None, automatic_resta ar = node.extra["scheduling"].get("automaticRestart") success = True + if on_host_maintenance not in [None, ohm]: success = False + if automatic_restart not in [None, ar]: success = False @@ -6786,24 +7126,32 @@ def attach_volume( :return: True if successful :rtype: ``bool`` """ + if volume is None and ex_source is None: raise ValueError( "Must supply either a StorageVolume or " "set `ex_source` URL for an existing disk." ) + if volume is None and device is None: raise ValueError("Must supply either a StorageVolume or " "set `device` name.") volume_data = {} + if ex_source: volume_data["source"] = ex_source + if ex_initialize_params: volume_data["initialzeParams"] = ex_initialize_params + if ex_licenses: volume_data["licenses"] = ex_licenses + if ex_interface: volume_data["interface"] = ex_interface + if ex_type: volume_data["type"] = ex_type + if ex_auto_delete: volume_data["autoDelete"] = ex_auto_delete @@ -6822,6 +7170,7 @@ def attach_volume( node.name, ) self.connection.async_request(request, method="POST", data=volume_data) + return True def ex_resize_volume(self, volume, size): @@ -6841,6 +7190,7 @@ def ex_resize_volume(self, volume, size): request_data = {"sizeGb": int(size)} self.connection.async_request(request, method="POST", data=request_data) + return True def detach_volume(self, volume, ex_node=None): @@ -6856,6 +7206,7 @@ def detach_volume(self, volume, ex_node=None): :return: True if successful :rtype: ``bool`` """ + if not ex_node: return False request = "/zones/{}/instances/{}/detachDisk?deviceName={}".format( @@ -6865,6 +7216,7 @@ def detach_volume(self, volume, ex_node=None): ) self.connection.async_request(request, method="POST", data="ignored") + return True def ex_set_volume_auto_delete(self, volume, node, auto_delete=True): @@ -6892,6 +7244,7 @@ def ex_set_volume_auto_delete(self, volume, node, auto_delete=True): "autoDelete": auto_delete, } self.connection.async_request(request, method="POST", params=delete_params) + return True def ex_destroy_address(self, address): @@ -6904,6 +7257,7 @@ def ex_destroy_address(self, address): :return: True if successful :rtype: ``bool`` """ + if not hasattr(address, "name"): address = self.ex_get_address(address) @@ -6913,6 +7267,7 @@ def ex_destroy_address(self, address): request = "/global/addresses/%s" % (address.name) self.connection.async_request(request, method="DELETE") + return True def ex_destroy_backendservice(self, backendservice): @@ -6928,6 +7283,7 @@ def ex_destroy_backendservice(self, backendservice): request = "/global/backendServices/%s" % backendservice.name self.connection.async_request(request, method="DELETE") + return True def ex_delete_image(self, image): @@ -6940,11 +7296,13 @@ def ex_delete_image(self, image): :return: True if successful :rtype: ``bool`` """ + if not hasattr(image, "name"): image = self.ex_get_image(image) request = "/global/images/%s" % (image.name) self.connection.async_request(request, method="DELETE") + return True def ex_deprecate_image( @@ -6980,6 +7338,7 @@ def ex_deprecate_image( :return: True if successful :rtype: ``bool`` """ + if not hasattr(image, "name"): image = self.ex_get_image(image) @@ -7001,6 +7360,7 @@ def ex_deprecate_image( "state": state, "replacement": replacement.extra["selfLink"], } + for attribute, value in [ ("deprecated", deprecated), ("obsolete", obsolete), @@ -7033,6 +7393,7 @@ def ex_destroy_healthcheck(self, healthcheck): """ request = "/global/httpHealthChecks/%s" % (healthcheck.name) self.connection.async_request(request, method="DELETE") + return True def ex_destroy_firewall(self, firewall): @@ -7047,6 +7408,7 @@ def ex_destroy_firewall(self, firewall): """ request = "/global/firewalls/%s" % (firewall.name) self.connection.async_request(request, method="DELETE") + return True def ex_destroy_forwarding_rule(self, forwarding_rule): @@ -7059,6 +7421,7 @@ def ex_destroy_forwarding_rule(self, forwarding_rule): :return: True if successful :rtype: ``bool`` """ + if forwarding_rule.region: request = "/regions/{}/forwardingRules/{}".format( forwarding_rule.region.name, @@ -7067,6 +7430,7 @@ def ex_destroy_forwarding_rule(self, forwarding_rule): else: request = "/global/forwardingRules/%s" % forwarding_rule.name self.connection.async_request(request, method="DELETE") + return True def ex_destroy_route(self, route): @@ -7081,6 +7445,7 @@ def ex_destroy_route(self, route): """ request = "/global/routes/%s" % (route.name) self.connection.async_request(request, method="DELETE") + return True def ex_destroy_network(self, network): @@ -7095,6 +7460,7 @@ def ex_destroy_network(self, network): """ request = "/global/networks/%s" % (network.name) self.connection.async_request(request, method="DELETE") + return True def ex_set_machine_type(self, node, machine_type="n1-standard-1"): @@ -7119,6 +7485,7 @@ def ex_set_machine_type(self, node, machine_type="n1-standard-1"): request = "{}/instances/{}/setMachineType".format(request, node.name) body = {"machineType": mt_url} self.connection.async_request(request, method="POST", data=body) + return True def start_node(self, node, ex_sync=True): @@ -7157,6 +7524,7 @@ def stop_node(self, node, ex_sync=True): :rtype: ``bool`` """ request = "/zones/{}/instances/{}/stop".format(node.extra["zone"].name, node.name) + if ex_sync: self.connection.async_request(request, method="POST") else: @@ -7168,12 +7536,14 @@ def ex_start_node(self, node, sync=True): # NOTE: This method is here for backward compatibility reasons after # this method was promoted to be part of the standard compute API in # Libcloud v2.7.0 + return self.start_node(node=node, ex_sync=sync) def ex_stop_node(self, node, sync=True): # NOTE: This method is here for backward compatibility reasons after # this method was promoted to be part of the standard compute API in # Libcloud v2.7.0 + return self.stop_node(node=node, ex_sync=sync) def ex_destroy_instancegroupmanager(self, manager): @@ -7193,6 +7563,7 @@ def ex_destroy_instancegroupmanager(self, manager): ) self.connection.async_request(request, method="DELETE") + return True def ex_destroy_instancetemplate(self, instancetemplate): @@ -7234,6 +7605,7 @@ def ex_destroy_autoscaler(self, autoscaler): request = "/zones/{}/autoscalers/{}".format(autoscaler.zone.name, autoscaler.name) self.connection.async_request(request, method="DELETE") + return True def destroy_node(self, node, destroy_boot_disk=False, ex_sync=True): @@ -7256,6 +7628,7 @@ def destroy_node(self, node, destroy_boot_disk=False, ex_sync=True): :rtype: ``bool`` """ request = "/zones/{}/instances/{}".format(node.extra["zone"].name, node.name) + if ex_sync: self.connection.async_request(request, method="DELETE") else: @@ -7263,6 +7636,7 @@ def destroy_node(self, node, destroy_boot_disk=False, ex_sync=True): if destroy_boot_disk and node.extra["boot_disk"]: node.extra["boot_disk"].destroy() + return True def ex_destroy_multiple_nodes( @@ -7301,6 +7675,7 @@ def ex_destroy_multiple_nodes( status_list = [] complete = False start_time = time.time() + for node in node_list: request = "/zones/{}/instances/{}".format(node.extra["zone"].name, node.name) try: @@ -7323,10 +7698,12 @@ def ex_destroy_multiple_nodes( if time.time() - start_time >= timeout: raise Exception("Timeout (%s sec) while waiting to delete " "multiple instances") complete = True + for status in status_list: # If one of the operations is running, check the status operation = status["node_response"] or status["disk_response"] delete_disk = False + if operation: no_errors = True try: @@ -7335,9 +7712,11 @@ def ex_destroy_multiple_nodes( self._catch_error(ignore_errors=ignore_errors) no_errors = False response = {"status": "DONE"} + if response["status"] == "DONE": # If a node was deleted, update status and indicate # that the disk is ready to be deleted. + if status["node_response"]: status["node_response"] = None status["node_success"] = no_errors @@ -7347,8 +7726,10 @@ def ex_destroy_multiple_nodes( status["disk_success"] = no_errors # If we are destroying disks, and the node has been deleted, # destroy the disk. + if delete_disk and destroy_boot_disk: boot_disk = status["node"].extra["boot_disk"] + if boot_disk: request = "/zones/{}/disks/{}".format( boot_disk.extra["zone"].name, @@ -7364,14 +7745,17 @@ def ex_destroy_multiple_nodes( else: # If there is no boot disk, ignore status["disk_success"] = True operation = status["node_response"] or status["disk_response"] + if operation: time.sleep(poll_interval) complete = False success = [] + for status in status_list: s = status["node_success"] and status["disk_success"] success.append(s) + return success def ex_destroy_targethttpproxy(self, targethttpproxy): @@ -7386,6 +7770,7 @@ def ex_destroy_targethttpproxy(self, targethttpproxy): """ request = "/global/targetHttpProxies/%s" % targethttpproxy.name self.connection.async_request(request, method="DELETE") + return True def ex_destroy_targethttpsproxy(self, targethttpsproxy): @@ -7425,6 +7810,7 @@ def ex_destroy_targetinstance(self, targetinstance): targetinstance.name, ) self.connection.async_request(request, method="DELETE") + return True def ex_destroy_targetpool(self, targetpool): @@ -7443,6 +7829,7 @@ def ex_destroy_targetpool(self, targetpool): ) self.connection.async_request(request, method="DELETE") + return True def ex_destroy_urlmap(self, urlmap): @@ -7458,6 +7845,7 @@ def ex_destroy_urlmap(self, urlmap): request = "/global/urlMaps/%s" % urlmap.name self.connection.async_request(request, method="DELETE") + return True def destroy_volume(self, volume): @@ -7472,6 +7860,7 @@ def destroy_volume(self, volume): """ request = "/zones/{}/disks/{}".format(volume.extra["zone"].name, volume.name) self.connection.async_request(request, method="DELETE") + return True def destroy_volume_snapshot(self, snapshot): @@ -7486,6 +7875,7 @@ def destroy_volume_snapshot(self, snapshot): """ request = "/global/snapshots/%s" % (snapshot.name) self.connection.async_request(request, method="DELETE") + return True def ex_get_license(self, project, name): @@ -7501,6 +7891,7 @@ def ex_get_license(self, project, name): :return: A License object for the name :rtype: :class:`GCELicense` """ + return GCELicense.lazy(name, project, self) def ex_get_disktype(self, name, zone=None): @@ -7534,6 +7925,7 @@ def ex_get_disktype(self, name, zone=None): for item in zone_item["diskTypes"]: if item["name"] == name: data = item + break else: data = response.object @@ -7559,6 +7951,7 @@ def ex_get_accelerator_type(self, name, zone=None): zone = self._set_zone(zone) request = "/zones/{}/acceleratorTypes/{}".format(zone.name, name) response = self.connection.request(request, method="GET").object + return self._to_accelerator_type(response) def ex_get_address(self, name, region=None): @@ -7575,6 +7968,7 @@ def ex_get_address(self, name, region=None): :return: An Address object for the address :rtype: :class:`GCEAddress` """ + if region == "global": request = "/global/addresses/%s" % (name) else: @@ -7583,6 +7977,7 @@ def ex_get_address(self, name, region=None): ) request = "/regions/{}/addresses/{}".format(region.name, name) response = self.connection.request(request, method="GET").object + return self._to_address(response) def ex_get_backendservice(self, name): @@ -7597,6 +7992,7 @@ def ex_get_backendservice(self, name): """ request = "/global/backendServices/%s" % name response = self.connection.request(request, method="GET").object + return self._to_backendservice(response) def ex_get_healthcheck(self, name): @@ -7611,6 +8007,7 @@ def ex_get_healthcheck(self, name): """ request = "/global/httpHealthChecks/%s" % (name) response = self.connection.request(request, method="GET").object + return self._to_healthcheck(response) def ex_get_firewall(self, name): @@ -7625,6 +8022,7 @@ def ex_get_firewall(self, name): """ request = "/global/firewalls/%s" % (name) response = self.connection.request(request, method="GET").object + return self._to_firewall(response) def ex_get_forwarding_rule(self, name, region=None, global_rule=False): @@ -7645,6 +8043,7 @@ def ex_get_forwarding_rule(self, name, region=None, global_rule=False): :return: A GCEForwardingRule object :rtype: :class:`GCEForwardingRule` """ + if global_rule: request = "/global/forwardingRules/%s" % name else: @@ -7654,6 +8053,7 @@ def ex_get_forwarding_rule(self, name, region=None, global_rule=False): request = "/regions/{}/forwardingRules/{}".format(region.name, name) response = self.connection.request(request, method="GET").object + return self._to_forwarding_rule(response) def ex_get_image(self, partial_name, ex_project_list=None, ex_standard_projects=True): @@ -7676,10 +8076,13 @@ def ex_get_image(self, partial_name, ex_project_list=None, ex_standard_projects= an image with that name is not found. :rtype: :class:`GCENodeImage` or raise ``ResourceNotFoundError`` """ + if partial_name.startswith("https://"): response = self.connection.request(partial_name, method="GET") + return self._to_node_image(response.object) image = self._match_images(ex_project_list, partial_name) + if not image and ex_standard_projects: for img_proj, short_list in self.IMAGE_PROJECTS.items(): for short_name in short_list: @@ -7688,6 +8091,7 @@ def ex_get_image(self, partial_name, ex_project_list=None, ex_standard_projects= if not image: raise ResourceNotFoundError("Could not find image '%s'" % (partial_name), None, None) + return image def ex_get_image_from_family( @@ -7716,6 +8120,7 @@ def ex_get_image_from_family( def _try_image_family(image_family, project=None): request = "/global/images/family/%s" % (image_family) save_request_path = self.connection.request_path + if project: new_request_path = save_request_path.replace(self.project, project) self.connection.request_path = new_request_path @@ -7730,8 +8135,10 @@ def _try_image_family(image_family, project=None): return image image = None + if image_family.startswith("https://"): response = self.connection.request(image_family, method="GET") + return self._to_node_image(response.object) if not ex_project_list: @@ -7739,6 +8146,7 @@ def _try_image_family(image_family, project=None): else: for img_proj in ex_project_list: image = _try_image_family(image_family, project=img_proj) + if image: break @@ -7747,6 +8155,7 @@ def _try_image_family(image_family, project=None): for short_name in short_list: if image_family.startswith(short_name): image = _try_image_family(image_family, project=img_proj) + if image: break @@ -7754,6 +8163,7 @@ def _try_image_family(image_family, project=None): raise ResourceNotFoundError( "Could not find image for family " "'%s'" % (image_family), None, None ) + return image def ex_get_route(self, name): @@ -7768,6 +8178,7 @@ def ex_get_route(self, name): """ request = "/global/routes/%s" % (name) response = self.connection.request(request, method="GET").object + return self._to_route(response) def ex_destroy_sslcertificate(self, sslcertificate): @@ -7807,6 +8218,7 @@ def ex_destroy_subnetwork(self, name, region=None): """ region_name = None subnet_name = None + if region: if isinstance(region, GCERegion): region_name = region.name @@ -7815,14 +8227,17 @@ def ex_destroy_subnetwork(self, name, region=None): region_name = region.split("/")[-1] else: region_name = region + if isinstance(name, GCESubnetwork): subnet_name = name.name + if not region_name: region_name = name.region.name else: if name.startswith("https://"): url_parts = self._get_components_from_path(name) subnet_name = url_parts["name"] + if not region_name: region_name = url_parts["region"] else: @@ -7830,6 +8245,7 @@ def ex_destroy_subnetwork(self, name, region=None): if not region_name: region = self._set_region(region) + if not region: raise ValueError("Could not determine region for subnetwork.") else: @@ -7837,6 +8253,7 @@ def ex_destroy_subnetwork(self, name, region=None): request = "/regions/{}/subnetworks/{}".format(region_name, subnet_name) self.connection.async_request(request, method="DELETE").object + return True def ex_get_subnetwork(self, name, region=None): @@ -7853,6 +8270,7 @@ def ex_get_subnetwork(self, name, region=None): :rtype: :class:`GCESubnetwork` """ region_name = None + if name.startswith("https://"): request = name else: @@ -7866,6 +8284,7 @@ def ex_get_subnetwork(self, name, region=None): if not region_name: region = self._set_region(region) + if not region: raise ValueError("Could not determine region for subnetwork.") else: @@ -7874,6 +8293,7 @@ def ex_get_subnetwork(self, name, region=None): request = "/regions/{}/subnetworks/{}".format(region_name, name) response = self.connection.request(request, method="GET").object + return self._to_subnetwork(response) def ex_get_network(self, name): @@ -7886,11 +8306,13 @@ def ex_get_network(self, name): :return: A Network object for the network :rtype: :class:`GCENetwork` """ + if name.startswith("https://"): request = name else: request = "/global/networks/%s" % (name) response = self.connection.request(request, method="GET").object + return self._to_network(response) def ex_get_node(self, name, zone=None): @@ -7911,6 +8333,7 @@ def ex_get_node(self, name, zone=None): zone = self._set_zone(zone) or self._find_zone_or_region(name, "instances", res_name="Node") request = "/zones/{}/instances/{}".format(zone.name, name) response = self.connection.request(request, method="GET").object + return self._to_node(response) def ex_get_project(self): @@ -7921,6 +8344,7 @@ def ex_get_project(self): :rtype: :class:`GCEProject` """ response = self.connection.request("", method="GET").object + return self._to_project(response) def ex_get_size(self, name, zone=None): @@ -7938,11 +8362,13 @@ def ex_get_size(self, name, zone=None): :rtype: :class:`GCENodeSize` """ zone = zone or self.zone + if not hasattr(zone, "name"): zone = self.ex_get_zone(zone) request = "/zones/{}/machineTypes/{}".format(zone.name, name) response = self.connection.request(request, method="GET").object instance_prices = get_pricing(driver_type="compute", driver_name="gce_instances") + return self._to_node_size(response, instance_prices) def ex_get_snapshot(self, name): @@ -7957,6 +8383,7 @@ def ex_get_snapshot(self, name): """ request = "/global/snapshots/%s" % (name) response = self.connection.request(request, method="GET").object + return self._to_snapshot(response) def ex_get_volume(self, name, zone=None, use_cache=False): @@ -7984,6 +8411,7 @@ def ex_get_volume(self, name, zone=None, use_cache=False): :return: A StorageVolume object for the volume :rtype: :class:`StorageVolume` """ + if not self._ex_volume_dict or use_cache is False: # Make the API call and build volume dictionary self._ex_populate_volume_dict() @@ -8006,6 +8434,7 @@ def ex_get_region(self, name): :return: A GCERegion object for the region :rtype: :class:`GCERegion` """ + if name.startswith("https://"): short_name = self._get_components_from_path(name)["name"] request = name @@ -8013,10 +8442,12 @@ def ex_get_region(self, name): short_name = name request = "/regions/%s" % (name) # Check region cache first + if short_name in self.region_dict: return self.region_dict[short_name] # Otherwise, look up region information response = self.connection.request(request, method="GET").object + return self._to_region(response) def ex_get_sslcertificate(self, name): @@ -8054,6 +8485,7 @@ def ex_get_targethttpproxy(self, name): """ request = "/global/targetHttpProxies/%s" % name response = self.connection.request(request, method="GET").object + return self._to_targethttpproxy(response) def ex_get_targethttpsproxy(self, name): @@ -8098,6 +8530,7 @@ def ex_get_targetinstance(self, name, zone=None): ) request = "/zones/{}/targetInstances/{}".format(zone.name, name) response = self.connection.request(request, method="GET").object + return self._to_targetinstance(response) def ex_get_targetpool(self, name, region=None): @@ -8119,6 +8552,7 @@ def ex_get_targetpool(self, name, region=None): ) request = "/regions/{}/targetPools/{}".format(region.name, name) response = self.connection.request(request, method="GET").object + return self._to_targetpool(response) def ex_get_urlmap(self, name): @@ -8133,6 +8567,7 @@ def ex_get_urlmap(self, name): """ request = "/global/urlMaps/%s" % name response = self.connection.request(request, method="GET").object + return self._to_urlmap(response) def ex_get_instancegroup(self, name, zone=None): @@ -8182,6 +8617,7 @@ def ex_get_instancegroupmanager(self, name, zone=None): ) request = "/zones/{}/instanceGroupManagers/{}".format(zone.name, name) response = self.connection.request(request, method="GET").object + return self._to_instancegroupmanager(response) def ex_get_instancetemplate(self, name): @@ -8196,6 +8632,7 @@ def ex_get_instancetemplate(self, name): """ request = "/global/instanceTemplates/%s" % (name) response = self.connection.request(request, method="GET").object + return self._to_instancetemplate(response) def ex_get_autoscaler(self, name, zone=None): @@ -8217,6 +8654,7 @@ def ex_get_autoscaler(self, name, zone=None): ) request = "/zones/{}/autoscalers/{}".format(zone.name, name) response = self.connection.request(request, method="GET").object + return self._to_autoscaler(response) def ex_get_zone(self, name): @@ -8229,6 +8667,7 @@ def ex_get_zone(self, name): :return: A GCEZone object for the zone or None if not found :rtype: :class:`GCEZone` or ``None`` """ + if name.startswith("https://"): short_name = self._get_components_from_path(name)["name"] request = name @@ -8236,6 +8675,7 @@ def ex_get_zone(self, name): short_name = name request = "/zones/%s" % (name) # Check zone cache first + if short_name in self.zone_dict: return self.zone_dict[short_name] # Otherwise, look up zone information @@ -8243,6 +8683,7 @@ def ex_get_zone(self, name): response = self.connection.request(request, method="GET").object except ResourceNotFoundError: return None + return self._to_zone(response) def _ex_connection_class_kwargs(self): @@ -8267,13 +8708,16 @@ def _build_volume_dict(self, zone_dict): :rtype: ``dict`` """ name_zone_dict = {} + for k, v in zone_dict.items(): zone_name = k.replace("zones/", "") disks = v.get("disks", []) + for disk in disks: n = disk["name"] name_zone_dict.setdefault(n, {}) name_zone_dict[n].update({zone_name: disk}) + return name_zone_dict def _ex_lookup_volume(self, volume_name, zone=None): @@ -8293,10 +8737,12 @@ def _ex_lookup_volume(self, volume_name, zone=None): :return: A StorageVolume object for the volume. :rtype: :class:`StorageVolume` or raise ``ResourceNotFoundError``. """ + if volume_name not in self._ex_volume_dict: # Possibly added through another thread/process, so re-populate # _volume_dict and try again. If still not found, raise exception. self._ex_populate_volume_dict() + if volume_name not in self._ex_volume_dict: raise ResourceNotFoundError( "Volume name: '{}' not found. Zone: {}".format(volume_name, zone), @@ -8306,14 +8752,17 @@ def _ex_lookup_volume(self, volume_name, zone=None): # Disk names are not unique across zones, so if zone is None or # 'all', we return the first one we find for that disk name. For # consistency, we sort by keys and set the zone to the first key. + if zone is None or zone == "all": zone = sorted(self._ex_volume_dict[volume_name])[0] volume = self._ex_volume_dict[volume_name].get(zone, None) + if not volume: raise ResourceNotFoundError( "Volume '{}' not found for zone {}.".format(volume_name, zone), None, None ) + return self._to_storage_volume(volume) def _ex_populate_volume_dict(self): @@ -8344,6 +8793,7 @@ def _catch_error(self, ignore_errors=False): :rtype: :class:`Exception` """ e = sys.exc_info()[1] + if ignore_errors: return e else: @@ -8365,6 +8815,7 @@ def _get_components_from_path(self, path): glob = False components = path.split("/") name = components[-1] + if components[-4] == "regions": region = components[-3] elif components[-4] == "zones": @@ -8386,12 +8837,14 @@ def _get_object_by_kind(self, url): :return: Object representation of the requested resource. "rtype: :class:`object` or ``None`` """ + if not url: return None # Relies on GoogleBaseConnection.morph_action_hook to rewrite # the URL to a request response = self.connection.request(url, method="GET").object + return GCENodeDriver.KIND_METHOD_MAP[response["kind"]](self, response) def _get_region_from_zone(self, zone): @@ -8404,8 +8857,10 @@ def _get_region_from_zone(self, zone): :return: Region object that contains the zone :rtype: :class:`GCERegion` """ + for region in self.region_list: zones = [z.name for z in region.zones] + if zone.name in zones: return region @@ -8430,6 +8885,7 @@ def _find_zone_or_region(self, name, res_type, region=False, res_name=None): :return: Zone/Region object for the zone/region for the resource. :rtype: :class:`GCEZone` or :class:`GCERegion` """ + if region: rz = "region" else: @@ -8437,17 +8893,21 @@ def _find_zone_or_region(self, name, res_type, region=False, res_name=None): rz_name = None res_name = res_name or res_type res_list = self.connection.request_aggregated_items(res_type) + for k, v in res_list["items"].items(): for res in v.get(res_type, []): if res["name"] == name: rz_name = k.replace("%ss/" % (rz), "") + break + if not rz_name: raise ResourceNotFoundError( "{} '{}' not found in any {}.".format(res_name, name, rz), None, None ) else: getrz = getattr(self, "ex_get_%s" % (rz)) + return getrz(rz_name) def _match_images(self, project, partial_name): @@ -8475,12 +8935,15 @@ def _match_images(self, project, partial_name): self.list_images, ex_project=project, ex_include_deprecated=True ) partial_match = [] + for page in project_images_list.page(): for image in page: if image.name == partial_name: return image + if image.name.startswith(partial_name): ts = timestamp_to_datetime(image.extra["creationTimestamp"]) + if not partial_match or partial_match[0] < ts: partial_match = [ts, image] @@ -8504,6 +8967,7 @@ def _set_region(self, region): if not hasattr(region, "name"): region = self.ex_get_region(region) + return region def _set_zone(self, zone): @@ -8517,11 +8981,13 @@ def _set_zone(self, zone): :rtype: :class:`GCEZone` or ``None`` """ zone = zone or self.zone + if zone == "all" or zone is None: return None if not hasattr(zone, "name"): zone = self.ex_get_zone(zone) + return zone def _create_node_req( @@ -8693,6 +9159,7 @@ def _create_node_req( """ # build disks + if not image and not boot_disk and not ex_disks_gce_struct: raise ValueError( "Missing root device or image. Must specify an " @@ -8708,6 +9175,7 @@ def _create_node_req( use_selflinks = True source = None + if boot_disk: source = boot_disk @@ -8741,6 +9209,7 @@ def _create_node_req( node_data["name"] = name request = "/zones/%s/instances" % (location.name) + return request, node_data def _multi_create_disk(self, status, node_attrs): @@ -8756,6 +9225,7 @@ def _multi_create_disk(self, status, node_attrs): """ disk = None # Check for existing disk + if node_attrs["use_existing_disk"]: try: disk = self.ex_get_volume(status["name"], node_attrs["location"]) @@ -8806,8 +9276,10 @@ def _multi_check_disk(self, status, node_attrs): error = e.value code = e.code response = {"status": "DONE"} + if response["status"] == "DONE": status["disk_response"] = None + if error: status["disk"] = GCEFailedDisk(status["name"], error, code) else: @@ -8881,8 +9353,10 @@ def _multi_check_node(self, status, node_attrs): error = e.value code = e.code response = {"status": "DONE"} + if response["status"] == "DONE": status["node_response"] = None + if error: status["node"] = GCEFailedNode(status["name"], error, code) else: @@ -8929,16 +9403,20 @@ def _create_vol_req( volume_data = {} params = None volume_data["name"] = name + if size: volume_data["sizeGb"] = str(size) + if image: if not hasattr(image, "name"): image = self.ex_get_image(image) params = {"sourceImage": image.extra["selfLink"]} volume_data["description"] = "Image: %s" % (image.extra["selfLink"]) + if snapshot: if not hasattr(snapshot, "name"): # Check for full URI to not break backward-compatibility + if snapshot.startswith("https"): snapshot = self._get_components_from_path(snapshot)["name"] snapshot = self.ex_get_snapshot(snapshot) @@ -8946,8 +9424,10 @@ def _create_vol_req( volume_data["sourceSnapshot"] = snapshot_link volume_data["description"] = "Snapshot: %s" % (snapshot_link) location = location or self.zone + if not hasattr(location, "name"): location = self.ex_get_zone(location) + if hasattr(ex_disk_type, "name"): # pylint: disable=no-member volume_data["type"] = ex_disk_type.extra["selfLink"] @@ -9038,6 +9518,7 @@ def _to_address(self, address): extra["selfLink"] = address.get("selfLink") extra["status"] = address.get("status") extra["description"] = address.get("description", None) + if address.get("users", None) is not None: extra["users"] = address.get("users") extra["creationTimestamp"] = address.get("creationTimestamp") @@ -9180,6 +9661,7 @@ def _to_forwarding_rule(self, forwarding_rule): extra["description"] = forwarding_rule.get("description") region = forwarding_rule.get("region") + if region: region = self.ex_get_region(region) target = self._get_object_by_kind(forwarding_rule["target"]) @@ -9206,6 +9688,7 @@ def _to_sslcertificate(self, sslcertificate): :rtype: :class:`GCESslCertificate` """ extra = {} + if "description" in sslcertificate: extra["description"] = sslcertificate["description"] extra["selfLink"] = sslcertificate["selfLink"] @@ -9276,6 +9759,7 @@ def _to_network(self, network): extra["routingConfig"] = network.get("routingConfig") # match Cloud SDK 'gcloud' + if "autoCreateSubnetworks" in network: if network["autoCreateSubnetworks"]: extra["mode"] = "auto" @@ -9312,12 +9796,16 @@ def _to_route(self, route): if "nextHopInstance" in route: extra["nextHopInstance"] = route["nextHopInstance"] + if "nextHopIp" in route: extra["nextHopIp"] = route["nextHopIp"] + if "nextHopNetwork" in route: extra["nextHopNetwork"] = route["nextHopNetwork"] + if "nextHopGateway" in route: extra["nextHopGateway"] = route["nextHopGateway"] + if "warnings" in route: extra["warnings"] = route["warnings"] @@ -9343,12 +9831,14 @@ def _to_node_image(self, image): :rtype: :class:`GCENodeImage` """ extra = {} + if "preferredKernel" in image: extra["preferredKernel"] = image.get("preferredKernel", None) extra["description"] = image.get("description", None) extra["family"] = image.get("family", None) extra["creationTimestamp"] = image.get("creationTimestamp") extra["selfLink"] = image.get("selfLink") + if "deprecated" in image: extra["deprecated"] = image.get("deprecated", None) extra["sourceType"] = image.get("sourceType", None) @@ -9356,12 +9846,16 @@ def _to_node_image(self, image): extra["status"] = image.get("status", None) extra["archiveSizeBytes"] = image.get("archiveSizeBytes", None) extra["diskSizeGb"] = image.get("diskSizeGb", None) + if "guestOsFeatures" in image: extra["guestOsFeatures"] = image.get("guestOsFeatures", []) + if "sourceDisk" in image: extra["sourceDisk"] = image.get("sourceDisk", None) + if "sourceDiskId" in image: extra["sourceDiskId"] = image.get("sourceDiskId", None) + if "licenses" in image: lic_objs = self._licenses_from_urls(licenses=image["licenses"]) extra["licenses"] = lic_objs @@ -9380,6 +9874,7 @@ def _to_node_location(self, location): :return: Location object :rtype: :class:`NodeLocation` """ + return NodeLocation( id=location["id"], name=location["name"], @@ -9445,12 +9940,14 @@ def _to_node(self, node, use_disk_cache=False): for network_interface in node.get("networkInterfaces", []): private_ips.append(network_interface.get("networkIP")) + for access_config in network_interface.get("accessConfigs", []): public_ips.append(access_config.get("natIP")) # For the node attributes, use just machine and image names, not full # paths. Full paths are available in the "extra" dict. image = None + if extra["image"]: image = self._get_components_from_path(extra["image"])["name"] else: @@ -9509,6 +10006,7 @@ def _to_node_size(self, machine_type, instance_prices): price = None except AttributeError: # no zone price = None + return GCENodeSize( id=machine_type["id"], name=machine_type["name"], @@ -9535,9 +10033,11 @@ def _to_project(self, project): extra["creationTimestamp"] = project.get("creationTimestamp") extra["description"] = project.get("description") metadata = project["commonInstanceMetadata"].get("items") + if "commonInstanceMetadata" in project: # add this struct to get 'fingerprint' too extra["commonInstanceMetadata"] = project["commonInstanceMetadata"] + if "usageExportLocation" in project: extra["usageExportLocation"] = project["usageExportLocation"] @@ -9597,14 +10097,19 @@ def _to_snapshot(self, snapshot): extra["selfLink"] = snapshot.get("selfLink") extra["creationTimestamp"] = snapshot.get("creationTimestamp") extra["sourceDisk"] = snapshot.get("sourceDisk") + if "description" in snapshot: extra["description"] = snapshot["description"] + if "sourceDiskId" in snapshot: extra["sourceDiskId"] = snapshot["sourceDiskId"] + if "storageBytes" in snapshot: extra["storageBytes"] = snapshot["storageBytes"] + if "storageBytesStatus" in snapshot: extra["storageBytesStatus"] = snapshot["storageBytesStatus"] + if "licenses" in snapshot: lic_objs = self._licenses_from_urls(licenses=snapshot["licenses"]) extra["licenses"] = lic_objs @@ -9698,6 +10203,7 @@ def _to_targethttpsproxy(self, targethttpsproxy): :rtype: :class:`GCETargetHttpsProxy` """ extra = {} + if "description" in targethttpsproxy: extra["description"] = targethttpsproxy["description"] extra["selfLink"] = targethttpsproxy["selfLink"] @@ -9733,6 +10239,7 @@ def _to_targetinstance(self, targetinstance): extra["description"] = targetinstance.get("description") extra["natPolicy"] = targetinstance.get("natPolicy") zone = self.ex_get_zone(targetinstance["zone"]) + if "instance" in targetinstance: node_name = targetinstance["instance"].split("/")[-1] try: @@ -9768,6 +10275,7 @@ def _to_targetpool(self, targetpool): self.ex_get_healthcheck(h.split("/")[-1]) for h in targetpool.get("healthChecks", []) ] node_list = [] + for n in targetpool.get("instances", []): # Nodes that do not exist can be part of a target pool. If the # node does not exist, use the URL of the node instead of the node @@ -9781,6 +10289,7 @@ def _to_targetpool(self, targetpool): if "failoverRatio" in targetpool: extra["failoverRatio"] = targetpool["failoverRatio"] + if "backupPool" in targetpool: tp_split = targetpool["backupPool"].split("/") extra["backupPool"] = self.ex_get_targetpool(tp_split[10], tp_split[8]) @@ -9812,6 +10321,7 @@ def _to_instancegroup(self, instancegroup): extra["fingerprint"] = instancegroup.get("fingerprint", None) zone = instancegroup.get("zone", None) + if zone: # Apparently zone attribute is not always present, see # https://github.com/apache/libcloud/issues/1346 for details @@ -9820,10 +10330,12 @@ def _to_instancegroup(self, instancegroup): # Note: network/subnetwork will not be available if the Instance Group # does not contain instances. network = instancegroup.get("network", None) + if network: network = self.ex_get_network(network) subnetwork = instancegroup.get("subnetwork", None) + if subnetwork: subnetwork = self.ex_get_subnetwork(subnetwork) @@ -9942,6 +10454,7 @@ def _format_guest_accelerators(self, accelerator_type, accelerator_count): accelerator_type = self._get_selflink_or_name( obj=accelerator_type, get_selflinks=True, objname="accelerator_type" ) + return [{"acceleratorType": accelerator_type, "acceleratorCount": accelerator_count}] def _format_metadata(self, fingerprint, metadata=None): @@ -9974,6 +10487,7 @@ def _format_metadata(self, fingerprint, metadata=None): :return: GCE-friendly metadata dict :rtype: ``dict`` """ + if not metadata: return {"fingerprint": fingerprint, "items": []} md = {"fingerprint": fingerprint} @@ -9981,11 +10495,14 @@ def _format_metadata(self, fingerprint, metadata=None): # Check `list` format. Can support / convert the following: # (a) [{'key': 'k1', 'value': 'v1'}, ...] # (b) [{'k1': 'v1'}, ...] + if isinstance(metadata, list): item_list = [] + for i in metadata: if isinstance(i, dict): # check (a) + if "key" in i and "value" in i and len(i) == 2: item_list.append(i) # check (b) @@ -10001,14 +10518,17 @@ def _format_metadata(self, fingerprint, metadata=None): # (c) {'key': 'k1', 'value': 'v1'} # (d) {'k1': 'v1', 'k2': 'v2', ...} # (e) {'items': [...]} + if isinstance(metadata, dict): # Check (c) + if "key" in metadata and "value" in metadata and len(metadata) == 2: md["items"] = [metadata] # check (d) elif len(metadata) == 1: if "items" in metadata: # check (e) + if isinstance(metadata["items"], list): md["items"] = metadata["items"] else: @@ -10023,11 +10543,13 @@ def _format_metadata(self, fingerprint, metadata=None): else: # check (d) md["items"] = [] + for k, v in metadata.items(): md["items"].append({"key": k, "value": v}) if "items" not in md: raise ValueError("Unsupported metadata format.") + return md def _to_urlmap(self, urlmap): @@ -10109,10 +10631,12 @@ def _set_project_metadata(self, metadata=None, force=False, current_keys=""): :return: GCE-friendly metadata dict :rtype: ``dict`` """ + if metadata is None: # User wants to delete metadata, but if 'force' is False # and we already have sshKeys, we should retain them. # Otherwise, delete ALL THE THINGS! + if not force and current_keys: new_md = [{"key": "sshKeys", "value": current_keys}] else: @@ -10122,15 +10646,18 @@ def _set_project_metadata(self, metadata=None, force=False, current_keys=""): # want to preserve existing sshKeys, otherwise 'force' is True # and the user wants to add/replace sshKeys. new_md = metadata["items"] + if not force and current_keys: # not sure how duplicate keys would be resolved, so ensure # existing 'sshKeys' entry is removed. updated_md = [] + for d in new_md: if d["key"] != "sshKeys": updated_md.append({"key": d["key"], "value": d["value"]}) new_md = updated_md new_md.append({"key": "sshKeys", "value": current_keys}) + return new_md def _licenses_from_urls(self, licenses): @@ -10145,11 +10672,13 @@ def _licenses_from_urls(self, licenses): :rtype: ``list`` """ return_list = [] + for license in licenses: selfLink_parts = license.split("/") lic_proj = selfLink_parts[6] lic_name = selfLink_parts[-1] return_list.append(self.ex_get_license(project=lic_proj, name=lic_name)) + return return_list KIND_METHOD_MAP = { @@ -10185,6 +10714,7 @@ def _verify_zone_is_set(self, zone=None): This check is mandatory for methods which rely on the location being set - e.g. create_node. """ + if self.zone: return True diff --git a/libcloud/compute/drivers/kubevirt.py b/libcloud/compute/drivers/kubevirt.py index c24ea21187..3de011e5fc 100644 --- a/libcloud/compute/drivers/kubevirt.py +++ b/libcloud/compute/drivers/kubevirt.py @@ -84,6 +84,7 @@ class KubeVirtNodeDriver(KubernetesDriverMixin, NodeDriver): def list_nodes(self, location=None): namespaces = [] + if location is not None: if isinstance(location, NodeLocation): namespaces.append(location.name) @@ -97,18 +98,22 @@ def list_nodes(self, location=None): dormant = [] live = [] + for ns in namespaces: req = KUBEVIRT_URL + "namespaces/" + ns + "/virtualmachines" result = self.connection.request(req) + if result.status != 200: continue result = result.object + for item in result["items"]: if not item["spec"]["running"]: dormant.append(item) else: live.append(item) vms = [] + for vm in dormant: vms.append(self._to_node(vm, is_stopped=True)) @@ -126,14 +131,22 @@ def get_node(self, id=None, name=None): :param name: name of the vm :type name: ``str`` """ + if not id and not name: raise ValueError("This method needs id or name to be specified") nodes = self.list_nodes() + + node_gen = None + if id: node_gen = filter(lambda x: x.id == id, nodes) + if name: node_gen = filter(lambda x: x.name == name, nodes) + if not node_gen: + raise ValueError("node_gen is not defined") + try: return next(node_gen) except StopIteration: @@ -149,6 +162,7 @@ def start_node(self, node): :rtype: ``bool`` """ # make sure it is stopped + if node.state is NodeState.RUNNING: return True name = node.name @@ -176,6 +190,7 @@ def stop_node(self, node): :rtype: ``bool`` """ # check if running + if node.state is NodeState.STOPPED: return True name = node.name @@ -215,6 +230,7 @@ def reboot_node(self, node): return result.status in VALID_RESPONSE_CODES except Exception: raise + return def destroy_node(self, node): @@ -231,6 +247,7 @@ def destroy_node(self, node): name = node.name # find and delete services for this VM only services = self.ex_list_services(namespace=namespace, node_name=name) + for service in services: service_name = service["metadata"]["name"] self.ex_delete_service(namespace=namespace, service_name=service_name) @@ -241,6 +258,7 @@ def destroy_node(self, node): KUBEVIRT_URL + "namespaces/" + namespace + "/virtualmachines/" + name, method="DELETE", ) + return result.status in VALID_RESPONSE_CODES except Exception: raise @@ -282,10 +300,13 @@ def _create_node_with_template(self, name: str, template: dict, namespace="defau :return: """ # k8s object checks + if template.get("apiVersion", "") != "kubevirt.io/v1alpha3": raise ValueError("The template must have an apiVersion: kubevirt.io/v1alpha3") + if template.get("kind", "") != "VirtualMachine": raise ValueError("The template must contain kind: VirtualMachine") + if name != template.get("metadata", {}).get("name"): raise ValueError( "The name of the VM must be the same as the name in the template. " @@ -293,6 +314,7 @@ def _create_node_with_template(self, name: str, template: dict, namespace="defau name, template.get("metadata", {}).get("name") ) ) + if template.get("spec", {}).get("running", False): warnings.warn( "The VM will be created in a stopped state, and then started. " @@ -316,9 +338,11 @@ def _create_node_with_template(self, name: str, template: dict, namespace="defau # Or self.get_node()? # I don't think a for loop over list_nodes is necessary. nodes = self.list_nodes(location=namespace) + for node in nodes: if node.name == name: self.start_node(node) + return node raise ValueError( @@ -339,8 +363,10 @@ def _base_vm_template(name=None): # type: (Optional[str]) -> dict :return: dict: A skeleton VM template. """ + if not name: name = uuid.uuid4() + return { "apiVersion": "kubevirt.io/v1alpha3", "kind": "VirtualMachine", @@ -407,6 +433,7 @@ def _create_node_vm_from_ex_template( ignored_non_none_param_keys = list( filter(lambda x: other_params[x] is not None, other_params) ) + if ignored_non_none_param_keys: warnings.warn( "ex_template is provided, ignoring the following non-None " @@ -473,11 +500,13 @@ def _create_node_size( def _format_memory(memory_value): # type: (int) -> str assert isinstance(memory_value, int), "memory must be an int in MiB" + return str(memory_value) + "Mi" if ex_memory_limit is not None: memory = _format_memory(ex_memory_limit) vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["memory"] = memory + if ex_memory_request is not None: memory = _format_memory(ex_memory_request) vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["memory"] = memory @@ -499,6 +528,7 @@ def _format_cpu(cpu_value): # type: (Union[int, str]) -> Union[str, float] if ex_cpu_limit is not None: cpu = _format_cpu(ex_cpu_limit) vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["cpu"] = cpu + if ex_cpu_request is not None: cpu = _format_cpu(ex_cpu_request) vm["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["cpu"] = cpu @@ -543,6 +573,7 @@ def _create_node_network(vm, ex_network, ex_ports): # type: (dict, dict, dict) :return: None """ # ex_network -> network and interface + if ex_network is not None: try: if isinstance(ex_network, dict): @@ -572,13 +603,17 @@ def _create_node_network(vm, ex_network, ex_ports): # type: (dict, dict, dict) # ex_ports -> network.ports ex_ports = ex_ports or {} + if ex_ports.get("ports_tcp"): ports_to_expose = [] + for port in ex_ports["ports_tcp"]: ports_to_expose.append({"port": port, "protocol": "TCP"}) interface_dict[interface]["ports"] = ports_to_expose + if ex_ports.get("ports_udp"): ports_to_expose = interface_dict[interface].get("ports", []) + for port in ex_ports.get("ports_udp"): ports_to_expose.append({"port": port, "protocol": "UDP"}) interface_dict[interface]["ports"] = ports_to_expose @@ -601,6 +636,7 @@ def _create_node_auth(vm, auth): # auth requires cloud-init, # and only one cloud-init volume is supported by kubevirt. # So if both auth and cloud-init are provided, raise an error. + for volume in vm["spec"]["template"]["spec"]["volumes"]: if "cloudInitNoCloud" in volume or "cloudInitConfigDrive" in volume: raise ValueError( @@ -617,6 +653,7 @@ def _create_node_auth(vm, auth): } # cloud_init_config reference: https://kubevirt.io/user-guide/virtual_machines/startup_scripts/#injecting-ssh-keys-with-cloud-inits-cloud-config + if isinstance(auth, NodeAuthSSHKey): public_key = auth.pubkey.strip() public_key = json.dumps(public_key) @@ -678,6 +715,7 @@ def _create_node_disks(self, vm, ex_disks, image, namespace, location): # depending on disk_type, in the future, # when more will be supported, # additional elif should be added + if disk_type == "containerDisk": try: image = disk["volume_spec"]["image"] @@ -688,6 +726,7 @@ def _create_node_disks(self, vm, ex_disks, image, namespace, location): elif disk_type == "persistentVolumeClaim": if "volume_spec" not in disk: raise KeyError("You must provide a volume_spec dictionary") + if "claim_name" not in disk["volume_spec"]: msg = ( "You must provide either a claim_name of an " @@ -760,6 +799,7 @@ def _create_node_image(vm, image): # type: (dict, NodeImage) -> None """ # image -> containerDisk # adding image in a container Disk + if isinstance(image, NodeImage): image = image.name @@ -1061,6 +1101,7 @@ def create_node( """ # location -> namespace + if isinstance(location, NodeLocation): if location.name not in map(lambda x: x.name, self.list_locations()): raise ValueError("The location must be one of the available namespaces") @@ -1069,10 +1110,12 @@ def create_node( namespace = "default" # ex_template exists, use it to create the vm, ignore other parameters + if ex_template is not None: vm = self._create_node_vm_from_ex_template( name=name, ex_template=ex_template, other_args=locals() ) + return self._create_node_with_template(name=name, template=vm, namespace=namespace) # else (ex_template is None): create a vm with other parameters vm = self._base_vm_template(name=name) @@ -1084,6 +1127,7 @@ def create_node( self._create_node_disks(vm, ex_disks, image, namespace, location) # auth -> cloud-init + if auth is not None: self._create_node_auth(vm, auth) @@ -1093,6 +1137,7 @@ def create_node( self._create_node_network(vm, ex_network, ex_ports) # terminationGracePeriodSeconds + if ex_termination_grace_period is not None: self._create_node_termination_grace_period(vm, ex_termination_grace_period) @@ -1104,11 +1149,13 @@ def list_images(self, location=None): in that location will be provided. Otherwise all of them. """ nodes = self.list_nodes() + if location: namespace = location.name nodes = list(filter(lambda x: x["extra"]["namespace"] == namespace, nodes)) name_set = set() images = [] + for node in nodes: if node.image.name in name_set: continue @@ -1125,20 +1172,24 @@ def list_locations(self): namespaces = [] result = self.connection.request(req).object + for item in result["items"]: name = item["metadata"]["name"] ID = item["metadata"]["uid"] namespaces.append( NodeLocation(id=ID, name=name, country="", driver=self.connection.driver) ) + return namespaces def list_sizes(self, location=None): namespace = "" + if location: namespace = location.name nodes = self.list_nodes() sizes = [] + for node in nodes: if not namespace: sizes.append(node.size) @@ -1205,6 +1256,7 @@ def create_volume( for awsElasticBlockStore volume_type {fsType: 'ext4', volumeID: "1234"} """ + if ex_dynamic: if location is None: msg = "Please provide a namespace for the PVC." @@ -1217,6 +1269,7 @@ def create_volume( volume_mode=ex_volume_mode, access_mode=ex_access_mode, ) + return vol else: if ex_volume_type is None or ex_volume_params is None: @@ -1252,6 +1305,7 @@ def create_volume( raise # make sure that the volume was created volumes = self.list_volumes() + for volume in volumes: if volume.name == name: return volume @@ -1323,6 +1377,7 @@ def _create_volume_dynamic( result = self.connection.request(req, method=method, data=data) except Exception: raise + if result.object["status"]["phase"] != "Bound": for _ in range(3): req = ROOT_URL + "namespaces/" + namespace + "/persistentvolumeclaims/" + name @@ -1330,12 +1385,14 @@ def _create_volume_dynamic( result = self.connection.request(req).object except Exception: raise + if result["status"]["phase"] == "Bound": break time.sleep(3) # check that the pv was created and bound volumes = self.list_volumes() + for volume in volumes: if volume.extra["pvc"]["name"] == name: return volume @@ -1346,6 +1403,7 @@ def _bind_volume(self, volume, namespace="default"): It will bind them to a pvc so they can be used by a kubernetes resource. """ + if volume.extra["is_bound"]: return # volume already bound @@ -1363,11 +1421,13 @@ def _bind_volume(self, volume, namespace="default"): namespace=namespace, access_mode=access_mode, ) + return vol def destroy_volume(self, volume): # first delete the pvc method = "DELETE" + if volume.extra["is_bound"]: pvc = volume.extra["pvc"]["name"] namespace = volume.extra["pvc"]["namespace"] @@ -1383,6 +1443,7 @@ def destroy_volume(self, volume): try: result = self.connection.request(req, method=method) + return result.status except Exception: raise @@ -1392,8 +1453,10 @@ def attach_volume(self, node, volume, device="disk", ex_bus="virtio", ex_name=No params: bus, name , device (disk or lun) """ # volume must be bound to a claim + if not volume.extra["is_bound"]: volume = self._bind_volume(volume, node.extra["namespace"]) + if volume is None: raise LibcloudError( "Selected Volume (PV) could not be bound " @@ -1402,6 +1465,7 @@ def attach_volume(self, node, volume, device="disk", ex_bus="virtio", ex_name=No ) claimName = volume.extra["pvc"]["name"] + if ex_name is None: name = claimName else: @@ -1410,6 +1474,7 @@ def attach_volume(self, node, volume, device="disk", ex_bus="virtio", ex_name=No # check if vm is stopped self.stop_node(node) # check if it is the same namespace + if node.extra["namespace"] != namespace: msg = "The PVC and the VM must be in the same namespace" raise ValueError(msg) @@ -1442,10 +1507,12 @@ def attach_volume(self, node, volume, device="disk", ex_bus="virtio", ex_name=No result = self.connection.request( req, method="PATCH", data=json.dumps(data), headers=headers ) + if "pvcs" in node.extra: node.extra["pvcs"].append(claimName) else: node.extra["pvcs"] = [claimName] + return result in VALID_RESPONSE_CODES except Exception: raise @@ -1472,12 +1539,15 @@ def detach_volume(self, volume, ex_node): disks = result["spec"]["template"]["spec"]["domain"]["devices"]["disks"] volumes = result["spec"]["template"]["spec"]["volumes"] to_delete = None + for volume in volumes: if "persistentVolumeClaim" in volume: if volume["persistentVolumeClaim"]["claimName"] == claimName: to_delete = volume["name"] volumes.remove(volume) + break + if not to_delete: msg = "The given volume is not attached to the given VM" raise ValueError(msg) @@ -1485,6 +1555,7 @@ def detach_volume(self, volume, ex_node): for disk in disks: if disk["name"] == to_delete: disks.remove(disk) + break # now patch the new volumes and disks lists into the resource data = { @@ -1502,6 +1573,7 @@ def detach_volume(self, volume, ex_node): req, method="PATCH", data=json.dumps(data), headers=headers ) ex_node.extra["pvcs"].remove(claimName) + return result in VALID_RESPONSE_CODES except Exception: raise @@ -1513,6 +1585,7 @@ def ex_list_persistent_volume_claims(self, namespace="default"): except Exception: raise pvcs = [item["metadata"]["name"] for item in result["items"]] + return pvcs def ex_list_storage_classes(self): @@ -1550,6 +1623,7 @@ def list_volumes(self): extra["is_bound"] = item["status"]["phase"] == "Bound" extra["access_modes"] = item["spec"]["accessModes"] extra["volume_mode"] = item["spec"]["volumeMode"] + if extra["is_bound"]: extra["pvc"]["name"] = item["spec"]["claimRef"]["name"] extra["pvc"]["namespace"] = item["spec"]["claimRef"]["namespace"] @@ -1566,10 +1640,13 @@ def list_volumes(self): def _ex_connection_class_kwargs(self): kwargs = {} + if hasattr(self, "key_file"): kwargs["key_file"] = self.key_file + if hasattr(self, "cert_file"): kwargs["cert_file"] = self.cert_file + return kwargs def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node @@ -1592,6 +1669,7 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node extra["pvcs"] = [] memory = 0 + if "limits" in vm["spec"]["template"]["spec"]["domain"]["resources"]: if "memory" in vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]: memory = vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["memory"] @@ -1607,12 +1685,14 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node .get("requests", {}) .get("memory", None) ) + if memory_req: memory_req = _memory_in_MB(memory_req) else: memory_req = memory cpu = 1 + if vm["spec"]["template"]["spec"]["domain"]["resources"].get("limits", None): if vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"].get("cpu", None): cpu = vm["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["cpu"] @@ -1623,6 +1703,7 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node cpu_req = cpu elif vm["spec"]["template"]["spec"]["domain"].get("cpu", None): cpu = vm["spec"]["template"]["spec"]["domain"]["cpu"].get("cores", 1) + if not isinstance(cpu, int): cpu = int(cpu.rstrip("m")) @@ -1631,12 +1712,13 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node .get("requests", {}) .get("cpu", None) ) + if cpu_req is None: cpu_req = cpu extra_size = {"cpu": cpu, "cpu_request": cpu_req, "ram": memory, "ram_request": memory_req} size_name = "{} vCPUs, {}MB Ram".format(str(cpu), str(memory)) - size_id = hashlib.md5(size_name.encode("utf-8")).hexdigest() + size_id = hashlib.md5(size_name.encode("utf-8")).hexdigest() # nosec size = NodeSize( id=size_id, name=size_name, @@ -1652,12 +1734,14 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node extra["cpu"] = cpu image_name = "undefined" + for volume in vm["spec"]["template"]["spec"]["volumes"]: for k, v in volume.items(): if type(v) is dict: if "image" in v: image_name = v["image"] image = NodeImage(image_name, image_name, driver) + if "volumes" in vm["spec"]["template"]["spec"]: for volume in vm["spec"]["template"]["spec"]["volumes"]: if "persistentVolumeClaim" in volume: @@ -1665,8 +1749,10 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node port_forwards = [] services = self.ex_list_services(namespace=extra["namespace"], node_name=name) + for service in services: service_type = service["spec"].get("type") + for port_pair in service["spec"]["ports"]: protocol = port_pair.get("protocol") public_port = port_pair.get("port") @@ -1689,6 +1775,7 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node state = NodeState.STOPPED public_ips = None private_ips = None + return Node( id=ID, name=name, @@ -1705,14 +1792,17 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node req = ROOT_URL + "namespaces/" + extra["namespace"] + "/pods" result = self.connection.request(req).object pod = None + for pd in result["items"]: if "metadata" in pd and "ownerReferences" in pd["metadata"]: if pd["metadata"]["ownerReferences"][0]["name"] == name: pod = pd + if pod is None or "containerStatuses" not in pod["status"]: state = NodeState.PENDING public_ips = None private_ips = None + return Node( id=ID, name=name, @@ -1725,8 +1815,10 @@ def _to_node(self, vm, is_stopped=False): # type: (dict, bool) -> Node extra=extra, ) extra["pod"] = {"name": pod["metadata"]["name"]} + for cont_status in pod["status"]["containerStatuses"]: # only 2 containers are present the launcher and the vmi + if cont_status["name"] != "compute": image = NodeImage(ID, cont_status["image"], driver) state = ( @@ -1759,16 +1851,21 @@ def ex_list_services(self, namespace="default", node_name=None, service_name=Non concern the node. """ params = None + if service_name is not None: params = {"fieldSelector": "metadata.name={}".format(service_name)} req = ROOT_URL + "/namespaces/{}/services".format(namespace) result = self.connection.request(req, params=params).object["items"] + if node_name: res = [] + for service in result: if node_name in service["metadata"].get("name", ""): res.append(service) + return res + return result def ex_create_service( @@ -1830,11 +1927,14 @@ def ex_create_service( ports_to_expose = [] # if ports has a falsey value like None or 0 + if not ports: ports = [] + for port_group in ports: if not port_group.get("target_port", None): port_group["target_port"] = port_group["port"] + if not port_group.get("name", ""): port_group["name"] = "port-{}".format(port_group["port"]) ports_to_expose.append( @@ -1847,18 +1947,22 @@ def ex_create_service( ) headers = None data = None + if len(service_list) > 0: if not ports: result = True + for service in service_list: service_name = service["metadata"]["name"] result = result and self.ex_delete_service( namespace=namespace, service_name=service_name ) + return result else: method = "PATCH" spec = {"ports": ports_to_expose} + if not override_existing_ports: existing_ports = service_list[0]["spec"]["ports"] spec = {"ports": existing_ports.extend(ports_to_expose)} @@ -1887,8 +1991,10 @@ def ex_create_service( } service["spec"]["ports"] = ports_to_expose service["spec"]["type"] = service_type + if cluster_ip is not None: service["spec"]["clusterIP"] = cluster_ip + if service_type == "LoadBalancer" and load_balancer_ip is not None: service["spec"]["loadBalancerIP"] = load_balancer_ip data = json.dumps(service) @@ -1897,6 +2003,7 @@ def ex_create_service( result = self.connection.request(req, method=method, data=data, headers=headers) except Exception: raise + return result.status in VALID_RESPONSE_CODES def ex_delete_service(self, namespace, service_name): @@ -1906,6 +2013,7 @@ def ex_delete_service(self, namespace, service_name): result = self.connection.request(req, method="DELETE", headers=headers) except Exception: raise + return result.status in VALID_RESPONSE_CODES @@ -1938,6 +2046,7 @@ def _deep_merge_dict(source: dict, destination: dict) -> dict: :return: dict: Updated destination. """ + for key, value in source.items(): if isinstance(value, dict): # recurse for dicts node = destination.setdefault(key, {}) # get node or create one @@ -1975,6 +2084,7 @@ def _memory_in_MB(memory): # type: (Union[str, int]) -> int try: mem_bytes = int(memory) + return mem_bytes // 1024 // 1024 except ValueError: pass @@ -2039,7 +2149,8 @@ def KubeVirtNodeSize( extra["ram_request"] = ram_request or ram name = "{} vCPUs, {}MB Ram".format(str(cpu), str(ram)) - size_id = hashlib.md5(name.encode("utf-8")).hexdigest() + size_id = hashlib.md5(name.encode("utf-8")).hexdigest() # nosec + return NodeSize( id=size_id, name=name, @@ -2065,5 +2176,6 @@ def KubeVirtNodeImage(name): # type: (str) -> NodeImage containerDisk image (e.g. ``"quay.io/containerdisks/ubuntu:22.04"``) :rtype: :class:`NodeImage` """ - image_id = hashlib.md5(name.encode("utf-8")).hexdigest() + image_id = hashlib.md5(name.encode("utf-8")).hexdigest() # nosec + return NodeImage(id=image_id, name=name, driver=KubeVirtNodeDriver) diff --git a/libcloud/compute/drivers/nttcis.py b/libcloud/compute/drivers/nttcis.py index d90574afba..e22dadf2ae 100644 --- a/libcloud/compute/drivers/nttcis.py +++ b/libcloud/compute/drivers/nttcis.py @@ -125,6 +125,7 @@ def __init__( ): if region not in API_ENDPOINTS and host is None: raise ValueError("Invalid region: %s, no host specified" % (region)) + if region is not None: self.selected_region = API_ENDPOINTS[region] @@ -150,6 +151,7 @@ def _ex_connection_class_kwargs(self): kwargs = super()._ex_connection_class_kwargs() kwargs["region"] = self.selected_region kwargs["api_version"] = self.api_version + return kwargs def _create_node_mcp1( @@ -217,6 +219,7 @@ def _create_node_mcp1( password = None image_needs_auth = self._image_needs_auth(image) + if image_needs_auth: if isinstance(auth, basestring): auth_obj = NodeAuthPassword(password=auth) @@ -231,6 +234,7 @@ def _create_node_mcp1( image_id = self._image_to_image_id(image) ET.SubElement(server_elm, "imageId").text = image_id ET.SubElement(server_elm, "start").text = str(ex_is_started).lower() + if password is not None: ET.SubElement(server_elm, "administratorPassword").text = password @@ -261,6 +265,7 @@ def _create_node_mcp1( ).object node_id = None + for info in findall(response, "info", TYPES_URN): if info.get("name") == "serverId": node_id = info.get("value") @@ -492,6 +497,7 @@ def create_node( """ # Neither legacy MCP1 network nor MCP2 network domain provided + if ex_network_domain is None and "ex_network" not in kwargs: raise ValueError( "You must provide either ex_network_domain " @@ -499,6 +505,7 @@ def create_node( ) # Ambiguous parameter provided. Can't determine if it is MCP 1 or 2. + if ex_network_domain is not None and "ex_network" in kwargs: raise ValueError( "You can only supply either " @@ -507,10 +514,12 @@ def create_node( ) # Set ex_is_started to False by default if none bool data type provided + if not isinstance(ex_is_started, bool): ex_is_started = True # Handle MCP1 legacy + if "ex_network" in kwargs: new_node = self._create_node_mcp1( name=name, @@ -530,6 +539,7 @@ def create_node( ) else: # Handle MCP2 legacy. CaaS api 2.2 or earlier + if "ex_vlan" in kwargs: ex_primary_nic_vlan = kwargs.get("ex_vlan") @@ -540,6 +550,7 @@ def create_node( if "ex_additional_nics_vlan" in kwargs: vlans = kwargs.get("ex_additional_nics_vlan") + if isinstance(vlans, (list, tuple)): for v in vlans: add_nic = NttCisNic(vlan=v) @@ -562,11 +573,13 @@ def create_node( ex_additional_nics = additional_nics # Handle MCP2 latest. CaaS API 2.3 onwards + if ex_network_domain is None: raise ValueError("ex_network_domain must be specified") password = None image_needs_auth = self._image_needs_auth(image) + if image_needs_auth: if isinstance(auth, basestring): auth_obj = NodeAuthPassword(password=auth) @@ -581,6 +594,7 @@ def create_node( image_id = self._image_to_image_id(image) ET.SubElement(server_elm, "imageId").text = image_id ET.SubElement(server_elm, "start").text = str(ex_is_started).lower() + if password is not None: ET.SubElement(server_elm, "administratorPassword").text = password @@ -685,6 +699,7 @@ def create_node( ).object node_id = None + for info in findall(response, "info", TYPES_URN): if info.get("name") == "serverId": node_id = info.get("value") @@ -713,6 +728,7 @@ def destroy_node(self, node): "server/deleteServer", method="POST", data=ET.tostring(request_elm) ).object response_code = findtext(body, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def reboot_node(self, node): @@ -731,6 +747,7 @@ def reboot_node(self, node): "server/rebootServer", method="POST", data=ET.tostring(request_elm) ).object response_code = findtext(body, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def list_nodes( @@ -796,6 +813,7 @@ def list_nodes( # This is a generator so we changed from the original # and if nodes is not empty, ie, the stop iteration confdition # Then set node_list to nodes and return. + for nodes in self.ex_list_nodes_paginated( location=ex_location, name=ex_name, @@ -810,6 +828,7 @@ def list_nodes( ): if nodes: node_list = nodes + return node_list def list_images(self, location=None): @@ -829,6 +848,7 @@ def list_images(self, location=None): """ params = {} + if location is not None: params["datacenterId"] = self._location_to_location_id(location) @@ -891,6 +911,7 @@ def list_locations(self, ex_id=None): """ params = {} + if ex_id is not None: params["id"] = ex_id @@ -912,6 +933,7 @@ def list_snapshot_windows(self, location, plan): params = {} params["datacenterId"] = self._location_to_location_id(location) params["servicePlan"] = plan + return self._to_windows( self.connection.request_with_orgId_api_2( "infrastructure/snapshotWindow", params=params @@ -932,6 +954,7 @@ def list_networks(self, location=None): """ url_ext = "" + if location is not None: url_ext = "/" + self._location_to_location_id(location) @@ -978,6 +1001,7 @@ def import_image( """ # Unsupported for version lower than 2.4 + if LooseVersion(self.connection.active_api_version) < LooseVersion("2.4"): raise Exception( "import image is feature is NOT supported in " "api version earlier than 2.4" @@ -1015,6 +1039,7 @@ def import_image( for k, v in tagkey_name_value_dictionaries.items(): tag_elem = ET.SubElement(import_image_elem, "urn:tag") ET.SubElement(tag_elem, "urn:tagKeyName").text = k + if v is not None: ET.SubElement(tag_elem, "urn:value").text = v @@ -1023,6 +1048,7 @@ def import_image( ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def start_node(self, node): @@ -1040,6 +1066,7 @@ def start_node(self, node): "server/startServer", method="POST", data=ET.tostring(request_elm) ).object response_code = findtext(body, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def stop_node(self, node): @@ -1059,6 +1086,7 @@ def stop_node(self, node): "server/shutdownServer", method="POST", data=ET.tostring(request_elm) ).object response_code = findtext(body, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_list_nodes_paginated( @@ -1122,26 +1150,37 @@ def ex_list_nodes_paginated( """ params = {} + if location is not None: params["datacenterId"] = self._location_to_location_id(location) + if ipv6 is not None: params["ipv6"] = ipv6 + if ipv4 is not None: params["privateIpv4"] = ipv4 + if state is not None: params["state"] = state + if started is not None: params["started"] = started + if deployed is not None: params["deployed"] = deployed + if name is not None: params["name"] = name + if network_domain is not None: params["networkDomainId"] = self._network_domain_to_network_domain_id(network_domain) + if network is not None: params["networkId"] = self._network_to_network_id(network) + if vlan is not None: params["vlanId"] = self._vlan_to_vlan_id(vlan) + if image is not None: params["sourceImageId"] = self._image_to_image_id(image) @@ -1155,22 +1194,27 @@ def ex_list_nodes_paginated( def ex_edit_metadata(self, node, name=None, description=None, drs_eligible=None): request_elem = ET.Element("editServerMetadata", {"xmlns": TYPES_URN, "id": node.id}) + if name is not None: ET.SubElement(request_elem, "name").text = name + if description is not None: ET.SubElement(request_elem, "description").text = description + if drs_eligible is not None: ET.SubElement(request_elem, "drsEligible").text = drs_eligible body = self.connection.request_with_orgId_api_2( "server/editServerMetadata", method="POST", data=ET.tostring(request_elem) ).object response_code = findtext(body, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_start_node(self, node): # NOTE: This method is here for backward compatibility reasons after # this method was promoted to be part of the standard compute API in # Libcloud v2.7.0 + return self.start_node(node=node) def ex_shutdown_graceful(self, node): @@ -1220,6 +1264,7 @@ def ex_reset(self, node): "server/resetServer", method="POST", data=ET.tostring(request_elm) ).object response_code = findtext(body, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_update_vm_tools(self, node): @@ -1238,6 +1283,7 @@ def ex_update_vm_tools(self, node): "server/updateVmwareTools", method="POST", data=ET.tostring(request_elm) ).object response_code = findtext(body, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_update_node(self, node, name=None, description=None, cpu_count=None, ram_mb=None): @@ -1263,18 +1309,23 @@ def ex_update_node(self, node, name=None, description=None, cpu_count=None, ram_ """ data = {} + if name is not None: data["name"] = name + if description is not None: data["description"] = description + if cpu_count is not None: data["cpuCount"] = str(cpu_count) + if ram_mb is not None: data["memory"] = str(ram_mb) body = self.connection.request_with_orgId_api_1( "server/%s" % (node.id), method="POST", data=urlencode(data, True) ).object response_code = findtext(body, "result", GENERAL_NS) + return response_code in ["IN_PROGRESS", "SUCCESS"] def ex_enable_snapshots(self, node, window, plan="ADVANCED", initiate="true"): @@ -1312,6 +1363,7 @@ def ex_enable_snapshots(self, node, window, plan="ADVANCED", initiate="true"): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def list_snapshots(self, node, page_size=None): @@ -1328,6 +1380,7 @@ def list_snapshots(self, node, page_size=None): params = {} params["serverId"] = self.list_nodes(ex_name=node)[0].id + if page_size is not None: params["pageSize"] = page_size @@ -1369,6 +1422,7 @@ def ex_disable_snapshots(self, node): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_initiate_manual_snapshot(self, name=None, server_id=None): @@ -1388,6 +1442,7 @@ def ex_initiate_manual_snapshot(self, name=None, server_id=None): if server_id is None: node = self.list_nodes(ex_name=name) + if len(node) > 1: raise RuntimeError( "Found more than one server Id, " @@ -1405,6 +1460,7 @@ def ex_initiate_manual_snapshot(self, name=None, server_id=None): data=ET.tostring(update_node), ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_create_snapshot_preview_server( @@ -1465,14 +1521,18 @@ def ex_create_snapshot_preview_server( {"xmlns": TYPES_URN, "snapshotId": snapshot_id}, ) ET.SubElement(create_preview, "serverName").text = server_name + if server_description is not None: ET.SubElement(create_preview, "serverDescription").text = server_description + if target_cluster_id is not None: ET.SubElement(create_preview, "targetClusterId").text = target_cluster_id ET.SubElement(create_preview, "serverStarted").text = server_started ET.SubElement(create_preview, "nicsConnected").text = nics_connected + if preserve_mac_addresses is not None: ET.SubElement(create_preview, "preserveMacAddresses").text = preserve_mac_addresses + if tag_key_name is not None: tag_elem = ET.SubElement(create_preview, "tag") ET.SubElement(tag_elem, "tagKeyName").text = tag_key_name @@ -1487,6 +1547,7 @@ def ex_create_snapshot_preview_server( data=ET.tostring(create_preview), ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_migrate_preview_server(self, preview_id): @@ -1499,6 +1560,7 @@ def ex_migrate_preview_server(self, preview_id): data=ET.tostring(migrate_preview), ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_create_anti_affinity_rule(self, node_list): @@ -1518,6 +1580,7 @@ def ex_create_anti_affinity_rule(self, node_list): if not isinstance(node_list, (list, tuple)): raise TypeError("Node list must be a list or a tuple.") anti_affinity_xml_request = ET.Element("createAntiAffinityRule", {"xmlns": TYPES_URN}) + for node in node_list: ET.SubElement(anti_affinity_xml_request, "serverId").text = self._node_to_node_id(node) result = self.connection.request_with_orgId_api_2( @@ -1526,6 +1589,7 @@ def ex_create_anti_affinity_rule(self, node_list): data=ET.tostring(anti_affinity_xml_request), ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "SUCCESS"] def ex_delete_anti_affinity_rule(self, anti_affinity_rule): @@ -1549,6 +1613,7 @@ def ex_delete_anti_affinity_rule(self, anti_affinity_rule): data=ET.tostring(update_node), ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "SUCCESS"] def ex_list_anti_affinity_rules( @@ -1587,18 +1652,24 @@ def ex_list_anti_affinity_rules( """ not_none_arguments = [key for key in (network, network_domain, node) if key is not None] + if len(not_none_arguments) != 1: raise ValueError("One and ONLY one of network, " "network_domain, or node must be set") params = {} + if network_domain is not None: params["networkDomainId"] = self._network_domain_to_network_domain_id(network_domain) + if network is not None: params["networkId"] = self._network_to_network_id(network) + if node is not None: params["serverId"] = self._node_to_node_id(node) + if filter_id is not None: params["id"] = filter_id + if filter_state is not None: params["state"] = filter_state @@ -1607,8 +1678,10 @@ def ex_list_anti_affinity_rules( ) rules = [] + for result in paged_result: rules.extend(self._to_anti_affinity_rules(result)) + return rules def ex_attach_node_to_vlan(self, node, vlan=None, private_ipv4=None): @@ -1648,6 +1721,7 @@ def ex_attach_node_to_vlan(self, node, vlan=None, private_ipv4=None): "server/addNic", method="POST", data=ET.tostring(request) ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_destroy_nic(self, nic_id): @@ -1666,6 +1740,7 @@ def ex_destroy_nic(self, nic_id): "server/removeNic", method="POST", data=ET.tostring(request) ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_list_networks(self, location=None): @@ -1703,6 +1778,7 @@ def ex_create_network(self, location, name, description=None): create_node = ET.Element("NewNetworkWithLocation", {"xmlns": NETWORK_NS}) ET.SubElement(create_node, "name").text = name + if description is not None: ET.SubElement(create_node, "description").text = description ET.SubElement(create_node, "location").text = network_location @@ -1730,6 +1806,7 @@ def ex_delete_network(self, network): "network/%s?delete" % network.id, method="GET" ).object response_code = findtext(response, "result", GENERAL_NS) + return response_code == "SUCCESS" def ex_rename_network(self, network, new_name): @@ -1749,6 +1826,7 @@ def ex_rename_network(self, network, new_name): "network/%s" % network.id, method="POST", data="name=%s" % new_name ).object response_code = findtext(response, "result", GENERAL_NS) + return response_code == "SUCCESS" def ex_get_network_domain(self, network_domain_id): @@ -1765,6 +1843,7 @@ def ex_get_network_domain(self, network_domain_id): net = self.connection.request_with_orgId_api_2( "network/networkDomain/%s" % network_domain_id ).object + return self._to_network_domain(net, locations) def ex_list_network_domains(self, location=None, name=None, service_plan=None, state=None): @@ -1790,18 +1869,23 @@ def ex_list_network_domains(self, location=None, name=None, service_plan=None, s """ params = {} + if location is not None: params["datacenterId"] = self._location_to_location_id(location) + if name is not None: params["name"] = name + if service_plan is not None: params["type"] = service_plan + if state is not None: params["state"] = state response = self.connection.request_with_orgId_api_2( "network/networkDomain", params=params ).object + return self._to_network_domains(response) def ex_create_network_domain(self, location, name, service_plan, description=None): @@ -1830,6 +1914,7 @@ def ex_create_network_domain(self, location, name, service_plan, description=Non ET.SubElement(create_node, "datacenterId").text = self._location_to_location_id(location) ET.SubElement(create_node, "name").text = name + if description is not None: ET.SubElement(create_node, "description").text = description ET.SubElement(create_node, "type").text = service_plan @@ -1867,6 +1952,7 @@ def ex_update_network_domain(self, network_domain): edit_node = ET.Element("editNetworkDomain", {"xmlns": TYPES_URN}) edit_node.set("id", network_domain.id) ET.SubElement(edit_node, "name").text = network_domain.name + if network_domain.description is not None: ET.SubElement(edit_node, "description").text = network_domain.description ET.SubElement(edit_node, "type").text = network_domain.plan @@ -1894,6 +1980,7 @@ def ex_delete_network_domain(self, network_domain): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_create_vlan( @@ -1931,6 +2018,7 @@ def ex_create_vlan( create_node = ET.Element("deployVlan", {"xmlns": TYPES_URN}) ET.SubElement(create_node, "networkDomainId").text = network_domain.id ET.SubElement(create_node, "name").text = name + if description is not None: ET.SubElement(create_node, "description").text = description ET.SubElement(create_node, "privateIpv4BaseAddress").text = private_ipv4_base_address @@ -1961,6 +2049,7 @@ def ex_get_vlan(self, vlan_id): locations = self.list_locations() vlan = self.connection.request_with_orgId_api_2("network/vlan/%s" % vlan_id).object + return self._to_vlan(vlan, locations) def ex_update_vlan(self, vlan): @@ -1978,6 +2067,7 @@ def ex_update_vlan(self, vlan): edit_node = ET.Element("editVlan", {"xmlns": TYPES_URN}) edit_node.set("id", vlan.id) ET.SubElement(edit_node, "name").text = vlan.name + if vlan.description is not None: ET.SubElement(edit_node, "description").text = vlan.description @@ -2028,6 +2118,7 @@ def ex_delete_vlan(self, vlan): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_list_vlans( @@ -2065,19 +2156,26 @@ def ex_list_vlans( """ params = {} + if location is not None: params["datacenterId"] = self._location_to_location_id(location) + if network_domain is not None: params["networkDomainId"] = self._network_domain_to_network_domain_id(network_domain) + if name is not None: params["name"] = name + if ipv4_address is not None: params["privateIpv4Address"] = ipv4_address + if ipv6_address is not None: params["ipv6Address"] = ipv6_address + if state is not None: params["state"] = state response = self.connection.request_with_orgId_api_2("network/vlan", params=params).object + return self._to_vlans(response) def ex_add_public_ip_block_to_network_domain(self, network_domain): @@ -2093,6 +2191,7 @@ def ex_add_public_ip_block_to_network_domain(self, network_domain): for info in findall(response, "info", TYPES_URN): if info.get("name") == "ipBlockId": block_id = info.get("value") + return self.ex_get_public_ip_block(block_id) def ex_list_public_ip_blocks(self, network_domain): @@ -2102,6 +2201,7 @@ def ex_list_public_ip_blocks(self, network_domain): response = self.connection.request_with_orgId_api_2( "network/publicIpBlock", params=params ).object + return self._to_ip_blocks(response) def ex_get_public_ip_block(self, block_id): @@ -2109,6 +2209,7 @@ def ex_get_public_ip_block(self, block_id): block = self.connection.request_with_orgId_api_2( "network/publicIpBlock/%s" % block_id ).object + return self._to_ip_block(block, locations) def ex_delete_public_ip_block(self, block): @@ -2119,19 +2220,24 @@ def ex_delete_public_ip_block(self, block): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] # 09/10/18 Adding private IPv4 and IPv6 addressing capability def ex_reserve_ip(self, vlan, ip, description): vlan_id = self._vlan_to_vlan_id(vlan) + if re.match(r"(\d+\.){3}", ip): private_ip = ET.Element("reservePrivateIpv4Address", {"xmlns": TYPES_URN}) resource = "network/reservePrivateIpv4Address" elif re.search(r":", ip): private_ip = ET.Element("reserveIpv6Address", {"xmlns": TYPES_URN}) resource = "network/reserveIpv6Address" + else: + raise ValueError("Unable to retrieve private ip in the provided ip argument") ET.SubElement(private_ip, "vlanId").text = vlan_id ET.SubElement(private_ip, "ipAddress").text = ip + if description is not None: ET.SubElement(private_ip, "description").text = description @@ -2139,22 +2245,27 @@ def ex_reserve_ip(self, vlan, ip, description): resource, method="POST", data=ET.tostring(private_ip) ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_unreserve_ip_addresses(self, vlan, ip): vlan_id = self._vlan_to_vlan_id(vlan) + if re.match(r"(\d+\.){3}", ip): private_ip = ET.Element("unreservePrivateIpv4Address", {"xmlns": TYPES_URN}) resource = "network/reservePrivateIpv4Address" elif re.search(r":", ip): private_ip = ET.Element("unreserveIpv6Address", {"xmlns": TYPES_URN}) resource = "network/unreserveIpv6Address" + else: + raise ValueError("Unable to parse private ip from ip argument") ET.SubElement(private_ip, "vlanId").text = vlan_id ET.SubElement(private_ip, "ipAddress").text = ip result = self.connection.request_with_orgId_api_2( resource, method="POST", data=ET.tostring(private_ip) ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_list_reserved_ipv4(self, vlan=None, datacenter_id=None): @@ -2175,6 +2286,7 @@ def ex_list_reserved_ipv4(self, vlan=None, datacenter_id=None): "network/reservedPrivateIpv4Address" ).object addresses = self._to_ipv4_addresses(response) + return addresses def ex_list_reserved_ipv6(self, vlan=None, datacenter_id=None): @@ -2195,10 +2307,12 @@ def ex_list_reserved_ipv6(self, vlan=None, datacenter_id=None): "network/reservedIpv6Address" ).object addresses = self._to_ipv6_addresses(response) + return addresses def ex_get_node_by_id(self, id): node = self.connection.request_with_orgId_api_2("server/server/%s" % id).object + return self._to_node(node) def ex_list_firewall_rules(self, network_domain, page_size=50, page_number=1): @@ -2208,6 +2322,7 @@ def ex_list_firewall_rules(self, network_domain, page_size=50, page_number=1): response = self.connection.request_with_orgId_api_2( "network/firewallRule", params=params ).object + return self._to_firewall_rules(response, network_domain) def ex_create_firewall_rule( @@ -2282,17 +2397,21 @@ def ex_create_firewall_rule( ET.SubElement(create_node, "protocol").text = protocol # Setup source port rule source = ET.SubElement(create_node, "source") + if source_addr.address_list_id is not None: source_ip = ET.SubElement(source, "ipAddressListId") source_ip.text = source_addr.address_list_id else: source_ip = ET.SubElement(source, "ip") + if source_addr.any_ip: source_ip.set("address", "ANY") else: source_ip.set("address", source_addr.ip_address) + if source_addr.ip_prefix_size is not None: source_ip.set("prefixSize", str(source_addr.ip_prefix_size)) + if source_addr.port_list_id is not None: source_port = ET.SubElement(source, "portListId") source_port.text = source_addr.port_list_id @@ -2300,21 +2419,26 @@ def ex_create_firewall_rule( if source_addr.port_begin is not None: source_port = ET.SubElement(source, "port") source_port.set("begin", source_addr.port_begin) + if source_addr.port_end is not None: source_port.set("end", source_addr.port_end) # Setup destination port rule dest = ET.SubElement(create_node, "destination") + if dest_addr.address_list_id is not None: dest_ip = ET.SubElement(dest, "ipAddressListId") dest_ip.text = dest_addr.address_list_id else: dest_ip = ET.SubElement(dest, "ip") + if dest_addr.any_ip: dest_ip.set("address", "ANY") else: dest_ip.set("address", dest_addr.ip_address) + if dest_addr.ip_prefix_size is not None: dest_ip.set("prefixSize", dest_addr.ip_prefix_size) + if dest_addr.port_list_id is not None: dest_port = ET.SubElement(dest, "portListId") dest_port.text = dest_addr.port_list_id @@ -2322,17 +2446,20 @@ def ex_create_firewall_rule( if dest_addr.port_begin is not None: dest_port = ET.SubElement(dest, "port") dest_port.set("begin", dest_addr.port_begin) + if dest_addr.port_end is not None: dest_port.set("end", dest_addr.port_end) # Set up positioning of rule ET.SubElement(create_node, "enabled").text = str(enabled) placement = ET.SubElement(create_node, "placement") + if position_relative_to_rule is not None: if position not in positions_with_rule: raise ValueError( "When position_relative_to_rule is specified" " position must be %s" % ", ".join(positions_with_rule) ) + if isinstance(position_relative_to_rule, NttCisFirewallRule): rule_name = position_relative_to_rule.name else: @@ -2351,10 +2478,12 @@ def ex_create_firewall_rule( ).object rule_id = None + for info in findall(response, "info", TYPES_URN): if info.get("name") == "firewallRuleId": rule_id = info.get("value") rule = self.ex_get_firewall_rule(network_domain, rule_id) + return rule def ex_edit_firewall_rule(self, rule, position=None, relative_rule_for_position=None): @@ -2426,19 +2555,23 @@ def ex_edit_firewall_rule(self, rule, position=None, relative_rule_for_position= # Source address source = ET.SubElement(edit_node, "source") + if rule.source.address_list_id is not None: source_ip = ET.SubElement(source, "ipAddressListId") source_ip.text = rule.source.address_list_id else: source_ip = ET.SubElement(source, "ip") + if rule.source.any_ip: source_ip.set("address", "ANY") else: source_ip.set("address", rule.source.ip_address) + if rule.source.ip_prefix_size is not None: source_ip.set("prefixSize", str(rule.source.ip_prefix_size)) # Setup source port rule + if rule.source.port_list_id is not None: source_port = ET.SubElement(source, "portListId") source_port.text = rule.source.port_list_id @@ -2446,21 +2579,26 @@ def ex_edit_firewall_rule(self, rule, position=None, relative_rule_for_position= if rule.source.port_begin is not None: source_port = ET.SubElement(source, "port") source_port.set("begin", rule.source.port_begin) + if rule.source.port_end is not None: source_port.set("end", rule.source.port_end) # Setup destination port rule dest = ET.SubElement(edit_node, "destination") + if rule.destination.address_list_id is not None: dest_ip = ET.SubElement(dest, "ipAddressListId") dest_ip.text = rule.destination.address_list_id else: dest_ip = ET.SubElement(dest, "ip") + if rule.destination.any_ip: dest_ip.set("address", "ANY") else: dest_ip.set("address", rule.destination.ip_address) + if rule.destination.ip_prefix_size is not None: dest_ip.set("prefixSize", rule.destination.ip_prefix_size) + if rule.destination.port_list_id is not None: dest_port = ET.SubElement(dest, "portListId") dest_port.text = rule.destination.port_list_id @@ -2468,19 +2606,23 @@ def ex_edit_firewall_rule(self, rule, position=None, relative_rule_for_position= if rule.destination.port_begin is not None: dest_port = ET.SubElement(dest, "port") dest_port.set("begin", rule.destination.port_begin) + if rule.destination.port_end is not None: dest_port.set("end", rule.destination.port_end) # Set up positioning of rule ET.SubElement(edit_node, "enabled").text = str(rule.enabled).lower() # changing placement to an option + if position is not None: placement = ET.SubElement(edit_node, "placement") + if relative_rule_for_position is not None: if position not in positions_with_rule: raise ValueError( "When position_relative_to_rule is specified" " position must be %s" % ", ".join(positions_with_rule) ) + if isinstance(relative_rule_for_position, NttCisFirewallRule): rule_name = relative_rule_for_position.name else: @@ -2499,11 +2641,13 @@ def ex_edit_firewall_rule(self, rule, position=None, relative_rule_for_position= ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_get_firewall_rule(self, network_domain, rule_id): locations = self.list_locations() rule = self.connection.request_with_orgId_api_2("network/firewallRule/%s" % rule_id).object + return self._to_firewall_rule(rule, locations, network_domain) def ex_set_firewall_rule_state(self, rule, state): @@ -2527,6 +2671,7 @@ def ex_set_firewall_rule_state(self, rule, state): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_delete_firewall_rule(self, rule): @@ -2546,6 +2691,7 @@ def ex_delete_firewall_rule(self, rule): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_create_nat_rule(self, network_domain, internal_ip, external_ip): @@ -2573,6 +2719,7 @@ def ex_create_nat_rule(self, network_domain, internal_ip, external_ip): ).object rule_id = None + for info in findall(result, "info", TYPES_URN): if info.get("name") == "natRuleId": rule_id = info.get("value") @@ -2599,6 +2746,7 @@ def ex_list_nat_rules(self, network_domain): params["networkDomainId"] = network_domain.id response = self.connection.request_with_orgId_api_2("network/natRule", params=params).object + return self._to_nat_rules(response, network_domain) def ex_get_nat_rule(self, network_domain, rule_id): @@ -2615,6 +2763,7 @@ def ex_get_nat_rule(self, network_domain, rule_id): """ rule = self.connection.request_with_orgId_api_2("network/natRule/%s" % rule_id).object + return self._to_nat_rule(rule, network_domain) def ex_delete_nat_rule(self, rule): @@ -2634,6 +2783,7 @@ def ex_delete_nat_rule(self, rule): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_get_location_by_id(self, id): @@ -2647,8 +2797,10 @@ def ex_get_location_by_id(self, id): """ location = None + if id is not None: location = self.list_locations(ex_id=id)[0] + return location def ex_wait_for_state(self, state, func, poll_interval=2, timeout=60, *args, **kwargs): @@ -2703,6 +2855,7 @@ def ex_enable_monitoring(self, node, service_plan="ESSENTIALS"): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_update_monitoring_plan(self, node, service_plan="ESSENTIALS"): @@ -2729,6 +2882,7 @@ def ex_update_monitoring_plan(self, node, service_plan="ESSENTIALS"): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_disable_monitoring(self, node): @@ -2750,6 +2904,7 @@ def ex_disable_monitoring(self, node): ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_add_scsi_controller_to_node(self, server_id, adapter_type, bus_number=None): @@ -2764,6 +2919,7 @@ def ex_add_scsi_controller_to_node(self, server_id, adapter_type, bus_number=Non update_node = ET.Element("addScsiController", {"xmlns": TYPES_URN}) ET.SubElement(update_node, "serverId").text = server_id ET.SubElement(update_node, "adapterType").text = adapter_type + if bus_number is not None: ET.SubElement(update_node, "busNumber").text = bus_number @@ -2771,6 +2927,7 @@ def ex_add_scsi_controller_to_node(self, server_id, adapter_type, bus_number=Non "server/addScsiController", method="POST", data=ET.tostring(update_node) ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_remove_scsi_controller(self, controller_id): @@ -2786,6 +2943,7 @@ def ex_remove_scsi_controller(self, controller_id): "server/removeScsiController", method="POST", data=ET.tostring(update_node) ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_add_storage_to_node( @@ -2823,6 +2981,7 @@ def ex_add_storage_to_node( raise RuntimeError("Either a node or a controller " "id must be specified") update_node = ET.Element("addDisk", {"xmlns": TYPES_URN}) + if node is not None: ET.SubElement(update_node, "serverId").text = node.id elif controller_id is not None: @@ -2831,6 +2990,7 @@ def ex_add_storage_to_node( update_node.insert(1, scsi_node) ET.SubElement(update_node, "sizeGb").text = str(amount) ET.SubElement(update_node, "speed").text = speed.upper() + if scsi_id is not None: ET.SubElement(update_node, "scsiId").text = str(scsi_id) @@ -2838,6 +2998,7 @@ def ex_add_storage_to_node( "server/addDisk", method="POST", data=ET.tostring(update_node) ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_remove_storage_from_node(self, node, scsi_id): @@ -2854,6 +3015,7 @@ def ex_remove_storage_from_node(self, node, scsi_id): """ disk = [disk for disk in node.extra["disks"] if disk.scsi_id == scsi_id][0] + return self.ex_remove_storage(disk.id) def ex_remove_storage(self, disk_id): @@ -2875,6 +3037,7 @@ def ex_remove_storage(self, disk_id): "server/removeDisk", method="POST", data=ET.tostring(remove_disk) ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_change_storage_speed(self, disk_id, speed, iops=None): @@ -2896,12 +3059,14 @@ def ex_change_storage_speed(self, disk_id, speed, iops=None): create_node = ET.Element("changeDiskSpeed", {"xmlns": TYPES_URN}) create_node.set("id", disk_id) ET.SubElement(create_node, "speed").text = speed + if iops is not None: ET.SubElement(create_node, "iops").text = str(iops) result = self.connection.request_with_orgId_api_2( "server/changeDiskSpeed", method="POST", data=ET.tostring(create_node) ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "SUCCESS"] def ex_change_storage_size(self, disk_id, size): @@ -2934,6 +3099,7 @@ def ex_change_storage_size(self, disk_id, size): "server/expandDisk", method="POST", data=ET.tostring(create_node) ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_reconfigure_node( @@ -2967,18 +3133,23 @@ def ex_reconfigure_node( update = ET.Element("reconfigureServer", {"xmlns": TYPES_URN}) update.set("id", node.id) + if memory_gb is not None: ET.SubElement(update, "memoryGb").text = str(memory_gb) + if cpu_count is not None: ET.SubElement(update, "cpuCount").text = str(cpu_count) + if cpu_performance is not None: ET.SubElement(update, "cpuSpeed").text = cpu_performance + if cores_per_socket is not None: ET.SubElement(update, "coresPerSocket").text = str(cores_per_socket) result = self.connection.request_with_orgId_api_2( "server/reconfigureServer", method="POST", data=ET.tostring(update) ).object response_code = findtext(result, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_clone_node_to_image( @@ -3072,6 +3243,7 @@ def ex_clean_failed_deployment(self, node): "server/cleanServer", method="POST", data=ET.tostring(request_elm) ).object response_code = findtext(body, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_list_customer_images(self, location=None): @@ -3085,6 +3257,7 @@ def ex_list_customer_images(self, location=None): """ params = {} + if location is not None: params["datacenterId"] = self._location_to_location_id(location) @@ -3104,6 +3277,7 @@ def ex_get_base_image_by_id(self, id): """ image = self.connection.request_with_orgId_api_2("image/osImage/%s" % id).object + return self._to_image(image) def ex_get_customer_image_by_id(self, id): @@ -3117,6 +3291,7 @@ def ex_get_customer_image_by_id(self, id): """ image = self.connection.request_with_orgId_api_2("image/customerImage/%s" % id).object + return self._to_image(image) def ex_get_image_by_id(self, id): @@ -3139,6 +3314,7 @@ def ex_get_image_by_id(self, id): except NttCisAPIException as e: if e.code != "RESOURCE_NOT_FOUND": raise e + return self.ex_get_customer_image_by_id(id) def ex_create_tag_key( @@ -3166,6 +3342,7 @@ def ex_create_tag_key( create_tag_key = ET.Element("createTagKey", {"xmlns": TYPES_URN}) ET.SubElement(create_tag_key, "name").text = name + if description is not None: ET.SubElement(create_tag_key, "description").text = description ET.SubElement(create_tag_key, "valueRequired").text = str(value_required).lower() @@ -3174,6 +3351,7 @@ def ex_create_tag_key( "tag/createTagKey", method="POST", data=ET.tostring(create_tag_key) ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_list_tag_keys(self, id=None, name=None, value_required=None, display_on_report=None): @@ -3198,12 +3376,16 @@ def ex_list_tag_keys(self, id=None, name=None, value_required=None, display_on_r """ params = {} + if id is not None: params["id"] = id + if name is not None: params["name"] = name + if value_required is not None: params["valueRequired"] = str(value_required).lower() + if display_on_report is not None: params["displayOnReport"] = str(display_on_report).lower() @@ -3212,8 +3394,10 @@ def ex_list_tag_keys(self, id=None, name=None, value_required=None, display_on_r ) tag_keys = [] + for result in paged_result: tag_keys.extend(self._to_tag_keys(result)) + return tag_keys def ex_get_tag_key_by_id(self, id): @@ -3227,6 +3411,7 @@ def ex_get_tag_key_by_id(self, id): """ tag_key = self.connection.request_with_orgId_api_2("tag/tagKey/%s" % id).object + return self._to_tag_key(tag_key) def ex_get_tag_key_by_name(self, name): @@ -3243,8 +3428,10 @@ def ex_get_tag_key_by_name(self, name): """ tag_keys = self.ex_list_tag_keys(name=name) + if len(tag_keys) != 1: raise ValueError("No tags found with name %s" % name) + return tag_keys[0] def ex_modify_tag_key( @@ -3280,12 +3467,16 @@ def ex_modify_tag_key( tag_key_id = self._tag_key_to_tag_key_id(tag_key) modify_tag_key = ET.Element("editTagKey", {"xmlns": TYPES_URN, "id": tag_key_id}) + if name is not None: ET.SubElement(modify_tag_key, "name").text = name + if description is not None: ET.SubElement(modify_tag_key, "description").text = description + if value_required is not None: ET.SubElement(modify_tag_key, "valueRequired").text = str(value_required).lower() + if display_on_report is not None: ET.SubElement(modify_tag_key, "displayOnReport").text = str(display_on_report).lower() @@ -3293,6 +3484,7 @@ def ex_modify_tag_key( "tag/editTagKey", method="POST", data=ET.tostring(modify_tag_key) ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_remove_tag_key(self, tag_key): @@ -3311,6 +3503,7 @@ def ex_remove_tag_key(self, tag_key): "tag/deleteTagKey", method="POST", data=ET.tostring(remove_tag_key) ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_apply_tag_to_asset(self, asset, tag_key, value=None): @@ -3343,6 +3536,7 @@ def ex_apply_tag_to_asset(self, asset, tag_key, value=None): tag_ele = ET.SubElement(apply_tags, "tag") ET.SubElement(tag_ele, "tagKeyName").text = tag_key_name + if value is not None: ET.SubElement(tag_ele, "value").text = value @@ -3350,6 +3544,7 @@ def ex_apply_tag_to_asset(self, asset, tag_key, value=None): "tag/applyTags", method="POST", data=ET.tostring(apply_tags) ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_remove_tag_from_asset(self, asset, tag_key): @@ -3379,6 +3574,7 @@ def ex_remove_tag_from_asset(self, asset, tag_key): "tag/removeTags", method="POST", data=ET.tostring(apply_tags) ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_list_tags( @@ -3425,20 +3621,28 @@ def ex_list_tags( """ params = {} + if asset_id is not None: params["assetId"] = asset_id + if asset_type is not None: params["assetType"] = asset_type + if location is not None: params["datacenterId"] = self._location_to_location_id(location) + if tag_key_name is not None: params["tagKeyName"] = tag_key_name + if tag_key_id is not None: params["tagKeyId"] = tag_key_id + if value is not None: params["value"] = value + if value_required is not None: params["valueRequired"] = str(value_required).lower() + if display_on_report is not None: params["displayOnReport"] = str(display_on_report).lower() @@ -3447,8 +3651,10 @@ def ex_list_tags( ) tags = [] + for result in paged_result: tags.extend(self._to_tags(result)) + return tags def ex_summary_usage_report(self, start_date, end_date): @@ -3467,6 +3673,7 @@ def ex_summary_usage_report(self, start_date, end_date): result = self.connection.raw_request_with_orgId_api_1( "report/usage?startDate={}&endDate={}".format(start_date, end_date) ) + return self._format_csv(result.response) def ex_detailed_usage_report(self, start_date, end_date): @@ -3485,6 +3692,7 @@ def ex_detailed_usage_report(self, start_date, end_date): result = self.connection.raw_request_with_orgId_api_1( "report/usageDetailed?startDate={}&endDate={}".format(start_date, end_date) ) + return self._format_csv(result.response) def ex_software_usage_report(self, start_date, end_date): @@ -3503,6 +3711,7 @@ def ex_software_usage_report(self, start_date, end_date): result = self.connection.raw_request_with_orgId_api_1( "report/usageSoftwareUnits?startDate={}&endDate={}".format(start_date, end_date) ) + return self._format_csv(result.response) def ex_audit_log_report(self, start_date, end_date): @@ -3521,6 +3730,7 @@ def ex_audit_log_report(self, start_date, end_date): result = self.connection.raw_request_with_orgId_api_1( "auditlog?startDate={}&endDate={}".format(start_date, end_date) ) + return self._format_csv(result.response) def ex_backup_usage_report(self, start_date, end_date, location): @@ -3545,6 +3755,7 @@ def ex_backup_usage_report(self, start_date, end_date, location): "backup/detailedUsageReport?datacenterId=%s&fromDate=%s&toDate=%s" % (datacenter_id, start_date, end_date) ) + return self._format_csv(result.response) def ex_list_ip_address_list(self, ex_network_domain): @@ -3586,6 +3797,7 @@ def ex_list_ip_address_list(self, ex_network_domain): response = self.connection.request_with_orgId_api_2( "network/ipAddressList", params=params ).object + return self._to_ip_address_lists(response) def ex_get_ip_address_list(self, ex_network_domain, ex_ip_address_list_name): @@ -3631,6 +3843,7 @@ def ex_get_ip_address_list(self, ex_network_domain, ex_ip_address_list_name): """ ip_address_lists = self.ex_list_ip_address_list(ex_network_domain) + return list(filter(lambda x: x.name == ex_ip_address_list_name, ip_address_lists)) def ex_create_ip_address_list( @@ -3759,6 +3972,7 @@ def ex_create_ip_address_list( ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_edit_ip_address_list( @@ -3830,6 +4044,7 @@ def ex_edit_ip_address_list( "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", }, ) + if description is not None: if description != "nil": ET.SubElement(edit_ip_address_list, "description").text = description @@ -3864,6 +4079,7 @@ def ex_edit_ip_address_list( ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_delete_ip_address_list(self, ex_ip_address_list): @@ -3907,6 +4123,7 @@ def ex_delete_ip_address_list(self, ex_ip_address_list): ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_list_portlist(self, ex_network_domain): @@ -3949,6 +4166,7 @@ def ex_list_portlist(self, ex_network_domain): response = self.connection.request_with_orgId_api_2( "network/portList", params=params ).object + return self._to_port_lists(response) def ex_get_portlist(self, ex_portlist_id): @@ -3979,6 +4197,7 @@ def ex_get_portlist(self, ex_portlist_id): url_path = "network/portList/%s" % ex_portlist_id response = self.connection.request_with_orgId_api_2(url_path).object + return self._to_port_list(response) def ex_create_portlist( @@ -4077,6 +4296,7 @@ def ex_create_portlist( ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_edit_portlist( @@ -4145,6 +4365,7 @@ def ex_edit_portlist( "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", }, ) + if description is not None: if description != "nil": ET.SubElement(existing_port_address_list, "description").text = description @@ -4174,6 +4395,7 @@ def ex_edit_portlist( ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_delete_portlist(self, ex_portlist): @@ -4211,6 +4433,7 @@ def ex_delete_portlist(self, ex_portlist): ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_exchange_nic_vlans(self, nic_id_1, nic_id_2): @@ -4236,6 +4459,7 @@ def ex_exchange_nic_vlans(self, nic_id_1, nic_id_2): ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_change_nic_network_adapter(self, nic_id, network_adapter_name): @@ -4260,6 +4484,7 @@ def ex_change_nic_network_adapter(self, nic_id, network_adapter_name): ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_create_node_uncustomized( @@ -4388,10 +4613,12 @@ def ex_create_node_uncustomized( """ # Unsupported for version lower than 2.4 + if LooseVersion(self.connection.active_api_version) < LooseVersion("2.4"): raise Exception("This feature is NOT supported in " "earlier api version of 2.4") # Default start to true if input is invalid + if not isinstance(ex_is_started, bool): ex_is_started = True print("Warning: ex_is_started input value is invalid. Default" "to True") @@ -4489,6 +4716,7 @@ def ex_create_node_uncustomized( raise TypeError("ex_disks must be None or tuple/list") # tagid and tagname value pair should not co-exists + if ex_tagid_value_pairs is not None and ex_tagname_value_pairs is not None: raise ValueError( "ex_tagid_value_pairs and ex_tagname_value_pairs" @@ -4496,6 +4724,7 @@ def ex_create_node_uncustomized( ) # Tag by ID + if ex_tagid_value_pairs is not None: if not isinstance(ex_tagid_value_pairs, dict): raise ValueError("ex_tagid_value_pairs must be a dictionary.") @@ -4535,6 +4764,7 @@ def ex_create_node_uncustomized( ).object node_id = None + for info in findall(response, "info", TYPES_URN): if info.get("name") == "serverId": node_id = info.get("value") @@ -4575,6 +4805,7 @@ def ex_create_consistency_group( consistency_group_elm = ET.Element("createConsistencyGroup", {"xmlns": TYPES_URN}) ET.SubElement(consistency_group_elm, "name").text = name + if description is not None: ET.SubElement(consistency_group_elm, "description").text = description ET.SubElement(consistency_group_elm, "journalSizeGb").text = journal_size_gb @@ -4587,6 +4818,7 @@ def ex_create_consistency_group( data=ET.tostring(consistency_group_elm), ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] @get_params @@ -4611,6 +4843,7 @@ def ex_list_consistency_groups(self, params={}): "consistencyGroup/consistencyGroup", params=params ).object cgs = self._to_consistency_groups(response) + return cgs def ex_get_consistency_group(self, consistency_group_id): @@ -4627,6 +4860,7 @@ def ex_get_consistency_group(self, consistency_group_id): "consistencyGroup/consistencyGroup/%s" % consistency_group_id ).object cg = self._to_process(response) + return cg def ex_list_consistency_group_snapshots( @@ -4659,6 +4893,7 @@ def ex_list_consistency_group_snapshots( :rtype: `list` of :class:`NttCisSnapshots` """ + params = {} if create_time_min is None and create_time_max is None: params = {"consistencyGroupId": consistency_group_id} @@ -4683,6 +4918,7 @@ def ex_list_consistency_group_snapshots( "consistencyGroup/snapshot", method="GET", params=params ).object snapshots = self._to_process(paged_result) + return snapshots def ex_expand_journal(self, consistency_group_id, size_gb): @@ -4708,6 +4944,7 @@ def ex_expand_journal(self, consistency_group_id, size_gb): data=ET.tostring(expand_elm), ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_start_drs_failover_preview(self, consistency_group_id, snapshot_id): @@ -4736,6 +4973,7 @@ def ex_start_drs_failover_preview(self, consistency_group_id, snapshot_id): data=ET.tostring(preview_elm), ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_stop_drs_failover_preview(self, consistency_group_id): @@ -4759,6 +4997,7 @@ def ex_stop_drs_failover_preview(self, consistency_group_id): data=ET.tostring(preview_elm), ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_initiate_drs_failover(self, consistency_group_id): @@ -4783,6 +5022,7 @@ def ex_initiate_drs_failover(self, consistency_group_id): data=ET.tostring(failover_elm), ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def ex_delete_consistency_group(self, consistency_group_id): @@ -4804,16 +5044,20 @@ def ex_delete_consistency_group(self, consistency_group_id): data=ET.tostring(delete_elm), ).object response_code = findtext(response, "responseCode", TYPES_URN) + return response_code in ["IN_PROGRESS", "OK"] def _to_consistency_groups(self, object): cgs = findall(object, "consistencyGroup", TYPES_URN) + return [self._to_process(el) for el in cgs] def _to_drs_snapshots(self, object): snapshots = [] + for element in object.findall(fixxpath("snapshot", TYPES_URN)): snapshots.append(self._to_process(element)) + return snapshots def _to_process(self, element): @@ -4822,27 +5066,33 @@ def _to_process(self, element): def _format_csv(self, http_response): text = http_response.read() lines = str.splitlines(ensure_string(text)) + return [line.split(",") for line in lines] @staticmethod def _get_tagging_asset_type(asset): objecttype = type(asset) + if objecttype.__name__ in OBJECT_TO_TAGGING_ASSET_TYPE_MAP: return OBJECT_TO_TAGGING_ASSET_TYPE_MAP[objecttype.__name__] raise TypeError("Asset type %s cannot be tagged" % objecttype.__name__) def _list_nodes_single_page(self, params={}): nodes = self.connection.request_with_orgId_api_2("server/server", params=params).object + return nodes def _to_tags(self, object): tags = [] + for element in object.findall(fixxpath("tag", TYPES_URN)): tags.append(self._to_tag(element)) + return tags def _to_tag(self, element): tag_key = self._to_tag_key(element, from_tag_api=True) + return NttCisTag( asset_type=findtext(element, "assetType", TYPES_URN), asset_id=findtext(element, "assetId", TYPES_URN), @@ -4854,8 +5104,10 @@ def _to_tag(self, element): def _to_tag_keys(self, object): keys = [] + for element in object.findall(fixxpath("tagKey", TYPES_URN)): keys.append(self._to_tag_key(element)) + return keys def _to_tag_key(self, element, from_tag_api=False): @@ -4888,6 +5140,7 @@ def _to_images(self, object, el_name="osImage"): for element in object.findall(fixxpath(el_name, TYPES_URN)): location_id = element.get("datacenterId") + if location_id in location_ids: images.append(self._to_image(element, locations)) @@ -4895,6 +5148,7 @@ def _to_images(self, object, el_name="osImage"): def _to_image(self, element, locations=None): location_id = element.get("datacenterId") + if locations is None: locations = self.list_locations(location_id) @@ -4931,6 +5185,7 @@ def _to_image(self, element, locations=None): def _to_nat_rules(self, object, network_domain): rules = [] + for element in findall(object, "natRule", TYPES_URN): rules.append(self._to_nat_rule(element, network_domain)) @@ -4947,19 +5202,24 @@ def _to_nat_rule(self, element, network_domain): def _to_anti_affinity_rules(self, object): rules = [] + for element in findall(object, "antiAffinityRule", TYPES_URN): rules.append(self._to_anti_affinity_rule(element)) + return rules def _to_anti_affinity_rule(self, element): node_list = [] + for node in findall(element, "serverSummary", TYPES_URN): node_list.append(node.get("id")) + return NttCisAntiAffinityRule(id=element.get("id"), node_list=node_list) def _to_firewall_rules(self, object, network_domain): rules = [] locations = self.list_locations() + for element in findall(object, "firewallRule", TYPES_URN): rules.append(self._to_firewall_rule(element, locations, network_domain)) @@ -4990,6 +5250,7 @@ def _to_firewall_address(self, element): port = element.find(fixxpath("port", TYPES_URN)) port_list = element.find(fixxpath("portList", TYPES_URN)) address_list = element.find(fixxpath("ipAddressList", TYPES_URN)) + if address_list is None: return NttCisFirewallAddress( any_ip=ip.get("address") == "ANY", @@ -5014,6 +5275,7 @@ def _to_firewall_address(self, element): def _to_ip_blocks(self, object): blocks = [] locations = self.list_locations() + for element in findall(object, "publicIpBlock", TYPES_URN): blocks.append(self._to_ip_block(element, locations)) @@ -5037,6 +5299,7 @@ def _to_ip_block(self, element, locations): def _to_networks(self, object): networks = [] locations = self.list_locations() + for element in findall(object, "network", NETWORK_NS): networks.append(self._to_network(element, locations)) @@ -5044,6 +5307,7 @@ def _to_networks(self, object): def _to_network(self, element, locations): multicast = False + if findtext(element, "multicast", NETWORK_NS) == "true": multicast = True @@ -5065,18 +5329,22 @@ def _to_network(self, element, locations): def _to_network_domains(self, object): network_domains = [] locations = self.list_locations() + for element in findall(object, "networkDomain", TYPES_URN): network_domains.append(self._to_network_domain(element, locations)) + return network_domains def _to_network_domain(self, element, locations): location_id = element.get("datacenterId") location = list(filter(lambda x: x.id == location_id, locations))[0] plan = findtext(element, "type", TYPES_URN) + if plan == "ESSENTIALS": plan_type = NetworkDomainServicePlan.ESSENTIALS else: plan_type = NetworkDomainServicePlan.ADVANCED + return NttCisNetworkDomain( id=element.get("id"), name=findtext(element, "name", TYPES_URN), @@ -5089,6 +5357,7 @@ def _to_network_domain(self, element, locations): def _to_vlans(self, object): vlans = [] locations = self.list_locations() + for element in findall(object, "vlan", TYPES_URN): vlans.append(self._to_vlan(element, locations=locations)) @@ -5101,6 +5370,7 @@ def _to_vlan(self, element, locations): ip6_range = element.find(fixxpath("ipv6Range", TYPES_URN)) network_domain_el = element.find(fixxpath("networkDomain", TYPES_URN)) network_domain = self.ex_get_network_domain(network_domain_el.get("id")) + return NttCisVlan( id=element.get("id"), name=findtext(element, "name", TYPES_URN), @@ -5118,6 +5388,7 @@ def _to_vlan(self, element, locations): def _to_locations(self, object): locations = [] + for element in object.findall(fixxpath("datacenter", TYPES_URN)): locations.append(self._to_location(element)) @@ -5130,6 +5401,7 @@ def _to_location(self, element): country=findtext(element, "country", TYPES_URN), driver=self, ) + return loc def _to_cpu_spec(self, element): @@ -5141,14 +5413,17 @@ def _to_cpu_spec(self, element): def _to_vmware_tools(self, element): status = None + if hasattr(element, "runningStatus"): status = element.get("runningStatus") version_status = None + if hasattr(element, "version_status"): version_status = element.get("version_status") api_version = None + if hasattr(element, "apiVersion"): api_version = element.get("apiVersion") @@ -5166,6 +5441,7 @@ def _to_scsi_controllers(self, elements): def _to_disks(self, object): disk_elements = object.findall(fixxpath("disk", TYPES_URN)) + return [self._to_disk(el) for el in disk_elements] def _to_disk(self, element): @@ -5191,6 +5467,7 @@ def _to_snapshots(self, object): """ snapshot_elements = object.findall(fixxpath("snapshot", TYPES_URN)) + return [self._to_snapshot(el) for el in snapshot_elements] def _to_snapshot(self, element): @@ -5205,10 +5482,12 @@ def _to_snapshot(self, element): def _to_ipv4_addresses(self, object): ipv4_address_elements = object.findall(fixxpath("ipv4", TYPES_URN)) + return [self._to_ipv4_6_address(el) for el in ipv4_address_elements] def _to_ipv6_addresses(self, object): ipv6_address_elements = object.findall(fixxpath("reservedIpv6Address", TYPES_URN)) + return [self._to_ipv4_6_address(el) for el in ipv6_address_elements] def _to_ipv4_6_address(self, element): @@ -5222,6 +5501,7 @@ def _to_ipv4_6_address(self, element): def _to_windows(self, object): snapshot_window_elements = object.findall(fixxpath("snapshotWindow", TYPES_URN)) + return [self._to_window(el) for el in snapshot_window_elements] def _to_window(self, element): @@ -5234,6 +5514,7 @@ def _to_window(self, element): def _to_nodes(self, object): node_elements = object.findall(fixxpath("server", TYPES_URN)) + return [self._to_node(el) for el in node_elements] def _to_node(self, element): @@ -5255,15 +5536,19 @@ def _to_node(self, element): has_ide = element.find(fixxpath("ideController")) is not None scsi_controllers = [] disks = [] + if has_scsi: scsi_controllers.append( self._to_scsi_controllers(element.find(fixxpath("scsiController", TYPES_URN))) ) + for scsi in element.findall(fixxpath("scsiController", TYPES_URN)): disks.extend(self._to_disks(scsi)) + if has_sata: for sata in element.findall(fixxpath("sataController", TYPES_URN)): disks.extend(self._to_disks(sata)) + if has_ide: for ide in element.findall(fixxpath("ideController", TYPES_URN)): disks.extend(self._to_snapshot(ide)) @@ -5271,6 +5556,7 @@ def _to_node(self, element): # Vmware Tools # Version 2.3 or earlier + if LooseVersion(self.connection.active_api_version) < LooseVersion("2.4"): vmware_tools = self._to_vmware_tools(element.find(fixxpath("vmwareTools", TYPES_URN))) operation_system = element.find(fixxpath("operatingSystem", TYPES_URN)) @@ -5278,6 +5564,7 @@ def _to_node(self, element): else: # vmtools_elm = fixxpath('guest/vmTools', TYPES_URN) vmtools_elm = element.find(fixxpath("guest/vmTools", TYPES_URN)) + if vmtools_elm is not None: vmware_tools = self._to_vmware_tools(vmtools_elm) else: @@ -5343,6 +5630,7 @@ def _to_node(self, element): driver=self.connection.driver, extra=extra, ) + return n def _to_status(self, element): @@ -5358,10 +5646,12 @@ def _to_status(self, element): step_percent_complete=findtext(element, "step/percentComplete", TYPES_URN), failure_reason=findtext(element, "failureReason", TYPES_URN), ) + return s def _to_ip_address_lists(self, object): ip_address_lists = [] + for element in findall(object, "ipAddressList", TYPES_URN): ip_address_lists.append(self._to_ip_address_list(element)) @@ -5369,10 +5659,12 @@ def _to_ip_address_lists(self, object): def _to_ip_address_list(self, element): ipAddresses = [] + for ip in findall(element, "ipAddress", TYPES_URN): ipAddresses.append(self._to_ip_address(ip)) child_ip_address_lists = [] + for child_ip_list in findall(element, "childIpAddressList", TYPES_URN): child_ip_address_lists.append(self._to_child_ip_list(child_ip_list)) @@ -5399,6 +5691,7 @@ def _to_ip_address(self, element): def _to_port_lists(self, object): port_lists = [] + for element in findall(object, "portList", TYPES_URN): port_lists.append(self._to_port_list(element)) @@ -5406,10 +5699,12 @@ def _to_port_lists(self, object): def _to_port_list(self, element): ports = [] + for port in findall(element, "port", TYPES_URN): ports.append(self._to_port(element=port)) child_portlist_list = [] + for child in findall(element, "childPortList", TYPES_URN): child_portlist_list.append(self._to_child_port_list(element=child)) @@ -5426,8 +5721,10 @@ def _to_port_list(self, element): def _image_needs_auth(self, image): if not isinstance(image, NodeImage): image = self.ex_get_image_by_id(image) + if image.extra["isCustomerImage"] and image.extra["OS_type"] == "UNIX": return False + return True @staticmethod diff --git a/libcloud/compute/drivers/opennebula.py b/libcloud/compute/drivers/opennebula.py index 737ebb9678..b67c9fa988 100644 --- a/libcloud/compute/drivers/opennebula.py +++ b/libcloud/compute/drivers/opennebula.py @@ -133,6 +133,7 @@ def success(self): :return: True is success, else False. """ i = int(self.status) + return 200 <= i <= 299 def parse_error(self): @@ -144,8 +145,10 @@ def parse_error(self): :rtype: :class:`ElementTree` :return: Contents of HTTP response body. """ + if int(self.status) == httplib.UNAUTHORIZED: raise InvalidCredsError(self.body) + return self.body @@ -179,13 +182,15 @@ def add_default_headers(self, headers): :rtype: ``dict`` :return: Dictionary containing updated headers. """ + if self.plain_auth: passwd = self.key else: - passwd = hashlib.sha1(b(self.key)).hexdigest() + passwd = hashlib.sha1(b(self.key)).hexdigest() # nosec headers["Authorization"] = "Basic %s" % b64encode( b("{}:{}".format(self.user_id, passwd)) ).decode("utf-8") + return headers @@ -278,7 +283,8 @@ def get_uuid(self): :rtype: ``str`` :return: Unique identifier for this instance. """ - return hashlib.sha1(b("{}:{}".format(self.id, self.driver.type))).hexdigest() + + return hashlib.sha1(b("{}:{}".format(self.id, self.driver.type))).hexdigest() # nosec def __repr__(self): return ( @@ -321,6 +327,7 @@ def __new__(cls, key, secret=None, api_version=DEFAULT_API_VERSION, **kwargs): cls = OpenNebula_3_6_NodeDriver elif api_version in ["3.8"]: cls = OpenNebula_3_8_NodeDriver + if "plain_auth" not in kwargs: kwargs["plain_auth"] = cls.plain_auth else: @@ -329,6 +336,7 @@ def __new__(cls, key, secret=None, api_version=DEFAULT_API_VERSION, **kwargs): raise NotImplementedError( "No OpenNebulaNodeDriver found for API version %s" % (api_version) ) + return super().__new__(cls) def create_node(self, name, size, image, networks=None): @@ -358,6 +366,7 @@ def create_node(self, name, size, image, networks=None): networks = [networks] networkGroup = ET.SubElement(compute, "NETWORK") + for network in networks: if network.address: ET.SubElement( @@ -394,6 +403,7 @@ def list_sizes(self, location=None): :return: List of compute node sizes supported by the cloud provider. :rtype: ``list`` of :class:`OpenNebulaNodeSize` """ + return [ NodeSize( id=1, @@ -439,6 +449,7 @@ def ex_list_networks(self, location=None): compute node. :rtype: ``list`` of :class:`OpenNebulaNetwork` """ + return self._to_networks(self.connection.request("/network").object) def ex_node_action(self, node, action): @@ -492,6 +503,7 @@ def _to_images(self, object): :return: List of images. """ images = [] + for element in object.findall("DISK"): image_id = element.attrib["href"].partition("/storage/")[2] image = self.connection.request("/storage/%s" % (image_id)).object @@ -510,6 +522,7 @@ def _to_image(self, image): :rtype: :class:`NodeImage` :return: The newly extracted :class:`NodeImage`. """ + return NodeImage( id=image.findtext("ID"), name=image.findtext("NAME"), @@ -530,6 +543,7 @@ def _to_networks(self, object): :return: List of virtual networks. """ networks = [] + for element in object.findall("NETWORK"): network_id = element.attrib["href"].partition("/network/")[2] network_element = self.connection.request("/network/%s" % (network_id)).object @@ -548,6 +562,7 @@ def _to_network(self, element): :rtype: :class:`OpenNebulaNetwork` :return: The newly extracted :class:`OpenNebulaNetwork`. """ + return OpenNebulaNetwork( id=element.findtext("ID"), name=element.findtext("NAME"), @@ -569,6 +584,7 @@ def _to_nodes(self, object): :return: A list of compute nodes. """ computes = [] + for element in object.findall("COMPUTE"): compute_id = element.attrib["href"].partition("/compute/")[2] compute = self.connection.request("/compute/%s" % (compute_id)).object @@ -621,6 +637,7 @@ def _extract_networks(self, compute): networks = list() network_list = compute.find("NETWORK") + for element in network_list.findall("NIC"): networks.append( OpenNebulaNetwork( @@ -650,6 +667,7 @@ def _extract_images(self, compute): disks = list() disk_list = compute.find("STORAGE") + if disk_list is not None: for element in disk_list.findall("DISK"): disks.append( @@ -663,6 +681,7 @@ def _extract_images(self, compute): # @TODO: Return all disks when the Node type accepts multiple # attached disks per node. + if len(disks) > 0: return disks[0] else: @@ -721,12 +740,14 @@ def create_node(self, name, size, image, networks=None, context=None): for network in networks: nic = ET.SubElement(compute, "NIC") ET.SubElement(nic, "NETWORK", {"href": "/network/%s" % (str(network.id))}) + if network.address: ip_line = ET.SubElement(nic, "IP") ip_line.text = network.address if context and isinstance(context, dict): contextGroup = ET.SubElement(compute, "CONTEXT") + for key, value in list(context.items()): context = ET.SubElement(contextGroup, key.upper()) context.text = value @@ -751,6 +772,7 @@ def list_sizes(self, location=None): :return: List of compute node sizes supported by the cloud provider. :rtype: ``list`` of :class:`OpenNebulaNodeSize` """ + return [ OpenNebulaNodeSize( id=1, @@ -807,6 +829,7 @@ def _to_images(self, object): :return: List of images. """ images = [] + for element in object.findall("STORAGE"): image_id = element.attrib["href"].partition("/storage/")[2] image = self.connection.request("/storage/%s" % (image_id)).object @@ -825,6 +848,7 @@ def _to_image(self, image): :rtype: :class:`NodeImage` :return: The newly extracted :class:`NodeImage`. """ + return NodeImage( id=image.findtext("ID"), name=image.findtext("NAME"), @@ -939,6 +963,7 @@ def _extract_images(self, compute): # Return all disks when the Node type accepts multiple attached disks # per node. + if len(disks) > 1: return disks elif len(disks) == 1: @@ -1056,6 +1081,7 @@ def _to_network(self, element): :return: The newly extracted :class:`OpenNebulaNetwork`. :rtype: :class:`OpenNebulaNetwork` """ + return OpenNebulaNetwork( id=element.findtext("ID"), name=element.findtext("NAME"), @@ -1085,6 +1111,7 @@ def list_sizes(self, location=None): :return: List of compute node sizes supported by the cloud provider. :rtype: ``list`` of :class:`OpenNebulaNodeSize` """ + return self._to_sizes(self.connection.request("/instance_type").object) def _to_sizes(self, object): @@ -1198,6 +1225,7 @@ def attach_volume(self, node, volume, device): url = "/compute/%s/action" % node.id resp = self.connection.request(url, method="POST", data=xml) + return resp.status == httplib.ACCEPTED def _do_detach_volume(self, node_id, disk_id): @@ -1215,20 +1243,24 @@ def _do_detach_volume(self, node_id, disk_id): url = "/compute/%s/action" % node_id resp = self.connection.request(url, method="POST", data=xml) + return resp.status == httplib.ACCEPTED def detach_volume(self, volume): # We need to find the node using this volume + for node in self.list_nodes(): if type(node.image) is not list: # This node has only one associated image. It is not the one we # are after. + continue for disk in node.image: if disk.id == volume.id: # Node found. We can now detach the volume disk_id = disk.extra["disk_id"] + return self._do_detach_volume(node.id, disk_id) return False @@ -1246,6 +1278,7 @@ def _to_volume(self, storage): def _to_volumes(self, object): volumes = [] + for storage in object.findall("STORAGE"): storage_id = storage.attrib["href"].partition("/storage/")[2] @@ -1301,6 +1334,7 @@ def _to_sizes(self, object): size = OpenNebulaNodeSize(**size_kwargs) sizes.append(size) size_id += 1 + return sizes def _ex_connection_class_kwargs(self): diff --git a/libcloud/compute/drivers/ovh.py b/libcloud/compute/drivers/ovh.py index ed0193c283..e62fceaa40 100644 --- a/libcloud/compute/drivers/ovh.py +++ b/libcloud/compute/drivers/ovh.py @@ -78,6 +78,7 @@ def __init__(self, key, secret, ex_project_id, ex_consumer_key=None, region=None def _get_project_action(self, suffix): base_url = "{}/cloud/project/{}/".format(API_ROOT, self.project_id) + return base_url + suffix @classmethod @@ -96,9 +97,11 @@ def list_nodes(self, location=None): """ action = self._get_project_action("instance") data = {} + if location: data["region"] = location.id response = self.connection.request(action, data=data) + return self._to_nodes(response.object) def ex_get_node(self, node_id): @@ -113,6 +116,7 @@ def ex_get_node(self, node_id): """ action = self._get_project_action("instance/%s" % node_id) response = self.connection.request(action, method="GET") + return self._to_node(response.object) def create_node(self, name, image, size, location, ex_keyname=None): @@ -144,23 +148,28 @@ def create_node(self, name, image, size, location, ex_keyname=None): "flavorId": size.id, "region": location.id, } + if ex_keyname: key_id = self.get_key_pair(ex_keyname, location).extra["id"] data["sshKeyId"] = key_id response = self.connection.request(action, data=data, method="POST") + return self._to_node(response.object) def destroy_node(self, node): action = self._get_project_action("instance/%s" % node.id) self.connection.request(action, method="DELETE") + return True def list_sizes(self, location=None): action = self._get_project_action("flavor") params = {} + if location: params["region"] = location.id response = self.connection.request(action, params=params) + return self._to_sizes(response.object) def ex_get_size(self, size_id): @@ -175,6 +184,7 @@ def ex_get_size(self, size_id): """ action = self._get_project_action("flavor/%s" % size_id) response = self.connection.request(action) + return self._to_size(response.object) def list_images(self, location=None, ex_size=None): @@ -192,21 +202,26 @@ def list_images(self, location=None, ex_size=None): """ action = self._get_project_action("image") params = {} + if location: params["region"] = location.id + if ex_size: params["flavorId"] = ex_size.id response = self.connection.request(action, params=params) + return self._to_images(response.object) def get_image(self, image_id): action = self._get_project_action("image/%s" % image_id) response = self.connection.request(action) + return self._to_image(response.object) def list_locations(self): action = self._get_project_action("region") data = self.connection.request(action) + return self._to_locations(data.object) def list_key_pairs(self, ex_location=None): @@ -221,9 +236,11 @@ def list_key_pairs(self, ex_location=None): """ action = self._get_project_action("sshkey") params = {} + if ex_location: params["region"] = ex_location.id response = self.connection.request(action, params=params) + return self._to_key_pairs(response.object) def get_key_pair(self, name, ex_location=None): @@ -241,8 +258,10 @@ def get_key_pair(self, name, ex_location=None): """ # Keys are indexed with ID keys = [key for key in self.list_key_pairs(ex_location) if key.name == name] + if not keys: raise Exception("No key named '%s'" % name) + return keys[0] def import_key_pair_from_string(self, name, key_material, ex_location): @@ -264,12 +283,14 @@ def import_key_pair_from_string(self, name, key_material, ex_location): action = self._get_project_action("sshkey") data = {"name": name, "publicKey": key_material, "region": ex_location.id} response = self.connection.request(action, data=data, method="POST") + return self._to_key_pair(response.object) def delete_key_pair(self, key_pair): action = self._get_project_action("sshkey/%s" % key_pair.extra["id"]) params = {"keyId": key_pair.extra["id"]} self.connection.request(action, params=params, method="DELETE") + return True def create_volume( @@ -313,14 +334,17 @@ def create_volume( "size": size, "type": ex_volume_type, } + if ex_description: data["description"] = ex_description response = self.connection.request(action, data=data, method="POST") + return self._to_volume(response.object) def destroy_volume(self, volume): action = self._get_project_action("volume/%s" % volume.id) self.connection.request(action, method="DELETE") + return True def list_volumes(self, ex_location=None): @@ -335,9 +359,11 @@ def list_volumes(self, ex_location=None): """ action = self._get_project_action("volume") data = {} + if ex_location: data["region"] = ex_location.id response = self.connection.request(action, data=data) + return self._to_volumes(response.object) def ex_get_volume(self, volume_id): @@ -352,6 +378,7 @@ def ex_get_volume(self, volume_id): """ action = self._get_project_action("volume/%s" % volume_id) response = self.connection.request(action) + return self._to_volume(response.object) def attach_volume(self, node, volume, device=None): @@ -372,6 +399,7 @@ def attach_volume(self, node, volume, device=None): action = self._get_project_action("volume/%s/attach" % volume.id) data = {"instanceId": node.id, "volumeId": volume.id} self.connection.request(action, data=data, method="POST") + return True def detach_volume(self, volume, ex_node=None): @@ -392,6 +420,7 @@ def detach_volume(self, volume, ex_node=None): node is attached to the volume """ action = self._get_project_action("volume/%s/detach" % volume.id) + if ex_node is None: if len(volume.extra["attachedTo"]) != 1: err_msg = ( @@ -401,6 +430,7 @@ def detach_volume(self, volume, ex_node=None): ex_node = self.ex_get_node(volume.extra["attachedTo"][0]) data = {"instanceId": ex_node.id} self.connection.request(action, data=data, method="POST") + return True def ex_list_snapshots(self, location=None): @@ -414,9 +444,11 @@ def ex_list_snapshots(self, location=None): """ action = self._get_project_action("volume/snapshot") params = {} + if location: params["region"] = location.id response = self.connection.request(action, params=params) + return self._to_snapshots(response.object) def ex_get_volume_snapshot(self, snapshot_id): @@ -431,6 +463,7 @@ def ex_get_volume_snapshot(self, snapshot_id): """ action = self._get_project_action("volume/snapshot/%s" % snapshot_id) response = self.connection.request(action) + return self._to_snapshot(response.object) def list_volume_snapshots(self, volume): @@ -438,6 +471,7 @@ def list_volume_snapshots(self, volume): params = {"region": volume.extra["region"]} response = self.connection.request(action, params=params) snapshots = self._to_snapshots(response.object) + return [snap for snap in snapshots if snap.extra["volume_id"] == volume.id] def create_volume_snapshot(self, volume, name=None, ex_description=None): @@ -457,22 +491,27 @@ def create_volume_snapshot(self, volume, name=None, ex_description=None): """ action = self._get_project_action("volume/%s/snapshot/" % volume.id) data = {} + if name: data["name"] = name + if ex_description: data["description"] = ex_description response = self.connection.request(action, data=data, method="POST") + return self._to_snapshot(response.object) def destroy_volume_snapshot(self, snapshot): action = self._get_project_action("volume/snapshot/%s" % snapshot.id) response = self.connection.request(action, method="DELETE") + return response.status == httplib.OK def ex_get_pricing(self, size_id, subsidiary="US"): action = "%s/cloud/subsidiaryPrice" % (API_ROOT) params = {"flavorId": size_id, "ovhSubsidiary": subsidiary} pricing = self.connection.request(action, params=params).object["instances"][0] + return { "hourly": pricing["price"]["value"], "monthly": pricing["monthlyPrice"]["value"], @@ -484,6 +523,7 @@ def _to_volume(self, obj): extra.pop("name") extra.pop("size") state = self.VOLUME_STATE_MAP.get(obj.pop("status", None), StorageVolumeState.UNKNOWN) + return StorageVolume( id=obj["id"], name=obj["name"], @@ -498,6 +538,7 @@ def _to_volumes(self, objs): def _to_location(self, obj): location = self.connectionCls.LOCATIONS[obj] + return NodeLocation(driver=self, **location) def _to_locations(self, objs): @@ -505,10 +546,14 @@ def _to_locations(self, objs): def _to_node(self, obj): extra = obj.copy() + if "ipAddresses" in extra: public_ips = [ip["ip"] for ip in extra["ipAddresses"]] + else: + public_ips = [] del extra["id"] del extra["name"] + return Node( id=obj["id"], name=obj["name"], @@ -524,6 +569,7 @@ def _to_nodes(self, objs): def _to_size(self, obj): extra = {"vcpus": obj["vcpus"], "type": obj["type"], "region": obj["region"]} + return NodeSize( id=obj["id"], name=obj["name"], @@ -540,6 +586,7 @@ def _to_sizes(self, objs): def _to_image(self, obj): extra = {"region": obj["region"], "visibility": obj["visibility"]} + return NodeImage(id=obj["id"], name=obj["name"], driver=self, extra=extra) def _to_images(self, objs): @@ -547,6 +594,7 @@ def _to_images(self, objs): def _to_key_pair(self, obj): extra = {"regions": obj["regions"], "id": obj["id"]} + return OpenStackKeyPair( name=obj["name"], public_key=obj["publicKey"], @@ -575,6 +623,7 @@ def _to_snapshot(self, obj): state=state, name=obj["name"], ) + return snapshot def _to_snapshots(self, objs): diff --git a/libcloud/compute/drivers/rackspace.py b/libcloud/compute/drivers/rackspace.py index fce843f86d..9a852be581 100644 --- a/libcloud/compute/drivers/rackspace.py +++ b/libcloud/compute/drivers/rackspace.py @@ -73,6 +73,7 @@ def get_endpoint(self): # Old US accounts can access UK API endpoint, but they don't # have this endpoint in the service catalog. Same goes for the # old UK accounts and US endpoint. + if self.region == "us": # Old UK account, which only have uk endpoint in the catalog public_url = public_url.replace("https://lon.servers.api", "https://servers.api") @@ -100,6 +101,7 @@ def __init__(self, key, secret=None, secure=True, host=None, port=None, region=" :param region: Region ID which should be used :type region: ``str`` """ + if region not in ["us", "uk"]: raise ValueError("Invalid region: %s" % (region)) @@ -122,16 +124,20 @@ def list_locations(self): @inherits: :class:`OpenStack_1_0_NodeDriver.list_locations` """ + if self.region == "us": locations = [NodeLocation(0, "Rackspace DFW1/ORD1", "US", self)] elif self.region == "uk": locations = [NodeLocation(0, "Rackspace UK London", "UK", self)] + else: + locations = [] return locations def _ex_connection_class_kwargs(self): kwargs = self.openstack_connection_kwargs() kwargs["region"] = self.region + return kwargs @@ -151,6 +157,7 @@ def __init__(self, *args, **kwargs): def get_service_name(self): if not self.get_endpoint_args: # if they used ex_force_base_url, assume the Rackspace default + return SERVICE_NAME_GEN2 return self.get_endpoint_args.get("name", SERVICE_NAME_GEN2) @@ -246,6 +253,7 @@ def _to_snapshot(self, api_node): state=state, name=api_node["displayName"], ) + return snapshot def _ex_connection_class_kwargs(self): @@ -253,4 +261,5 @@ def _ex_connection_class_kwargs(self): kwargs = self.openstack_connection_kwargs() kwargs["region"] = self.region kwargs["get_endpoint_args"] = endpoint_args + return kwargs diff --git a/libcloud/compute/drivers/vcloud.py b/libcloud/compute/drivers/vcloud.py index f65f0f6811..913cca45c0 100644 --- a/libcloud/compute/drivers/vcloud.py +++ b/libcloud/compute/drivers/vcloud.py @@ -1463,21 +1463,23 @@ def ex_set_control_access(self, node, control_access): if control_access.subjects: access_settings_elem = ET.SubElement(xml, "AccessSettings") - for subject in control_access.subjects: - setting = ET.SubElement(access_settings_elem, "AccessSetting") - - if subject.id: - href = subject.id - else: - res = self.ex_query(type=subject.type, filter="name==" + subject.name) - - if not res: - raise LibcloudError( - 'Specified subject "{} {}" not found '.format(subject.type, subject.name) - ) - href = res[0]["href"] - ET.SubElement(setting, "Subject", {"href": href}) - ET.SubElement(setting, "AccessLevel").text = subject.access_level + for subject in control_access.subjects: + setting = ET.SubElement(access_settings_elem, "AccessSetting") + + if subject.id: + href = subject.id + else: + res = self.ex_query(type=subject.type, filter="name==" + subject.name) + + if not res: + raise LibcloudError( + 'Specified subject "{} {}" not found '.format( + subject.type, subject.name + ) + ) + href = res[0]["href"] + ET.SubElement(setting, "Subject", {"href": href}) + ET.SubElement(setting, "AccessLevel").text = subject.access_level headers = {"Content-Type": "application/vnd.vmware.vcloud.controlAccess+xml"} self.connection.request( diff --git a/libcloud/compute/drivers/vsphere.py b/libcloud/compute/drivers/vsphere.py index 98e6659479..d6958ac404 100644 --- a/libcloud/compute/drivers/vsphere.py +++ b/libcloud/compute/drivers/vsphere.py @@ -55,14 +55,17 @@ def recurse_snapshots(snapshot_list): ret = [] + for s in snapshot_list: ret.append(s) ret += recurse_snapshots(getattr(s, "childSnapshotList", [])) + return ret def format_snapshots(snapshot_list): ret = [] + for s in snapshot_list: ret.append( { @@ -73,6 +76,7 @@ def format_snapshots(snapshot_list): "state": s.state, } ) + return ret @@ -92,6 +96,7 @@ def __init__(self, host, username, password, port=443, ca_cert=None): """Initialize a connection by providing a hostname, username and password """ + if pyvmomi is None: raise ImportError( 'Missing "pyvmomi" dependency. ' @@ -119,15 +124,19 @@ def __init__(self, host, username, password, port=443, ca_cert=None): atexit.register(connect.Disconnect, self.connection) except Exception as exc: error_message = str(exc).lower() + if "incorrect user name" in error_message: raise InvalidCredsError("Check your username and " "password are valid") + if "connection refused" in error_message or "is not a vim server" in error_message: raise LibcloudError( "Check that the host provided is a " "vSphere installation", driver=self, ) + if "name or service not known" in error_message: raise LibcloudError("Check that the vSphere host is accessible", driver=self) + if "certificate verify failed" in error_message: # bypass self signed certificates try: @@ -170,12 +179,14 @@ def list_locations(self, ex_show_hosts_in_drs=True): locations = [] hosts_all = [] clusters = [] + for location in potential_locations: if isinstance(location, vim.HostSystem): hosts_all.append(location) elif isinstance(location, vim.ClusterComputeResource): if location.configuration.drsConfig.enabled: clusters.append(location) + if ex_show_hosts_in_drs: hosts = hosts_all else: @@ -184,11 +195,14 @@ def list_locations(self, ex_show_hosts_in_drs=True): for cluster in clusters: locations.append(self._to_location(cluster)) + for host in hosts: locations.append(self._to_location(host)) + return locations def _to_location(self, data): + extra = {} try: if isinstance(data, vim.HostSystem): extra = { @@ -217,6 +231,7 @@ def _to_location(self, data): except AttributeError as exc: logger.error("Cannot convert location {}: {!r}".format(data.name, exc)) extra = {} + return NodeLocation(id=data.name, name=data.name, country=None, extra=extra, driver=self) def ex_list_networks(self): @@ -238,12 +253,14 @@ def _to_network(self, data): "ip_pool_id": summary.ipPoolId, "accessible": summary.accessible, } + return VSphereNetwork(id=data.name, name=data.name, extra=extra) def list_sizes(self): """ Returns sizes """ + return [] def list_images(self, location=None, folder_ids=None): @@ -254,8 +271,10 @@ def list_images(self, location=None, folder_ids=None): """ images = [] + if folder_ids: vms = [] + for folder_id in folder_ids: folder_object = self._get_item_by_moid("Folder", folder_id) vms.extend(folder_object.childEntity) @@ -279,6 +298,7 @@ def _to_image(self, data): cpus = summary.config.numCpu operating_system = summary.config.guestFullName os_type = "unix" + if "Microsoft" in str(operating_system): os_type = "windows" annotation = summary.config.annotation @@ -296,8 +316,10 @@ def _to_image(self, data): } boot_time = summary.runtime.bootTime + if boot_time: extra["boot_time"] = boot_time.isoformat() + if annotation: extra["annotation"] = annotation @@ -360,8 +382,10 @@ def _collect_properties(self, content, view_ref, obj_type, path_set=None, includ props = collector.RetrieveContents([filter_spec]) data = [] + for obj in props: properties = {} + for prop in obj.propSet: properties[prop.name] = prop.val @@ -369,6 +393,7 @@ def _collect_properties(self, content, view_ref, obj_type, path_set=None, includ properties["obj"] = obj.obj data.append(properties) + return data def list_nodes(self, enhance=True, max_properties=20): @@ -400,6 +425,7 @@ def list_nodes(self, enhance=True, max_properties=20): ) i = 0 vm_dict = {} + while i < len(vm_properties): vm_list = self._collect_properties( content, @@ -409,6 +435,7 @@ def list_nodes(self, enhance=True, max_properties=20): include_mors=True, ) i += max_properties + for vm in vm_list: if not vm_dict.get(vm["obj"]): vm_dict[vm["obj"]] = vm @@ -419,6 +446,7 @@ def list_nodes(self, enhance=True, max_properties=20): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) nodes = loop.run_until_complete(self._to_nodes(vm_list)) + if enhance: nodes = self._enhance_metadata(nodes, content) @@ -432,10 +460,12 @@ def list_nodes_recursive(self, enhance=True): content = self.connection.RetrieveContent() children = content.rootFolder.childEntity # this will be needed for custom VM metadata + if content.customFieldsManager: self.custom_fields = content.customFieldsManager.field else: self.custom_fields = [] + for child in children: if hasattr(child, "vmFolder"): datacenter = child @@ -450,6 +480,7 @@ def list_nodes_recursive(self, enhance=True): def _enhance_metadata(self, nodes, content): nodemap = {} + for node in nodes: node.extra["vSphere version"] = content.about.version nodemap[node.id] = node @@ -457,11 +488,13 @@ def _enhance_metadata(self, nodes, content): # Get VM deployment events to extract creation dates & images filter_spec = vim.event.EventFilterSpec(eventTypeId=["VmBeingDeployedEvent"]) deploy_events = content.eventManager.QueryEvent(filter_spec) + for event in deploy_events: try: uuid = event.vm.vm.config.instanceUuid except Exception: continue + if uuid in nodemap: node = nodemap[uuid] try: # Get source template as image @@ -479,16 +512,19 @@ def _enhance_metadata(self, nodes, content): async def _to_nodes(self, vm_list): vms = [] + for vm in vm_list: if vm.get("config.template"): continue # Do not include templates in node list vms.append(vm) loop = asyncio.get_event_loop() vms = [loop.run_in_executor(None, self._to_node, vms[i]) for i in range(len(vms))] + return await asyncio.gather(*vms) def _to_nodes_recursive(self, vm_list): nodes = [] + for virtual_machine in vm_list: if hasattr(virtual_machine, "childEntity"): # If this is a group it will have children. @@ -505,6 +541,7 @@ def _to_nodes_recursive(self, vm_list): ): continue # Do not include templates in node list nodes.append(self._to_node_recursive(virtual_machine)) + return nodes def _to_node(self, vm): @@ -514,7 +551,7 @@ def _to_node(self, vm): cpus = vm.get("summary.config.numCpu") disk = vm.get("summary.storage.committed", 0) // (1024**3) id_to_hash = str(memory) + str(cpus) + str(disk) - size_id = hashlib.md5(id_to_hash.encode("utf-8")).hexdigest() + size_id = hashlib.md5(id_to_hash.encode("utf-8")).hexdigest() # nosec size_name = name + "-size" size_extra = {"cpus": cpus} driver = self @@ -532,11 +569,13 @@ def _to_node(self, vm): host = vm.get("summary.runtime.host") os_type = "unix" + if "Microsoft" in str(operating_system): os_type = "windows" uuid = vm.get("summary.config.instanceUuid") or ( vm.get("obj").config and vm.get("obj").config.instanceUuid ) + if not uuid: logger.error("No uuid for vm: {}".format(vm)) annotation = vm.get("summary.config.annotation") @@ -545,6 +584,7 @@ def _to_node(self, vm): boot_time = vm.get("summary.runtime.bootTime") ip_addresses = [] + if vm.get("summary.guest.ipAddress"): ip_addresses.append(vm.get("summary.guest.ipAddress")) @@ -569,14 +609,17 @@ def _to_node(self, vm): if host: extra["host"] = host.name parent = host.parent + while parent: if isinstance(parent, vim.ClusterComputeResource): extra["cluster"] = parent.name + break parent = parent.parent if boot_time: extra["boot_time"] = boot_time.isoformat() + if annotation: extra["annotation"] = annotation @@ -589,6 +632,7 @@ def _to_node(self, vm): except Exception: # IPV6 not supported pass + if vm.get("snapshot"): extra["snapshots"] = format_snapshots( recurse_snapshots(vm.get("snapshot").rootSnapshotList) @@ -610,6 +654,7 @@ def _to_node(self, vm): extra=extra, ) node._uuid = uuid + return node def _to_node_recursive(self, virtual_machine): @@ -619,10 +664,11 @@ def _to_node_recursive(self, virtual_machine): memory = summary.config.memorySizeMB cpus = summary.config.numCpu disk = "" + if summary.storage.committed: disk = summary.storage.committed / (1024**3) id_to_hash = str(memory) + str(cpus) + str(disk) - size_id = hashlib.md5(id_to_hash.encode("utf-8")).hexdigest() + size_id = hashlib.md5(id_to_hash.encode("utf-8")).hexdigest() # nosec size_name = name + "-size" size_extra = {"cpus": cpus} driver = self @@ -641,6 +687,7 @@ def _to_node_recursive(self, virtual_machine): # mist.io needs this metadata os_type = "unix" + if "Microsoft" in str(operating_system): os_type = "windows" uuid = summary.config.instanceUuid @@ -649,6 +696,7 @@ def _to_node_recursive(self, virtual_machine): status = self.NODE_STATE_MAP.get(state, NodeState.UNKNOWN) boot_time = summary.runtime.bootTime ip_addresses = [] + if summary.guest is not None: ip_addresses.append(summary.guest.ipAddress) @@ -673,13 +721,17 @@ def _to_node_recursive(self, virtual_machine): if host: extra["host"] = host.name parent = host.parent + while parent: if isinstance(parent, vim.ClusterComputeResource): extra["cluster"] = parent.name + break parent = parent.parent + if boot_time: extra["boot_time"] = boot_time.isoformat() + if annotation: extra["annotation"] = annotation @@ -692,6 +744,7 @@ def _to_node_recursive(self, virtual_machine): except Exception: # IPV6 not supported pass + if virtual_machine.snapshot: snapshots = [ { @@ -721,28 +774,34 @@ def _to_node_recursive(self, virtual_machine): extra=extra, ) node._uuid = uuid + return node def reboot_node(self, node): """ """ vm = self.find_by_uuid(node.id) + return self.wait_for_task(vm.RebootGuest()) def destroy_node(self, node): """ """ vm = self.find_by_uuid(node.id) + if node.state != NodeState.STOPPED: self.stop_node(node) + return self.wait_for_task(vm.Destroy()) def stop_node(self, node): """ """ vm = self.find_by_uuid(node.id) + return self.wait_for_task(vm.PowerOff()) def start_node(self, node): """ """ vm = self.find_by_uuid(node.id) + return self.wait_for_task(vm.PowerOn()) def ex_list_snapshots(self, node): @@ -750,8 +809,10 @@ def ex_list_snapshots(self, node): List node snapshots """ vm = self.find_by_uuid(node.id) + if not vm.snapshot: return [] + return format_snapshots(recurse_snapshots(vm.snapshot.rootSnapshotList)) def ex_create_snapshot( @@ -761,6 +822,7 @@ def ex_create_snapshot( Create node snapshot """ vm = self.find_by_uuid(node.id) + return WaitForTask(vm.CreateSnapshot(snapshot_name, description, dump_memory, quiesce)) def ex_remove_snapshot(self, node, snapshot_name=None, remove_children=True): @@ -769,21 +831,25 @@ def ex_remove_snapshot(self, node, snapshot_name=None, remove_children=True): If snapshot_name is not defined remove the last one. """ vm = self.find_by_uuid(node.id) + if not vm.snapshot: raise LibcloudError( "Remove snapshot failed. No snapshots for node %s" % node.name, driver=self, ) snapshots = recurse_snapshots(vm.snapshot.rootSnapshotList) + if not snapshot_name: snapshot = snapshots[-1].snapshot else: for s in snapshots: if snapshot_name == s.name: snapshot = s.snapshot + break else: raise LibcloudError("Snapshot `%s` not found" % snapshot_name, driver=self) + return self.wait_for_task(snapshot.RemoveSnapshot_Task(removeChildren=remove_children)) def ex_revert_to_snapshot(self, node, snapshot_name=None): @@ -792,20 +858,24 @@ def ex_revert_to_snapshot(self, node, snapshot_name=None): If snapshot_name is not defined revert to the last one. """ vm = self.find_by_uuid(node.id) + if not vm.snapshot: raise LibcloudError( "Revert failed. No snapshots " "for node %s" % node.name, driver=self ) snapshots = recurse_snapshots(vm.snapshot.rootSnapshotList) + if not snapshot_name: snapshot = snapshots[-1].snapshot else: for s in snapshots: if snapshot_name == s.name: snapshot = s.snapshot + break else: raise LibcloudError("Snapshot `%s` not found" % snapshot_name, driver=self) + return self.wait_for_task(snapshot.RevertToSnapshot_Task()) def _find_template_by_uuid(self, template_uuid): @@ -823,6 +893,7 @@ def _find_template_by_uuid(self, template_uuid): template = vm except Exception as exc: raise LibcloudError("Error while searching for template: %s" % exc, driver=self) + if not template: raise LibcloudError("Unable to locate VirtualMachine.", driver=self) @@ -833,24 +904,31 @@ def find_by_uuid(self, node_uuid): returns pyVmomi.VmomiSupport.vim.VirtualMachine """ vm = self.connection.content.searchIndex.FindByUuid(None, node_uuid, True, True) + if not vm: # perhaps it is a moid vm = self._get_item_by_moid("VirtualMachine", node_uuid) + if not vm: raise LibcloudError("Unable to locate VirtualMachine.", driver=self) + return vm def find_custom_field_key(self, key_id): """Return custom field key name, provided it's id""" + if not hasattr(self, "custom_fields"): content = self.connection.RetrieveContent() + if content.customFieldsManager: self.custom_fields = content.customFieldsManager.field else: self.custom_fields = [] + for k in self.custom_fields: if k.key == key_id: return k.name + return None def get_obj(self, vimtype, name): @@ -861,29 +939,36 @@ def get_obj(self, vimtype, name): obj = None content = self.connection.RetrieveContent() container = content.viewManager.CreateContainerView(content.rootFolder, vimtype, True) + for c in container.view: if name: if c.name == name: obj = c + break else: obj = c + break + return obj def wait_for_task(self, task, timeout=1800, interval=10): """wait for a vCenter task to finish""" start_time = time.time() task_done = False + while not task_done: if time.time() - start_time >= timeout: raise LibcloudError( "Timeout while waiting " "for import task Id %s" % task.info.id, driver=self, ) + if task.info.state == "success": if task.info.result and str(task.info.result) != "success": return task.info.result + return True if task.info.state == "error": @@ -912,20 +997,25 @@ def create_node( """ template = self._find_template_by_uuid(image.id) + if ex_cluster: cluster_name = ex_cluster else: cluster_name = location.name cluster = self.get_obj([vim.ClusterComputeResource], cluster_name) + if not cluster: # It is a host go with it cluster = self.get_obj([vim.HostSystem], cluster_name) datacenter = None + if not ex_datacenter: # Get datacenter from cluster parent = cluster.parent + while parent: if isinstance(parent, vim.Datacenter): datacenter = parent + break parent = parent.parent @@ -934,6 +1024,7 @@ def create_node( if ex_folder: folder = self.get_obj([vim.Folder], ex_folder) + if folder is None: folder = self._get_item_by_moid("Folder", ex_folder) else: @@ -955,6 +1046,7 @@ def create_node( datastore = None pod = None podsel = vim.storageDrs.PodSelectionSpec() + if ex_datastore_cluster: pod = self.get_obj([vim.StoragePod], ex_datastore_cluster) else: @@ -962,6 +1054,7 @@ def create_node( pods = content.viewManager.CreateContainerView( content.rootFolder, [vim.StoragePod], True ).view + for pod in pods: if cluster.name.lower() in pod.name: break @@ -985,11 +1078,13 @@ def create_node( if ex_datastore: datastore = self.get_obj([vim.Datastore], ex_datastore) + if datastore is None: datastore = self._get_item_by_moid("Datastore", ex_datastore) elif not datastore: datastore = self.get_obj([vim.Datastore], template.datastore[0].info.name) add_network = True + if ex_network and len(template.network) > 0: for nets in template.network: if template in nets.vm: @@ -1003,6 +1098,7 @@ def create_node( nicspec.device.deviceInfo = vim.Description() portgroup = self.get_obj([vim.dvs.DistributedVirtualPortgroup], ex_network) + if portgroup: dvs_port_connection = vim.dvs.PortConnection() dvs_port_connection.portgroupKey = portgroup.key @@ -1039,8 +1135,10 @@ def create_node( relospec = vim.vm.RelocateSpec() relospec.datastore = datastore relospec.pool = resource_pool + if location: host = self.get_obj([vim.HostSystem], location.name) + if host: relospec.host = host clonespec.location = relospec @@ -1077,6 +1175,7 @@ def ex_connect_network(self, vm, network_name): def _get_item_by_moid(self, type_, moid): vm_obj = VmomiSupport.templateOf(type_)(moid, self.connection._stub) + return vm_obj def ex_list_folders(self): @@ -1085,11 +1184,13 @@ def ex_list_folders(self): content.rootFolder, [vim.Folder], True ).view folders = [] + for folder in folders_raw: to_add = {"type": list(folder.childType)} to_add["name"] = folder.name to_add["id"] = folder._moId folders.append(to_add) + return folders def ex_list_datastores(self): @@ -1112,10 +1213,12 @@ def ex_list_datastores(self): def ex_open_console(self, vm_uuid): vm = self.find_by_uuid(vm_uuid) ticket = vm.AcquireTicket(ticketType="webmks") + return "wss://{}:{}/ticket/{}".format(ticket.host, ticket.port, ticket.ticket) def _get_version(self): content = self.connection.RetrieveContent() + return content.about.version @@ -1141,7 +1244,9 @@ def parse_error(self): if self.body: message = self.body message += "-- code: {}".format(self.status) + return message + return self.body @@ -1157,12 +1262,14 @@ def add_default_headers(self, headers): """ headers["Content-Type"] = "application/json" headers["Accept"] = "application/json" + if self.session_token is None: to_encode = "{}:{}".format(self.key, self.secret) b64_user_pass = base64.b64encode(to_encode.encode()) headers["Authorization"] = "Basic {}".format(b64_user_pass.decode()) else: headers["vmware-api-session-id"] = self.session_token + return headers @@ -1204,9 +1311,11 @@ def __init__(self, key, secret=None, secure=True, host=None, port=443, ca_cert=N raise InvalidCredsError("Please provide both username " "(key) and password (secret).") super().__init__(key=key, secure=secure, host=host, port=port) prefixes = ["http://", "https://"] + for prefix in prefixes: if host.startswith(prefix): host = host.lstrip(prefix) + if ca_cert: self.connection.connection.ca_cert = ca_cert else: @@ -1278,6 +1387,7 @@ def list_nodes( "filter.resource_pools": ex_filter_resource_pools, } params = {} + for param, value in kwargs.items(): if value: params[param] = value @@ -1285,8 +1395,10 @@ def list_nodes( vm_ids = [[item["vm"]] for item in result] vms = [] interfaces = self._list_interfaces() + for vm_id in vm_ids: vms.append(self._to_node(vm_id, interfaces)) + return vms def async_list_nodes(self): @@ -1300,6 +1412,7 @@ def async_list_nodes(self): result = loop.run_until_complete(self._get_all_vms()) vm_ids = [(item["vm"], item["host"]) for item in result] interfaces = self._list_interfaces() + return loop.run_until_complete(self._list_nodes_async(vm_ids, interfaces)) async def _list_nodes_async(self, vm_ids, interfaces): @@ -1337,6 +1450,7 @@ async def _get_all_vms(self): vm_resp = await asyncio.gather(*vm_resp_futures) # return a flat list + return [item for vm_list in vm_resp for item in vm_list] def _get_vms_with_host(self, host): @@ -1344,8 +1458,10 @@ def _get_vms_with_host(self, host): host_id = host["host"] response = self._request(req, params={"filter.hosts": host_id}) vms = response.object["value"] + for vm in vms: vm["host"] = host + return vms def list_locations(self, ex_show_hosts_in_drs=True): @@ -1365,6 +1481,7 @@ def list_locations(self, ex_show_hosts_in_drs=True): clusters = self.ex_list_clusters() hosts_all = self.ex_list_hosts() hosts = [] + if ex_show_hosts_in_drs: hosts = hosts_all else: @@ -1373,6 +1490,7 @@ def list_locations(self, ex_show_hosts_in_drs=True): hosts = [host for host in hosts_all if host not in filter_hosts] driver = self.connection.driver locations = [] + for cluster in clusters: if cluster["drs_enabled"]: extra = {"type": "cluster", "drs": True, "ha": cluster["ha_enabled"]} @@ -1385,6 +1503,7 @@ def list_locations(self, ex_show_hosts_in_drs=True): extra=extra, ) ) + for host in hosts: extra = { "type": "host", @@ -1401,6 +1520,7 @@ def list_locations(self, ex_show_hosts_in_drs=True): extra=extra, ) ) + return locations def stop_node(self, node): @@ -1411,6 +1531,7 @@ def stop_node(self, node): req = "/rest/vcenter/vm/{}/power/stop".format(node.id) result = self._request(req, method=method) + return result.status in self.VALID_RESPONSE_CODES def start_node(self, node): @@ -1423,6 +1544,7 @@ def start_node(self, node): method = "POST" req = "/rest/vcenter/vm/{}/power/start".format(node_id) result = self._request(req, method=method) + return result.status in self.VALID_RESPONSE_CODES def reboot_node(self, node): @@ -1432,10 +1554,12 @@ def reboot_node(self, node): method = "POST" req = "/rest/vcenter/vm/{}/power/reset".format(node.id) result = self._request(req, method=method) + return result.status in self.VALID_RESPONSE_CODES def destroy_node(self, node): # make sure the machine is stopped + if node.state is not NodeState.STOPPED: self.stop_node(node) # wait to make sure it stopped @@ -1446,6 +1570,7 @@ def destroy_node(self, node): # time.sleep(10) req = "/rest/vcenter/vm/{}".format(node.id) resp = self._request(req, method="DELETE") + return resp.status in self.VALID_RESPONSE_CODES def ex_suspend_node(self, node): @@ -1455,6 +1580,7 @@ def ex_suspend_node(self, node): method = "POST" req = "/rest/vcenter/vm/{}/power/suspend".format(node.id) result = self._request(req, method=method) + return result.status in self.VALID_RESPONSE_CODES def _list_interfaces(self): @@ -1469,6 +1595,7 @@ def _list_interfaces(self): } for interface in response ] + return interfaces def _to_node(self, vm_id_host, interfaces): @@ -1485,12 +1612,15 @@ def _to_node(self, vm_id_host, interfaces): # IP's private_ips = [] nic_macs = set() + for nic in vm["nics"]: nic_macs.add(nic["value"]["mac_address"]) + for interface in interfaces: if interface["mac"] in nic_macs: private_ips.append(interface["ip"]) nic_macs.remove(interface["mac"]) + if len(nic_macs) == 0: break public_ips = [] # should default_getaway be the public? @@ -1500,12 +1630,13 @@ def _to_node(self, vm_id_host, interfaces): # size total_size = 0 gb_converter = 1024**3 + for disk in vm["disks"]: total_size += int(int(disk["value"]["capacity"] / gb_converter)) ram = int(vm["memory"]["size_MiB"]) cpus = int(vm["cpu"]["count"]) id_to_hash = str(ram) + str(cpus) + str(total_size) - size_id = hashlib.md5(id_to_hash.encode("utf-8")).hexdigest() + size_id = hashlib.md5(id_to_hash.encode("utf-8")).hexdigest() # nosec size_name = name + "-size" size_extra = {"cpus": cpus} size = NodeSize( @@ -1525,8 +1656,10 @@ def _to_node(self, vm_id_host, interfaces): image_extra = {"type": "guest_OS"} image = NodeImage(id=image_id, name=image_name, driver=driver, extra=image_extra) extra = {"snapshots": []} + if len(vm_id_host) > 1: extra["host"] = vm_id_host[1].get("name", "") + return Node( id=vm_id, name=name, @@ -1560,11 +1693,13 @@ def ex_list_hosts( } params = {} + for param, value in kwargs.items(): if value: params[param] = value req = "/rest/vcenter/host" result = self._request(req, params=params).object["value"] + return result def ex_list_clusters( @@ -1581,11 +1716,13 @@ def ex_list_clusters( "filter.clusters": ex_filter_clusters, } params = {} + for param, value in kwargs.items(): if value: params[param] = value req = "/rest/vcenter/cluster" result = self._request(req, params=params).object["value"] + return result def ex_list_datacenters( @@ -1598,6 +1735,7 @@ def ex_list_datacenters( "filter.datacenters": ex_filter_datacenters, } params = {} + for param, value in kwargs.items(): if value: params[param] = value @@ -1605,12 +1743,14 @@ def ex_list_datacenters( to_return = [ {"name": item["name"], "id": item["datacenter"]} for item in result.object["value"] ] + return to_return def ex_list_content_libraries(self): req = "/rest/com/vmware/content/library" try: result = self._request(req).object + return result["value"] except BaseHTTPError: return [] @@ -1620,6 +1760,7 @@ def ex_list_content_library_items(self, library_id): params = {"library_id": library_id} try: result = self._request(req, params=params).object + return result["value"] except BaseHTTPError: logger.error( @@ -1627,14 +1768,17 @@ def ex_list_content_library_items(self, library_id): " most probably the VCenter service " "is stopped" ) + return [] def ex_list_folders(self): req = "/rest/vcenter/folder" response = self._request(req).object folders = response["value"] + for folder in folders: folder["id"] = folder["folder"] + return folders def ex_list_datastores( @@ -1654,12 +1798,15 @@ def ex_list_datastores( "filter.datastores": ex_filter_datastores, } params = {} + for param, value in kwargs.items(): if value: params[param] = value result = self._request(req, params=params).object["value"] + for datastore in result: datastore["id"] = datastore["datastore"] + return result def ex_update_memory(self, node, ram): @@ -1667,6 +1814,7 @@ def ex_update_memory(self, node, ram): :param ram: The amount of ram in MB. :type ram: `str` or `int` """ + if isinstance(node, str): node_id = node else: @@ -1676,6 +1824,7 @@ def ex_update_memory(self, node, ram): body = {"spec": {"size_MiB": ram}} response = self._request(request, method="PATCH", data=json.dumps(body)) + return response.status in self.VALID_RESPONSE_CODES def ex_update_cpu(self, node, cores): @@ -1684,6 +1833,7 @@ def ex_update_cpu(self, node, cores): :param cores: Integer or string indicating number of cores :type cores: `int` or `str` """ + if isinstance(node, str): node_id = node else: @@ -1692,6 +1842,7 @@ def ex_update_cpu(self, node, cores): cores = int(cores) body = {"spec": {"count": cores}} response = self._request(request, method="PATCH", data=json.dumps(body)) + return response.status in self.VALID_RESPONSE_CODES def ex_update_capacity(self, node, capacity): @@ -1703,6 +1854,7 @@ def ex_add_nic(self, node, network): Creates a network adapter that will connect to the specified network for the given node. Returns a boolean indicating success or not. """ + if isinstance(node, str): node_id = node else: @@ -1718,22 +1870,33 @@ def ex_add_nic(self, node, network): req = "/rest/vcenter/vm/{}/hardware/ethernet".format(node_id) method = "POST" resp = self._request(req, method=method, data=data) + return resp.status def _get_library_item(self, item_id): req = "/rest/com/vmware/content/library/item/id:{}".format(item_id) result = self._request(req).object + return result["value"] def _get_resource_pool(self, host_id=None, cluster_id=None, name=None): + pms = None + if host_id: pms = {"filter.hosts": host_id} + if cluster_id: pms = {"filter.clusters": cluster_id} + if name: pms = {"filter.names": name} + + if not pms: + raise ValueError("Either host_id, cluster_id or name must be provided") + rp_request = "/rest/vcenter/resource-pool" resource_pool = self._request(rp_request, params=pms).object + return resource_pool["value"][0]["resource_pool"] def _request(self, req, method="GET", params=None, data=None): @@ -1748,43 +1911,53 @@ def _request(self, req, method="GET", params=None, data=None): raise except Exception: raise + return result def list_images(self, **kwargs): libraries = self.ex_list_content_libraries() item_ids = [] + if libraries: for library in libraries: item_ids.extend(self.ex_list_content_library_items(library)) items = [] + if item_ids: for item_id in item_ids: items.append(self._get_library_item(item_id)) images = [] names = set() + if items: driver = self.connection.driver + for item in items: names.add(item["name"]) extra = {"type": item["type"]} + if item["type"] == "vm-template": capacity = item["size"] // (1024**3) extra["disk_size"] = capacity images.append( NodeImage(id=item["id"], name=item["name"], driver=driver, extra=extra) ) + if self.driver_soap is None: self._get_soap_driver() templates_in_hosts = self.driver_soap.list_images() + for template in templates_in_hosts: if template.name not in names: images += [template] + return images def ex_list_networks(self): request = "/rest/vcenter/network" response = self._request(request).object["value"] networks = [] + for network in response: networks.append( VSphereNetwork( @@ -1793,6 +1966,7 @@ def ex_list_networks(self): extra={"type": network["type"]}, ) ) + return networks def create_node( @@ -1815,7 +1989,11 @@ def create_node( will attempt to put the VM in a random folder and a warning about it will be issued in case the value remains `None`. """ + create_request = None + data = {} + # image is in the host then need the 6.5 driver + if image.extra["type"] == "template_6_5": kwargs = {} kwargs["name"] = name @@ -1823,14 +2001,18 @@ def create_node( kwargs["size"] = size kwargs["ex_network"] = ex_network kwargs["location"] = location + for dstore in self.ex_list_datastores(): if dstore["id"] == ex_datastore: kwargs["ex_datastore"] = dstore["name"] + break kwargs["folder"] = ex_folder + if self.driver_soap is None: self._get_soap_driver() result = self.driver_soap.create_node(**kwargs) + return result # post creation checks @@ -1839,11 +2021,13 @@ def create_node( update_cpu = False create_disk = False update_capacity = False + if image.extra["type"] == "guest_OS": spec = {} spec["guest_OS"] = image.name spec["name"] = name spec["placement"] = {} + if ex_folder is None: warn = ( "The API(6.7) requires the folder to be given, I will" @@ -1853,13 +2037,16 @@ def create_node( ) warnings.warn(warn) folders = self.ex_list_folders() + for folder in folders: if folder["type"] == "VIRTUAL_MACHINE": ex_folder = folder["folder"] + if ex_folder is None: msg = "No suitable folder vor VMs found, please create one" raise ProviderError(msg, 404) spec["placement"]["folder"] = ex_folder + if location.extra["type"] == "host": spec["placement"]["host"] = location.id elif location.extra["type"] == "cluster": @@ -1870,11 +2057,13 @@ def create_node( cpu = size.extra.get("cpu", 1) spec["cpu"] = {"count": cpu} spec["memory"] = {"size_MiB": size.ram} + if size.disk: disk = {} disk["new_vmdk"] = {} disk["new_vmdk"]["capacity"] = size.disk * (1024**3) spec["disks"] = [disk] + if ex_network: nic = {} nic["mac_type"] = "GENERATED" @@ -1892,10 +2081,12 @@ def create_node( ) spec = {} spec["target"] = {} + if location.extra.get("type") == "resource-pool": spec["target"]["resource_pool_id"] = location.id elif location.extra.get("type") == "host": resource_pool = self._get_resource_pool(host_id=location.id) + if not resource_pool: msg = ( "Could not find resource-pool for given location " @@ -1906,6 +2097,7 @@ def create_node( spec["target"]["host_id"] = location.id elif location.extra.get("type") == "cluster": resource_pool = self._get_resource_pool(cluster_id=location.id) + if not resource_pool: msg = ( "Could not find resource-pool for given location " @@ -1920,6 +2112,7 @@ def create_node( # assuming that since you want to make a vm you don't need reminder spec["deployment_spec"]["accept_all_EULA"] = True # network + if ex_network and ovf["networks"]: spec["deployment_spec"]["network_mappings"] = [ {"key": ovf["networks"][0], "value": ex_network} @@ -1927,17 +2120,22 @@ def create_node( elif not ovf["networks"] and ex_network: create_nic = True # storage + if ex_datastore: spec["deployment_spec"]["storage_mappings"] = [] store_map = {"type": "DATASTORE", "datastore_id": ex_datastore} spec["deployment_spec"]["storage_mappings"].append(store_map) + if size and size.ram: update_memory = True + if size and size.extra and size.extra.get("cpu"): update_cpu = True + if size and size.disk: # TODO Should update capacity but it is not possible with 6.7 pass + if ex_disks: create_disk = True @@ -1955,12 +2153,14 @@ def create_node( spec["name"] = name # storage + if ex_datastore: spec["disk_storage"] = {} spec["disk_storage"]["datastore"] = ex_datastore # location :: folder,resource group, datacenter, host spec["placement"] = {} + if not ex_folder: warn = ( "The API(6.7) requires the folder to be given, I will" @@ -1970,13 +2170,16 @@ def create_node( ) warnings.warn(warn) folders = self.ex_list_folders() + for folder in folders: if folder["type"] == "VIRTUAL_MACHINE": ex_folder = folder["folder"] + if ex_folder is None: msg = "No suitable folder vor VMs found, please create one" raise ProviderError(msg, 404) spec["placement"]["folder"] = ex_folder + if location.extra["type"] == "host": spec["placement"]["host"] = location.id elif location.extra["type"] == "cluster": @@ -1987,8 +2190,10 @@ def create_node( # after the creation finishes # only one network atm?? spec["hardware_customization"] = {} + if ex_network: nics = template["nics"] + if len(nics) > 0: nic = nics[0] spec["hardware_customization"]["nics"] = [ @@ -1998,17 +2203,21 @@ def create_node( create_nic = True spec["powered_on"] = False # hardware + if size: if size.ram: spec["hardware_customization"]["memory_update"] = {"memory": int(size.ram)} + if size.extra.get("cpu"): spec["hardware_customization"]["cpu_update"] = {"num_cpus": size.extra["cpu"]} + if size.disk: if not len(template["disks"]) > 0: create_disk = True else: capacity = size.disk * 1024 * 1024 * 1024 dsk = template["disks"][0]["key"] + if template["disks"][0]["value"]["capacity"] < capacity: update = {"capacity": capacity} spec["hardware_customization"]["disks_to_update"] = [ @@ -2019,26 +2228,38 @@ def create_node( image.id ) data = json.dumps({"spec": spec}) + + if not create_request: + raise ValueError("Missing create_request") + # deploy the node result = self._request(create_request, method="POST", data=data) # wait until the node is up and then add extra config node_id = result.object["value"] + if image.extra["type"] == "ovf": node_id = node_id["resource_id"]["id"] node = self.list_nodes(ex_filter_vms=node_id)[0] + if create_nic: self.ex_add_nic(node, ex_network) + if update_memory: self.ex_update_memory(node, size.ram) + if update_cpu: self.ex_update_cpu(node, size.extra["cpu"]) + if create_disk: pass # until volumes are added + if update_capacity: pass # until API method is added + if ex_turned_on: self.start_node(node) + return node # TODO As soon as snapshot support gets added to the REST api @@ -2047,8 +2268,10 @@ def ex_list_snapshots(self, node): """ List node snapshots """ + if self.driver_soap is None: self._get_soap_driver() + return self.driver_soap.ex_list_snapshots(node) def ex_create_snapshot( @@ -2057,8 +2280,10 @@ def ex_create_snapshot( """ Create node snapshot """ + if self.driver_soap is None: self._get_soap_driver() + return self.driver_soap.ex_create_snapshot( node, snapshot_name, @@ -2072,8 +2297,10 @@ def ex_remove_snapshot(self, node, snapshot_name=None, remove_children=True): Remove a snapshot from node. If snapshot_name is not defined remove the last one. """ + if self.driver_soap is None: self._get_soap_driver() + return self.driver_soap.ex_remove_snapshot( node, snapshot_name=snapshot_name, remove_children=remove_children ) @@ -2083,11 +2310,14 @@ def ex_revert_to_snapshot(self, node, snapshot_name=None): Revert node to a specific snapshot. If snapshot_name is not defined revert to the last one. """ + if self.driver_soap is None: self._get_soap_driver() + return self.driver_soap.ex_revert_to_snapshot(node, snapshot_name=snapshot_name) def ex_open_console(self, vm_id): if self.driver_soap is None: self._get_soap_driver() + return self.driver_soap.ex_open_console(vm_id) diff --git a/libcloud/container/drivers/kubernetes.py b/libcloud/container/drivers/kubernetes.py index 498a9439d7..78c89757f0 100644 --- a/libcloud/container/drivers/kubernetes.py +++ b/libcloud/container/drivers/kubernetes.py @@ -59,10 +59,13 @@ def to_n_bytes(memory_str: str) -> int: """Convert memory string to number of bytes (e.g. '1234Mi'-> 1293942784) """ + if memory_str.startswith("0"): return 0 + if memory_str.isnumeric(): return int(memory_str) + for unit, multiplier in K8S_UNIT_MAP.items(): if memory_str.endswith(unit): return int(memory_str.strip(unit)) * multiplier @@ -72,19 +75,23 @@ def to_memory_str(n_bytes: int, unit: Optional[str] = None) -> str: """Convert number of bytes to k8s memory string (e.g. 1293942784 -> '1234Mi') """ + if n_bytes == 0: return "0K" n_bytes = int(n_bytes) memory_str = None + if unit is None: for unit, multiplier in reversed(K8S_UNIT_MAP.items()): converted_n_bytes_float = n_bytes / multiplier converted_n_bytes = n_bytes // multiplier memory_str = f"{converted_n_bytes}{unit}" + if converted_n_bytes_float % 1 == 0: break elif K8S_UNIT_MAP.get(unit): memory_str = f"{n_bytes // K8S_UNIT_MAP[unit]}{unit}" + return memory_str @@ -92,15 +99,19 @@ def to_cpu_str(n_cpus: Union[int, float]) -> str: """Convert number of cpus to cpu string (e.g. 0.5 -> '500m') """ + if n_cpus == 0: return "0" millicores = n_cpus * 1000 + if millicores % 1 == 0: return f"{int(millicores)}m" microcores = n_cpus * 1000000 + if microcores % 1 == 0: return f"{int(microcores)}u" nanocores = n_cpus * 1000000000 + return f"{int(nanocores)}n" @@ -108,6 +119,7 @@ def to_n_cpus(cpu_str: str) -> Union[int, float]: """Convert cpu string to number of cpus (e.g. '500m' -> 0.5, '2000000000n' -> 2) """ + if cpu_str.endswith("n"): return int(cpu_str.strip("n")) / 1000000000 elif cpu_str.endswith("u"): @@ -123,9 +135,11 @@ def to_n_cpus(cpu_str: str) -> Union[int, float]: def sum_resources(*resource_dicts): total_cpu = 0 total_memory = 0 + for rd in resource_dicts: total_cpu += to_n_cpus(rd.get("cpu", "0m")) total_memory += to_n_bytes(rd.get("memory", "0K")) + return {"cpu": to_cpu_str(total_cpu), "memory": to_memory_str(total_memory)} @@ -222,6 +236,7 @@ def list_containers(self, image=None, all=True) -> List[Container]: result = self.connection.request(ROOT_URL + "v1/pods").object except Exception as exc: errno = getattr(exc, "errno", None) + if errno == 111: raise KubernetesException( errno, @@ -231,8 +246,10 @@ def list_containers(self, image=None, all=True) -> List[Container]: pods = [self._to_pod(value) for value in result["items"]] containers = [] + for pod in pods: containers.extend(pod.containers) + return containers def get_container(self, id: str) -> Container: @@ -246,6 +263,7 @@ def get_container(self, id: str) -> Container: """ containers = self.list_containers() match = [container for container in containers if container.id == id] + return match[0] def list_namespaces(self) -> List[KubernetesNamespace]: @@ -258,6 +276,7 @@ def list_namespaces(self) -> List[KubernetesNamespace]: result = self.connection.request(ROOT_URL + "v1/namespaces/").object except Exception as exc: errno = getattr(exc, "errno", None) + if errno == 111: raise KubernetesException( errno, @@ -266,6 +285,7 @@ def list_namespaces(self) -> List[KubernetesNamespace]: raise namespaces = [self._to_namespace(value) for value in result["items"]] + return namespaces def get_namespace(self, id: str) -> KubernetesNamespace: @@ -291,6 +311,7 @@ def delete_namespace(self, namespace: KubernetesNamespace) -> bool: self.connection.request( ROOT_URL + "v1/namespaces/%s" % namespace.id, method="DELETE" ).object + return True def create_namespace(self, name: str) -> KubernetesNamespace: @@ -306,6 +327,7 @@ def create_namespace(self, name: str) -> KubernetesNamespace: result = self.connection.request( ROOT_URL + "v1/namespaces", method="POST", data=json.dumps(request) ).object + return self._to_namespace(result) def deploy_container( @@ -338,6 +360,7 @@ def deploy_container( :rtype: :class:`.Container` """ + if namespace is None: namespace = "default" else: @@ -351,6 +374,7 @@ def deploy_container( method="POST", data=json.dumps(request), ).object + return self._to_namespace(result) def destroy_container(self, container: Container) -> bool: @@ -363,6 +387,7 @@ def destroy_container(self, container: Container) -> bool: :rtype: ``bool`` """ + return self.ex_destroy_pod(container.extra["namespace"], container.extra["pod"]) def ex_list_pods(self, fetch_metrics: bool = False) -> List[KubernetesPod]: @@ -376,6 +401,7 @@ def ex_list_pods(self, fetch_metrics: bool = False) -> List[KubernetesPod]: """ result = self.connection.request(ROOT_URL + "v1/pods").object metrics = None + if fetch_metrics: try: metrics = { @@ -407,6 +433,7 @@ def ex_destroy_pod(self, namespace: str, pod_name: str) -> bool: ROOT_URL + "v1/namespaces/{}/pods/{}".format(namespace, pod_name), method="DELETE", ).object + return True def ex_list_nodes(self) -> List[Node]: @@ -416,6 +443,7 @@ def ex_list_nodes(self) -> List[Node]: :rtype: ``list`` of :class:`.Node` """ result = self.connection.request(ROOT_URL + "v1/nodes").object + return [self._to_node(node) for node in result["items"]] def ex_destroy_node(self, node_name: str) -> bool: @@ -428,6 +456,7 @@ def ex_destroy_node(self, node_name: str) -> bool: :rtype: ``bool`` """ self.connection.request(ROOT_URL + f"v1/nodes/{node_name}", method="DELETE").object + return True def ex_get_version(self) -> str: @@ -435,6 +464,7 @@ def ex_get_version(self) -> str: :rtype: ``str`` """ + return self.connection.request("/version").object["gitVersion"] def ex_list_nodes_metrics(self) -> List[Dict[str, Any]]: @@ -442,6 +472,7 @@ def ex_list_nodes_metrics(self) -> List[Dict[str, Any]]: :rtype: ``list`` of ``dict`` """ + return self.connection.request("/apis/metrics.k8s.io/v1beta1/nodes").object["items"] def ex_list_pods_metrics(self) -> List[Dict[str, Any]]: @@ -449,6 +480,7 @@ def ex_list_pods_metrics(self) -> List[Dict[str, Any]]: :rtype: ``list`` of ``dict`` """ + return self.connection.request("/apis/metrics.k8s.io/v1beta1/pods").object["items"] def ex_list_services(self) -> List[Dict[str, Any]]: @@ -456,6 +488,7 @@ def ex_list_services(self) -> List[Dict[str, Any]]: :rtype: ``list`` of ``dict`` """ + return self.connection.request(ROOT_URL + "v1/services").object["items"] def ex_list_deployments(self) -> List[KubernetesDeployment]: @@ -464,6 +497,7 @@ def ex_list_deployments(self) -> List[KubernetesDeployment]: :rtype: ``list`` of :class:`.KubernetesDeployment` """ items = self.connection.request("/apis/apps/v1/deployments").object["items"] + return [self._to_deployment(item) for item in items] def _to_deployment(self, data): @@ -483,6 +517,7 @@ def _to_deployment(self, data): "available_replicas": data["status"]["availableReplicas"], "conditions": data["status"]["conditions"], } + return KubernetesDeployment( id=id_, name=name, @@ -502,12 +537,13 @@ def _to_node(self, data): driver = self.connection.driver memory = data["status"].get("capacity", {}).get("memory", "0K") cpu = data["status"].get("capacity", {}).get("cpu", "1") + if isinstance(cpu, str) and not cpu.isnumeric(): cpu = to_n_cpus(cpu) image_name = data["status"]["nodeInfo"].get("osImage") image = NodeImage(image_name, image_name, driver) size_name = f"{cpu} vCPUs, {memory} Ram" - size_id = hashlib.md5(size_name.encode("utf-8")).hexdigest() + size_id = hashlib.md5(size_name.encode("utf-8")).hexdigest() # nosec extra_size = {"cpus": cpu} size = NodeSize( id=size_id, @@ -523,13 +559,16 @@ def _to_node(self, data): extra["os"] = data["status"]["nodeInfo"].get("operatingSystem") extra["kubeletVersion"] = data["status"]["nodeInfo"]["kubeletVersion"] extra["provider_id"] = data.get("spec", {}).get("providerID") + for condition in data["status"]["conditions"]: if condition["type"] == "Ready" and condition["status"] == "True": state = NodeState.RUNNING + break else: state = NodeState.UNKNOWN public_ips, private_ips = [], [] + for address in data["status"]["addresses"]: if address["type"] == "InternalIP": private_ips.append(address["address"]) @@ -538,6 +577,7 @@ def _to_node(self, data): created_at = datetime.datetime.strptime( data["metadata"]["creationTimestamp"], "%Y-%m-%dT%H:%M:%SZ" ) + return Node( id=ID, name=name, @@ -563,6 +603,7 @@ def _to_pod(self, data, metrics=None): container_statuses = data["status"].get("containerStatuses", {}) containers = [] extra = {"resources": {}} + if metrics: try: extra["metrics"] = metrics[name, namespace] @@ -570,6 +611,7 @@ def _to_pod(self, data, metrics=None): pass # response contains the status of the containers in a separate field + for container in data["spec"]["containers"]: if container_statuses: spec = list(filter(lambda i: i["name"] == container["name"], container_statuses))[0] @@ -609,8 +651,10 @@ def _to_container(self, data, container_status, pod_data): """ state = container_status.get("state") created_at = None + if state: started_at = list(state.values())[0].get("startedAt") + if started_at: created_at = datetime.datetime.strptime(started_at, "%Y-%m-%dT%H:%M:%SZ") extra = { @@ -618,8 +662,10 @@ def _to_container(self, data, container_status, pod_data): "namespace": pod_data["metadata"]["namespace"], } resources = data.get("resources") + if resources: extra["resources"] = resources + return Container( id=container_status.get("containerID") or data["name"], name=data["name"], @@ -641,6 +687,7 @@ def _to_namespace(self, data): """ Convert an API node data object to a `KubernetesNamespace` object """ + return KubernetesNamespace( id=data["metadata"]["name"], name=data["metadata"]["name"], @@ -655,4 +702,5 @@ def ts_to_str(timestamp): """ date = datetime.datetime.fromtimestamp(timestamp) date_string = date.strftime("%d/%m/%Y %H:%M %Z") + return date_string diff --git a/libcloud/dns/drivers/google.py b/libcloud/dns/drivers/google.py index a7a99050a6..376186debc 100644 --- a/libcloud/dns/drivers/google.py +++ b/libcloud/dns/drivers/google.py @@ -78,6 +78,7 @@ def __init__(self, user_id, key, project=None, auth_type=None, scopes=None, **kw self.auth_type = auth_type self.project = project self.scopes = scopes + if not self.project: raise ValueError("Project name must be specified using " '"project" keyword.') super().__init__(user_id, key, **kwargs) @@ -88,6 +89,7 @@ def iterate_zones(self): :rtype: ``generator`` of :class:`Zone` """ + return self._get_more("zones") def iterate_records(self, zone): @@ -99,6 +101,7 @@ def iterate_records(self, zone): :rtype: ``generator`` of :class:`Record` """ + return self._get_more("records", zone=zone) def get_zone(self, zone_id): @@ -147,6 +150,7 @@ def get_record(self, zone_id, record_id): if len(response["rrsets"]) > 0: zone = self.get_zone(zone_id) + return self._to_record(response["rrsets"][0], zone) raise RecordDoesNotExistError(value="", driver=self.connection.driver, record_id=record_id) @@ -188,6 +192,7 @@ def create_zone(self, domain, type="master", ttl=None, extra=None): request = "/managedZones" response = self.connection.request(request, method="POST", data=data).object + return self._to_zone(response) def create_record(self, name, zone, type, data, extra=None): @@ -217,6 +222,7 @@ def create_record(self, name, zone, type, data, extra=None): data = {"additions": [{"name": name, "type": type, "ttl": int(ttl), "rrdatas": rrdatas}]} request = "/managedZones/%s/changes" % (zone.id) response = self.connection.request(request, method="POST", data=data).object + return self._to_record(response["additions"][0], zone) def delete_zone(self, zone): @@ -232,6 +238,7 @@ def delete_zone(self, zone): """ request = "/managedZones/%s" % (zone.id) response = self.connection.request(request, method="DELETE") + return response.success() def delete_record(self, record): @@ -255,6 +262,7 @@ def delete_record(self, record): } request = "/managedZones/%s/changes" % (record.zone.id) response = self.connection.request(request, method="POST", data=data) + return response.success() def ex_bulk_record_changes(self, zone, records): @@ -296,6 +304,7 @@ def ex_bulk_record_changes(self, zone, records): def _get_more(self, rtype, **kwargs): last_key = None exhausted = False + while not exhausted: items, last_key, exhausted = self._get_data(rtype, last_key, **kwargs) yield from items @@ -315,6 +324,8 @@ def _get_data(self, rtype, last_key, **kwargs): request = "/managedZones/%s/rrsets" % (zone.id) transform_func = self._to_records r_key = "rrsets" + else: + raise ValueError(f"Unsupported rtype: {rtype}") response = self.connection.request( request, @@ -326,6 +337,7 @@ def _get_data(self, rtype, last_key, **kwargs): nextpage = response.object.get("nextPageToken", None) items = transform_func(response.object.get(r_key), **kwargs) exhausted = False if nextpage is not None else True + return items, nextpage, exhausted else: return [], None, True @@ -339,8 +351,10 @@ def _ex_connection_class_kwargs(self): def _to_zones(self, response): zones = [] + for r in response: zones.append(self._to_zone(r)) + return zones def _to_zone(self, r): @@ -364,12 +378,15 @@ def _to_zone(self, r): def _to_records(self, response, zone): records = [] + for r in response: records.append(self._to_record(r, zone)) + return records def _to_record(self, r, zone): record_id = "{}:{}".format(r["type"], r["name"]) + return Record( id=record_id, name=r["name"], @@ -384,6 +401,8 @@ def _to_record(self, r, zone): def _cleanup_domain(self, domain): # name can only contain lower case alphanumeric characters and hyphens domain = re.sub(r"[^a-zA-Z0-9-]", "-", domain) + if domain[-1] == "-": domain = domain[:-1] + return domain diff --git a/libcloud/dns/drivers/onapp.py b/libcloud/dns/drivers/onapp.py index 7d0dfbfb63..322e812b3c 100644 --- a/libcloud/dns/drivers/onapp.py +++ b/libcloud/dns/drivers/onapp.py @@ -53,6 +53,7 @@ def list_zones(self): response = self.connection.request("/dns_zones.json") zones = self._to_zones(response.object) + return zones def get_zone(self, zone_id): @@ -66,6 +67,7 @@ def get_zone(self, zone_id): """ response = self.connection.request("/dns_zones/%s.json" % zone_id) zone = self._to_zone(response.object) + return zone def create_zone(self, domain, type="master", ttl=None, extra=None): @@ -91,6 +93,7 @@ def create_zone(self, domain, type="master", ttl=None, extra=None): https://docs.onapp.com/display/52API/Add+DNS+Zone """ dns_zone = {"name": domain} + if extra is not None: dns_zone.update(extra) dns_zone_data = json.dumps({"dns_zone": dns_zone}) @@ -101,6 +104,7 @@ def create_zone(self, domain, type="master", ttl=None, extra=None): data=dns_zone_data, ) zone = self._to_zone(response.object) + return zone def delete_zone(self, zone): @@ -115,6 +119,7 @@ def delete_zone(self, zone): :rtype: ``bool`` """ self.connection.request("/dns_zones/%s.json" % zone.id, method="DELETE") + return True def list_records(self, zone): @@ -129,6 +134,7 @@ def list_records(self, zone): response = self.connection.request("/dns_zones/%s/records.json" % zone.id) dns_records = response.object["dns_zone"]["records"] records = self._to_records(dns_records, zone) + return records def get_record(self, zone_id, record_id): @@ -147,6 +153,7 @@ def get_record(self, zone_id, record_id): "/dns_zones/{}/records/{}.json".format(zone_id, record_id) ) record = self._to_record(response.object, zone_id=zone_id) + return record def create_record(self, name, zone, type, data, extra=None): @@ -186,6 +193,7 @@ def create_record(self, name, zone, type, data, extra=None): data=dns_record_data, ) record = self._to_record(response.object, zone=zone) + return record def update_record(self, record, name, type, data, extra=None): @@ -226,6 +234,7 @@ def update_record(self, record, name, type, data, extra=None): data=dns_record_data, ) record = self.get_record(zone.id, record.id) + return record def delete_record(self, record): @@ -244,6 +253,7 @@ def delete_record(self, record): self.connection.request( "/dns_zones/{}/records/{}.json".format(zone_id, record.id), method="DELETE" ) + return True # @@ -253,6 +263,7 @@ def delete_record(self, record): def _format_record(self, name, type, data, extra): if name == "": name = "@" + if extra is None: extra = {} record_type = self.RECORD_TYPE_MAP[type] @@ -261,6 +272,7 @@ def _format_record(self, name, type, data, extra): "ttl": extra.get("ttl", DEFAULT_ZONE_TTL), "type": record_type, } + if type == RecordType.MX: additions = { "priority": extra.get("priority", 1), @@ -283,12 +295,16 @@ def _format_record(self, name, type, data, extra): additions = {"txt": extra.get("txt")} elif type == RecordType.NS: additions = {"hostname": extra.get("hostname")} + else: + additions = {} new_record.update(additions) + return new_record def _to_zones(self, data): zones = [] + for zone in data: _zone = self._to_zone(zone) zones.append(_zone) @@ -320,11 +336,13 @@ def _to_zone(self, data): def _to_records(self, data, zone): records = [] data = data.values() + for data_type in data: for item in data_type: record = self._to_record(item, zone=zone) records.append(record) records.sort(key=lambda x: x.id, reverse=False) + return records def _to_record(self, data, zone_id=None, zone=None): @@ -335,6 +353,7 @@ def _to_record(self, data, zone_id=None, zone=None): name = record.get("name") type = record.get("type") ttl = record.get("ttl", None) + return Record( id=id, name=name, diff --git a/libcloud/dns/drivers/rcodezero.py b/libcloud/dns/drivers/rcodezero.py index 854fbc9ac7..21a79a87ee 100644 --- a/libcloud/dns/drivers/rcodezero.py +++ b/libcloud/dns/drivers/rcodezero.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - RcodeZero DNS Driver +RcodeZero DNS Driver """ import re import json @@ -35,6 +35,7 @@ class RcodeZeroResponse(JsonResponse): def success(self): i = int(self.status) + return 200 <= i <= 299 def parse_error(self): @@ -64,6 +65,7 @@ class RcodeZeroConnection(ConnectionKey): def add_default_headers(self, headers): headers["Authorization"] = "Bearer " + self.key headers["Accept"] = "application/json" + return headers @@ -183,6 +185,7 @@ def create_record(self, name, zone, type, data, extra=None): ttl = extra["ttl"] else: ttl = None + return Record(id=None, name=name, data=data, type=type, zone=zone, ttl=ttl, driver=self) def create_zone(self, domain, type="master", ttl=None, extra={}): @@ -207,6 +210,7 @@ def create_zone(self, domain, type="master", ttl=None, extra={}): :rtype: :class:`Zone` """ action = "%s/zones" % (self.api_root) + if type.lower() == "slave" and (extra is None or extra.get("masters", None) is None): msg = "Master IPs required for slave zones" raise ValueError(msg) @@ -221,6 +225,7 @@ def create_zone(self, domain, type="master", ttl=None, extra={}): ): raise ZoneAlreadyExistsError(zone_id=zone_id, driver=self, value=e.message) raise e + return Zone( id=zone_id, domain=domain, @@ -254,6 +259,7 @@ def update_zone(self, zone, domain, type=None, ttl=None, extra=None): :rtype: :class:`Zone` """ action = "{}/zones/{}".format(self.api_root, domain) + if type is None: type = zone.type @@ -261,6 +267,7 @@ def update_zone(self, zone, domain, type=None, ttl=None, extra=None): msg = "Master IPs required for slave zones" raise ValueError(msg) payload = {"domain": domain.lower(), "type": type.lower()} + if extra is not None: payload.update(extra) try: @@ -271,6 +278,7 @@ def update_zone(self, zone, domain, type=None, ttl=None, extra=None): ): raise ZoneAlreadyExistsError(zone_id=zone.id, driver=self, value=e.message) raise e + return Zone(id=zone.id, domain=domain, type=type, ttl=None, driver=self, extra=extra) def delete_record(self, record): @@ -321,6 +329,7 @@ def delete_zone(self, zone): self.connection.request(action=action, method="DELETE") except BaseHTTPError: return False + return True def get_zone(self, zone_id): @@ -385,6 +394,7 @@ def list_records(self, zone): ): raise ZoneDoesNotExistError(zone_id=zone.id, driver=self, value=e.message) raise e + return self._to_records(response.object["data"], zone) def list_zones(self): @@ -395,6 +405,7 @@ def list_zones(self): """ action = "%s/zones?page_size=-1" % (self.api_root) response = self.connection.request(action=action, method="GET") + return self._to_zones(response.object["data"]) def update_record(self, record, name, type, data, extra=None): @@ -434,13 +445,14 @@ def update_record(self, record, name, type, data, extra=None): ): raise ZoneDoesNotExistError(zone_id=record.zone.id, driver=self, value=e.message) raise e + if not (extra is None or extra.get("ttl", None) is None): ttl = extra["ttl"] else: ttl = record.ttl return Record( - id=hashlib.md5(str(name + " " + data).encode("utf-8")).hexdigest(), + id=hashlib.md5(str(name + " " + data).encode("utf-8")).hexdigest(), # nosec name=name, data=data, type=type, @@ -452,6 +464,7 @@ def update_record(self, record, name, type, data, extra=None): def _to_zone(self, item): extra = {} + for e in [ "dnssec_status", "dnssec_status_detail", @@ -468,6 +481,7 @@ def _to_zone(self, item): ]: if e in item: extra[e] = item[e] + return Zone( id=item["domain"], domain=item["domain"], @@ -479,12 +493,15 @@ def _to_zone(self, item): def _to_zones(self, items): zones = [] + for item in items: zones.append(self._to_zone(item)) + return zones def _to_records(self, items, zone): records = [] + for item in items: for record in item["records"]: extra = {} @@ -493,7 +510,7 @@ def _to_records(self, items, zone): recordname = re.sub("." + zone.id + "$", "", item["name"][:-1]) records.append( Record( - id=hashlib.md5( + id=hashlib.md5( # nosec str(recordname + " " + record["content"]).encode("utf-8") ).hexdigest(), name=recordname, @@ -505,6 +522,7 @@ def _to_records(self, items, zone): extra=extra, ) ) + return records # rcodezero supports only rrset, so we must create rrsets from the given @@ -524,22 +542,27 @@ def _to_patchrequest(self, zone, record, name, type, data, extra, action): rrset["type"] = type rrset["changetype"] = action rrset["records"] = [] + if not (extra is None or extra.get("ttl", None) is None): rrset["ttl"] = extra["ttl"] content = {} + if not action == "delete": content["content"] = data + if not (extra is None or extra.get("disabled", None) is None): content["disabled"] = extra["disabled"] rrset["records"].append(content) - id = hashlib.md5(str(name + " " + data).encode("utf-8")).hexdigest() + id = hashlib.md5(str(name + " " + data).encode("utf-8")).hexdigest() # nosec # check if rrset contains more than one record. if yes we need to create an # update request + for r in cur_records: if action == "update" and r.id == record.id: # do not include records which should be updated in the update # request + continue if name == r.name and r.id != id: @@ -548,9 +571,11 @@ def _to_patchrequest(self, zone, record, name, type, data, extra, action): rrset["changetype"] = "update" content = {} content["content"] = r.data + if not (r.extra is None or r.extra.get("disabled", None) is None): content["disabled"] = r.extra["disabled"] rrset["records"].append(content) request = list() request.append(rrset) + return request diff --git a/libcloud/dns/drivers/route53.py b/libcloud/dns/drivers/route53.py index c4c8451349..6140cc6cb6 100644 --- a/libcloud/dns/drivers/route53.py +++ b/libcloud/dns/drivers/route53.py @@ -124,11 +124,13 @@ def get_zone(self, zone_id): uri = API_ROOT + "hostedzone/" + zone_id data = self.connection.request(uri).object elem = findall(element=data, xpath="HostedZone", namespace=NAMESPACE)[0] + return self._to_zone(elem) def get_record(self, zone_id, record_id): zone = self.get_zone(zone_id=zone_id) record_type, name = record_id.split(":", 1) + if name: full_name = ".".join((name, zone.domain)) else: @@ -144,6 +146,7 @@ def get_record(self, zone_id, record_id): # hints than filters!! # So will return a result even if its not what you asked for. record_type_num = self._string_to_record_type(record_type) + if record.name != name or record.type != record_type_num: raise RecordDoesNotExistError(value="", driver=self, record_id=record_id) @@ -163,6 +166,7 @@ def create_zone(self, domain, type="master", ttl=None, extra=None): rsp = self.connection.request(uri, method="POST", data=data).object elem = findall(element=rsp, xpath="HostedZone", namespace=NAMESPACE)[0] + return self._to_zone(elem=elem) def delete_zone(self, zone, ex_delete_records=False): @@ -173,6 +177,7 @@ def delete_zone(self, zone, ex_delete_records=False): uri = API_ROOT + "hostedzone/%s" % (zone.id) response = self.connection.request(uri, method="DELETE") + return response.status in [httplib.OK] def create_record(self, name, zone, type, data, extra=None): @@ -182,6 +187,7 @@ def create_record(self, name, zone, type, data, extra=None): batch = [("CREATE", name, type, data, extra)] self._post_changeset(zone, batch) id = ":".join((self.RECORD_TYPE_MAP[type], name)) + return Record( id=id, name=name, @@ -216,6 +222,7 @@ def update_record(self, record, name=None, type=None, data=None, extra=None): ) id = ":".join((self.RECORD_TYPE_MAP[type], name)) + return Record( id=id, name=name, @@ -234,6 +241,7 @@ def delete_record(self, record): self._post_changeset(record.zone, batch) except InvalidChangeBatch: raise RecordDoesNotExistError(value="", driver=self, record_id=r.id) + return True def ex_create_multi_value_record(self, name, zone, type, data, extra=None): @@ -275,6 +283,7 @@ def ex_create_multi_value_record(self, name, zone, type, data, extra=None): id = ":".join((self.RECORD_TYPE_MAP[type], name)) records = [] + for value in values: record = Record( id=id, @@ -298,6 +307,7 @@ def ex_delete_all_records(self, zone): :type zone: :class:`Zone` """ deletions = [] + for r in zone.list_records(): if r.type in (RecordType.NS, RecordType.SOA): continue @@ -400,6 +410,7 @@ def _post_changeset(self, zone, changes_list): rrecs = ET.SubElement(rrs, "ResourceRecords") rrec = ET.SubElement(rrecs, "ResourceRecord") + if "priority" in extra: data = "{} {}".format(extra["priority"], data) ET.SubElement(rrec, "Value").text = data @@ -413,6 +424,7 @@ def _post_changeset(self, zone, changes_list): def _to_zones(self, data): zones = [] + for element in data.findall(fixxpath(xpath="HostedZones/HostedZone", namespace=NAMESPACE)): zones.append(self._to_zone(element)) @@ -429,6 +441,7 @@ def _to_zone(self, elem): extra = {"Comment": comment, "ResourceRecordSetCount": resource_record_count} zone = Zone(id=id, domain=name, type="master", ttl=0, driver=self, extra=extra) + return zone def _to_records(self, data, zone): @@ -436,6 +449,7 @@ def _to_records(self, data, zone): elems = data.findall( fixxpath(xpath="ResourceRecordSets/ResourceRecordSet", namespace=NAMESPACE) ) + for elem in elems: record_set = elem.findall( fixxpath(xpath="ResourceRecords/ResourceRecord", namespace=NAMESPACE) @@ -457,6 +471,7 @@ def _to_records(self, data, zone): record_set_records.append(record) # Store reference to other records so update works correctly + if multiple_value_record: for index in range(0, len(record_set_records)): record = record_set_records[index] @@ -464,6 +479,7 @@ def _to_records(self, data, zone): for other_index, other_record in enumerate(record_set_records): if index == other_index: # Skip current record + continue extra = copy.deepcopy(other_record.extra) @@ -490,6 +506,7 @@ def _to_record(self, elem, zone, index=0): findtext(element=elem, xpath="Type", namespace=NAMESPACE) ) ttl = findtext(element=elem, xpath="TTL", namespace=NAMESPACE) + if ttl is not None: ttl = int(ttl) @@ -522,17 +539,20 @@ def _to_record(self, elem, zone, index=0): ttl=extra.get("ttl", None), extra=extra, ) + return record def _get_more(self, rtype, **kwargs): exhausted = False last_key = None + while not exhausted: items, last_key, exhausted = self._get_data(rtype, last_key, **kwargs) yield from items def _get_data(self, rtype, last_key, **kwargs): params = {} + if last_key: params["name"] = last_key path = API_ROOT + "hostedzone" @@ -546,6 +566,8 @@ def _get_data(self, rtype, last_key, **kwargs): self.connection.set_context({"zone_id": zone.id}) response = self.connection.request(path, params=params) transform_func = self._to_records + else: + raise ValueError(f"Unsupported rtype: {rtype}") if response.status == httplib.OK: is_truncated = findtext( @@ -556,6 +578,7 @@ def _get_data(self, rtype, last_key, **kwargs): element=response.object, xpath="NextRecordName", namespace=NAMESPACE ) items = transform_func(data=response.object, **kwargs) + return items, last_key, exhausted else: return [], None, True @@ -563,9 +586,11 @@ def _get_data(self, rtype, last_key, **kwargs): def _ex_connection_class_kwargs(self): kwargs = super()._ex_connection_class_kwargs() kwargs["token"] = self.token + return kwargs def _quote_data(self, data): if data[0] == '"' and data[-1] == '"': return data + return '"{}"'.format(data.replace('"', '"')) diff --git a/libcloud/dns/drivers/vultr.py b/libcloud/dns/drivers/vultr.py index bdff9bdfcd..e0fae847f6 100644 --- a/libcloud/dns/drivers/vultr.py +++ b/libcloud/dns/drivers/vultr.py @@ -89,6 +89,7 @@ def __new__( raise NotImplementedError( "No Vultr driver found for API version: %s" % (api_version) ) + return super().__new__(cls) @@ -130,6 +131,7 @@ def list_records(self, zone): :rtype: list of :class: `Record` """ + if not isinstance(zone, Zone): raise ZoneRequiredException("zone should be of type Zone") @@ -212,13 +214,17 @@ def create_zone(self, domain, type="master", ttl=None, extra=None): (e.g. {'serverip':'127.0.0.1'}) """ extra = extra or {} + if extra and extra.get("serverip"): serverip = extra["serverip"] + else: + raise ValueError("Missing servertip key in extra") params = {"api_key": self.key} data = urlencode({"domain": domain, "serverip": serverip}) action = "/v1/dns/create_domain" zones = self.list_zones() + if self.ex_zone_exists(domain, zones): raise ZoneAlreadyExistsError(value="", driver=self, zone_id=domain) @@ -257,6 +263,7 @@ def create_record(self, name, zone, type, data, extra=None): old_records_list = self.list_records(zone=zone) # check if record already exists # if exists raise RecordAlreadyExistsError + for record in old_records_list: if record.name == name and record.data == data: raise RecordAlreadyExistsError(value="", driver=self, record_id=record.id) @@ -266,6 +273,8 @@ def create_record(self, name, zone, type, data, extra=None): if extra and extra.get("priority"): priority = int(extra["priority"]) + else: + priority = None post_data = { "domain": zone.domain, @@ -275,6 +284,8 @@ def create_record(self, name, zone, type, data, extra=None): } if type == MX or type == SRV: + if priority is None: + raise ValueError("Missing priority argument for MX record type") post_data["priority"] = priority encoded_data = urlencode(post_data) @@ -305,6 +316,7 @@ def delete_zone(self, zone): params = {"api_key": self.key} data = urlencode({"domain": zone.domain}) zones = self.list_zones() + if not self.ex_zone_exists(zone.domain, zones): raise ZoneDoesNotExistError(value="", driver=self, zone_id=zone.domain) @@ -326,6 +338,7 @@ def delete_record(self, record): data = urlencode({"RECORDID": record.id, "domain": record.zone.domain}) zone_records = self.list_records(record.zone) + if not self.ex_record_exists(record.id, zone_records): raise RecordDoesNotExistError(value="", driver=self, record_id=record.id) @@ -347,6 +360,7 @@ def ex_zone_exists(self, zone_id, zones_list): """ zone_ids = [] + for zone in zones_list: zone_ids.append(zone.domain) @@ -363,6 +377,7 @@ def ex_record_exists(self, record_id, records_list): :rtype: ``bool`` """ record_ids = [] + for record in records_list: record_ids.append(record.id) @@ -400,6 +415,7 @@ def _to_zones(self, items): :type items: ``list`` """ zones = [] + for item in items: zones.append(self._to_zone(item)) @@ -426,6 +442,7 @@ def _to_record(self, item, zone): def _to_records(self, items, zone): records = [] + for item in items: records.append(self._to_record(item, zone=zone)) @@ -453,6 +470,7 @@ def list_zones(self) -> List[Zone]: :return: ``list`` of :class:`Zone` """ data = self._paginated_request("/v2/domains", "domains") + return [self._to_zone(item) for item in data] def get_zone(self, zone_id: str) -> Zone: @@ -464,6 +482,7 @@ def get_zone(self, zone_id: str) -> Zone: :rtype: :class:`Zone` """ resp = self.connection.request("/v2/domains/%s" % zone_id) + return self._to_zone(resp.object["domain"]) def create_zone( @@ -496,6 +515,7 @@ def create_zone( } extra = extra or {} + if "ip" in extra: data["ip"] = extra["ip"] @@ -503,6 +523,7 @@ def create_zone( data["dns_sec"] = "enabled" if extra["dns_sec"] is True else "disabled" resp = self.connection.request("/v2/domains", data=json.dumps(data), method="POST") + return self._to_zone(resp.object["domain"]) def delete_zone(self, zone: Zone) -> bool: @@ -516,6 +537,7 @@ def delete_zone(self, zone: Zone) -> bool: :rtype: ``bool`` """ resp = self.connection.request("/v2/domains/%s" % zone.domain, method="DELETE") + return resp.success() def list_records(self, zone: Zone) -> List[Record]: @@ -527,6 +549,7 @@ def list_records(self, zone: Zone) -> List[Record]: :return: ``list`` of :class:`Record` """ data = self._paginated_request("/v2/domains/%s/records" % zone.domain, "records") + return [self._to_record(item, zone) for item in data] def get_record(self, zone_id: str, record_id: str) -> Record: @@ -586,6 +609,7 @@ def create_record( "data": data, } extra = extra or {} + if "ttl" in extra: data["ttl"] = int(extra["ttl"]) @@ -631,6 +655,7 @@ def update_record( :rtype: ``bool`` """ body = {} + if name: body["name"] = name @@ -638,6 +663,7 @@ def update_record( body["data"] = data extra = extra or {} + if "ttl" in extra: body["ttl"] = int(extra["ttl"]) @@ -673,6 +699,7 @@ def _to_zone(self, data: Dict[str, Any]) -> Zone: extra = { "date_created": data["date_created"], } + return Zone(id=domain, domain=domain, driver=self, type=type_, ttl=None, extra=extra) def _to_record(self, data: Dict[str, Any], zone: Zone) -> Record: @@ -721,8 +748,10 @@ def _paginated_request( resp = self.connection.request(url, params=params).object data = list(resp.get(key, [])) objects = data + while True: next_page = resp["meta"]["links"]["next"] + if next_page: params["cursor"] = next_page resp = self.connection.request(url, params=params).object diff --git a/libcloud/dns/drivers/worldwidedns.py b/libcloud/dns/drivers/worldwidedns.py index 5b8c72a11c..80593b499c 100644 --- a/libcloud/dns/drivers/worldwidedns.py +++ b/libcloud/dns/drivers/worldwidedns.py @@ -101,9 +101,11 @@ def list_zones(self): https://www.worldwidedns.net/dns_api_protocol_list_reseller.asp """ action = "/api_dns_list.asp" + if self.reseller_id is not None: action = "/api_dns_list_reseller.asp" zones = self.connection.request(action) + if len(zones.body) == 0: return [] else: @@ -132,10 +134,12 @@ def get_zone(self, zone_id): """ zones = self.list_zones() zone = [zone for zone in zones if zone.id == zone_id] + if len(zone) == 0: raise ZoneDoesNotExistError( driver=self, value="The zone doesn't exists", zone_id=zone_id ) + return zone[0] def get_record(self, zone_id, record_id): @@ -164,6 +168,7 @@ def get_record(self, zone_id, record_id): type = zone.extra.get("T%s" % record_id) data = zone.extra.get("D%s" % record_id) record = self._to_record(record_id, subdomain, type, data, zone) + return record def update_zone(self, zone, domain, type="master", ttl=None, extra=None, ex_raw=False): @@ -204,6 +209,7 @@ def update_zone(self, zone, domain, type="master", ttl=None, extra=None, ex_raw= or https://www.worldwidedns.net/dns_api_protocol_list_domain_raw_reseller.asp """ + if extra is not None: not_specified = [key for key in zone.extra.keys() if key not in extra.keys()] else: @@ -216,20 +222,25 @@ def update_zone(self, zone, domain, type="master", ttl=None, extra=None, ex_raw= for key in not_specified: params[key] = zone.extra[key] + if extra is not None: params.update(extra) + if ex_raw: action = "/api_dns_modify_raw.asp" + if self.reseller_id is not None: action = "/api_dns_modify_raw_reseller.asp" method = "POST" else: action = "/api_dns_modify.asp" + if self.reseller_id is not None: action = "/api_dns_modify_reseller.asp" method = "GET" response = self.connection.request(action, params=params, method=method) # noqa zone = self.get_zone(zone.id) + return zone def update_record(self, record, name, type, data, extra=None): @@ -256,11 +267,14 @@ def update_record(self, record, name, type, data, extra=None): :rtype: :class:`Record` """ + if (extra is None) or ("entry" not in extra): raise WorldWideDNSError(value="You must enter 'entry' parameter", driver=self) record_id = extra.get("entry") + if name == "": name = "@" + if type not in self.RECORD_TYPE_MAP: raise RecordError( value="Record type is not allowed", @@ -275,6 +289,7 @@ def update_record(self, record, name, type, data, extra=None): } zone = self.update_zone(zone, zone.domain, extra=extra) record = self.get_record(zone.id, record_id) + return record def create_zone(self, domain, type="master", ttl=None, extra=None): @@ -302,23 +317,30 @@ def create_zone(self, domain, type="master", ttl=None, extra=None): or https://www.worldwidedns.net/dns_api_protocol_new_domain_reseller.asp """ + if type == "master": _type = 0 elif type == "slave": _type = 1 + else: + raise ValueError(f"Unsupported type: {type}") + if extra: dyn = extra.get("DYN") or 1 else: dyn = 1 params = {"DOMAIN": domain, "TYPE": _type} action = "/api_dns_new_domain.asp" + if self.reseller_id is not None: params["DYN"] = dyn action = "/api_dns_new_domain_reseller.asp" self.connection.request(action, params=params) zone = self.get_zone(domain) + if ttl is not None: zone = self.update_zone(zone, zone.domain, ttl=ttl) + return zone def create_record(self, name, zone, type, data, extra=None): @@ -348,16 +370,20 @@ def create_record(self, name, zone, type, data, extra=None): :rtype: :class:`Record` """ + if (extra is None) or ("entry" not in extra): # If no entry is specified, we look for an available one. If all # are full, raise error. record_id = self._get_available_record_entry(zone) + if not record_id: raise WorldWideDNSError(value="All record entries are full", driver=zone.driver) else: record_id = extra.get("entry") + if name == "": name = "@" + if type not in self.RECORD_TYPE_MAP: raise RecordError( value="Record type is not allowed", @@ -371,6 +397,7 @@ def create_record(self, name, zone, type, data, extra=None): } zone = self.update_zone(zone, zone.domain, extra=extra) record = self.get_record(zone.id, record_id) + return record def delete_zone(self, zone): @@ -391,9 +418,11 @@ def delete_zone(self, zone): """ params = {"DOMAIN": zone.domain} action = "/api_dns_delete_domain.asp" + if self.reseller_id is not None: action = "/api_dns_delete_domain_reseller.asp" response = self.connection.request(action, params=params) + return response.success() def delete_record(self, record): @@ -406,12 +435,15 @@ def delete_record(self, record): :rtype: ``bool`` """ zone = record.zone + for index in range(MAX_RECORD_ENTRIES): if record.name == zone.extra["S%s" % (index + 1)]: entry = index + 1 + break extra = {"S%s" % entry: "", "T%s" % entry: "NONE", "D%s" % entry: ""} self.update_zone(zone, zone.domain, extra=extra) + return True def ex_view_zone(self, domain, name_server): @@ -433,9 +465,11 @@ def ex_view_zone(self, domain, name_server): """ params = {"DOMAIN": domain, "NS": name_server} action = "/api_dns_viewzone.asp" + if self.reseller_id is not None: action = "/api_dns_viewzone_reseller.asp" response = self.connection.request(action, params=params) + return response.object def ex_transfer_domain(self, domain, user_id): @@ -455,26 +489,32 @@ def ex_transfer_domain(self, domain, user_id): For more info, please see: https://www.worldwidedns.net/dns_api_protocol_transfer.asp """ + if self.reseller_id is None: raise WorldWideDNSError("This is not a reseller account", driver=self) params = {"DOMAIN": domain, "NEW_ID": user_id} response = self.connection.request("/api_dns_transfer.asp", params=params) + return response.success() def _get_available_record_entry(self, zone): """Return an available entry to store a record.""" entries = zone.extra + for entry in range(1, MAX_RECORD_ENTRIES + 1): subdomain = entries.get("S%s" % entry) _type = entries.get("T%s" % entry) data = entries.get("D%s" % entry) + if not any([subdomain, _type, data]): return entry + return None def _to_zones(self, data): domain_list = re.split("\r?\n", data) zones = [] + for line in domain_list: zone = self._to_zone(line) zones.append(zone) @@ -484,6 +524,7 @@ def _to_zones(self, data): def _to_zone(self, line): data = line.split("\x1f") name = data[0] + if data[1] == "P": type = "master" domain_data = self._get_domain_data(name) @@ -498,6 +539,7 @@ def _to_zone(self, line): "SECURE": soa_block[5], } ttl = soa_block[4] + for line in range(MAX_RECORD_ENTRIES): line_data = zone_data[line].split("\x1f") extra["S%s" % (line + 1)] = line_data[0] @@ -511,22 +553,29 @@ def _to_zone(self, line): type = "slave" extra = {} ttl = 0 + else: + ttl = 0 + return Zone(id=name, domain=name, type=type, ttl=ttl, driver=self, extra=extra) def _get_domain_data(self, name): params = {"DOMAIN": name} data = self.connection.request("/api_dns_list_domain.asp", params=params) + return data def _to_records(self, zone): records = [] + for record_id in range(1, MAX_RECORD_ENTRIES + 1): subdomain = zone.extra["S%s" % (record_id)] type = zone.extra["T%s" % (record_id)] data = zone.extra["D%s" % (record_id)] + if subdomain and type and data: record = self._to_record(record_id, subdomain, type, data, zone) records.append(record) + return records def _to_record(self, _id, subdomain, type, data, zone): diff --git a/libcloud/dns/drivers/zerigo.py b/libcloud/dns/drivers/zerigo.py index 77e2cda08b..0f37e0f357 100644 --- a/libcloud/dns/drivers/zerigo.py +++ b/libcloud/dns/drivers/zerigo.py @@ -67,6 +67,7 @@ def parse_error(self): raise InvalidCredsError(self.body) elif status == 404: context = self.connection.context + if context["resource"] == "zone": raise ZoneDoesNotExistError(value="", driver=self, zone_id=context["id"]) elif context["resource"] == "record": @@ -78,6 +79,7 @@ def parse_error(self): raise MalformedResponseError("Failed to parse XML", body=self.body) errors = [] + for error in findall(element=body, xpath="error"): errors.append(error.text) @@ -94,16 +96,19 @@ class ZerigoDNSConnection(ConnectionUserAndKey): def add_default_headers(self, headers): auth_b64 = base64.b64encode(b("{}:{}".format(self.user_id, self.key))) headers["Authorization"] = "Basic %s" % (auth_b64.decode("utf-8")) + return headers def request(self, action, params=None, data="", headers=None, method="GET"): if not headers: headers = {} + if not params: params = {} if method in ("POST", "PUT"): headers = {"Content-Type": "application/xml; charset=UTF-8"} + return super().request( action=action, params=params, data=data, method=method, headers=headers ) @@ -142,6 +147,7 @@ def get_zone(self, zone_id): self.connection.set_context({"resource": "zone", "id": zone_id}) data = self.connection.request(path).object zone = self._to_zone(elem=data) + return zone def get_record(self, zone_id, record_id): @@ -150,6 +156,7 @@ def get_record(self, zone_id, record_id): path = API_ROOT + "hosts/%s.xml" % (record_id) data = self.connection.request(path).object record = self._to_record(elem=data, zone=zone) + return record def create_zone(self, domain, type="master", ttl=None, extra=None): @@ -167,6 +174,7 @@ def create_zone(self, domain, type="master", ttl=None, extra=None): action=path, data=ET.tostring(zone_elem), method="POST" ).object zone = self._to_zone(elem=data) + return zone def update_zone(self, zone, domain=None, type=None, ttl=None, extra=None): @@ -178,6 +186,7 @@ def update_zone(self, zone, domain=None, type=None, ttl=None, extra=None): @inherits: :class:`DNSDriver.update_zone` """ + if domain: raise LibcloudError("Domain cannot be changed", driver=self) @@ -194,6 +203,7 @@ def update_zone(self, zone, domain=None, type=None, ttl=None, extra=None): updated_zone = get_new_obj( obj=zone, klass=Zone, attributes={"type": type, "ttl": ttl, "extra": merged} ) + return updated_zone def create_record(self, name, zone, type, data, extra=None): @@ -212,6 +222,7 @@ def create_record(self, name, zone, type, data, extra=None): ) assert response.status == httplib.CREATED record = self._to_record(elem=response.object, zone=zone) + return record def update_record(self, record, name=None, type=None, data=None, extra=None): @@ -230,18 +241,21 @@ def update_record(self, record, name=None, type=None, data=None, extra=None): klass=Record, attributes={"type": type, "data": data, "extra": merged}, ) + return updated_record def delete_zone(self, zone): path = API_ROOT + "zones/%s.xml" % (zone.id) self.connection.set_context({"resource": "zone", "id": zone.id}) response = self.connection.request(action=path, method="DELETE") + return response.status == httplib.OK def delete_record(self, record): path = API_ROOT + "hosts/%s.xml" % (record.id) self.connection.set_context({"resource": "record", "id": record.id}) response = self.connection.request(action=path, method="DELETE") + return response.status == httplib.OK def ex_get_zone_by_domain(self, domain): @@ -257,6 +271,7 @@ def ex_get_zone_by_domain(self, domain): self.connection.set_context({"resource": "zone", "id": domain}) data = self.connection.request(path).object zone = self._to_zone(elem=data) + return zone def ex_force_slave_axfr(self, zone): @@ -272,6 +287,7 @@ def ex_force_slave_axfr(self, zone): self.connection.set_context({"resource": "zone", "id": zone.id}) response = self.connection.request(path, method="POST") assert response.status == httplib.ACCEPTED + return zone def _to_zone_elem(self, domain=None, type=None, ttl=None, extra=None): @@ -299,6 +315,7 @@ def _to_zone_elem(self, domain=None, type=None, ttl=None, extra=None): elif type == "std_master": # TODO: Each driver should provide supported zone types # Slave name servers are elsewhere + if not extra or "slave-nameservers" not in extra: raise LibcloudError( "slave-nameservers extra " @@ -390,6 +407,7 @@ def _to_zone(self, elem): "tags": tags, } zone = Zone(id=str(id), domain=domain, type=type, ttl=int(ttl), driver=self, extra=extra) + return zone def _to_records(self, elem, zone): @@ -438,6 +456,7 @@ def _to_record(self, elem, zone): ttl=ttl, extra=extra, ) + return record def _get_more(self, rtype, **kwargs): @@ -467,6 +486,8 @@ def _get_data(self, rtype, last_key, **kwargs): self.connection.set_context({"resource": "zone", "id": zone.id}) response = self.connection.request(path, params=params) transform_func = self._to_records + else: + raise ValueError(f"Unsupported rtype: {rtype}") exhausted = False result_count = int(response.headers.get("x-query-count", 0)) @@ -476,6 +497,7 @@ def _get_data(self, rtype, last_key, **kwargs): if response.status == httplib.OK: items = transform_func(elem=response.object, **kwargs) + return items, params["page"], exhausted else: return [], None, True diff --git a/libcloud/storage/drivers/atmos.py b/libcloud/storage/drivers/atmos.py index cf7d771cee..c9d29d326e 100644 --- a/libcloud/storage/drivers/atmos.py +++ b/libcloud/storage/drivers/atmos.py @@ -72,6 +72,7 @@ def add_default_headers(self, headers): if "Content-Type" not in headers: headers["Content-Type"] = "application/octet-stream" + if "Accept" not in headers: headers["Accept"] = "*/*" @@ -85,8 +86,10 @@ def pre_connect_hook(self, params, headers): def _calculate_signature(self, params, headers): pathstring = urlunquote(self.action) driver_path = self.driver.path # pylint: disable=no-member + if pathstring.startswith(driver_path): pathstring = pathstring[len(driver_path) :] + if params: if type(params) is dict: params = list(params.items()) @@ -107,6 +110,7 @@ def _calculate_signature(self, params, headers): signature = "\n".join(signature) key = base64.b64decode(self.key) signature = hmac.new(b(key), b(signature), hashlib.sha1).digest() + return base64.b64encode(b(signature)).decode("utf-8") @@ -129,6 +133,7 @@ def __init__(self, key, secret=None, secure=True, host=None, port=None): def iterate_containers(self): result = self.connection.request(self._namespace_path("")) entries = self._list_objects(result.object, object_type="directory") + for entry in entries: extra = {"object_id": entry["id"]} yield Container(entry["name"], extra, self) @@ -143,6 +148,7 @@ def get_container(self, container_name): raise ContainerDoesNotExistError(e, self, container_name) meta = self._emc_meta(result) extra = {"object_id": meta["objectid"]} + return Container(container_name, extra, self) def create_container(self, container_name): @@ -153,6 +159,7 @@ def create_container(self, container_name): if e.code != 1016: raise raise ContainerAlreadyExistsError(e, self, container_name) + return self.get_container(container_name) def delete_container(self, container): @@ -163,6 +170,7 @@ def delete_container(self, container): raise ContainerDoesNotExistError(e, self, container.name) elif e.code == 1023: raise ContainerIsNotEmptyError(e, self, container.name) + return True def get_object(self, container_name, object_name): @@ -185,6 +193,7 @@ def get_object(self, container_name, object_name): last_modified = time.strftime("%a, %d %b %Y %H:%M:%S GMT", last_modified) extra = {"object_id": system_meta["objectid"], "last_modified": last_modified} data_hash = user_meta.pop("md5", "") + return Object( object_name, int(system_meta["size"]), @@ -263,7 +272,7 @@ def upload_object_via_stream(self, iterator, container, object_name, extra=None, iterator = iter(iterator) extra_headers = headers or {} - data_hash = hashlib.md5() + data_hash = hashlib.md5() # nosec generator = read_in_chunks(iterator, CHUNK_SIZE, True) bytes_transferred = 0 try: @@ -295,7 +304,7 @@ def upload_object_via_stream(self, iterator, container, object_name, extra=None, headers.update( { - "x-emc-meta": "md5=" + data_hash.hexdigest(), + "x-emc-meta": "md5=" + data_hash.hexdigest(), # nosec "Content-Type": content_type, } ) @@ -311,10 +320,11 @@ def upload_object_via_stream(self, iterator, container, object_name, extra=None, chunk = next(generator) except StopIteration: break + if len(chunk) == 0: break - data_hash = data_hash.hexdigest() + data_hash = data_hash.hexdigest() # nosec if extra is None: meta_data = {} @@ -376,6 +386,7 @@ def delete_object(self, obj): if e.code != 1003: raise raise ObjectDoesNotExistError(e, self, obj.name) + return True def enable_object_cdn(self, obj): @@ -396,6 +407,7 @@ def get_object_cdn_url(self, obj, expiry=None, use_object=False): :rtype: ``str`` """ + if use_object: path = "/rest/objects" + obj.meta_data["object_id"] else: @@ -415,6 +427,7 @@ def get_object_cdn_url(self, obj, expiry=None, use_object=False): params = urlencode(params) path = self.path + path + return urlparse.urlunparse((protocol, self.host, path, "", params, "")) def _cdn_signature(self, path, params, expiry): @@ -427,8 +440,10 @@ def _cdn_signature(self, path, params, expiry): def _list_objects(self, tree, object_type=None): listing = tree.find(self._emc_tag("DirectoryList")) entries = [] + for entry in listing.findall(self._emc_tag("DirectoryEntry")): file_type = entry.find(self._emc_tag("FileType")).text + if object_type is not None and object_type != file_type: continue entries.append( @@ -438,6 +453,7 @@ def _list_objects(self, tree, object_type=None): "name": entry.find(self._emc_tag("Filename")).text, } ) + return entries def _clean_object_name(self, name): @@ -455,9 +471,11 @@ def _emc_tag(tag): def _emc_meta(self, response): meta = response.headers.get("x-emc-meta", "") + if len(meta) == 0: return {} meta = meta.split(", ") + return dict([x.split("=", 1) for x in meta]) def _entries_to_objects(self, container, entries): @@ -490,4 +508,5 @@ def iterate_container_objects(self, container, prefix=None, ex_prefix=None): result = self.connection.request(path, headers=headers) entries = self._list_objects(result.object, object_type="regular") objects = self._entries_to_objects(container, entries) + return self._filter_listed_container_objects(objects, prefix) diff --git a/libcloud/storage/drivers/backblaze_b2.py b/libcloud/storage/drivers/backblaze_b2.py index 21690bf505..37ee54336a 100644 --- a/libcloud/storage/drivers/backblaze_b2.py +++ b/libcloud/storage/drivers/backblaze_b2.py @@ -81,6 +81,7 @@ def authenticate(self, force=False): token. :type force: ``bool`` """ + if not self._is_authentication_needed(force=force): return self @@ -147,6 +148,7 @@ def download_request(self, action, params=None): response = self._request( auth_conn=auth_conn, action=action, params=params, method=method, raw=raw ) + return response def upload_request(self, action, headers, upload_host, auth_token, data): @@ -168,6 +170,7 @@ def upload_request(self, action, headers, upload_host, auth_token, data): raw=raw, auth_token=auth_token, ) + return response def request( @@ -190,10 +193,12 @@ def request( self._set_host(host=auth_conn.api_host) # Include Content-Type + if not raw and data: headers["Content-Type"] = "application/json" # Include account id + if include_account_id: if method == "GET": params["accountId"] = auth_conn.account_id @@ -202,6 +207,7 @@ def request( data["accountId"] = auth_conn.account_id action = API_PATH + action + if data: data = json.dumps(data) @@ -214,6 +220,7 @@ def request( headers=headers, raw=raw, ) + return response def _request( @@ -244,6 +251,7 @@ def _request( headers=headers, raw=raw, ) + return response def _set_host(self, host): @@ -272,6 +280,7 @@ def iterate_containers(self): action="b2_list_buckets", method="GET", include_account_id=True ) containers = self._to_containers(data=resp.object) + return containers def iterate_container_objects(self, container, prefix=None, ex_prefix=None): @@ -298,11 +307,13 @@ def iterate_container_objects(self, container, prefix=None, ex_prefix=None): params = {"bucketId": container.extra["id"]} resp = self.connection.request(action="b2_list_file_names", method="GET", params=params) objects = self._to_objects(data=resp.object, container=container) + return self._filter_listed_container_objects(objects, prefix) def get_container(self, container_name): containers = self.iterate_containers() container = next((c for c in containers if c.name == container_name), None) + if container: return container else: @@ -328,6 +339,7 @@ def create_container(self, container_name, ex_type="allPrivate"): action="b2_create_bucket", data=data, method="POST", include_account_id=True ) container = self._to_container(item=resp.object) + return container def delete_container(self, container): @@ -337,6 +349,7 @@ def delete_container(self, container): resp = self.connection.request( action="b2_delete_bucket", data=data, method="POST", include_account_id=True ) + return resp.status == httplib.OK def download_object( @@ -347,6 +360,7 @@ def download_object( response = self.connection.download_request(action=action) # TODO: Include metadata from response headers + return self._get_object( obj=obj, callback=self._save_object, @@ -438,6 +452,7 @@ def delete_object(self, obj): data["fileName"] = obj.name data["fileId"] = obj.extra["fileId"] resp = self.connection.request(action="b2_delete_file_version", data=data, method="POST") + return resp.status == httplib.OK def ex_get_object(self, object_id): @@ -445,6 +460,7 @@ def ex_get_object(self, object_id): params["fileId"] = object_id resp = self.connection.request(action="b2_get_file_info", method="GET", params=params) obj = self._to_object(item=resp.object, container=None) + return obj def ex_hide_object(self, container_id, object_name): @@ -453,6 +469,7 @@ def ex_hide_object(self, container_id, object_name): data["fileName"] = object_name resp = self.connection.request(action="b2_hide_file", data=data, method="POST") obj = self._to_object(item=resp.object, container=None) + return obj def ex_list_object_versions( @@ -476,6 +493,7 @@ def ex_list_object_versions( resp = self.connection.request(action="b2_list_file_versions", params=params, method="GET") objects = self._to_objects(data=resp.object, container=None) + return objects def ex_get_upload_data(self, container_id): @@ -489,6 +507,7 @@ def ex_get_upload_data(self, container_id): params = {} params["bucketId"] = container_id response = self.connection.request(action="b2_get_upload_url", method="GET", params=params) + return response.object def ex_get_upload_url(self, container_id): @@ -499,10 +518,12 @@ def ex_get_upload_url(self, container_id): """ result = self.ex_get_upload_data(container_id=container_id) upload_url = result["uploadUrl"] + return upload_url def _to_containers(self, data): result = [] + for item in data["buckets"]: container = self._to_container(item=item) result.append(container) @@ -514,10 +535,12 @@ def _to_container(self, item): extra["id"] = item["bucketId"] extra["bucketType"] = item["bucketType"] container = Container(name=item["bucketName"], extra=extra, driver=self) + return container def _to_objects(self, data, container): result = [] + for item in data["files"]: obj = self._to_object(item=item, container=container) result.append(obj) @@ -540,6 +563,7 @@ def _to_object(self, item, container=None): container=container, driver=self, ) + return obj def _get_object_download_path(self, container, obj): @@ -549,6 +573,7 @@ def _get_object_download_path(self, container, obj): :rtype: ``str`` """ path = container.name + "/" + obj.name + return path def _perform_upload( @@ -569,11 +594,12 @@ def _perform_upload( headers["X-Bz-File-Name"] = object_name headers["Content-Type"] = content_type - sha1 = hashlib.sha1() + sha1 = hashlib.sha1() # nosec sha1.update(b(data)) headers["X-Bz-Content-Sha1"] = sha1.hexdigest() # Include optional meta-data (up to 10 items) + for key, value in meta_data.items(): # TODO: Encode / escape key headers["X-Bz-Info-%s" % (key)] = value @@ -596,6 +622,7 @@ def _perform_upload( if response.status == httplib.OK: obj = self._to_object(item=response.object, container=container) + return obj else: body = response.response.read() diff --git a/libcloud/storage/drivers/dummy.py b/libcloud/storage/drivers/dummy.py index 695b42993e..34ecb08a25 100644 --- a/libcloud/storage/drivers/dummy.py +++ b/libcloud/storage/drivers/dummy.py @@ -43,6 +43,7 @@ def read(self, size): def _get_chunk(self, chunk_len): chunk = [str(x) for x in random.randint(97, 120)] + return chunk def __len__(self): @@ -51,12 +52,12 @@ def __len__(self): class DummyIterator: def __init__(self, data=None): - self.hash = hashlib.md5() + self.hash = hashlib.md5() # nosec self._data = data or [] self._current_item = 0 def get_md5_hash(self): - return self.hash.hexdigest() + return self.hash.hexdigest() # nosec def next(self): if self._current_item == len(self._data): @@ -65,6 +66,7 @@ def next(self): value = self._data[self._current_item] self.hash.update(b(value)) self._current_item += 1 + return value def __next__(self): @@ -137,8 +139,10 @@ def get_meta_data(self): ) bytes_used = 0 + for container in self._containers: objects = self._containers[container]["objects"] + for _, obj in objects.items(): bytes_used += obj.size @@ -183,6 +187,7 @@ def iterate_container_objects(self, container, prefix=None, ex_prefix=None): container = self.get_container(container.name) objects = self._containers[container.name]["objects"].values() + return self._filter_listed_container_objects(objects, prefix) def get_container(self, container_name): @@ -258,6 +263,7 @@ def get_object(self, container_name, object_name): self.get_container(container_name) container_objects = self._containers[container_name]["objects"] + if object_name not in container_objects: raise ObjectDoesNotExistError(object_name=object_name, value=None, driver=self) @@ -283,6 +289,7 @@ def get_object_cdn_url(self, obj): container_name = obj.container.name container_objects = self._containers[container_name]["objects"] + if obj.name not in container_objects: raise ObjectDoesNotExistError(object_name=obj.name, value=None, driver=self) @@ -317,6 +324,7 @@ def create_container(self, container_name): "objects": {}, "cdn_url": "http://www.test.com/container/%s" % (container_name.replace(" ", "_")), } + return container def delete_container(self, container): @@ -352,14 +360,17 @@ def delete_container(self, container): """ container_name = container.name + if container_name not in self._containers: raise ContainerDoesNotExistError(container_name=container_name, value=None, driver=self) container = self._containers[container_name] + if len(container["objects"]) > 0: raise ContainerIsNotEmptyError(container_name=container_name, value=None, driver=self) del self._containers[container_name] + return True def download_object( @@ -425,6 +436,7 @@ def upload_object( raise LibcloudError(value="File %s does not exist" % (file_path), driver=self) size = os.path.getsize(file_path) + return self._add_object( container=container, object_name=object_name, size=size, extra=extra ) @@ -445,6 +457,7 @@ def upload_object_via_stream(self, iterator, container, object_name, extra=None, """ size = len(iterator) + return self._add_object( container=container, object_name=object_name, size=size, extra=extra ) @@ -476,6 +489,7 @@ def delete_object(self, obj): obj = self.get_object(container_name=container_name, object_name=object_name) del self._containers[container_name]["objects"][object_name] + return True def _add_object(self, container, object_name, size, extra=None): @@ -497,6 +511,7 @@ def _add_object(self, container, object_name, size, extra=None): ) self._containers[container.name]["objects"][object_name] = obj + return obj diff --git a/libcloud/test/compute/test_ssh_client.py b/libcloud/test/compute/test_ssh_client.py index c64e0b173b..9a70cce92e 100644 --- a/libcloud/test/compute/test_ssh_client.py +++ b/libcloud/test/compute/test_ssh_client.py @@ -565,7 +565,7 @@ def test_consume_stdout_chunk_contains_part_of_multi_byte_utf8_character(self): chan = Mock() chan.recv_ready.side_effect = [True, True, True, True, False] - chan.recv.side_effect = ["\xF0", "\x90", "\x8D", "\x88"] + chan.recv.side_effect = ["\xf0", "\x90", "\x8d", "\x88"] stdout = client._consume_stdout(chan).getvalue() self.assertEqual("ð\x90\x8d\x88", stdout) @@ -578,7 +578,7 @@ def test_consume_stderr_chunk_contains_part_of_multi_byte_utf8_character(self): chan = Mock() chan.recv_stderr_ready.side_effect = [True, True, True, True, False] - chan.recv_stderr.side_effect = ["\xF0", "\x90", "\x8D", "\x88"] + chan.recv_stderr.side_effect = ["\xf0", "\x90", "\x8d", "\x88"] stderr = client._consume_stderr(chan).getvalue() self.assertEqual("ð\x90\x8d\x88", stderr) diff --git a/libcloud/test/storage/test_base.py b/libcloud/test/storage/test_base.py index e9dc72c840..ced1f9de74 100644 --- a/libcloud/test/storage/test_base.py +++ b/libcloud/test/storage/test_base.py @@ -30,10 +30,12 @@ class BaseMockRawResponse(MockHttp): def _(self, method, url, body, headers): body = "ab" + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) def root(self, method, url, body, headers): body = "ab" + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) @@ -161,7 +163,7 @@ def test_upload_object_hash_calculation_is_efficient( object_name="test1", content_type=None, request_path="/", stream=iterator ) - hasher = hashlib.md5() + hasher = hashlib.md5() # nosec hasher.update(b("a") * size) expected_hash = hasher.hexdigest() @@ -196,7 +198,7 @@ def test_upload_object_hash_calculation_is_efficient( object_name="test2", content_type=None, request_path="/", stream=iterator ) - hasher = hashlib.md5() + hasher = hashlib.md5() # nosec hasher.update(b("b") * size) expected_hash = hasher.hexdigest() @@ -226,7 +228,7 @@ def test_upload_object_via_stream_illegal_seek_errors_are_ignored(self): object_name="test1", content_type=None, request_path="/", stream=iterator ) - hasher = hashlib.md5() + hasher = hashlib.md5() # nosec hasher.update(b("a") * size) expected_hash = hasher.hexdigest() @@ -284,6 +286,7 @@ class SecondException(Exception): def raise_on_second(*_, **__): nonlocal count count += 1 + if count > 1: raise SecondException() else: @@ -307,9 +310,11 @@ def test_should_retry_rate_limited_errors_until_success(self): def succeed_on_second(*_, **__) -> mock.MagicMock: nonlocal count count += 1 + if count > 1: successful_response = mock.MagicMock() successful_response.status_code = 200 + return successful_response else: raise RateLimitReachedError() diff --git a/libcloud/test/storage/test_cloudfiles.py b/libcloud/test/storage/test_cloudfiles.py index df4a73e5de..05db499e0c 100644 --- a/libcloud/test/storage/test_cloudfiles.py +++ b/libcloud/test/storage/test_cloudfiles.py @@ -524,6 +524,7 @@ def upload_file( def func(*args, **kwargs): self.assertEqual(kwargs["headers"]["Content-Length"], 0) func.called = True + return old_request(*args, **kwargs) self.driver.connection.request = func @@ -880,6 +881,7 @@ def _v1_MossoCloudFS_py3_img_or_vid2(self, method, url, body, headers): headers = {"etag": "d79fb00c27b50494a463e680d459c90c"} headers.update(self.base_headers) _201 = httplib.CREATED + return _201, "", headers, httplib.responses[_201] self.driver_klass.connectionCls.rawResponseCls = InterceptResponse @@ -1068,11 +1070,13 @@ class CloudFilesMockHttp(BaseRangeDownloadMockHttp, unittest.TestCase): def _v2_0_tokens(self, method, url, body, headers): headers = copy.deepcopy(self.base_headers) body = self.fixtures.load("_v2_0__auth.json") + return (httplib.OK, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_MALFORMED_JSON(self, method, url, body, headers): # test_invalid_json_throws_exception body = 'broken: json /*"' + return ( httplib.NO_CONTENT, body, @@ -1090,6 +1094,7 @@ def _v1_MossoCloudFS_EMPTY(self, method, url, body, headers): def _v1_MossoCloudFS(self, method, url, body, headers): headers = copy.deepcopy(self.base_headers) + if method == "GET": # list_containers body = self.fixtures.load("list_containers.json") @@ -1108,10 +1113,12 @@ def _v1_MossoCloudFS(self, method, url, body, headers): elif method == "POST": body = "" status_code = httplib.NO_CONTENT + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_not_found(self, method, url, body, headers): # test_get_object_not_found + if method == "HEAD": body = "" else: @@ -1126,16 +1133,20 @@ def _v1_MossoCloudFS_not_found(self, method, url, body, headers): def _v1_MossoCloudFS_test_container_EMPTY(self, method, url, body, headers): body = self.fixtures.load("list_container_objects_empty.json") + return (httplib.OK, body, self.base_headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_test_20container_201_EMPTY(self, method, url, body, headers): body = self.fixtures.load("list_container_objects_empty.json") + return (httplib.OK, body, self.base_headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_test_container(self, method, url, body, headers): headers = copy.deepcopy(self.base_headers) + if method == "GET": # list_container_objects + if url.find("marker") == -1: body = self.fixtures.load("list_container_objects.json") status_code = httplib.OK @@ -1147,11 +1158,13 @@ def _v1_MossoCloudFS_test_container(self, method, url, body, headers): body = self.fixtures.load("list_container_objects_empty.json") status_code = httplib.NO_CONTENT headers.update({"x-container-object-count": "800", "x-container-bytes-used": "1234568"}) + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_test_container_ITERATOR(self, method, url, body, headers): headers = copy.deepcopy(self.base_headers) # list_container_objects + if url.find("foo-test-3") != -1: body = self.fixtures.load("list_container_objects_not_exhausted2.json") status_code = httplib.OK @@ -1167,6 +1180,7 @@ def _v1_MossoCloudFS_test_container_ITERATOR(self, method, url, body, headers): def _v1_MossoCloudFS_test_container_not_found(self, method, url, body, headers): # test_get_container_not_found + if method == "HEAD": body = "" else: @@ -1181,6 +1195,7 @@ def _v1_MossoCloudFS_test_container_not_found(self, method, url, body, headers): def _v1_MossoCloudFS_test_container_test_object(self, method, url, body, headers): headers = copy.deepcopy(self.base_headers) + if method == "HEAD": # get_object body = self.fixtures.load("list_container_objects_empty.json") @@ -1195,10 +1210,12 @@ def _v1_MossoCloudFS_test_container_test_object(self, method, url, body, headers "content-type": "application/zip", } ) + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_test_container__7E_test_object(self, method, url, body, headers): headers = copy.deepcopy(self.base_headers) + if method == "HEAD": # get_object_name_encoding body = self.fixtures.load("list_container_objects_empty.json") @@ -1213,6 +1230,7 @@ def _v1_MossoCloudFS_test_container__7E_test_object(self, method, url, body, hea "content-type": "application/zip", } ) + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_test_create_container(self, method, url, body, headers): @@ -1222,6 +1240,7 @@ def _v1_MossoCloudFS_test_create_container(self, method, url, body, headers): headers = copy.deepcopy(self.base_headers) headers.update({"content-length": "18", "date": "Mon, 28 Feb 2011 07:52:57 GMT"}) status_code = httplib.CREATED + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_speci_40l_name(self, method, url, body, headers): @@ -1236,6 +1255,7 @@ def _v1_MossoCloudFS_speci_40l_name(self, method, url, body, headers): headers = copy.deepcopy(self.base_headers) headers.update({"content-length": "18", "date": "Mon, 28 Feb 2011 07:52:57 GMT"}) status_code = httplib.CREATED + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_test_create_container_ALREADY_EXISTS(self, method, url, body, headers): @@ -1244,6 +1264,7 @@ def _v1_MossoCloudFS_test_create_container_ALREADY_EXISTS(self, method, url, bod body = self.fixtures.load("list_container_objects_empty.json") headers.update({"content-type": "text/plain"}) status_code = httplib.ACCEPTED + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container(self, method, url, body, headers): @@ -1257,6 +1278,7 @@ def _v1_MossoCloudFS_foo_bar_container(self, method, url, body, headers): body = "" headers = self.base_headers status_code = httplib.ACCEPTED + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_object_PURGE_SUCCESS(self, method, url, body, headers): @@ -1264,6 +1286,7 @@ def _v1_MossoCloudFS_foo_bar_container_object_PURGE_SUCCESS(self, method, url, b # test_ex_purge_from_cdn headers = self.base_headers status_code = httplib.NO_CONTENT + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_object_PURGE_SUCCESS_EMAIL( @@ -1274,6 +1297,7 @@ def _v1_MossoCloudFS_foo_bar_container_object_PURGE_SUCCESS_EMAIL( self.assertEqual(headers["X-Purge-Email"], "test@test.com") headers = self.base_headers status_code = httplib.NO_CONTENT + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_NOT_FOUND(self, method, url, body, headers): @@ -1282,6 +1306,7 @@ def _v1_MossoCloudFS_foo_bar_container_NOT_FOUND(self, method, url, body, header body = self.fixtures.load("list_container_objects_empty.json") headers = self.base_headers status_code = httplib.NOT_FOUND + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_NOT_EMPTY(self, method, url, body, headers): @@ -1290,6 +1315,7 @@ def _v1_MossoCloudFS_foo_bar_container_NOT_EMPTY(self, method, url, body, header body = self.fixtures.load("list_container_objects_empty.json") headers = self.base_headers status_code = httplib.CONFLICT + return (status_code, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_bar_object(self, method, url, body, headers): @@ -1298,9 +1324,11 @@ def _v1_MossoCloudFS_foo_bar_container_foo_bar_object(self, method, url, body, h body = self.fixtures.load("list_container_objects_empty.json") headers = self.base_headers status_code = httplib.NO_CONTENT + return (status_code, body, headers, httplib.responses[httplib.OK]) elif method == "GET": body = generate_random_data(1000) + return (httplib.OK, body, self.base_headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_range(self, method, url, body, headers): @@ -1325,6 +1353,7 @@ def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_range(self, method, url, b def _v1_MossoCloudFS_py3_img_or_vid(self, method, url, body, headers): headers = {"etag": "e2378cace8712661ce7beec3d9362ef6"} headers.update(self.base_headers) + return httplib.CREATED, "", headers, httplib.responses[httplib.CREATED] def _v1_MossoCloudFS_foo_bar_container_foo_test_upload(self, method, url, body, headers): @@ -1334,6 +1363,7 @@ def _v1_MossoCloudFS_foo_bar_container_foo_test_upload(self, method, url, body, headers = {} headers.update(self.base_headers) headers["etag"] = "hash343hhash89h932439jsaa89" + return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_speci_40l_name_m_40obj_E2_82_ACct(self, method, url, body, headers): @@ -1345,6 +1375,7 @@ def _v1_MossoCloudFS_speci_40l_name_m_40obj_E2_82_ACct(self, method, url, body, headers = copy.deepcopy(self.base_headers) body = "" headers["etag"] = "hash343hhash89h932439jsaa89" + return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_empty(self, method, url, body, headers): @@ -1353,6 +1384,7 @@ def _v1_MossoCloudFS_foo_bar_container_empty(self, method, url, body, headers): headers = {} headers.update(self.base_headers) headers["etag"] = "hash343hhash89h932439jsaa89" + return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_test_upload_INVALID_HASH( @@ -1363,6 +1395,7 @@ def _v1_MossoCloudFS_foo_bar_container_foo_test_upload_INVALID_HASH( headers = {} headers.update(self.base_headers) headers["etag"] = "foobar" + return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_INVALID_SIZE( @@ -1370,12 +1403,14 @@ def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_INVALID_SIZE( ): # test_download_object_invalid_file_size body = generate_random_data(100) + return (httplib.OK, body, self.base_headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_NOT_FOUND( self, method, url, body, headers ): body = "" + return ( httplib.NOT_FOUND, body, @@ -1385,7 +1420,7 @@ def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_NOT_FOUND( def _v1_MossoCloudFS_foo_bar_container_foo_test_stream_data(self, method, url, body, headers): # test_upload_object_via_stream_success - hasher = hashlib.md5() + hasher = hashlib.md5() # nosec hasher.update(b"235") hash_value = hasher.hexdigest() @@ -1393,13 +1428,14 @@ def _v1_MossoCloudFS_foo_bar_container_foo_test_stream_data(self, method, url, b headers.update(self.base_headers) headers["etag"] = hash_value body = "test" + return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_test_stream_data_seek( self, method, url, body, headers ): # test_upload_object_via_stream_stream_seek_at_end - hasher = hashlib.md5() + hasher = hashlib.md5() # nosec hasher.update(b"123456789") hash_value = hasher.hexdigest() @@ -1407,6 +1443,7 @@ def _v1_MossoCloudFS_foo_bar_container_foo_test_stream_data_seek( headers.update(self.base_headers) headers["etag"] = hash_value body = "test" + return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_NO_BUFFER( @@ -1417,6 +1454,7 @@ def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_NO_BUFFER( headers.update(self.base_headers) headers["etag"] = "577ef1154f3240ad5b9b413aa7346a1e" body = generate_random_data(1000) + return (httplib.OK, body, headers, httplib.responses[httplib.OK]) diff --git a/libcloud/utils/publickey.py b/libcloud/utils/publickey.py index e882c430fb..c3a479c75d 100644 --- a/libcloud/utils/publickey.py +++ b/libcloud/utils/publickey.py @@ -33,12 +33,14 @@ def _to_md5_fingerprint(data): - hashed = hashlib.md5(data).digest() + hashed = hashlib.md5(data).digest() # nosec + return ":".join(hexadigits(hashed)) def get_pubkey_openssh_fingerprint(pubkey): # We import and export the key to make sure it is in OpenSSH format + if not cryptography_available: raise RuntimeError("cryptography is not available") public_key = serialization.load_ssh_public_key(b(pubkey), backend=default_backend()) @@ -48,12 +50,14 @@ def get_pubkey_openssh_fingerprint(pubkey): )[ 7: ] # strip ssh-rsa prefix + return _to_md5_fingerprint(base64_decode_string(pub_openssh)) def get_pubkey_ssh2_fingerprint(pubkey): # This is the format that EC2 shows for public key fingerprints in its # KeyPair mgmt API + if not cryptography_available: raise RuntimeError("cryptography is not available") public_key = serialization.load_ssh_public_key(b(pubkey), backend=default_backend()) @@ -61,13 +65,16 @@ def get_pubkey_ssh2_fingerprint(pubkey): encoding=serialization.Encoding.DER, format=serialization.PublicFormat.SubjectPublicKeyInfo, ) + return _to_md5_fingerprint(pub_der) def get_pubkey_comment(pubkey, default=None): if pubkey.startswith("ssh-"): # This is probably an OpenSSH key + return pubkey.strip().split(" ", 3)[2] + if default: return default raise ValueError("Public key is not in a supported format") diff --git a/pyproject.toml b/pyproject.toml index 7b63889b17..d4e8467f01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,11 +42,11 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", @@ -107,7 +107,7 @@ universal = true [tool.black] line_length = 100 -target_version = ['py37', 'py38', 'py39', 'py310', 'py311'] +target_version = ['py39', 'py310', 'py311', 'py312', 'py313'] include = '\.pyi?$' exclude = ''' ( diff --git a/requirements-docs.txt b/requirements-docs.txt index 55b62dcae6..6c6829a84c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,4 @@ -rstcheck==6.2.1; python_version >= '3.7' +rstcheck==6.2.4; python_version >= '3.7' fasteners sphinx_rtd_theme==2.0.0 sphinx==6.2.1 diff --git a/requirements-lint.txt b/requirements-lint.txt index 5110684050..a273f6be4b 100644 --- a/requirements-lint.txt +++ b/requirements-lint.txt @@ -1,12 +1,12 @@ pep8==1.7.1 flake8==5.0.4 -astroid==3.1.0; python_version >= '3.8' -pylint==3.1.0; python_version >= '3.8' +astroid==3.3.8; python_version >= '3.8' +pylint==3.3.4; python_version >= '3.8' bandit[toml]==1.7.8; python_version >= '3.7' -black==24.4.0; python_version >= '3.7' and implementation_name == "cpython" -isort[pyproject]==5.12.0; python_version >= '3.8' +black==25.1.0; python_version >= '3.7' and implementation_name == "cpython" +isort[pyproject]==6.0.1; python_version >= '3.8' pyupgrade==3.3.1 -rstcheck==6.2.1; python_version >= '3.7' +rstcheck==6.2.4; python_version >= '3.7' codespell==2.2.5 requests>=2.27.1 diff --git a/requirements-mypy.txt b/requirements-mypy.txt index 4259287993..c101179bae 100644 --- a/requirements-mypy.txt +++ b/requirements-mypy.txt @@ -1,6 +1,6 @@ typing # Mypy requires typed-ast, which is broken on PyPy 3.7 (could work in PyPy 3.8). -mypy==1.10.0; implementation_name == "cpython" +mypy==1.15.0; implementation_name == "cpython" types-simplejson types-certifi types-requests diff --git a/requirements-tests.txt b/requirements-tests.txt index ef45f4af6a..a1efcf8b2c 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,7 +5,7 @@ pytest==8.1.1 pytest-xdist==3.5.0 pytest-timeout==2.2.0 pytest-benchmark[histogram]==4.0.0 -cryptography==44.0.0 +cryptography==44.0.2 # NOTE: Only needed by nttcis loadbalancer driver # We need to use >= 25.0.0 to be compatible with cryptography >= 43 diff --git a/scripts/time_imports.sh b/scripts/time_imports.sh index a1002d52a0..314ae8aa93 100755 --- a/scripts/time_imports.sh +++ b/scripts/time_imports.sh @@ -26,7 +26,7 @@ find . -name "*.pyc" -print0 | xargs -0 rm # Example line: # import time: 1112 | 70127 | libcloud -LIBCLOUD_IMPORT_TIMINGS=$(python3.8 -X importtime -c "import libcloud" 2>&1) +LIBCLOUD_IMPORT_TIMINGS=$(python3.9 -X importtime -c "import libcloud" 2>&1) LIBCLOUD_IMPORT_TIME_CUMULATIVE_US=$(echo -e "${LIBCLOUD_IMPORT_TIMINGS}" | tail -1 | grep "| libcloud" | awk '{print $5}') echo "Import timings for \"libcloud\" module" @@ -40,7 +40,7 @@ fi # Clean up any cached files to ensure consistent and clean environment find . -name "*.pyc" -print0 | xargs -0 rm -EC2_DRIVER_IMPORT_TIMINGS=$(python3.8 -X importtime -c "import libcloud.compute.drivers.ec2" 2>&1) +EC2_DRIVER_IMPORT_TIMINGS=$(python3.9 -X importtime -c "import libcloud.compute.drivers.ec2" 2>&1) EC2_DRIVER_IMPORT_TIME_CUMULATIVE_US=$(echo -e "$EC2_DRIVER_IMPORT_TIMINGS}" | tail -1 | grep "| libcloud.compute.drivers.ec2" | awk '{print $5}') echo "" diff --git a/tox.ini b/tox.ini index 64f6bbb1da..7c2ebbb491 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{pypy3,3.8,3.9,3.10,3.11,pyjion},checks,lint,pylint,pyupgrade,isort,black,mypy,docs,coverage,integration-storage +envlist = py{pypy3,3.9,3.10,3.11,pyjion},checks,lint,pylint,pyupgrade,isort,black,mypy,docs,coverage,integration-storage skipsdist = true requires = wheel @@ -21,16 +21,18 @@ allowlist_externals = /bin/bash scripts/*.sh basepython = - py3.12-dev: python3.12 pypypy3: pypy3 - pypypy-3.8: pypy3.8 + pypypy3.9: pypy3.9 + pypypy-3.9: pypy3.9 + pypypy3.10: pypy3.10 + pypypy-3.10: pypy3.10 pypyjion: pyjion - {docs,checks,black,black-check,lint,pylint,bandit,mypy,micro-benchmarks,coverage,import-timings,isort,isort-check,pyupgrade}: python3.8 - {py3.8,py3.8-windows,integration-storage,py3.8-dist,py3.8-dist-wheel}: python3.8 - {py3.9,py3.9-dist,py3.9-dist-wheel}: python3.9 + {docs,checks,black,black-check,lint,pylint,bandit,mypy,micro-benchmarks,coverage,import-timings,isort,isort-check,pyupgrade}: python3.9 + {py3.9,py3.9-dist,py3.9-dist-wheel,py3.9-windows,integration-storage}: python3.9 {py3.10,py3.10-dist,py3.10-dist-wheel}: python3.10 {py3.11,py3.11-dist,py3.11-dist-wheel}: python3.11 - {py3.12-dev,py3.12-dev-dist,py3.12-dev-dist-wheel}: python3.12 + {py3.12,py3.12-dist,py3.12-dist-wheel}: python3.12 + {py3.13-dev,py3.13-dev-dist,py3.13-dev-dist-wheel}: python3.13 setenv = CRYPTOGRAPHY_ALLOW_OPENSSL_102=1 # NOTE: By default we run tests on CI in parallel to speed up the build @@ -43,7 +45,7 @@ commands = cp libcloud/test/secrets.py-dist libcloud/test/secrets.py pytest --color=yes -rsx -vvv --capture=tee-sys -o log_cli=True --durations=10 --timeout=15 -n auto --dist loadfile --ignore libcloud/test/benchmarks/ --ignore-glob "*test_list_objects_filtering_performance*" -m "not serial" pytest --color=yes -rsx -vvv --capture=tee-sys -o log_cli=True --durations=10 --timeout=15 --ignore libcloud/test/benchmarks/ --ignore-glob "*test_list_objects_filtering_performance*" -m "serial" -[testenv:py3.8-dist] +[testenv:py3.9-dist] # Verify library installs without any dependencies when using python setup.py # install skipdist = True @@ -53,7 +55,7 @@ recreate = True deps = commands = bash -c "./scripts/dist_install_check.sh" -[testenv:py3.8-dist-wheel] +[testenv:py3.9-dist-wheel] # Verify library installs without any dependencies when using built wheel skipdist = True recreate = True @@ -62,7 +64,7 @@ recreate = True deps = commands = bash -c "./scripts/dist_wheel_install_check.sh" -[testenv:py3.9-dist] +[testenv:py3.10-dist] # Verify library installs without any dependencies when using python setup.py # install skipdist = True @@ -72,7 +74,7 @@ recreate = True deps = commands = bash -c "./scripts/dist_install_check.sh" -[testenv:py3.9-dist-wheel] +[testenv:py3.10-dist-wheel] # Verify library installs without any dependencies when using built wheel skipdist = True recreate = True @@ -81,7 +83,7 @@ recreate = True deps = commands = bash -c "./scripts/dist_wheel_install_check.sh" -[testenv:py3.10-dist] +[testenv:py3.11-dist] # Verify library installs without any dependencies when using python setup.py # install skipdist = True @@ -91,7 +93,7 @@ recreate = True deps = commands = bash -c "./scripts/dist_install_check.sh" -[testenv:py3.10-dist-wheel] +[testenv:py3.11-dist-wheel] # Verify library installs without any dependencies when using built wheel skipdist = True recreate = True @@ -100,7 +102,7 @@ recreate = True deps = commands = bash -c "./scripts/dist_wheel_install_check.sh" -[testenv:py3.11-dist] +[testenv:py3.12-dist] # Verify library installs without any dependencies when using python setup.py # install skipdist = True @@ -110,7 +112,7 @@ recreate = True deps = commands = bash -c "./scripts/dist_install_check.sh" -[testenv:py3.11-dist-wheel] +[testenv:py3.12-dist-wheel] # Verify library installs without any dependencies when using built wheel skipdist = True recreate = True @@ -119,7 +121,7 @@ recreate = True deps = commands = bash -c "./scripts/dist_wheel_install_check.sh" -[testenv:py3.12-dev-dist] +[testenv:py3.13-dist] # Verify library installs without any dependencies when using python setup.py # install skipdist = True @@ -129,7 +131,7 @@ recreate = True deps = commands = bash -c "./scripts/dist_install_check.sh" -[testenv:py3.12-dev-dist-wheel] +[testenv:py3.13-dist-wheel] # Verify library installs without any dependencies when using built wheel skipdist = True recreate = True @@ -150,11 +152,11 @@ commands = rstcheck --report-level warning ../README.rst sphinx-build -j auto -b html -d {envtmpdir}/doctrees . _build/html [testenv:provider-tables] -basepython: python3.8 +basepython: python3.9 commands = python ./contrib/generate_provider_feature_matrix_table.py [testenv:scrape-and-publish-provider-prices] -basepython: python3.8 +basepython: python3.9 # Needed to avoid urllib3 errors related to old openssl version # https://github.com/urllib3/urllib3/issues/2168 deps = urllib3==1.26.6 @@ -188,7 +190,7 @@ commands = echo "https://libcloud-pricing-data.s3.amazonaws.com/pricing.json.sha512" [testenv:scrape-provider-prices] -basepython: python3.8 +basepython: python3.9 # Needed to avoid urllib3 errors related to old openssl version # https://github.com/urllib3/urllib3/issues/2168 deps = urllib3==1.26.6 @@ -211,7 +213,7 @@ commands = bash -c "(cd libcloud/data/ ; sha512sum pricing.json > {toxinidir}/libcloud/data/pricing.json.sha512)" [testenv:scrape-ec2-prices] -basepython: python3.8 +basepython: python3.9 # Needed to avoid urllib3 errors related to old openssl version # https://github.com/urllib3/urllib3/issues/2168 deps = urllib3==1.26.6 @@ -222,7 +224,7 @@ deps = urllib3==1.26.6 commands = python contrib/scrape-ec2-prices.py [testenv:scrape-ec2-sizes] -basepython: python3.8 +basepython: python3.9 # Needed to avoid urllib3 errors related to old openssl version # https://github.com/urllib3/urllib3/issues/2168 deps = urllib3==1.26.6 @@ -365,13 +367,13 @@ commands = deps = -r{toxinidir}/requirements-lint.txt commands = - bash -c "find libcloud/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only" - bash -c "find contrib/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only" - bash -c "find demos/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only" - bash -c "find pylint_plugins/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only" - bash -c "find integration/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only" - bash -c "find scripts/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only" - bash -c "find docs/examples/ -name '*.py' -print0 | xargs -0 pyupgrade --py37-plus --py3-only" + bash -c "find libcloud/ -name '*.py' -print0 | xargs -0 pyupgrade --py39-plus --py3-only" + bash -c "find contrib/ -name '*.py' -print0 | xargs -0 pyupgrade --py39-plus --py3-only" + bash -c "find demos/ -name '*.py' -print0 | xargs -0 pyupgrade --py39-plus --py3-only" + bash -c "find pylint_plugins/ -name '*.py' -print0 | xargs -0 pyupgrade --py39-plus --py3-only" + bash -c "find integration/ -name '*.py' -print0 | xargs -0 pyupgrade --py39-plus --py3-only" + bash -c "find scripts/ -name '*.py' -print0 | xargs -0 pyupgrade --py39-plus --py3-only" + bash -c "find docs/examples/ -name '*.py' -print0 | xargs -0 pyupgrade --py39-plus --py3-only" isort {toxinidir}