diff --git a/src/coldfront_plugin_cloud/base.py b/src/coldfront_plugin_cloud/base.py index 00a4807d..aad8ff58 100644 --- a/src/coldfront_plugin_cloud/base.py +++ b/src/coldfront_plugin_cloud/base.py @@ -77,3 +77,7 @@ def assign_role_on_user(self, username, project_id): @abc.abstractmethod def remove_role_from_user(self, username, project_id): pass + + @abc.abstractmethod + def get_usage(self, project_id): + pass diff --git a/src/coldfront_plugin_cloud/management/commands/validate_allocations.py b/src/coldfront_plugin_cloud/management/commands/validate_allocations.py index 4c6e46eb..85d07e3a 100644 --- a/src/coldfront_plugin_cloud/management/commands/validate_allocations.py +++ b/src/coldfront_plugin_cloud/management/commands/validate_allocations.py @@ -1,5 +1,4 @@ import logging -import re from coldfront_plugin_cloud import attributes from coldfront_plugin_cloud import openstack @@ -7,7 +6,7 @@ from coldfront_plugin_cloud import utils from coldfront_plugin_cloud import tasks -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from coldfront.core.resource.models import Resource from coldfront.core.allocation.models import ( Allocation, @@ -242,51 +241,9 @@ def handle(self, *args, **options): expected_value = allocation.get_attribute(attr) current_value = quota.get(key, None) - - PATTERN = r"([0-9]+)(m|Ki|Mi|Gi|Ti|Pi|Ei|K|M|G|T|P|E)?" - - suffix = { - "Ki": 2**10, - "Mi": 2**20, - "Gi": 2**30, - "Ti": 2**40, - "Pi": 2**50, - "Ei": 2**60, - "m": 10**-3, - "K": 10**3, - "M": 10**6, - "G": 10**9, - "T": 10**12, - "P": 10**15, - "E": 10**18, - } - - if current_value and current_value != "0": - result = re.search(PATTERN, current_value) - - if result is None: - raise CommandError( - f"Unable to parse current_value = '{current_value}' for {attr}" - ) - - value = int(result.groups()[0]) - unit = result.groups()[1] - - # Convert to number i.e. without any unit suffix - - if unit is not None: - current_value = value * suffix[unit] - else: - current_value = value - - # Convert some attributes to units that coldfront uses - - if "RAM" in attr: - current_value = round(current_value / suffix["Mi"]) - elif "Storage" in attr: - current_value = round(current_value / suffix["Gi"]) - elif current_value and current_value == "0": - current_value = 0 + current_value = openshift.parse_openshift_quota_value( + attr, current_value + ) if expected_value is None and current_value is not None: msg = ( diff --git a/src/coldfront_plugin_cloud/openshift.py b/src/coldfront_plugin_cloud/openshift.py index 5bc910f4..9726b8a7 100644 --- a/src/coldfront_plugin_cloud/openshift.py +++ b/src/coldfront_plugin_cloud/openshift.py @@ -2,6 +2,7 @@ import json import logging import os +import re import requests from requests.auth import HTTPBasicAuth import time @@ -44,6 +45,54 @@ def clean_openshift_metadata(obj): return obj +def parse_openshift_quota_value(attr_name, quota_value): + PATTERN = r"([0-9]+)(m|Ki|Mi|Gi|Ti|Pi|Ei|K|M|G|T|P|E)?" + + suffix = { + "Ki": 2**10, + "Mi": 2**20, + "Gi": 2**30, + "Ti": 2**40, + "Pi": 2**50, + "Ei": 2**60, + "m": 10**-3, + "K": 10**3, + "M": 10**6, + "G": 10**9, + "T": 10**12, + "P": 10**15, + "E": 10**18, + } + + if quota_value and quota_value != "0": + result = re.search(PATTERN, quota_value) + + if result is None: + raise ValueError( + f"Unable to parse quota_value = '{quota_value}' for {attr_name}" + ) + + value = int(result.groups()[0]) + unit = result.groups()[1] + + # Convert to number i.e. without any unit suffix + + if unit is not None: + quota_value = value * suffix[unit] + else: + quota_value = value + + # Convert some attributes to units that coldfront uses + + if "RAM" in attr_name: + return round(quota_value / suffix["Mi"]) + elif "Storage" in attr_name: + return round(quota_value / suffix["Gi"]) + return quota_value + elif quota_value and quota_value == "0": + return 0 + + class ApiException(Exception): def __init__(self, message): self.message = message @@ -69,6 +118,11 @@ class OpenShiftResourceAllocator(base.ResourceAllocator): attributes.QUOTA_PVC: lambda x: {"persistentvolumeclaims": f"{x}"}, } + CF_QUOTA_KEY_MAPPING = { + list(quota_lambda_func(0).keys())[0]: cf_attr + for cf_attr, quota_lambda_func in QUOTA_KEY_MAPPING.items() + } + resource_type = "openshift" project_name_max_length = 63 @@ -200,6 +254,21 @@ def get_quota(self, project_id): return combined_quota + def get_usage(self, project_id): + cloud_quotas = self._openshift_get_resourcequotas(project_id) + combined_quota_used = {} + for cloud_quota in cloud_quotas: + combined_quota_used.update( + { + self.CF_QUOTA_KEY_MAPPING[quota_key]: parse_openshift_quota_value( + self.CF_QUOTA_KEY_MAPPING[quota_key], value + ) + for quota_key, value in cloud_quota["status"]["used"].items() + } + ) + + return combined_quota_used + def create_project_defaults(self, project_id): pass diff --git a/src/coldfront_plugin_cloud/openstack.py b/src/coldfront_plugin_cloud/openstack.py index b47ec84b..64ab7e7c 100644 --- a/src/coldfront_plugin_cloud/openstack.py +++ b/src/coldfront_plugin_cloud/openstack.py @@ -470,3 +470,6 @@ def get_users(self, project_id): role_assignment.user["name"] for role_assignment in role_assignments ) return user_names + + def get_usage(self, project_id): + raise NotImplementedError diff --git a/src/coldfront_plugin_cloud/tasks.py b/src/coldfront_plugin_cloud/tasks.py index fca94a31..4f157afc 100644 --- a/src/coldfront_plugin_cloud/tasks.py +++ b/src/coldfront_plugin_cloud/tasks.py @@ -191,3 +191,27 @@ def remove_user_from_allocation(allocation_user_pk): allocator.remove_role_from_user(username, project_id) else: logger.warning("No project has been created. Nothing to disable.") + + +def get_allocation_cloud_usage(allocation_pk): + """ + Obtains the current quota usage for the allocation. + + For example, the output for an Openshift quota would be: + + { + "limits.cpu": "1", + "limits.memory": "2Gi", + "limits.ephemeral-storage": "10Gi", + } + """ + allocation = Allocation.objects.get(pk=allocation_pk) + if allocator := find_allocator(allocation): + if project_id := allocation.get_attribute(attributes.ALLOCATION_PROJECT_ID): + try: + return allocator.get_usage(project_id) + except NotImplemented: + return + else: + logger.warning("No project has been created. No quota to check.") + return diff --git a/src/coldfront_plugin_cloud/tests/unit/openshift/test_quota.py b/src/coldfront_plugin_cloud/tests/unit/openshift/test_quota.py index e44bd89a..17ec895d 100644 --- a/src/coldfront_plugin_cloud/tests/unit/openshift/test_quota.py +++ b/src/coldfront_plugin_cloud/tests/unit/openshift/test_quota.py @@ -109,3 +109,25 @@ def test_get_moc_quota(self, fake_get_quota): ] res = self.allocator.get_quota("fake-project") self.assertEqual(res, expected_quota) + + @mock.patch( + "coldfront_plugin_cloud.openshift.OpenShiftResourceAllocator._openshift_get_resourcequotas" + ) + def test_get_moc_usage(self, fake_get_quota): + fake_usage = { + "limits.cpu": "2000m", + "limits.memory": "4096Mi", + } + fake_get_quota.return_value = [ + { + "status": {"used": fake_usage}, + } + ] + res = self.allocator.get_usage("fake-project") + self.assertEqual( + res, + { + "OpenShift Limit on CPU Quota": 2.0, + "OpenShift Limit on RAM Quota (MiB)": 4096, + }, + ) diff --git a/src/coldfront_plugin_cloud/tests/unit/test_utils.py b/src/coldfront_plugin_cloud/tests/unit/test_utils.py index 86ae535e..498f32fc 100644 --- a/src/coldfront_plugin_cloud/tests/unit/test_utils.py +++ b/src/coldfront_plugin_cloud/tests/unit/test_utils.py @@ -29,3 +29,51 @@ def test_env_safe_name(self): self.assertEqual(utils.env_safe_name(42), "42") self.assertEqual(utils.env_safe_name(None), "NONE") self.assertEqual(utils.env_safe_name("hello"), "HELLO") + + +class TestCheckIfQuotaAttr(base.TestCase): + def test_valid_quota_attr(self): + self.assertTrue(utils.check_if_quota_attr("OpenShift Limit on CPU Quota")) + + def test_invalid_quota_attr(self): + self.assertFalse(utils.check_if_quota_attr("Test")) + self.assertFalse(utils.check_if_quota_attr("Allocated Project ID")) + + +class TestGetNewCloudQuota(base.TestCase): + def test_get_requested_quota(self): + data = [ + {"name": "OpenShift Limit on CPU Quota", "new_value": "2"}, + {"name": "OpenShift Limit on RAM Quota (MiB)", "new_value": ""}, + ] + + result = utils.get_new_cloud_quota(data) + self.assertEqual(result, {"OpenShift Limit on CPU Quota": "2"}) + + +class TestCheckChangeRequests(base.TestBase): + def test_check_usage(self): + # No error case, usage is lower + test_quota_usage = { + "OpenShift Limit on CPU Quota": 2.0, + "OpenShift Limit on RAM Quota (MiB)": 2048, + "OpenShift Limit on Ephemeral Storage Quota (GiB)": 10, # Other quotas should be ignored + } + test_requested_quota = {"OpenShift Limit on CPU Quota": "2"} + + self.assertEqual( + [], utils.check_cloud_usage_is_lower(test_requested_quota, test_quota_usage) + ) + + # Requested cpu (2) lower than current used, should return errors + test_quota_usage["OpenShift Limit on CPU Quota"] = 16 + self.assertEqual( + [ + ( + "Current quota usage for OpenShift Limit on CPU Quota " + "(16) is higher than " + "the requested amount (2)." + ) + ], + utils.check_cloud_usage_is_lower(test_requested_quota, test_quota_usage), + ) diff --git a/src/coldfront_plugin_cloud/utils.py b/src/coldfront_plugin_cloud/utils.py index d3114816..972616bf 100644 --- a/src/coldfront_plugin_cloud/utils.py +++ b/src/coldfront_plugin_cloud/utils.py @@ -244,3 +244,53 @@ def get_included_duration( total_interval_duration -= (e_interval_end - e_interval_start).total_seconds() return math.ceil(total_interval_duration) + + +def check_if_quota_attr(attr_name: str): + for quota_attr in attributes.ALLOCATION_QUOTA_ATTRIBUTES: + if attr_name == quota_attr.name: + return True + return False + + +def get_new_cloud_quota(change_request_data: list[dict[str, str]]): + """ + Converts change request data to a dictionary of requested quota changes. + Ignores attributes with empty `new_value` str, meaning no change requested for them + Input typically looks like: + [ + { + "name": "OpenShift Limit on CPU Quota", + "new_value": "2", + ... + }, + { + "name": "OpenShift Limit on RAM Quota (MiB)", + "new_value": "", + ... + } + ] + """ + requested_quota = {} + for form in change_request_data: + if check_if_quota_attr(form["name"]) and form["new_value"]: + requested_quota[form["name"]] = form["new_value"] + return requested_quota + + +def check_cloud_usage_is_lower( + requested_quota: dict[str, str], cloud_quota_usage: dict[str, str] +): + usage_errors = [] + for quota_name, requested_quota_value in requested_quota.items(): + current_usage_value = cloud_quota_usage[quota_name] + if int(requested_quota_value) < current_usage_value: + usage_errors.append( + ( + f"Current quota usage for {quota_name} " + f"({current_usage_value}) is higher than " + f"the requested amount ({requested_quota_value})." + ) + ) + + return usage_errors