From 3e16e514c6162f4d2e31dc932c1b81ee5c9b6ca2 Mon Sep 17 00:00:00 2001 From: Quan Pham Date: Thu, 20 Nov 2025 13:16:45 -0500 Subject: [PATCH] Add internal cluster name as a resource attribute The internal cluster name (as opposed to the name seen by users) is used invoicing. For now, it is used for fetching outage data from `nerc_rates` --- src/coldfront_plugin_cloud/attributes.py | 2 ++ .../commands/add_openshift_resource.py | 15 +++++++++++ .../commands/add_openstack_resource.py | 15 +++++++++++ .../commands/calculate_storage_gb_hours.py | 27 +++++++------------ src/coldfront_plugin_cloud/tests/base.py | 5 +++- .../tests/unit/test_attribute_migration.py | 4 ++- .../unit/test_calculate_quota_unit_hours.py | 12 +++------ 7 files changed, 52 insertions(+), 28 deletions(-) diff --git a/src/coldfront_plugin_cloud/attributes.py b/src/coldfront_plugin_cloud/attributes.py index 19d827f8..db379dc3 100644 --- a/src/coldfront_plugin_cloud/attributes.py +++ b/src/coldfront_plugin_cloud/attributes.py @@ -34,6 +34,7 @@ class CloudAllocationAttribute: RESOURCE_DEFAULT_NETWORK_CIDR = "OpenStack Default Network CIDR" RESOURCE_EULA_URL = "EULA URL" +RESOURCE_CLUSTER_NAME = "Internal Cluster Name" RESOURCE_ATTRIBUTES = [ CloudResourceAttribute(name=RESOURCE_AUTH_URL), @@ -48,6 +49,7 @@ class CloudAllocationAttribute: CloudResourceAttribute(name=RESOURCE_EULA_URL), CloudResourceAttribute(name=RESOURCE_DEFAULT_PUBLIC_NETWORK), CloudResourceAttribute(name=RESOURCE_DEFAULT_NETWORK_CIDR), + CloudResourceAttribute(name=RESOURCE_CLUSTER_NAME), ] # TODO: Migration to rename the OpenStack specific prefix out of these attrs diff --git a/src/coldfront_plugin_cloud/management/commands/add_openshift_resource.py b/src/coldfront_plugin_cloud/management/commands/add_openshift_resource.py index 8e05ef4c..128c00fc 100644 --- a/src/coldfront_plugin_cloud/management/commands/add_openshift_resource.py +++ b/src/coldfront_plugin_cloud/management/commands/add_openshift_resource.py @@ -24,6 +24,12 @@ def add_arguments(self, parser): parser.add_argument( "--name", type=str, required=True, help="Name of OpenShift resource" ) + parser.add_argument( + "--internal-name", + type=str, + required=False, + help="Internal name of cluster used for invoicing. Defaults to public name", + ) parser.add_argument( "--api-url", type=str, @@ -99,3 +105,12 @@ def handle(self, *args, **options): resource=openshift, value="true" if options["ibm_storage_available"] else "false", ) + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get( + name=attributes.RESOURCE_CLUSTER_NAME + ), + resource=openshift, + value=options["internal_name"] + if options["internal_name"] + else options["name"], + ) diff --git a/src/coldfront_plugin_cloud/management/commands/add_openstack_resource.py b/src/coldfront_plugin_cloud/management/commands/add_openstack_resource.py index 0d6940b3..606eea67 100644 --- a/src/coldfront_plugin_cloud/management/commands/add_openstack_resource.py +++ b/src/coldfront_plugin_cloud/management/commands/add_openstack_resource.py @@ -17,6 +17,12 @@ def add_arguments(self, parser): parser.add_argument( "--name", type=str, required=True, help="Name of OpenStack resource" ) + parser.add_argument( + "--internal-name", + type=str, + required=False, + help="Internal name of cluster used for invoicing. Defaults to public name", + ) parser.add_argument( "--auth-url", type=str, @@ -133,6 +139,15 @@ def handle(self, *args, **options): resource=openstack, value=options["role"], ) + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get( + name=attributes.RESOURCE_CLUSTER_NAME + ), + resource=openstack, + value=options["internal_name"] + if options["internal_name"] + else options["name"], + ) # Quantity values do not make sense for an ESI allocation if not options["esi"]: diff --git a/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py b/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py index f700738e..11a16ce2 100644 --- a/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py +++ b/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py @@ -20,12 +20,6 @@ _RATES = None -RESOURCE_NAME_TO_NERC_SERVICE = { - "NERC": "stack", - "NERC-OCP": "ocp-prod", - "NERC-OCP-EDU": "academic", -} - def get_rates(): # nerc-rates doesn't work with Python 3.9, which is what ColdFront is currently @@ -197,23 +191,22 @@ def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month): logger.info(f"Uploaded to {secondary_location}.") def handle(self, *args, **options): - def get_outages_for_service(resource_name: str): + def get_outages_for_service(cluster_name: str): """Get outages for a service from nerc-rates. - :param resource_name: Name of the resource to get outages for. + :param cluster_name: Name of the cluster to get outages for. :return: List of excluded intervals or None. """ - service_name = RESOURCE_NAME_TO_NERC_SERVICE.get(resource_name) - if service_name: - return utils.load_outages_from_nerc_rates( - options["start"], options["end"], service_name - ) - return None + return utils.load_outages_from_nerc_rates( + options["start"], options["end"], cluster_name + ) def process_invoice_row(allocation, attrs, su_name, rate): """Calculate the value and write the bill using the writer.""" - resource_name = allocation.resources.first().name - excluded_intervals_list = get_outages_for_service(resource_name) + internal_cluster_name = allocation.resources.first().get_attribute( + attributes.RESOURCE_CLUSTER_NAME + ) + excluded_intervals_list = get_outages_for_service(internal_cluster_name) time = 0 for attribute in attrs: @@ -234,7 +227,7 @@ def process_invoice_row(allocation, attrs, su_name, rate): attributes.ALLOCATION_PROJECT_ID ), PI=allocation.project.pi.email, - Cluster_Name=allocation.resources.first().name, + Cluster_Name=internal_cluster_name, Institution_Specific_Code=allocation.get_attribute( attributes.ALLOCATION_INSTITUTION_SPECIFIC_CODE ) diff --git a/src/coldfront_plugin_cloud/tests/base.py b/src/coldfront_plugin_cloud/tests/base.py index 03d35d62..7cccb1bd 100644 --- a/src/coldfront_plugin_cloud/tests/base.py +++ b/src/coldfront_plugin_cloud/tests/base.py @@ -63,12 +63,15 @@ def new_esi_resource(name=None, auth_url=None) -> Resource: return Resource.objects.get(name=resource_name) @staticmethod - def new_openstack_resource(name=None, auth_url=None) -> Resource: + def new_openstack_resource( + name=None, internal_name=None, auth_url=None + ) -> Resource: resource_name = name or uuid.uuid4().hex call_command( "add_openstack_resource", name=resource_name, + internal_name=internal_name, auth_url=auth_url or f"https://{resource_name}/identity/v3", projects_domain="default", users_domain="default", diff --git a/src/coldfront_plugin_cloud/tests/unit/test_attribute_migration.py b/src/coldfront_plugin_cloud/tests/unit/test_attribute_migration.py index 509ee738..1a5a12c5 100644 --- a/src/coldfront_plugin_cloud/tests/unit/test_attribute_migration.py +++ b/src/coldfront_plugin_cloud/tests/unit/test_attribute_migration.py @@ -124,7 +124,9 @@ def test_rename_identity_url(self): new_resource_attrs, ): call_command("register_cloud_attributes") - resource = self.new_openstack_resource("Example", auth_url_val) + resource = self.new_openstack_resource( + "Example", auth_url=auth_url_val + ) self.assertEqual( resource.get_attribute(new_auth_url_name), diff --git a/src/coldfront_plugin_cloud/tests/unit/test_calculate_quota_unit_hours.py b/src/coldfront_plugin_cloud/tests/unit/test_calculate_quota_unit_hours.py index c2328817..f71e12a5 100644 --- a/src/coldfront_plugin_cloud/tests/unit/test_calculate_quota_unit_hours.py +++ b/src/coldfront_plugin_cloud/tests/unit/test_calculate_quota_unit_hours.py @@ -92,10 +92,6 @@ def test_new_allocation_quota(self, mock_load_outages): "2020-03", ) - # Verify that load_outages_from_nerc_rates is not called when resource name - # doesn't match NERC service mapping - mock_load_outages.assert_not_called() - def test_new_allocation_quota_expired(self): """Test that expiration doesn't affect invoicing.""" self.resource = self.new_openshift_resource( @@ -607,10 +603,6 @@ def test_load_excluded_intervals_invalid(self): with self.assertRaises(AssertionError): utils.load_excluded_intervals(invalid_interval) - @patch( - "coldfront_plugin_cloud.management.commands.calculate_storage_gb_hours.RESOURCE_NAME_TO_NERC_SERVICE", - {"TEST-RESOURCE": "test-service"}, - ) @patch( "coldfront_plugin_cloud.management.commands.calculate_storage_gb_hours.get_rates" ) @@ -633,7 +625,9 @@ def test_nerc_outages_integration(self, mock_rates_loader): with freezegun.freeze_time("2020-03-01"): user = self.new_user() project = self.new_project(pi=user) - resource = self.new_openstack_resource(name="TEST-RESOURCE") + resource = self.new_openstack_resource( + name="TEST-RESOURCE", internal_name="test-service" + ) allocation = self.new_allocation(project, resource, 100) for attr, val in [ (attributes.ALLOCATION_PROJECT_NAME, "test"),