From c21d6def238893e76923af860134495f4e66e26a Mon Sep 17 00:00:00 2001 From: Kristi Nikolla Date: Tue, 2 Dec 2025 11:33:34 -0500 Subject: [PATCH 1/2] Add columns and file for daily billable usage - Add the columns Report Start Time, Report End Time, Generated At - Uploads the file to the daily location "NERC Storage .csv" --- .../commands/calculate_storage_gb_hours.py | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py b/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py index f700738e..4d59ee93 100644 --- a/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py +++ b/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py @@ -1,7 +1,7 @@ import csv from decimal import Decimal, ROUND_HALF_UP import dataclasses -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import logging import os @@ -44,6 +44,8 @@ def get_rates(): @dataclasses.dataclass class InvoiceRow: InvoiceMonth: str = "" + Report_Start_Time: str = "" + Report_End_Time: str = "" Project_Name: str = "" Project_ID: str = "" PI: str = "" @@ -56,12 +58,15 @@ class InvoiceRow: Invoice_Type: str = "" Rate: Decimal = 0 Cost: Decimal = 0 + Generated_At: str = "" @classmethod def get_headers(cls): """Returns all headers for display.""" return [ "Invoice Month", + "Report Start Time", + "Report End Time", "Project - Allocation", "Project - Allocation ID", "Manager (PI)", @@ -74,6 +79,7 @@ def get_headers(cls): "SU Type", "Rate", "Cost", + "Generated At", ] def get_value(self, field: str): @@ -162,7 +168,7 @@ def default_end_argument(): return pytz.utc.localize(d) @staticmethod - def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month): + def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month, end_time): s3_key_id = os.getenv("S3_INVOICING_ACCESS_KEY_ID") s3_secret = os.getenv("S3_INVOICING_SECRET_ACCESS_KEY") @@ -188,7 +194,18 @@ def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month): s3.upload_file(file_location, Bucket=s3_bucket, Key=primary_location) logger.info(f"Uploaded to {primary_location}.") - timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + # Upload daily copy + # End time is exclusive, subtract one second to find the inclusive end date + invoice_date = end_time - timedelta(seconds=1) + invoice_date = invoice_date.strftime("%Y-%m-%d") + daily_location = ( + f"Invoices/{invoice_month}/Service Invoices/NERC Storage {invoice_date}.csv" + ) + s3.upload_file(file_location, Bucket=s3_bucket, Key=daily_location) + logger.info(f"Uploaded to {daily_location}.") + + # Archival copy + timestamp = datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ") secondary_location = ( f"Invoices/{invoice_month}/" f"Archive/NERC Storage {invoice_month} {timestamp}.csv" @@ -197,6 +214,8 @@ def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month): logger.info(f"Uploaded to {secondary_location}.") def handle(self, *args, **options): + generated_at = datetime.now(tz=timezone.utc).isoformat(timespec="seconds") + def get_outages_for_service(resource_name: str): """Get outages for a service from nerc-rates. @@ -227,6 +246,8 @@ def process_invoice_row(allocation, attrs, su_name, rate): if time > 0: row = InvoiceRow( InvoiceMonth=options["invoice_month"], + Report_Start_Time=options["start"].isoformat(), + Report_End_Time=options["end"].isoformat(), Project_Name=allocation.get_attribute( attributes.ALLOCATION_PROJECT_NAME ), @@ -243,6 +264,7 @@ def process_invoice_row(allocation, attrs, su_name, rate): Invoice_Type=su_name, Rate=rate, Cost=(time * rate).quantize(Decimal(".01"), rounding=ROUND_HALF_UP), + Generated_At=generated_at, ) csv_invoice_writer.writerow(row.get_values()) @@ -338,4 +360,5 @@ def process_invoice_row(allocation, attrs, su_name, rate): options["s3_bucket_name"], options["output"], options["invoice_month"], + options["end"], ) From e369213514756945ce270f7c8029df573788fc66 Mon Sep 17 00:00:00 2001 From: Kristi Nikolla Date: Tue, 2 Dec 2025 15:00:22 -0500 Subject: [PATCH 2/2] Update calculate_storage_gb_hours.py --- .../management/commands/calculate_storage_gb_hours.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py b/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py index 4575abc0..f2f75fea 100644 --- a/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py +++ b/src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py @@ -210,7 +210,7 @@ def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month, end_time) def handle(self, *args, **options): generated_at = datetime.now(tz=timezone.utc).isoformat(timespec="seconds") - def get_outages_for_service(resource_name: str): + def get_outages_for_service(cluster_name: str): """Get outages for a service from nerc-rates. :param cluster_name: Name of the cluster to get outages for.