MAINT: Update values

OpenStack-Rally-Tester/usr/local/bin/rally_extract_results.py

@@ -4,6 +4,7 @@
 """
 Parses results from the rally task report and sends them to influxdb
 """
+
 import json
 import sys
 import time

OpenStack-accounting/usr/local/sbin/accountinglib.py

@@ -3,7 +3,8 @@
 # Copyright (c) 2023 United Kingdom Research and Innovation
 import time
 import datetime
-#from datetime import datetime,time
+
+# from datetime import datetime,time
 import json
 import requests
 import sys
@@ -13,22 +14,27 @@
 from sqlalchemy.orm import sessionmaker
 import configparser
 
+
 def get_logger(component):
-    logging.basicConfig(filename="/var/log/thecount.log",
-                    format=f'%(asctime)s {component} %(message)s',
-                    filemode='a')
+    logging.basicConfig(
+        filename="/var/log/thecount.log",
+        format=f"%(asctime)s {component} %(message)s",
+        filemode="a",
+    )
     logger = logging.getLogger()
     logger.setLevel(logging.INFO)
     return logger
 
+
 def ifnull(var, val):
-    '''Returns the second argument if the first argument is Null/None'''
+    """Returns the second argument if the first argument is Null/None"""
     if var is None:
         return val
     return var
 
+
 def project_to_department(result):
-    '''Returns an appropriate department for a project'''
+    """Returns an appropriate department for a project"""
     if "rally" in result["Project"]:
         department = "STFC Cloud"
     elif "efault" in result["Department"]:
@@ -37,51 +43,57 @@ def project_to_department(result):
         department = result["Department"]
     return department
 
+
 def send_to_influx(datastring, logger):
-    '''Takes a datastring formatted to send to InfluxDBs rest api. Loads necessary config, sends and returns the response'''
+    """Takes a datastring formatted to send to InfluxDBs rest api. Loads necessary config, sends and returns the response"""
     # Read from config file
     influx_parser = configparser.SafeConfigParser()
     try:
-        influx_parser.read('/etc/influxdb.conf')
+        influx_parser.read("/etc/influxdb.conf")
     except Exceptions as exp:
-        logger.info(f'Unable to read from influx config file - {str(exp)}')
+        logger.info(f"Unable to read from influx config file - {str(exp)}")
         sys.exit(1)
     try:
-        host = influx_parser.get('db', 'host')
-        database = influx_parser.get('db', 'database')
-        username = influx_parser.get('auth', 'username')
-        password = influx_parser.get('auth', 'password')
-        instance = influx_parser.get('cloud','instance')
+        host = influx_parser.get("db", "host")
+        database = influx_parser.get("db", "database")
+        username = influx_parser.get("auth", "username")
+        password = influx_parser.get("auth", "password")
+        instance = influx_parser.get("cloud", "instance")
     except Exceptions as exp:
-        logger.info(f'Unable to parse influx config file - {str(exp)}')
+        logger.info(f"Unable to parse influx config file - {str(exp)}")
         sys.exit(1)
-    finaldatastring = datastring.replace("Accounting,","Accounting,instance="+instance+",")
+    finaldatastring = datastring.replace(
+        "Accounting,", "Accounting,instance=" + instance + ","
+    )
     logger.info(finaldatastring)
-    url = f'http://{host}/write?db={database}&precision=s'
-    response = requests.post(url,data=finaldatastring,auth=(username,password))
+    url = f"http://{host}/write?db={database}&precision=s"
+    response = requests.post(url, data=finaldatastring, auth=(username, password))
     return response
 
-def get_accounting_data(database,starttime,endtime, logger):
-    '''Takes a db name and a start and end time as arguments. Loads db config, creates a db connection and runs a stored procedure. Returns the results of the stored procedure'''
+
+def get_accounting_data(database, starttime, endtime, logger):
+    """Takes a db name and a start and end time as arguments. Loads db config, creates a db connection and runs a stored procedure. Returns the results of the stored procedure"""
     thecount_parser = configparser.RawConfigParser(strict=False)
 
     try:
-        thecount_parser.read('/etc/thecount/thecount.conf')
+        thecount_parser.read("/etc/thecount/thecount.conf")
     except Exceptions as exp:
-        logger.info(f'Unable to read from thecount config file - {str(exp)}')
+        logger.info(f"Unable to read from thecount config file - {str(exp)}")
         sys.exit(1)
 
     try:
-        connectionstring = thecount_parser.get('database','connection') + '/' + database
+        connectionstring = (
+            thecount_parser.get("database", "connection") + "/" + database
+        )
     except Exceptions as exp:
-        logger.info(f'Unable to parse thecount config file - {str(exp)}')
+        logger.info(f"Unable to parse thecount config file - {str(exp)}")
         sys.exit(1)
 
-    engine = sqlalchemy.create_engine(connectionstring, encoding='utf-8')
+    engine = sqlalchemy.create_engine(connectionstring, encoding="utf-8")
     connection = engine.connect()
     sess = sessionmaker(bind=engine)()
     query = f'call get_accounting_data( "{starttime}","{endtime}")'
 
     logger.info(query)
-    results = sess.execute(query, { 'p1': starttime, 'p2': endtime })
+    results = sess.execute(query, {"p1": starttime, "p2": endtime})
     return results

OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py

@@ -8,48 +8,50 @@
 import datetime
 import logging
 
+
 def main():
-    '''Processes accounting from Cinder'''
+    """Processes accounting from Cinder"""
     nowtime = time.localtime()
     logger = accountinglib.get_logger("cinder")
 
     logger.info("Cinder Accounting run start")
-    starttime=sys.argv[1]
+    starttime = sys.argv[1]
     logger.info("Start Time = " + starttime)
-    endtime=sys.argv[2]
+    endtime = sys.argv[2]
     logger.info("End Time = " + endtime)
-    endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m')
-    endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple())
-
+    endyyyymm = datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").strftime("%Y-%m")
+    endtimestamp = time.mktime(
+        datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()
+    )
 
     results = accountinglib.get_accounting_data("cinder", starttime, endtime, logger)
-    datastring = ''
+    datastring = ""
     logger.info(results)
     for result in results:
         logger.info(result)
         department = accountinglib.project_to_department(result)
 
-
         datastring += "Accounting"
 
-        datastring += ",AvailabilityZone="+result["AvailabilityZone"]
-        datastring += ",Project="+result["Project"].replace(' ','\ ')
-        datastring += ",Department="+department.replace(' ','\ ')
-        datastring += ",CinderType="+result["CinderType"]
-        datastring += ",YYYY-MM="+ endyyyymm
-        datastring += " Volumes="+str(result["Volumes"])
-        datastring += ",Volume_Seconds="+str(result["Volume_Seconds"])
-        datastring += ",CinderGBs="+str(result["Volume_GB"] * result['Volume_Seconds'] * result["Volumes"])
-
-
-
-        datastring += " "+str(int(endtimestamp))
+        datastring += ",AvailabilityZone=" + result["AvailabilityZone"]
+        datastring += ",Project=" + result["Project"].replace(" ", "\ ")
+        datastring += ",Department=" + department.replace(" ", "\ ")
+        datastring += ",CinderType=" + result["CinderType"]
+        datastring += ",YYYY-MM=" + endyyyymm
+        datastring += " Volumes=" + str(result["Volumes"])
+        datastring += ",Volume_Seconds=" + str(result["Volume_Seconds"])
+        datastring += ",CinderGBs=" + str(
+            result["Volume_GB"] * result["Volume_Seconds"] * result["Volumes"]
+        )
+
+        datastring += " " + str(int(endtimestamp))
         datastring += "\n"
 
     r = accountinglib.send_to_influx(datastring, logger)
 
     logger.info(r.text)
     logger.info(r)
 
+
 if __name__ == "__main__":
     main()

OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py

@@ -8,47 +8,50 @@
 import datetime
 import logging
 
+
 def main():
-    '''Processes accounting from Glance'''
+    """Processes accounting from Glance"""
     nowtime = time.localtime()
     logger = accountinglib.get_logger("glance")
 
     logger.info("Glance Accounting run start")
-    starttime=sys.argv[1]
+    starttime = sys.argv[1]
     logger.info("Start Time = " + starttime)
-    endtime=sys.argv[2]
+    endtime = sys.argv[2]
     logger.info("End Time = " + endtime)
-    endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m')
-    endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple())
-
+    endyyyymm = datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").strftime("%Y-%m")
+    endtimestamp = time.mktime(
+        datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()
+    )
 
     results = accountinglib.get_accounting_data("glance", starttime, endtime, logger)
-    datastring = ''
+    datastring = ""
     logger.info(results)
     for result in results:
         logger.info(result)
         department = accountinglib.project_to_department(result)
 
         datastring += "Accounting"
 
-        datastring += ",Project="+result["Project"].replace(' ','\ ')
-        datastring += ",Department="+department.replace(' ','\ ')
-        datastring += ",StorageBackend="+result["StorageBackend"]
-        datastring += ",GlanceType="+result["GlanceType"]
-        datastring += ",YYYY-MM="+ endyyyymm
-        datastring += " Images="+str(result["Images"])
-        datastring += ",Image_Seconds="+str(result["Image_Seconds"])
-        datastring += ",GlanceGBSeconds="+str(result["Glance_GB"] * result['Image_Seconds'] * result['Images'])
-
-
-
-        datastring += " "+str(int(endtimestamp))
+        datastring += ",Project=" + result["Project"].replace(" ", "\ ")
+        datastring += ",Department=" + department.replace(" ", "\ ")
+        datastring += ",StorageBackend=" + result["StorageBackend"]
+        datastring += ",GlanceType=" + result["GlanceType"]
+        datastring += ",YYYY-MM=" + endyyyymm
+        datastring += " Images=" + str(result["Images"])
+        datastring += ",Image_Seconds=" + str(result["Image_Seconds"])
+        datastring += ",GlanceGBSeconds=" + str(
+            result["Glance_GB"] * result["Image_Seconds"] * result["Images"]
+        )
+
+        datastring += " " + str(int(endtimestamp))
         datastring += "\n"
 
     r = accountinglib.send_to_influx(datastring, logger)
 
     logger.info(r.text)
     logger.info(r)
 
+
 if __name__ == "__main__":
     main()

OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py

@@ -8,42 +8,44 @@
 import datetime
 import logging
 
+
 def main():
-    '''Processes accounting from Manila'''
+    """Processes accounting from Manila"""
     nowtime = time.localtime()
     logger = accountinglib.get_logger("manila")
 
     logger.info("Manila Accounting run start")
-    starttime=sys.argv[1]
+    starttime = sys.argv[1]
     logger.info("Start Time = " + starttime)
-    endtime=sys.argv[2]
+    endtime = sys.argv[2]
     logger.info("End Time = " + endtime)
-    endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m')
-    endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple())
-
+    endyyyymm = datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").strftime("%Y-%m")
+    endtimestamp = time.mktime(
+        datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()
+    )
 
     results = accountinglib.get_accounting_data("manila", starttime, endtime, logger)
-    datastring = ''
+    datastring = ""
     logger.info(results)
     for result in results:
         logger.info(result)
         department = accountinglib.project_to_department(result)
 
         datastring += "Accounting"
 
-        datastring += ",AvailabilityZone="+result["Availability_zone"]
-        datastring += ",Project="+result["Project"].replace(' ','\ ')
-        datastring += ",Department="+department.replace(' ','\ ')
-        datastring += ",ManilaType="+result["ManilaType"]
-        datastring += ",ManilaShareType="+result["Share_type"]
-        datastring += ",YYYY-MM="+ endyyyymm
-        datastring += " Shares="+str(result["Shares"])
-        datastring += ",Share_Seconds="+str(result["Share_Seconds"])
-        datastring += ",ManilaGBs="+str(result["Share_GB"] * result['Share_Seconds'] * result["Shares"])
-
-
-
-        datastring += " "+str(int(endtimestamp))
+        datastring += ",AvailabilityZone=" + result["Availability_zone"]
+        datastring += ",Project=" + result["Project"].replace(" ", "\ ")
+        datastring += ",Department=" + department.replace(" ", "\ ")
+        datastring += ",ManilaType=" + result["ManilaType"]
+        datastring += ",ManilaShareType=" + result["Share_type"]
+        datastring += ",YYYY-MM=" + endyyyymm
+        datastring += " Shares=" + str(result["Shares"])
+        datastring += ",Share_Seconds=" + str(result["Share_Seconds"])
+        datastring += ",ManilaGBs=" + str(
+            result["Share_GB"] * result["Share_Seconds"] * result["Shares"]
+        )
+
+        datastring += " " + str(int(endtimestamp))
         datastring += "\n"
 
     r = accountinglib.send_to_influx(datastring, logger)