fix default retry and add rate limit code

osc_sdk_python/call.py

@@ -7,54 +7,85 @@
 import json
 import os
 
+
 class Call(object):
     def __init__(self, logger=None, **kwargs):
-        self.version = kwargs.pop('version', 'latest')
-        self.host = kwargs.pop('host', None)
-        self.ssl = kwargs.pop('_ssl', True)
-        self.user_agent = kwargs.pop('user_agent', DEFAULT_USER_AGENT)
-        self.max_retries = kwargs.pop('max_retries', 0)
+        self.version = kwargs.pop("version", "latest")
+        self.host = kwargs.pop("host", None)
+        self.ssl = kwargs.pop("_ssl", True)
+        self.user_agent = kwargs.pop("user_agent", DEFAULT_USER_AGENT)
         self.logger = logger
-        self.update_credentials(access_key=kwargs.pop('access_key', None),
-                                secret_key=kwargs.pop('secret_key', None),
-                                region=kwargs.pop('region', None),
-                                profile=kwargs.pop('profile', None),
-                                email=kwargs.pop('email', None),
-                                password=kwargs.pop('password', None),
-                                proxy=kwargs.pop('proxy', None),
-                                x509_client_cert=kwargs.pop('x509_client_cert', None))
+        self.update_credentials(
+            access_key=kwargs.pop("access_key", None),
+            secret_key=kwargs.pop("secret_key", None),
+            region=kwargs.pop("region", None),
+            profile=kwargs.pop("profile", None),
+            email=kwargs.pop("email", None),
+            password=kwargs.pop("password", None),
+            proxy=kwargs.pop("proxy", None),
+            x509_client_cert=kwargs.pop("x509_client_cert", None),
+            max_retries=kwargs.pop("max_retries", None),
+            retry_backoff_factor=kwargs.pop("retry_backoff_factor", None),
+            retry_backoff_jitter=kwargs.pop("retry_backoff_jitter", None)
+        )
 
-    def update_credentials(self, region=None, profile=None, access_key=None,
-                           secret_key=None, email=None, password=None, proxy=None,
-                           x509_client_cert=None):
+    def update_credentials(
+        self,
+        region=None,
+        profile=None,
+        access_key=None,
+        secret_key=None,
+        email=None,
+        password=None,
+        proxy=None,
+        x509_client_cert=None,
+        max_retries=None,
+        retry_backoff_factor=None,
+        retry_backoff_jitter=None,
+    ):
         self.credentials = {
-            'access_key': access_key,
-            'secret_key': secret_key,
-            'region': region,
-            'profile': profile,
-            'email': email,
-            'password': password,
-            'x509_client_cert': x509_client_cert,
-            'proxy': proxy
+            "access_key": access_key,
+            "secret_key": secret_key,
+            "region": region,
+            "profile": profile,
+            "email": email,
+            "password": password,
+            "x509_client_cert": x509_client_cert,
+            "max_retries": max_retries,
+            "retry_backoff_factor": retry_backoff_factor,
+            "retry_backoff_jitter": retry_backoff_jitter,
         }
 
     def api(self, action, **data):
         try:
             credentials = Credentials(**self.credentials)
-            host = (self.host if self.host
-                    else 'api.{}.outscale.{}'.format(credentials.region, credentials.get_url_extension()))
-            uri = '/api/{}/{}'.format(self.version, action)
-            protocol = 'https' if self.ssl else 'http'
+            host = (
+                self.host
+                if self.host
+                else "api.{}.outscale.{}".format(
+                    credentials.region, credentials.get_url_extension()
+                )
+            )
+            uri = "/api/{}/{}".format(self.version, action)
+            protocol = "https" if self.ssl else "http"
 
-            endpoint = os.environ.get('OSC_ENDPOINT_API')
+            endpoint = os.environ.get("OSC_ENDPOINT_API")
             if endpoint is None:
-                endpoint = '{}://{}{}'.format(protocol, host, uri)
+                endpoint = "{}://{}{}".format(protocol, host, uri)
             else:
-                endpoint = '{}{}'.format(endpoint, uri)
+                endpoint = "{}{}".format(endpoint, uri)
 
-            requester = Requester(Authentication(credentials, host, user_agent=self.user_agent), endpoint, self.max_retries)
+            requester = Requester(
+                Authentication(credentials, host, user_agent=self.user_agent),
+                endpoint,
+                credentials.max_retries,
+                credentials.retry_backoff_factor,
+                credentials.retry_backoff_jitter,
+            )
             if self.logger != None:
-                self.logger.do_log("uri: " + uri + "\npayload:\n" + json.dumps(data, indent=2))
+                self.logger.do_log(
+                    "uri: " + uri + "\npayload:\n" + json.dumps(data, indent=2)
+                )
             return requester.send(uri, json.dumps(data))
         except Exception as err:
             raise err

osc_sdk_python/credentials.py

@@ -1,32 +1,51 @@
 import json
 import os
 
-ORIGINAL_PATH = os.path.join(os.path.expanduser('~'),'.oapi_credentials')
-STD_PATH = os.path.join(os.path.expanduser('~'),'.osc/config.json')
-DEFAULT_REGION="eu-west-2"
-DEFAULT_PROFILE="default"
+ORIGINAL_PATH = os.path.join(os.path.expanduser("~"), ".oapi_credentials")
+STD_PATH = os.path.join(os.path.expanduser("~"), ".osc/config.json")
+DEFAULT_REGION = "eu-west-2"
+DEFAULT_PROFILE = "default"
+MAX_RETRIES = 3
+RETRY_BACKOFF_FACTOR = 1
+RETRY_BACKOFF_JITTER = 3
+
 
 class Credentials:
-    def __init__(self, region, profile, access_key, secret_key, email, password,
-                 x509_client_cert=None, proxy=None):
+    def __init__(
+        self,
+        region,
+        profile,
+        access_key,
+        secret_key,
+        email,
+        password,
+        x509_client_cert=None,
+        proxy=None,
+        max_retries=None,
+        retry_backoff_factor=None,
+        retry_backoff_jitter=None,
+    ):
         self.region = None
         self.access_key = access_key
         self.secret_key = secret_key
         self.email = email
         self.password = password
-        self.x509_client_cert=x509_client_cert
-        self.proxy=proxy
+        self.x509_client_cert = x509_client_cert
+        self.proxy = proxy
+        self.max_retries = max_retries
+        self.retry_backoff_factor = retry_backoff_factor
+        self.retry_backoff_jitter = retry_backoff_jitter
 
         if profile is None:
-            profile = os.environ.get('OSC_PROFILE')
+            profile = os.environ.get("OSC_PROFILE")
         else:
-            # Overide with environmental configuration if available
+            # Override with environmental configuration if available
             self.load_credentials_from_env()
-        # Overide with old configuration if available
+        # Override with old configuration if available
         self.load_credentials_from_file(profile, ORIGINAL_PATH)
-        # Overide with standard configuration if available
+        # Override with standard configuration if available
         self.load_credentials_from_file(profile, STD_PATH)
-        # Overide with environmental configuration if available
+        # Override with environmental configuration if available
         if profile is None:
             profile = DEFAULT_PROFILE
             self.load_credentials_from_env()
@@ -39,43 +58,76 @@ def __init__(self, region, profile, access_key, secret_key, email, password,
             self.region = DEFAULT_REGION
 
         self.profile = profile
-        # Overide with app parameters if provided
+        # Override with app parameters if provided
         if access_key is not None:
             self.access_key = access_key
         if secret_key is not None:
             self.secret_key = secret_key
 
+        if self.max_retries is None:
+            self.max_retries = MAX_RETRIES
+        if self.retry_backoff_factor is None:
+            self.retry_backoff_factor = RETRY_BACKOFF_FACTOR
+        if self.retry_backoff_jitter is None:
+            self.retry_backoff_jitter = RETRY_BACKOFF_JITTER
+
         self.check_options()
 
     def load_credentials_from_file(self, profile, file_path):
         try:
             with open(file_path) as f:
                 config = json.load(f)
                 profile = config.get(profile)
+
                 if profile is None:
                     return
+
                 ak = profile.get("access_key")
                 if ak is not None:
                     self.access_key = ak
+
                 sk = profile.get("secret_key")
                 if sk is not None:
                     self.secret_key = sk
+
                 region = profile.get("region")
                 if region is not None:
                     self.region = region
+
+                max_retries = profile.get("max_retries")
+                if max_retries is not None:
+                    self.max_retries = max_retries
+
+                retry_backoff_factor = profile.get("retry_backoff_factor")
+                if retry_backoff_factor is not None:
+                    self.retry_backoff_factor = retry_backoff_factor
+
+                retry_backoff_jitter = profile.get("retry_backoff_jitter")
+                if retry_backoff_jitter is not None:
+                    self.retry_backoff_jitter = retry_backoff_jitter
+
         except IOError:
             pass
 
     def load_credentials_from_env(self):
-        ak = os.environ.get('OSC_ACCESS_KEY')
+        ak = os.environ.get("OSC_ACCESS_KEY")
         if ak is not None:
             self.access_key = ak
-        sk = os.environ.get('OSC_SECRET_KEY')
+        sk = os.environ.get("OSC_SECRET_KEY")
         if sk is not None:
             self.secret_key = sk
-        region = os.environ.get('OSC_REGION')
+        region = os.environ.get("OSC_REGION")
         if region is not None:
             self.region = region
+        max_retries = os.environ.get("OSC_MAX_RETRIES")
+        if max_retries is not None:
+            self.max_retires = int(max_retries)
+        retry_backoff_factor = os.environ.get("OSC_RETRY_BACKOFF_FACTOR")
+        if retry_backoff_factor is not None:
+            self.retry_backoff_factor = float(retry_backoff_factor)
+        retry_backoff_jitter = os.environ.get("OSC_RETRY_BACKOFF_JITTER")
+        if retry_backoff_jitter is not None:
+            self.retry_backoff_factor = float(retry_backoff_jitter)
 
     def check_options(self):
         if self.access_key is not None or self.secret_key is not None:
@@ -92,4 +144,4 @@ def check_options(self):
             raise Exception("Invalid Outscale region")
 
     def get_url_extension(self):
-        return 'hk' if 'cn' in self.region else 'com'
+        return "hk" if "cn" in self.region else "com"