diff --git a/osc_sdk_python/call.py b/osc_sdk_python/call.py index f354442..df15c81 100644 --- a/osc_sdk_python/call.py +++ b/osc_sdk_python/call.py @@ -7,54 +7,85 @@ import json import os + class Call(object): def __init__(self, logger=None, **kwargs): - self.version = kwargs.pop('version', 'latest') - self.host = kwargs.pop('host', None) - self.ssl = kwargs.pop('_ssl', True) - self.user_agent = kwargs.pop('user_agent', DEFAULT_USER_AGENT) - self.max_retries = kwargs.pop('max_retries', 0) + self.version = kwargs.pop("version", "latest") + self.host = kwargs.pop("host", None) + self.ssl = kwargs.pop("_ssl", True) + self.user_agent = kwargs.pop("user_agent", DEFAULT_USER_AGENT) self.logger = logger - self.update_credentials(access_key=kwargs.pop('access_key', None), - secret_key=kwargs.pop('secret_key', None), - region=kwargs.pop('region', None), - profile=kwargs.pop('profile', None), - email=kwargs.pop('email', None), - password=kwargs.pop('password', None), - proxy=kwargs.pop('proxy', None), - x509_client_cert=kwargs.pop('x509_client_cert', None)) + self.update_credentials( + access_key=kwargs.pop("access_key", None), + secret_key=kwargs.pop("secret_key", None), + region=kwargs.pop("region", None), + profile=kwargs.pop("profile", None), + email=kwargs.pop("email", None), + password=kwargs.pop("password", None), + proxy=kwargs.pop("proxy", None), + x509_client_cert=kwargs.pop("x509_client_cert", None), + max_retries=kwargs.pop("max_retries", None), + retry_backoff_factor=kwargs.pop("retry_backoff_factor", None), + retry_backoff_jitter=kwargs.pop("retry_backoff_jitter", None) + ) - def update_credentials(self, region=None, profile=None, access_key=None, - secret_key=None, email=None, password=None, proxy=None, - x509_client_cert=None): + def update_credentials( + self, + region=None, + profile=None, + access_key=None, + secret_key=None, + email=None, + password=None, + proxy=None, + x509_client_cert=None, + max_retries=None, + retry_backoff_factor=None, + retry_backoff_jitter=None, + ): self.credentials = { - 'access_key': access_key, - 'secret_key': secret_key, - 'region': region, - 'profile': profile, - 'email': email, - 'password': password, - 'x509_client_cert': x509_client_cert, - 'proxy': proxy + "access_key": access_key, + "secret_key": secret_key, + "region": region, + "profile": profile, + "email": email, + "password": password, + "x509_client_cert": x509_client_cert, + "max_retries": max_retries, + "retry_backoff_factor": retry_backoff_factor, + "retry_backoff_jitter": retry_backoff_jitter, } def api(self, action, **data): try: credentials = Credentials(**self.credentials) - host = (self.host if self.host - else 'api.{}.outscale.{}'.format(credentials.region, credentials.get_url_extension())) - uri = '/api/{}/{}'.format(self.version, action) - protocol = 'https' if self.ssl else 'http' + host = ( + self.host + if self.host + else "api.{}.outscale.{}".format( + credentials.region, credentials.get_url_extension() + ) + ) + uri = "/api/{}/{}".format(self.version, action) + protocol = "https" if self.ssl else "http" - endpoint = os.environ.get('OSC_ENDPOINT_API') + endpoint = os.environ.get("OSC_ENDPOINT_API") if endpoint is None: - endpoint = '{}://{}{}'.format(protocol, host, uri) + endpoint = "{}://{}{}".format(protocol, host, uri) else: - endpoint = '{}{}'.format(endpoint, uri) + endpoint = "{}{}".format(endpoint, uri) - requester = Requester(Authentication(credentials, host, user_agent=self.user_agent), endpoint, self.max_retries) + requester = Requester( + Authentication(credentials, host, user_agent=self.user_agent), + endpoint, + credentials.max_retries, + credentials.retry_backoff_factor, + credentials.retry_backoff_jitter, + ) if self.logger != None: - self.logger.do_log("uri: " + uri + "\npayload:\n" + json.dumps(data, indent=2)) + self.logger.do_log( + "uri: " + uri + "\npayload:\n" + json.dumps(data, indent=2) + ) return requester.send(uri, json.dumps(data)) except Exception as err: raise err diff --git a/osc_sdk_python/credentials.py b/osc_sdk_python/credentials.py index 9062ce6..31f0334 100644 --- a/osc_sdk_python/credentials.py +++ b/osc_sdk_python/credentials.py @@ -1,32 +1,51 @@ import json import os -ORIGINAL_PATH = os.path.join(os.path.expanduser('~'),'.oapi_credentials') -STD_PATH = os.path.join(os.path.expanduser('~'),'.osc/config.json') -DEFAULT_REGION="eu-west-2" -DEFAULT_PROFILE="default" +ORIGINAL_PATH = os.path.join(os.path.expanduser("~"), ".oapi_credentials") +STD_PATH = os.path.join(os.path.expanduser("~"), ".osc/config.json") +DEFAULT_REGION = "eu-west-2" +DEFAULT_PROFILE = "default" +MAX_RETRIES = 3 +RETRY_BACKOFF_FACTOR = 1 +RETRY_BACKOFF_JITTER = 3 + class Credentials: - def __init__(self, region, profile, access_key, secret_key, email, password, - x509_client_cert=None, proxy=None): + def __init__( + self, + region, + profile, + access_key, + secret_key, + email, + password, + x509_client_cert=None, + proxy=None, + max_retries=None, + retry_backoff_factor=None, + retry_backoff_jitter=None, + ): self.region = None self.access_key = access_key self.secret_key = secret_key self.email = email self.password = password - self.x509_client_cert=x509_client_cert - self.proxy=proxy + self.x509_client_cert = x509_client_cert + self.proxy = proxy + self.max_retries = max_retries + self.retry_backoff_factor = retry_backoff_factor + self.retry_backoff_jitter = retry_backoff_jitter if profile is None: - profile = os.environ.get('OSC_PROFILE') + profile = os.environ.get("OSC_PROFILE") else: - # Overide with environmental configuration if available + # Override with environmental configuration if available self.load_credentials_from_env() - # Overide with old configuration if available + # Override with old configuration if available self.load_credentials_from_file(profile, ORIGINAL_PATH) - # Overide with standard configuration if available + # Override with standard configuration if available self.load_credentials_from_file(profile, STD_PATH) - # Overide with environmental configuration if available + # Override with environmental configuration if available if profile is None: profile = DEFAULT_PROFILE self.load_credentials_from_env() @@ -39,12 +58,19 @@ def __init__(self, region, profile, access_key, secret_key, email, password, self.region = DEFAULT_REGION self.profile = profile - # Overide with app parameters if provided + # Override with app parameters if provided if access_key is not None: self.access_key = access_key if secret_key is not None: self.secret_key = secret_key + if self.max_retries is None: + self.max_retries = MAX_RETRIES + if self.retry_backoff_factor is None: + self.retry_backoff_factor = RETRY_BACKOFF_FACTOR + if self.retry_backoff_jitter is None: + self.retry_backoff_jitter = RETRY_BACKOFF_JITTER + self.check_options() def load_credentials_from_file(self, profile, file_path): @@ -52,30 +78,56 @@ def load_credentials_from_file(self, profile, file_path): with open(file_path) as f: config = json.load(f) profile = config.get(profile) + if profile is None: return + ak = profile.get("access_key") if ak is not None: self.access_key = ak + sk = profile.get("secret_key") if sk is not None: self.secret_key = sk + region = profile.get("region") if region is not None: self.region = region + + max_retries = profile.get("max_retries") + if max_retries is not None: + self.max_retries = max_retries + + retry_backoff_factor = profile.get("retry_backoff_factor") + if retry_backoff_factor is not None: + self.retry_backoff_factor = retry_backoff_factor + + retry_backoff_jitter = profile.get("retry_backoff_jitter") + if retry_backoff_jitter is not None: + self.retry_backoff_jitter = retry_backoff_jitter + except IOError: pass def load_credentials_from_env(self): - ak = os.environ.get('OSC_ACCESS_KEY') + ak = os.environ.get("OSC_ACCESS_KEY") if ak is not None: self.access_key = ak - sk = os.environ.get('OSC_SECRET_KEY') + sk = os.environ.get("OSC_SECRET_KEY") if sk is not None: self.secret_key = sk - region = os.environ.get('OSC_REGION') + region = os.environ.get("OSC_REGION") if region is not None: self.region = region + max_retries = os.environ.get("OSC_MAX_RETRIES") + if max_retries is not None: + self.max_retires = int(max_retries) + retry_backoff_factor = os.environ.get("OSC_RETRY_BACKOFF_FACTOR") + if retry_backoff_factor is not None: + self.retry_backoff_factor = float(retry_backoff_factor) + retry_backoff_jitter = os.environ.get("OSC_RETRY_BACKOFF_JITTER") + if retry_backoff_jitter is not None: + self.retry_backoff_factor = float(retry_backoff_jitter) def check_options(self): if self.access_key is not None or self.secret_key is not None: @@ -92,4 +144,4 @@ def check_options(self): raise Exception("Invalid Outscale region") def get_url_extension(self): - return 'hk' if 'cn' in self.region else 'com' + return "hk" if "cn" in self.region else "com" diff --git a/osc_sdk_python/outscale_gateway.py b/osc_sdk_python/outscale_gateway.py index bcbe9d7..1d36f97 100644 --- a/osc_sdk_python/outscale_gateway.py +++ b/osc_sdk_python/outscale_gateway.py @@ -6,10 +6,7 @@ import ruamel.yaml from osc_sdk_python import __version__ -type_mapping = {'boolean': 'bool', - 'string': 'str', - 'integer': 'int', - 'array': 'list'} +type_mapping = {"boolean": "bool", "string": "str", "integer": "int", "array": "list"} # Logs Output Options LOG_NONE = 0 @@ -21,6 +18,7 @@ LOG_ALL = 0 LOG_KEEP_ONLY_LAST_REQ = 1 + class ActionNotExists(NotImplementedError): pass @@ -36,19 +34,23 @@ class ParameterIsRequired(NotImplementedError): class ParameterHasWrongType(NotImplementedError): pass + class Logger: string = "" type = LOG_NONE what = LOG_ALL + def config(self, type=None, what=None): if type is not None: - self.type=type + self.type = type if what is not None: - self.what=what + self.what = what + def str(self): if self.type == LOG_MEMORY: return self.string return None + def do_log(self, s): if self.type & LOG_MEMORY: if self.what == LOG_KEEP_ONLY_LAST_REQ: @@ -63,27 +65,20 @@ def do_log(self, s): class OutscaleGateway: - - def __init__(self, retry=True, **kwargs): + def __init__(self, **kwargs): self._load_gateway_structure() self._load_errors() self.log = Logger() - if retry: - kwargs['max_retries'] = 5 self.call = Call(logger=self.log, **kwargs) - def update_credentials(self, region=None, profile=None, access_key=None, - secret_key=None, email=None, password=None, - x509_client_cert=None, proxy=None): + def update_credentials(self, **kwargs): """ destroy and create a new credential map use for each call. so you can change your ak/sk, region without having to recreate the whole Gateway as the object is recreate, you can't expect to keep parameter from the old configuration example: just updating the password, without renter the login will fail """ - self.call.update_credentials(region=region, profile=profile, access_key=access_key, - secret_key=secret_key, email=email, password=password, - x509_client_cert=x509_client_cert, proxy=x509_client_cert) + self.call.update_credentials(**kwargs) def access_key(self): return Credentials(**self.call.credentials).access_key @@ -103,70 +98,97 @@ def password(self): def _convert(self, input_file): structure = {} try: - with open(input_file, 'r') as fi: - yaml = ruamel.yaml.YAML(typ='safe') + with open(input_file, "r") as fi: + yaml = ruamel.yaml.YAML(typ="safe") content = yaml.load(fi.read()) except Exception as err: - print('Problem reading {}:{}'.format(input_file, str(err))) - self.api_version = content['info']['version'] - for action, params in content['components']['schemas'].items(): - if action.endswith('Request'): - action_name = action.split('Request')[0] + print("Problem reading {}:{}".format(input_file, str(err))) + self.api_version = content["info"]["version"] + for action, params in content["components"]["schemas"].items(): + if action.endswith("Request"): + action_name = action.split("Request")[0] structure[action_name] = {} - for propertie_name, properties in params['properties'].items(): - if propertie_name == 'DryRun': + for propertie_name, properties in params["properties"].items(): + if propertie_name == "DryRun": continue - if 'type' not in properties.keys(): + if "type" not in properties.keys(): action_type = None else: - action_type = type_mapping[properties['type']] - structure[action_name][propertie_name] = {'type': action_type, 'required': False} - - if 'required' in params.keys(): - for required in params['required']: - structure[action_name][required]['required'] = True + action_type = type_mapping[properties["type"]] + structure[action_name][propertie_name] = { + "type": action_type, + "required": False, + } + + if "required" in params.keys(): + for required in params["required"]: + structure[action_name][required]["required"] = True return structure def _load_gateway_structure(self): dir_path = os.path.join(os.path.dirname(__file__)) - yaml_file = os.path.abspath('{}/osc-api/outscale.yaml'.format(dir_path)) + yaml_file = os.path.abspath("{}/osc-api/outscale.yaml".format(dir_path)) self.gateway_structure = self._convert(yaml_file) def _load_errors(self): dir_path = os.path.join(os.path.dirname(__file__)) - yaml_file = os.path.abspath('{}/resources/gateway_errors.yaml'.format(dir_path)) - with open(yaml_file, 'r') as yam: - yaml=ruamel.yaml.YAML(typ='safe') + yaml_file = os.path.abspath("{}/resources/gateway_errors.yaml".format(dir_path)) + with open(yaml_file, "r") as yam: + yaml = ruamel.yaml.YAML(typ="safe") self.gateway_errors = yaml.load(yam.read()) def _check_parameters_type(self, action_structure, input_structure): for i_param, i_value in input_structure.items(): - if i_param != 'Filters' and \ - action_structure[i_param]['type'] is not None and \ - action_structure[i_param]['type'] != i_value.__class__.__name__: - raise ParameterHasWrongType('{} is <{}> instead of <{}>'.format(i_param, i_value.__class__.__name__, - action_structure[i_param]['type'])) + if ( + i_param != "Filters" + and action_structure[i_param]["type"] is not None + and action_structure[i_param]["type"] != i_value.__class__.__name__ + ): + raise ParameterHasWrongType( + "{} is <{}> instead of <{}>".format( + i_param, + i_value.__class__.__name__, + action_structure[i_param]["type"], + ) + ) def _check_parameters_required(self, action_structure, input_structure): - action_mandatory_params = [param for param in action_structure if action_structure[param]['required']] - difference = set(action_mandatory_params).difference(set(input_structure.keys())) + action_mandatory_params = [ + param for param in action_structure if action_structure[param]["required"] + ] + difference = set(action_mandatory_params).difference( + set(input_structure.keys()) + ) if difference: - raise ParameterIsRequired('Missing {}. Required parameters are {}'.format(', '.join(list(difference)) - ,', '.join(action_mandatory_params))) + raise ParameterIsRequired( + "Missing {}. Required parameters are {}".format( + ", ".join(list(difference)), ", ".join(action_mandatory_params) + ) + ) def _check_parameters_valid(self, action_name, params): structure_parameters = self.gateway_structure[action_name].keys() input_parameters = set(params) - different_parameters = list(input_parameters.difference(set(structure_parameters))) + different_parameters = list( + input_parameters.difference(set(structure_parameters)) + ) if different_parameters: - raise ParameterNotValid("""{}. Available parameters on sdk: {} api: {} are: {}.""".format( - ', '.join(different_parameters), - __version__, self.api_version, - ', '.join(structure_parameters))) + raise ParameterNotValid( + """{}. Available parameters on sdk: {} api: {} are: {}.""".format( + ", ".join(different_parameters), + __version__, + self.api_version, + ", ".join(structure_parameters), + ) + ) def _check(self, action_name, **params): if action_name not in self.gateway_structure: - raise ActionNotExists('Action {} does not exists for python sdk: {} with api: {}'.format(action_name, __version__, self.api_version)) + raise ActionNotExists( + "Action {} does not exists for python sdk: {} with api: {}".format( + action_name, __version__, self.api_version + ) + ) self._check_parameters_valid(action_name, params) self._check_parameters_required(self.gateway_structure[action_name], params) self._check_parameters_type(self.gateway_structure[action_name], params) @@ -183,8 +205,9 @@ def _get_action(self, action_name): def action(**kwargs): kwargs = self._remove_none_parameters(**kwargs) self._check(action_name, **kwargs) - result = self.call.api(action_name,**kwargs) + result = self.call.api(action_name, **kwargs) return result + return action def __getattr__(self, attr): @@ -196,38 +219,62 @@ def raw(self, action_name, **kwargs): def test(): a = OutscaleGateway() - a.CreateVms(ImageId='ami-xx', BlockDeviceMappings=[{'/dev/sda1': {'Size': 10}}], SecurityGroupIds=['sg-aaa', 'sg-bbb']) + a.CreateVms( + ImageId="ami-xx", + BlockDeviceMappings=[{"/dev/sda1": {"Size": 10}}], + SecurityGroupIds=["sg-aaa", "sg-bbb"], + ) try: - a.CreateVms(ImageId='ami-xx', BlockDeviceMappings=[{'/dev/sda1': {'Size': 10}}], SecurityGroupIds=['sg-aaa', 'sg-bbb'], Wrong='wrong') + a.CreateVms( + ImageId="ami-xx", + BlockDeviceMappings=[{"/dev/sda1": {"Size": 10}}], + SecurityGroupIds=["sg-aaa", "sg-bbb"], + Wrong="wrong", + ) except ParameterNotValid: pass else: raise AssertionError() try: - a.CreateVms(BlockDeviceMappings=[{'/dev/sda1': {'Size': 10}}], SecurityGroupIds=['sg-aaa', 'sg-bbb']) - except ParameterIsRequired : + a.CreateVms( + BlockDeviceMappings=[{"/dev/sda1": {"Size": 10}}], + SecurityGroupIds=["sg-aaa", "sg-bbb"], + ) + except ParameterIsRequired: pass else: raise AssertionError() try: - a.CreateVms(ImageId=['ami-xxx'], BlockDeviceMappings=[{'/dev/sda1': {'Size': 10}}], SecurityGroupIds=['sg-aaa', 'sg-bbb']) - except ParameterHasWrongType : + a.CreateVms( + ImageId=["ami-xxx"], + BlockDeviceMappings=[{"/dev/sda1": {"Size": 10}}], + SecurityGroupIds=["sg-aaa", "sg-bbb"], + ) + except ParameterHasWrongType: pass else: raise AssertionError() try: - a.CreateVms(ImageId='ami-xxx', BlockDeviceMappings=[{'/dev/sda1': {'Size': 10}}], SecurityGroupIds='wrong') - except ParameterHasWrongType : + a.CreateVms( + ImageId="ami-xxx", + BlockDeviceMappings=[{"/dev/sda1": {"Size": 10}}], + SecurityGroupIds="wrong", + ) + except ParameterHasWrongType: pass else: raise AssertionError() try: - a.CreateVms(ImageId=['ami-wrong'], BlockDeviceMappings=[{'/dev/sda1': {'Size': 10}}], SecurityGroupIds='wrong') - except ParameterHasWrongType : + a.CreateVms( + ImageId=["ami-wrong"], + BlockDeviceMappings=[{"/dev/sda1": {"Size": 10}}], + SecurityGroupIds="wrong", + ) + except ParameterHasWrongType: pass else: raise AssertionError() -if __name__ == '__main__': +if __name__ == "__main__": test() diff --git a/osc_sdk_python/requester.py b/osc_sdk_python/requester.py index 7b605dc..e290b5f 100644 --- a/osc_sdk_python/requester.py +++ b/osc_sdk_python/requester.py @@ -4,15 +4,32 @@ from requests.exceptions import JSONDecodeError from urllib3.util.retry import Retry +HTTP_CODE_RETRY = (400, 429, 500, 503) +METHODS_RETRY = ("POST", "GET") + + class Requester: - def __init__(self, auth, endpoint, max_retries=0, backoff_factor=0.5, status_forcelist=[400, 500]): + def __init__( + self, + auth, + endpoint, + max_retries=0, + backoff_factor=0, + backoff_jitter=0, + status_forcelist=HTTP_CODE_RETRY, + allowed_methods=METHODS_RETRY, + ): self.auth = auth self.endpoint = endpoint - if max_retries > 0: - retry = Retry(total=max_retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist) - self.adapter = HTTPAdapter(max_retries=retry) - else: - self.adapter = HTTPAdapter() + self.adapter = HTTPAdapter( + max_retries=Retry( + total=max_retries, + backoff_factor=backoff_factor, + backoff_jitter=backoff_jitter, + status_forcelist=status_forcelist, + allowed_methods=allowed_methods, + ) + ) def send(self, uri, payload): headers = None @@ -22,22 +39,28 @@ def send(self, uri, payload): headers = self.auth.forge_headers_signed(uri, payload) if self.auth.x509_client_cert is not None: - cert_file=self.auth.x509_client_cert + cert_file = self.auth.x509_client_cert else: - cert_file=None + cert_file = None if self.auth.proxy: if self.auth.proxy.startswith("https"): - proxy= { "https": self.auth.proxy } + proxy = {"https": self.auth.proxy} else: - proxy= { "http": self.auth.proxy } + proxy = {"http": self.auth.proxy} else: - proxy=None + proxy = None with Session() as session: session.mount("https://", self.adapter) session.mount("http://", self.adapter) - response = session.post(self.endpoint, data=payload, headers=headers, verify=True, - proxies=proxy, cert=cert_file) + response = session.post( + self.endpoint, + data=payload, + headers=headers, + verify=True, + proxies=proxy, + cert=cert_file, + ) self.raise_for_status(response) return response.json() @@ -82,9 +105,7 @@ def raise_for_status(self, response): f"url = {response.url}" ) else: - http_error_msg = ( - f"{response.status_code} Client Error: {reason} for url: {response.url}" - ) + http_error_msg = f"{response.status_code} Client Error: {reason} for url: {response.url}" elif 500 <= response.status_code < 600: if error_code and request_id: @@ -98,9 +119,7 @@ def raise_for_status(self, response): f"url = {response.url}" ) else: - http_error_msg = ( - f"{response.status_code} Server Error: {reason} for url: {response.url}" - ) + http_error_msg = f"{response.status_code} Server Error: {reason} for url: {response.url}" if http_error_msg: raise HTTPError(http_error_msg, response=response) diff --git a/tests/test_exeptions.py b/tests/test_exceptions.py similarity index 67% rename from tests/test_exeptions.py rename to tests/test_exceptions.py index 7f9a109..c726c30 100644 --- a/tests/test_exeptions.py +++ b/tests/test_exceptions.py @@ -2,14 +2,14 @@ import sys sys.path.append("..") from osc_sdk_python import Gateway -from requests import HTTPError +from requests.exceptions import RetryError -class TestExept(unittest.TestCase): +class TestExcept(unittest.TestCase): def test_listing(self): gw = Gateway() # a is not a valide argument - with self.assertRaises(HTTPError): + with self.assertRaises(RetryError): gw.ReadVms(Filters="a") if __name__ == '__main__': diff --git a/tests/test_exeptions_500.py b/tests/test_exceptions_500.py similarity index 94% rename from tests/test_exeptions_500.py rename to tests/test_exceptions_500.py index 5e95ca2..c5eeb3b 100644 --- a/tests/test_exeptions_500.py +++ b/tests/test_exceptions_500.py @@ -7,7 +7,7 @@ import time sys.path.append("..") from osc_sdk_python import Gateway -from requests import HTTPError +from requests.exceptions import RetryError class Send500(http.server.BaseHTTPRequestHandler): def do_POST(self): @@ -48,7 +48,7 @@ def test_server_error(self): os.environ['OSC_ENDPOINT_API'] = "http://127.0.0.1:8000" gw = Gateway() # a is not a valide argument - with self.assertRaises(HTTPError): + with self.assertRaises(RetryError): gw.ReadVms() if __name__ == '__main__':