Merge pull request #2112 from allmightyspiff/threads

Added a thread capable client.cf_call() function

Author: allmightyspiff

SoftLayer/API.py

@@ -9,11 +9,12 @@
 import time
 import warnings
 
+import concurrent.futures as cf
 import json
 import logging
+import math
 import requests
 
-
 from SoftLayer import auth as slauth
 from SoftLayer import config
 from SoftLayer import consts
@@ -289,13 +290,6 @@ def call(self, service, method, *args, **kwargs):
             request.verify = kwargs.get('verify')
 
         if self.auth:
-            extra_headers = self.auth.get_headers()
-            if extra_headers:
-                warnings.warn("auth.get_headers() is deprecated and will be "
-                              "removed in the next major version",
-                              DeprecationWarning)
-                request.headers.update(extra_headers)
-
             request = self.auth.get_request(request)
 
         request.headers.update(kwargs.get('headers', {}))
@@ -352,6 +346,42 @@ def iter_call(self, service, method, *args, **kwargs):
 
             offset += limit
 
+    def cf_call(self, service, method, *args, **kwargs):
+        """Uses threads to iterate through API calls.
+
+        :param service: the name of the SoftLayer API service
+        :param method: the method to call on the service
+        :param integer limit: result size for each API call (defaults to 100)
+        :param \\*args: same optional arguments that ``Service.call`` takes
+        :param \\*\\*kwargs: same optional keyword arguments that ``Service.call`` takes
+        """
+        limit = kwargs.pop('limit', 100)
+        offset = kwargs.pop('offset', 0)
+
+        if limit <= 0:
+            raise AttributeError("Limit size should be greater than zero.")
+        # This initial API call is to determine how many API calls we need to make after this first one.
+        first_call = self.call(service, method, offset=offset, limit=limit, *args, **kwargs)
+
+        # This was not a list result, just return it.
+        if not isinstance(first_call, transports.SoftLayerListResult):
+            return first_call
+        # How many more API calls we have to make
+        api_calls = math.ceil((first_call.total_count - limit) / limit)
+
+        def this_api(offset):
+            """Used to easily call executor.map() on this fuction"""
+            return self.call(service, method, offset=offset, limit=limit, *args, **kwargs)
+
+        with cf.ThreadPoolExecutor(max_workers=10) as executor:
+            future_results = {}
+            offset_map = [x * limit for x in range(1, api_calls)]
+            future_results = list(executor.map(this_api, offset_map))
+        # Append the results in the order they were called
+        for call_result in future_results:
+            first_call = first_call + call_result
+        return first_call
+
     def __repr__(self):
         return "Client(transport=%r, auth=%r)" % (self.transport, self.auth)
 

SoftLayer/CLI/vlan/list.py

@@ -25,17 +25,13 @@
 
 
 @click.command(cls=SoftLayer.CLI.command.SLCommand, )
-@click.option('--sortby',
-              help='Column to sort by',
-              type=click.Choice(COLUMNS))
+@click.option('--sortby', help='Column to sort by', type=click.Choice(COLUMNS))
 @click.option('--datacenter', '-d',
               help='Filter by datacenter shortname (sng01, dal05, ...)')
 @click.option('--number', '-n', help='Filter by VLAN number')
 @click.option('--name', help='Filter by VLAN name')
-@click.option('--limit', '-l',
-              help='How many results to get in one api call, default is 100',
-              default=100,
-              show_default=True)
+@click.option('--limit', '-l', default=100, show_default=True,
+              help='How many results to get in one api call, default is 100')
 @environment.pass_env
 def cli(env, sortby, datacenter, number, name, limit):
     """List VLANs.

SoftLayer/managers/network.py

@@ -515,46 +515,37 @@ def list_subnets(self, identifier=None, datacenter=None, version=0,
         kwargs['iter'] = True
         return self.client.call('Account', 'getSubnets', **kwargs)
 
-    def list_vlans(self, datacenter=None, vlan_number=None, name=None, limit=100, **kwargs):
+    def list_vlans(self, datacenter=None, vlan_number=None, name=None, limit=100, mask=None, _filter=None):
         """Display a list of all VLANs on the account.
 
         This provides a quick overview of all VLANs including information about
         data center residence and the number of devices attached.
 
-        :param string datacenter: If specified, the list will only contain
-                                    VLANs in the specified data center.
-        :param int vlan_number: If specified, the list will only contain the
-                                  VLAN matching this VLAN number.
-        :param int name: If specified, the list will only contain the
-                                  VLAN matching this VLAN name.
+        :param string datacenter: If specified, the list will only contain  VLANs in the specified data center.
+        :param int vlan_number: If specified, the list will only contain the VLAN matching this VLAN number.
+        :param int name: If specified, the list will only contain the VLAN matching this VLAN name.
         :param dict \\*\\*kwargs: response-level options (mask, limit, etc.)
 
         """
-        _filter = utils.NestedDict(kwargs.get('filter') or {})
+        _filter = utils.NestedDict(_filter or {})
 
         _filter['networkVlans']['id'] = utils.query_filter_orderby()
 
         if vlan_number:
-            _filter['networkVlans']['vlanNumber'] = (
-                utils.query_filter(vlan_number))
+            _filter['networkVlans']['vlanNumber'] = utils.query_filter(vlan_number)
 
         if name:
             _filter['networkVlans']['name'] = utils.query_filter(name)
 
         if datacenter:
-            _filter['networkVlans']['primaryRouter']['datacenter']['name'] = (
-                utils.query_filter(datacenter))
-
-        kwargs['filter'] = _filter.to_dict()
+            _filter['networkVlans']['primaryRouter']['datacenter']['name'] = utils.query_filter(datacenter)
 
-        if 'mask' not in kwargs:
-            kwargs['mask'] = DEFAULT_VLAN_MASK
+        if mask is None:
+            mask = DEFAULT_VLAN_MASK
 
-        kwargs['iter'] = True
-        if limit > 0:
-            return self.account.getNetworkVlans(mask=kwargs['mask'], filter=_filter.to_dict(), limit=limit, iter=True)
-        else:
-            return self.account.getNetworkVlans(mask=kwargs['mask'], filter=_filter.to_dict(), iter=True)
+        # cf_call uses threads to get all results.
+        return self.client.cf_call('SoftLayer_Account', 'getNetworkVlans',
+                                   mask=mask, filter=_filter.to_dict(), limit=limit)
 
     def list_securitygroups(self, **kwargs):
         """List security groups."""

tests/CLI/modules/vlan_tests.py

@@ -147,9 +147,15 @@ def test_vlan_list_get_pod_with_closed_announcement(self, ngb_mock):
         ngb_mock.return_value = True
         vlan_mock = self.set_mock('SoftLayer_Account', 'getNetworkVlans')
         gpods_mock = self.set_mock('SoftLayer_Network_Pod', 'getAllObjects')
-        vlan_mock.return_value = {'primaryRouter': {'fullyQualifiedDomainName': 'bcr01a.fra02.softlayer.com',
-                                                    'id': 462912},
-                                  'tagReferences': ''}
+        vlan_mock.return_value = [
+            {
+                "primaryRouter": {
+                    "fullyQualifiedDomainName": "bcr01a.fra02.softlayer.com",
+                    "id": 462912
+                },
+                "tagReferences": ""
+            }
+        ]
         gpods_mock.return_value = [{'backendRouterId': 462912,
                                     'backendRouterName': 'bcr01a.fra02',
                                     'datacenterId': 449506,
@@ -166,9 +172,15 @@ def test_vlan_list_get_pod_with_closed_announcement_no_closure(self, ngb_mock):
         ngb_mock.return_value = True
         vlan_mock = self.set_mock('SoftLayer_Account', 'getNetworkVlans')
         gpods_mock = self.set_mock('SoftLayer_Network_Pod', 'getAllObjects')
-        vlan_mock.return_value = {'primaryRouter': {'fullyQualifiedDomainName': 'bcr01a.fra02.softlayer.com',
-                                                    'id': 462912},
-                                  'tagReferences': ''}
+        vlan_mock.return_value = [
+            {
+                "primaryRouter": {
+                    "fullyQualifiedDomainName": "bcr01a.fra02.softlayer.com",
+                    "id": 462912
+                },
+                "tagReferences": ""
+            }
+        ]
         gpods_mock.return_value = [{'backendRouterId': 462912,
                                     'backendRouterName': 'bcr01a.fra02',
                                     'datacenterId': 449506,