Merge pull request #2 from bgpkit/v2-broker

support broker api version 2

Author: digizeph

README.md

@@ -62,7 +62,7 @@ Example:
 ```python
 import bgpkit
 broker = bgpkit.Broker()
-items = broker.query(start_ts=1634693400, end_ts=1634693400)
+items = broker.query(ts_start="1634693400", ts_end="2021-10-20T01:30:00")
 for item in items:
     print(item)
 print(len(items))
@@ -72,14 +72,13 @@ assert len(items) == 58
 Available fields:
 
 - `Broker()`
-  - `api_url`: the base URL for the BGPKIT Broker instance. Default: `https://api.broker.bgpkit.com/v1`
+  - `api_url`: the base URL for the BGPKIT Broker instance. Default: `https://api.broker.bgpkit.com/v2`
   - `page_size`: the number of items per API call (no need to change it). Default: 100.
 - `query()`
-  - `start_ts`: start timestamp for MRT file, UNIX timestamp format
-  - `end_ts`: end timestamp for MRT file, UNIX timestamp format
+  - `ts_start`: start timestamp for MRT file, UNIX timestamp format
+  - `ts_end`: end timestamp for MRT file, UNIX timestamp format
   - `collector`: collector name, e.g. `rrc00` or `route-views2`
   - `data_type`: `rib` or `update`
-  - `order`: order by timestamp, `asc` or `desc`, default
 
 ### BGPKIT ROAS Lookup
 

bgpkit/bgpkit_broker.py

@@ -13,46 +13,43 @@ def check_type(value: any, ty: type) -> bool:
 
 @dataclass
 class BrokerItem:
+    ts_start: str
+    ts_end: str
     collector_id: str
     data_type: str
-    timestamp: int
     url: str
+    rough_size: int
+    exact_size: int
 
 
 class Broker:
 
-    def __init__(self, api_url: str = "https://api.broker.bgpkit.com/v1", page_size: int = 100):
+    def __init__(self, api_url: str = "https://api.broker.bgpkit.com/v2", page_size: int = 100):
         self.base_url = api_url.strip()
         self.page_size = int(page_size)
 
     def query(self,
-              start_ts: int = None,
-              end_ts: int = None,
-              collector: str = None,
+              ts_start: str = None,
+              ts_end: str = None,
+              collector_id: str = None,
               project: str = None,
               data_type: str = None,
-              order: str = None,
               print_url: bool = False,
               ) -> [BrokerItem]:
         params = []
-        if start_ts:
-            check_type(start_ts, int)
-            params.append(f"start_ts={start_ts}")
-        if end_ts:
-            check_type(end_ts, int)
-            params.append(f"end_ts={end_ts}")
-        if collector:
-            check_type(collector, str)
-            params.append(f"collector={collector}")
+        if ts_start:
+            params.append(f"ts_start={ts_start}")
+        if ts_end:
+            params.append(f"ts_end={ts_end}")
+        if collector_id:
+            check_type(collector_id, str)
+            params.append(f"collector={collector_id}")
         if project:
             check_type(project, str)
             params.append(f"project={project}")
         if data_type:
             check_type(data_type, str)
             params.append(f"data_type={data_type}")
-        if order:
-            check_type(order, str)
-            params.append(f"order={order}")
         params.append(f"page_size={self.page_size}")
 
         api_url = f"{self.base_url}/search?" + "&".join(params)
@@ -61,10 +58,10 @@ def query(self,
         data_items = []
         if print_url:
             print(api_url)
-        res = requests.get(api_url).json()["data"]
+        res = requests.get(api_url).json()
         while res:
             if res["count"] > 0:
-                data_items.extend([BrokerItem(**i) for i in res["items"]])
+                data_items.extend([BrokerItem(**i) for i in res["data"]])
 
                 if res["count"] < res["page_size"]:
                     break
@@ -73,7 +70,7 @@ def query(self,
                 query_url = f"{api_url}&page={page}"
                 if print_url:
                     print(query_url)
-                res = requests.get(query_url).json()["data"]
+                res = requests.get(query_url).json()
             else:
                 break
 

bgpkit/bgpkit_parser.py

@@ -1 +1 @@
-from pybgpkit_parser import Parser
+from pybgpkit_parser import Parser

bgpkit/test_integration.py

@@ -1,3 +1,4 @@
+import json
 import unittest
 
 import bgpkit
@@ -13,11 +14,11 @@ def test_parser(self):
 
     def test_broker(self):
         broker = bgpkit.Broker()
-        items = broker.query(start_ts=1643760000, end_ts=1643761200)
+        items = broker.query(ts_start="1643760000", ts_end="2022-02-02T00:20:00")
         for item in items:
-            print(item)
+            print(json.dumps(item.__dict__))
         print(len(items))
-        assert len(items) == 58
+        assert len(items) == 290
 
         # elems = bgpkit.Parser(items[0].url).parse_all()
         # assert len(elems) == 222467

setup.py

@@ -7,7 +7,7 @@
 
 setuptools.setup(
     name='pybgpkit',
-    version='0.0.2.post1',
+    version='0.0.4',
     description='BGPKIT tools Python bindings',
     url='https://github.com/bgpkit/pybgpkit',
     author='Mingwei Zhang',
@@ -18,9 +18,13 @@
     long_description_content_type='text/markdown',
     install_requires=[
         # available on pip
-        'pybgpkit-parser==0.0.1',
+        'dataclasses_json',
+        'pybgpkit-parser==0.1.0',
         'requests',
     ],
-    entry_points={'console_scripts': [
-    ]}
+    entry_points={
+        'console_scripts': [
+            "pybgpkit=bgpkit.cli:main"
+        ]
+    }
 )