feat: Use AS-SETs correctly and hint irrd with source for performance

wanda/as_filter/as_filter.py

@@ -2,7 +2,7 @@
 from functools import cached_property
 
 from wanda.autonomous_system.autonomous_system import AutonomousSystem
-from wanda.irrd_client import IRRDClient
+from wanda.irrd_client import IRRDClient, InvalidASSETException
 from wanda.logger import Logger
 
 l = Logger("as_filter.py")
@@ -21,20 +21,38 @@ def prefix_lists(self):
         v4_set = set()
         v6_set = set()
 
+        irr_v4_set = set()
+        irr_v6_set = set()
+
         irr_names = self.autos.get_irr_names()
 
-        if not irr_names:
+        for irr_name in irr_names:
+            try:
+                result_entries_v4, result_entries_v6 = self.irrd_client.generate_prefix_lists(irr_name)
+
+                irr_v4_set.update(result_entries_v4)
+                irr_v6_set.update(result_entries_v6)
+            except InvalidASSETException:
+                l.warning(f"{irr_name} is not a valid AS-SET, ignoring...")
+
+        enforce_as_based_filtering = len(irr_names) > 0 and len(irr_v4_set) == 0 and len(irr_v6_set) == 0
+
+        if not irr_names or enforce_as_based_filtering:
+            # Print a warning to notify the user, that we will filter ASN-based
+            if enforce_as_based_filtering:
+                l.warning(f"AS {self.autos.asn} has not a single valid AS-SET, falling back to AS-based prefix filter lists...")
+
+            # Using the ASN-based filtering
             result_entries_v4, result_entries_v6 = self.irrd_client.generate_prefix_lists_for_asn(self.autos.asn)
 
             v4_set.update(result_entries_v4)
             v6_set.update(result_entries_v6)
         else:
-            for irr_name in irr_names:
-                result_entries_v4, result_entries_v6 = self.irrd_client.generate_prefix_lists(irr_name)
-
-                v4_set.update(result_entries_v4)
-                v6_set.update(result_entries_v6)
+            # Using the AS-SET based filtering
+            v4_set = irr_v4_set
+            v6_set = irr_v6_set
 
+        # If the ASN is a customer, we forbid entirely empty filter lists.
         if len(v4_set) == 0 and len(v6_set) == 0 and self.is_customer:
             raise Exception(f"{self.autos} has neither IPv4, nor IPv6 filter lists. Since AS is our customer, we forbid this for security reasons.")
 
@@ -45,11 +63,16 @@ def get_filter_lists(self, enable_extended_filters=False):
         irr_names = self.autos.get_irr_names()
         filters = {}
 
-        if irr_names:
-            filters['origin_asns'] = sorted(self.irrd_client.generate_input_aspath_access_list(self.autos.asn, irr_names[0]))
-        else:
-            filters['origin_asns'] = [self.autos.asn]
+        default_origin_asns = [self.autos.asn]
+
+        try:
+            if len(irr_names) > 0:
+                filters['origin_asns'] = sorted(self.irrd_client.generate_input_aspath_access_list(irr_names[0]))
+        except InvalidASSETException:
+            l.warning(f"{irr_names[0]} is not a valid AS-SET, falling back to AS-based as-path filter lists..")
 
+        if 'origin_asns' not in filters:
+            filters['origin_asns'] = default_origin_asns
 
         if enable_extended_filters:
             v4_set, v6_set = self.prefix_lists

wanda/autonomous_system/autonomous_system.py

@@ -23,10 +23,11 @@ def get_irr_names(self):
         if self.irr_as_set:
             set_elements = self.irr_as_set.upper().split(" ")
             for set_element in set_elements:
-                match_result = re.findall(r'(AS[^\s,]*)', set_element)
+                match_result = re.findall(r'^(?:([A-Z]+)::)?((?:AS[0-9]+[:]+)?(?:AS-[A-Z0-9-]+))$', set_element)
 
                 if len(match_result) != 0:
-                    return_elements.append(match_result[0])
+                    matches = match_result[0]
+                    return_elements.append(matches[1])
 
         # Note: If there is no IRR names, we fall back to AS1234, but we do this later in the code.
 

wanda/irrd_client.py

@@ -6,6 +6,10 @@
 
 l = Logger("irrd_client.py")
 
+class InvalidASSETException(Exception):
+    def __init__(self, as_set, asn=None):
+        super().__init__(f"AS-SET {as_set} did not resolve, probably invalid AS-SET..")
+
 
 class IRRDClient:
 
@@ -17,14 +21,17 @@ def fetch_graphql_data(self, query):
         response.raise_for_status()
         return response.json()["data"]
 
-    def generate_input_aspath_access_list(self, asn, irr_name):
+    def generate_input_aspath_access_list(self, irr_name):
         body = f"""
           {{
               recursiveSetMembers(setNames: ["{irr_name}"], depth: 8) {{ members }}
           }}
         """
         result = self.fetch_graphql_data(body)
 
+        if len(result['recursiveSetMembers']) == 0:
+            raise InvalidASSETException(irr_name)
+
         # return unique members that are ASNs
         members = set(result["recursiveSetMembers"][0]["members"])
         return [int(i[2:]) for i in members if re.match(r"^AS\d+$", i)]
@@ -47,5 +54,8 @@ def generate_prefix_lists(self, irr_name):
           }}
         """
         result = self.fetch_graphql_data(body)
+
+        if len(result['v4']) == 0 and len(result['v6']) == 0:
+            raise InvalidASSETException(irr_name)
 
         return set(result["v4"][0]["prefixes"]), set(result["v6"][0]["prefixes"])

wanda/tests/test_as.py

@@ -11,7 +11,8 @@ class TestAutonomousSystem:
         [
             (9136, "WOBCOM", "AS-WOBCOM", ["AS-WOBCOM"]),
             (208395, "WDZ", "", []),
-            (1299, "Twelve99", "RIPE::AS-TELIANET RIPE::AS-TELIANET-V6", ["AS-TELIANET", "AS-TELIANET-V6"])
+            (1299, "Twelve99", "RIPE::AS-TELIANET RIPE::AS-TELIANET-V6", ["AS-TELIANET", "AS-TELIANET-V6"]),
+            (64496, "FAKE", "RIPE::AS64496:AS-FAKE RIPE::AS64496:AS-FAKE-V6", ["AS64496:AS-FAKE", "AS64496:AS-FAKE-V6"])
         ]
     )
     def test_irr_name(self, asn, name, irr_names, expected_irr_names):

wanda/tests/test_bgp_device_group.py

@@ -10,7 +10,7 @@
 def get_bgp_device_group_4(**kwargs):
     group = BGPDeviceGroup(
         name="TEST",
-        asn=1234,
+        asn=64496,
         ip_version=4,
         max_prefixes=69,
         **kwargs
@@ -31,7 +31,7 @@ def get_bgp_device_group_4(**kwargs):
 def get_bgp_device_group_6(**kwargs):
     group = BGPDeviceGroup(
         name="TEST",
-        asn=1234,
+        asn=64496,
         ip_version=6,
         max_prefixes=69,
         **kwargs
@@ -63,7 +63,7 @@ def test_junos_closure(self, bdg, ip_version):
         junos_closure = bdg.to_junos()
 
         assert junos_closure['name'] is "TEST"
-        assert junos_closure['peer_as'] is 1234
+        assert junos_closure['peer_as'] is 64496
         assert isinstance(junos_closure['export'], list)
         assert isinstance(junos_closure['import'], list)
         assert junos_closure["family"][f"ipv{ip_version}_unicast"]["max_prefixes"] == 69
@@ -141,34 +141,34 @@ def test_additional_export_policies(self, bdg, expected_export_policy):
         [
             (get_bgp_device_group_4(),
              ['FILTER_BOGONS_V4', 'FILTER_OWN_V4', 'BOGON_ASN_FILTERING', 'SCRUB_COMMUNITIES', 'TIER1_FILTERING',
-              'RPKI_FILTERING', 'POLICY_AS1234_V4', 'PEERING_IMPORT_V4']),
+              'RPKI_FILTERING', 'POLICY_AS64496_V4', 'PEERING_IMPORT_V4']),
             (get_bgp_device_group_4(is_route_server=True),
              ['FILTER_BOGONS_V4', 'FILTER_OWN_V4', 'BOGON_ASN_FILTERING', 'SCRUB_COMMUNITIES', 'TIER1_FILTERING',
               'RPKI_FILTERING', 'PEERING_IMPORT_V4']),
             (get_bgp_device_group_4(policy_type="customer"),
              ['FILTER_BOGONS_V4', 'FILTER_OWN_V4', 'BOGON_ASN_FILTERING', 'TIER1_FILTERING', 'RPKI_FILTERING',
-              'POLICY_AS1234_V4', 'CUSTOMER_IMPORT_V4']),
+              'POLICY_AS64496_V4', 'CUSTOMER_IMPORT_V4']),
             (get_bgp_device_group_4(policy_type="transit"),
              ['FILTER_BOGONS_V4', 'FILTER_OWN_V4', 'BOGON_ASN_FILTERING', 'SCRUB_COMMUNITIES', 'RPKI_FILTERING',
               'UPSTREAM_IMPORT_V4']),
             (get_bgp_device_group_4(policy_type="pni"),
              ['FILTER_BOGONS_V4', 'FILTER_OWN_V4', 'BOGON_ASN_FILTERING', 'SCRUB_COMMUNITIES', 'TIER1_FILTERING',
-              'RPKI_FILTERING', 'POLICY_AS1234_V4', 'PNI_IMPORT_V4']),
+              'RPKI_FILTERING', 'POLICY_AS64496_V4', 'PNI_IMPORT_V4']),
             (get_bgp_device_group_6(),
              ['FILTER_BOGONS_V6', 'FILTER_OWN_V6', 'BOGON_ASN_FILTERING', 'SCRUB_COMMUNITIES', 'TIER1_FILTERING',
-              'RPKI_FILTERING', 'POLICY_AS1234_V6', 'PEERING_IMPORT_V6']),
+              'RPKI_FILTERING', 'POLICY_AS64496_V6', 'PEERING_IMPORT_V6']),
             (get_bgp_device_group_6(is_route_server=True),
              ['FILTER_BOGONS_V6', 'FILTER_OWN_V6', 'BOGON_ASN_FILTERING', 'SCRUB_COMMUNITIES', 'TIER1_FILTERING',
               'RPKI_FILTERING', 'PEERING_IMPORT_V6']),
             (get_bgp_device_group_6(policy_type="customer"),
              ['FILTER_BOGONS_V6', 'FILTER_OWN_V6', 'BOGON_ASN_FILTERING', 'TIER1_FILTERING', 'RPKI_FILTERING',
-              'POLICY_AS1234_V6', 'CUSTOMER_IMPORT_V6']),
+              'POLICY_AS64496_V6', 'CUSTOMER_IMPORT_V6']),
             (get_bgp_device_group_6(policy_type="transit"),
              ['FILTER_BOGONS_V6', 'FILTER_OWN_V6', 'BOGON_ASN_FILTERING', 'SCRUB_COMMUNITIES', 'RPKI_FILTERING',
               'UPSTREAM_IMPORT_V6']),
             (get_bgp_device_group_6(policy_type="pni"),
              ['FILTER_BOGONS_V6', 'FILTER_OWN_V6', 'BOGON_ASN_FILTERING', 'SCRUB_COMMUNITIES', 'TIER1_FILTERING',
-              'RPKI_FILTERING', 'POLICY_AS1234_V6', 'PNI_IMPORT_V6']),
+              'RPKI_FILTERING', 'POLICY_AS64496_V6', 'PNI_IMPORT_V6']),
         ]
     )
     def test_basic_import_policies(self, bdg, expected_policies):

wanda/tests/test_irrd_client.py

@@ -3,10 +3,19 @@
 
 import pytest
 
-from wanda.irrd_client import IRRDClient
+from wanda.irrd_client import IRRDClient, InvalidASSETException
+
+FAKE_PREFIX_LIST_MOCK_V4 = [
+    "198.51.100.0/25"
+]
+
+FAKE_PREFIX_LIST_MOCK_V6 = [
+    "2001:db8::23/32"
+]
+
 
 WDZ_PREFIX_LIST_MOCK_V4 = [
-    "198.51.100.0/24"
+    "198.51.100.128/25"
 ]
 
 WDZ_PREFIX_LIST_MOCK_V6 = [
@@ -147,6 +156,9 @@
     "AS208395"
 ]
 
+AS_PATH_FAKE = [
+    "AS64496"
+]
 
 # We mock each response and threat this as a unit test since bgpq4 is considered stable.
 # We might test an additional integration test later on.
@@ -164,6 +176,7 @@ def irrd_instance(self):
     @pytest.mark.parametrize(
         "irr_name,prefix_num,prefix_list_v4,prefix_list_v6",
         [
+            ("RIPE::AS64496:AS-FAKE", (1, 1), FAKE_PREFIX_LIST_MOCK_V4, FAKE_PREFIX_LIST_MOCK_V6),
             ("AS-WOBCOM", (4, 4), WOBCOM_PREFIX_LIST_MOCK_V4, WOBCOM_PREFIX_LIST_MOCK_V6),
             ("AS208395", (1, 1), WDZ_PREFIX_LIST_MOCK_V4, WDZ_PREFIX_LIST_MOCK_V6),
         ]
@@ -195,9 +208,28 @@ def test_prefix_lists(self, mocker, irrd_instance, irr_name, prefix_num, prefix_
         assert all(ipaddress.IPv4Network(ip, strict=False) for ip in prefix_list_4)
         assert all(ipaddress.IPv6Network(ip, strict=False) for ip in prefix_list_6)
 
+    @pytest.mark.parametrize(
+        "irr_name,",
+        [
+            ("RIPE::AS64496:AS-FAKE",),
+        ]
+    )
+    def test_invalid_prefix_lists(self, mocker, irrd_instance, irr_name, ):
+        mocker.patch(
+            'wanda.irrd_client.IRRDClient.fetch_graphql_data',
+            return_value={
+                "v4": [],
+                "v6": []
+            }
+        )
+
+        with pytest.raises(InvalidASSETException):
+            irrd_instance.generate_prefix_lists(irr_name)
+
     @pytest.mark.parametrize(
         "irr_name,asn,as_path_output",
         [
+            ("RIPE::AS64496:AS-FAKE", 64496, AS_PATH_FAKE),
             ("AS-WOBCOM", 9136, AS_PATH_WOBCOM),
             ("AS208395", 208395, AS_PATH_WDZ),
         ]
@@ -214,11 +246,28 @@ def test_input_as_path_access_list(self, mocker, irrd_instance, irr_name, asn, a
             }
         )
 
-        access_list = irrd_instance.generate_input_aspath_access_list(asn, irr_name)
+        access_list = irrd_instance.generate_input_aspath_access_list(irr_name)
 
         assert asn in access_list
         assert all([isinstance(x, int) for x in access_list])
 
+    @pytest.mark.parametrize(
+        "irr_name",
+        [
+            ("RIPE::AS64497:AS-FAKE",),
+        ]
+    )
+    def test_input_invalid_as_path_access_list(self, mocker, irrd_instance, irr_name):
+        mocker.patch(
+            'wanda.irrd_client.IRRDClient.fetch_graphql_data',
+            return_value={
+                "recursiveSetMembers": [],
+            }
+        )
+
+        with pytest.raises(InvalidASSETException):
+            irrd_instance.generate_input_aspath_access_list(irr_name)
+
 
     def test_invalid_bgpq4_prefix_lists(self, irrd_instance):
         with pytest.raises(Exception):