From f9cd70361db159479635ec51924e41828c1c94de Mon Sep 17 00:00:00 2001 From: Jeremy Cloarec Date: Mon, 16 Jun 2025 18:25:47 +0200 Subject: [PATCH 1/4] [client] report expection when element is ignored during import --- pycti/connector/opencti_connector_helper.py | 1 + pycti/utils/opencti_stix2.py | 8 ++++++- pycti/utils/opencti_stix2_splitter.py | 9 ++++--- .../utils/test_opencti_stix2_splitter.py | 24 +++++++++---------- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/pycti/connector/opencti_connector_helper.py b/pycti/connector/opencti_connector_helper.py index d8983f896..939354aa1 100644 --- a/pycti/connector/opencti_connector_helper.py +++ b/pycti/connector/opencti_connector_helper.py @@ -1921,6 +1921,7 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list: ( expectations_number, bundles, + _ ) = stix2_splitter.split_bundle_with_expectations( bundle=bundle, use_json=True, diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index a303844c3..be43b9d10 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -2748,9 +2748,15 @@ def import_bundle( ) stix2_splitter = OpenCTIStix2Splitter() - _, bundles = stix2_splitter.split_bundle_with_expectations( + _, nb_incompatible_elements, bundles = stix2_splitter.split_bundle_with_expectations( stix_bundle, False, event_version ) + + # Report every element ignored during bundle splitting + if work_id is not None: + for i in range(nb_incompatible_elements): + self.opencti.work.report_expectation(work_id, None) + # Import every element in a specific order imported_elements = [] for bundle in bundles: diff --git a/pycti/utils/opencti_stix2_splitter.py b/pycti/utils/opencti_stix2_splitter.py index 10a655908..f422baf06 100644 --- a/pycti/utils/opencti_stix2_splitter.py +++ b/pycti/utils/opencti_stix2_splitter.py @@ -35,6 +35,7 @@ def __init__(self): self.cache_index = {} self.cache_refs = {} self.elements = [] + self.nb_incompatible_items = 0 def get_internal_ids_in_extension(self, item): ids = [] @@ -189,6 +190,8 @@ def enlist_element( is_compatible = is_id_supported(item_id) if is_compatible: self.elements.append(item) + else: + self.nb_incompatible_items = self.nb_incompatible_items + 1 self.cache_index[item_id] = item for internal_id in self.get_internal_ids_in_extension(item): self.cache_index[internal_id] = item @@ -201,7 +204,7 @@ def split_bundle_with_expectations( use_json=True, event_version=None, cleanup_inconsistent_bundle=False, - ) -> Tuple[int, list]: + ) -> Tuple[int, int, list]: """splits a valid stix2 bundle into a list of bundles""" if use_json: try: @@ -251,11 +254,11 @@ def by_dep_size(elem): ) ) - return number_expectations, bundles + return number_expectations, self.nb_incompatible_items, bundles @deprecated("Use split_bundle_with_expectations instead") def split_bundle(self, bundle, use_json=True, event_version=None) -> list: - expectations, bundles = self.split_bundle_with_expectations( + _, _, bundles = self.split_bundle_with_expectations( bundle, use_json, event_version ) return bundles diff --git a/tests/01-unit/utils/test_opencti_stix2_splitter.py b/tests/01-unit/utils/test_opencti_stix2_splitter.py index 59cd33354..146182873 100644 --- a/tests/01-unit/utils/test_opencti_stix2_splitter.py +++ b/tests/01-unit/utils/test_opencti_stix2_splitter.py @@ -10,7 +10,7 @@ def test_split_bundle(): stix_splitter = OpenCTIStix2Splitter() with open("./tests/data/enterprise-attack.json") as file: content = file.read() - expectations, bundles = stix_splitter.split_bundle_with_expectations(content) + expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content) assert expectations == 7016 @@ -18,7 +18,7 @@ def test_split_test_bundle(): stix_splitter = OpenCTIStix2Splitter() with open("./tests/data/DATA-TEST-STIX2_v2.json") as file: content = file.read() - expectations, bundles = stix_splitter.split_bundle_with_expectations(content) + expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content) assert expectations == 59 base_bundles = json.loads(content)["objects"] for base in base_bundles: @@ -40,13 +40,13 @@ def test_split_mono_entity_bundle(): stix_splitter = OpenCTIStix2Splitter() with open("./tests/data/mono-bundle-entity.json") as file: content = file.read() - expectations, bundles = stix_splitter.split_bundle_with_expectations(content) + expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content) assert expectations == 1 json_bundle = json.loads(bundles[0])["objects"][0] assert json_bundle["created_by_ref"] == "fa42a846-8d90-4e51-bc29-71d5b4802168" # Split with cleanup_inconsistent_bundle stix_splitter = OpenCTIStix2Splitter() - expectations, bundles = stix_splitter.split_bundle_with_expectations( + expectations, _, bundles = stix_splitter.split_bundle_with_expectations( bundle=content, cleanup_inconsistent_bundle=True ) assert expectations == 1 @@ -58,11 +58,11 @@ def test_split_mono_relationship_bundle(): stix_splitter = OpenCTIStix2Splitter() with open("./tests/data/mono-bundle-relationship.json") as file: content = file.read() - expectations, bundles = stix_splitter.split_bundle_with_expectations(content) + expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content) assert expectations == 1 # Split with cleanup_inconsistent_bundle stix_splitter = OpenCTIStix2Splitter() - expectations, bundles = stix_splitter.split_bundle_with_expectations( + expectations, _, bundles = stix_splitter.split_bundle_with_expectations( bundle=content, cleanup_inconsistent_bundle=True ) assert expectations == 0 @@ -72,7 +72,7 @@ def test_split_capec_bundle(): stix_splitter = OpenCTIStix2Splitter() with open("./tests/data/mitre_att_capec.json") as file: content = file.read() - expectations, bundles = stix_splitter.split_bundle_with_expectations(content) + expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content) assert expectations == 2610 @@ -80,11 +80,11 @@ def test_split_internal_ids_bundle(): stix_splitter = OpenCTIStix2Splitter() with open("./tests/data/bundle_with_internal_ids.json") as file: content = file.read() - expectations, bundles = stix_splitter.split_bundle_with_expectations(content) + expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content) assert expectations == 4 # Split with cleanup_inconsistent_bundle stix_splitter = OpenCTIStix2Splitter() - expectations, bundles = stix_splitter.split_bundle_with_expectations( + expectations, _, bundles = stix_splitter.split_bundle_with_expectations( bundle=content, cleanup_inconsistent_bundle=True ) assert expectations == 4 @@ -101,11 +101,11 @@ def test_split_missing_refs_bundle(): stix_splitter = OpenCTIStix2Splitter() with open("./tests/data/missing_refs.json") as file: content = file.read() - expectations, bundles = stix_splitter.split_bundle_with_expectations(content) + expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content) assert expectations == 4 # Split with cleanup_inconsistent_bundle stix_splitter = OpenCTIStix2Splitter() - expectations, bundles = stix_splitter.split_bundle_with_expectations( + expectations, _, bundles = stix_splitter.split_bundle_with_expectations( bundle=content, cleanup_inconsistent_bundle=True ) assert expectations == 3 @@ -115,7 +115,7 @@ def test_split_cyclic_bundle(): stix_splitter = OpenCTIStix2Splitter() with open("./tests/data/cyclic-bundle.json") as file: content = file.read() - expectations, bundles = stix_splitter.split_bundle_with_expectations(content) + expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content) assert expectations == 6 for bundle in bundles: json_bundle = json.loads(bundle) From 65e638891f7da0e01f7bfc99d3897f86d80e713e Mon Sep 17 00:00:00 2001 From: Jeremy Cloarec Date: Mon, 16 Jun 2025 18:28:47 +0200 Subject: [PATCH 2/4] [client] report expection when element is ignored during import --- pycti/connector/opencti_connector_helper.py | 16 +++++++--------- pycti/utils/opencti_stix2.py | 6 ++++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pycti/connector/opencti_connector_helper.py b/pycti/connector/opencti_connector_helper.py index 939354aa1..50443da2c 100644 --- a/pycti/connector/opencti_connector_helper.py +++ b/pycti/connector/opencti_connector_helper.py @@ -1918,15 +1918,13 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list: os.rename(write_file, final_write_file) stix2_splitter = OpenCTIStix2Splitter() - ( - expectations_number, - bundles, - _ - ) = stix2_splitter.split_bundle_with_expectations( - bundle=bundle, - use_json=True, - event_version=event_version, - cleanup_inconsistent_bundle=cleanup_inconsistent_bundle, + (expectations_number, bundles, _) = ( + stix2_splitter.split_bundle_with_expectations( + bundle=bundle, + use_json=True, + event_version=event_version, + cleanup_inconsistent_bundle=cleanup_inconsistent_bundle, + ) ) if len(bundles) == 0: diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index be43b9d10..9b0b95f24 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -2748,8 +2748,10 @@ def import_bundle( ) stix2_splitter = OpenCTIStix2Splitter() - _, nb_incompatible_elements, bundles = stix2_splitter.split_bundle_with_expectations( - stix_bundle, False, event_version + _, nb_incompatible_elements, bundles = ( + stix2_splitter.split_bundle_with_expectations( + stix_bundle, False, event_version + ) ) # Report every element ignored during bundle splitting From cf01faa374f98ffa22942f6c222f0f046250f9c1 Mon Sep 17 00:00:00 2001 From: Jeremy Cloarec Date: Mon, 16 Jun 2025 18:30:30 +0200 Subject: [PATCH 3/4] [client] report expection when element is ignored during import --- pycti/connector/opencti_connector_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycti/connector/opencti_connector_helper.py b/pycti/connector/opencti_connector_helper.py index 50443da2c..db7c0480e 100644 --- a/pycti/connector/opencti_connector_helper.py +++ b/pycti/connector/opencti_connector_helper.py @@ -1918,7 +1918,7 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list: os.rename(write_file, final_write_file) stix2_splitter = OpenCTIStix2Splitter() - (expectations_number, bundles, _) = ( + (expectations_number, _, bundles) = ( stix2_splitter.split_bundle_with_expectations( bundle=bundle, use_json=True, From 0d4910d014625ef810135617f8cd3a8b69115290 Mon Sep 17 00:00:00 2001 From: Jeremy Cloarec Date: Tue, 17 Jun 2025 10:04:13 +0200 Subject: [PATCH 4/4] [client] report error expection when element is ignored during import --- pycti/utils/opencti_stix2.py | 14 +++++++++++--- pycti/utils/opencti_stix2_splitter.py | 8 ++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/pycti/utils/opencti_stix2.py b/pycti/utils/opencti_stix2.py index 9b0b95f24..9604388cf 100644 --- a/pycti/utils/opencti_stix2.py +++ b/pycti/utils/opencti_stix2.py @@ -2748,7 +2748,7 @@ def import_bundle( ) stix2_splitter = OpenCTIStix2Splitter() - _, nb_incompatible_elements, bundles = ( + _, incompatible_elements, bundles = ( stix2_splitter.split_bundle_with_expectations( stix_bundle, False, event_version ) @@ -2756,8 +2756,16 @@ def import_bundle( # Report every element ignored during bundle splitting if work_id is not None: - for i in range(nb_incompatible_elements): - self.opencti.work.report_expectation(work_id, None) + for incompatible_element in incompatible_elements: + self.opencti.work.report_expectation( + work_id, + { + "error": "Incompatible element in bundle", + "source": "Element " + + incompatible_element["id"] + + " is incompatible and couldn't be processed", + }, + ) # Import every element in a specific order imported_elements = [] diff --git a/pycti/utils/opencti_stix2_splitter.py b/pycti/utils/opencti_stix2_splitter.py index f422baf06..73c321b1e 100644 --- a/pycti/utils/opencti_stix2_splitter.py +++ b/pycti/utils/opencti_stix2_splitter.py @@ -35,7 +35,7 @@ def __init__(self): self.cache_index = {} self.cache_refs = {} self.elements = [] - self.nb_incompatible_items = 0 + self.incompatible_items = [] def get_internal_ids_in_extension(self, item): ids = [] @@ -191,7 +191,7 @@ def enlist_element( if is_compatible: self.elements.append(item) else: - self.nb_incompatible_items = self.nb_incompatible_items + 1 + self.incompatible_items.append(item) self.cache_index[item_id] = item for internal_id in self.get_internal_ids_in_extension(item): self.cache_index[internal_id] = item @@ -204,7 +204,7 @@ def split_bundle_with_expectations( use_json=True, event_version=None, cleanup_inconsistent_bundle=False, - ) -> Tuple[int, int, list]: + ) -> Tuple[int, list, list]: """splits a valid stix2 bundle into a list of bundles""" if use_json: try: @@ -254,7 +254,7 @@ def by_dep_size(elem): ) ) - return number_expectations, self.nb_incompatible_items, bundles + return number_expectations, self.incompatible_items, bundles @deprecated("Use split_bundle_with_expectations instead") def split_bundle(self, bundle, use_json=True, event_version=None) -> list: