From 3b07a1a0b597ee4873cc7e1f1c833100eb392273 Mon Sep 17 00:00:00 2001 From: Martin Pavella Date: Thu, 29 Jan 2026 14:44:41 +0100 Subject: [PATCH 1/2] NXP backend: Fix Neutron IR node support check. --- .../backend/ir/converter/node_converter.py | 17 ++++-- backends/nxp/tests/executors.py | 11 +++- .../test_constant_pad_nd_converter.py | 59 ++++++++++++++----- .../node_converter/test_softmax_converter.py | 32 ++++++---- 4 files changed, 86 insertions(+), 33 deletions(-) diff --git a/backends/nxp/backend/ir/converter/node_converter.py b/backends/nxp/backend/ir/converter/node_converter.py index b653718e643..2330f38977f 100755 --- a/backends/nxp/backend/ir/converter/node_converter.py +++ b/backends/nxp/backend/ir/converter/node_converter.py @@ -173,13 +173,22 @@ def assert_convertible(self, node): """Assert that the call `_is_supported_in_IR()` returns `True`. Otherwise, raise an exception and print an error message. """ - assert self._is_supported_in_IR( + supported_in_ir = self._is_supported_in_IR( node, self.context.parameters_mapping, self.context.custom_delegation_options, - ), ( - f"Node `{node}` is not convertible to the intermediate representation. " - "There is an error in the partitioner." + ) + + supported_on_target = self._is_supported_on_target( + node, + self.neutron_target_spec, + self.context.parameters_mapping, + self.context.custom_delegation_options, + ) + + assert supported_in_ir and supported_on_target, ( + f"Node `{node}` was selected for delegation to Neutron, but it is not convertible to the intermediate " + "representation. There is an error in the NXP partitioner,]. Please report this." ) @property diff --git a/backends/nxp/tests/executors.py b/backends/nxp/tests/executors.py index fa99046ff33..2dd7b1735ba 100644 --- a/backends/nxp/tests/executors.py +++ b/backends/nxp/tests/executors.py @@ -1,7 +1,8 @@ -# Copyright 2023-2025 NXP +# Copyright 2023-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. + import warnings from typing import Callable, Dict, Union @@ -392,10 +393,16 @@ def __init__( ): self._converter_class = converter_class self.new_target_support_check = new_target_support_check + + # Store the original target check method. NOTE: The staticmethod must be unwrapped to get the underlying + # function, therefore the stored function is not a staticmethod anymore. self.old_target_support_check = converter_class._is_supported_on_target def __enter__(self): self._converter_class._is_supported_on_target = self.new_target_support_check def __exit__(self, exc_type, exc_val, exc_tb): - self._converter_class._is_supported_on_target = self.old_target_support_check + # The stored `old_target_support_check` is a plain function, so it needs to be wrapped back as a staticmethod. + self._converter_class._is_supported_on_target = staticmethod( + self.old_target_support_check + ) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py index a2c9526a508..1fb0b808523 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025 NXP +# Copyright 2024-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -6,8 +6,10 @@ import numpy as np import pytest import torch - from executorch.backends.nxp.backend.ir.conversion_config import ConversionConfig +from executorch.backends.nxp.backend.ir.converter.node_converters.ops_converters.constant_pad_nd_converter import ( + ConstantPadNDConverter, +) from executorch.backends.nxp.tests.executorch_pipeline import ( to_edge_program, to_quantized_edge_program, @@ -23,6 +25,7 @@ ConstantPadNDModule, ) from executorch.backends.nxp.tests.use_qat import * # noqa F403 +from executorch.backends.nxp.tests.executors import OverrideTargetSupportCheck from executorch.exir.dialects._ops import ops as exir_ops @@ -43,7 +46,14 @@ def test_constant_pad_nd_conversion__specific_constant(constant): input_data = np.random.random(input_shape).astype(np.float32) - convert_run_compare(edge_program, input_data) + # Ignore the target requirement, as this test is target agnostic. + def supported_target(*_): + return True + + with OverrideTargetSupportCheck( + ConstantPadNDConverter, new_target_support_check=supported_target + ): + convert_run_compare(edge_program, input_data) def test_constant_pad_nd_conversion__default_constant(): @@ -56,7 +66,14 @@ def test_constant_pad_nd_conversion__default_constant(): input_data = np.random.random(input_shape).astype(np.float32) - convert_run_compare(edge_program, input_data) + # Ignore the target requirement, as this test is target agnostic. + def supported_target(*_): + return True + + with OverrideTargetSupportCheck( + ConstantPadNDConverter, new_target_support_check=supported_target + ): + convert_run_compare(edge_program, input_data) @pytest.mark.parametrize( @@ -80,7 +97,14 @@ def test_constant_pad_nd_conversion__format_less(input_shape, paddings): input_data = np.random.random(input_shape).astype(np.float32) - convert_run_compare(edge_program, input_data) + # Ignore the target requirement, as this test is target agnostic. + def supported_target(*_): + return True + + with OverrideTargetSupportCheck( + ConstantPadNDConverter, new_target_support_check=supported_target + ): + convert_run_compare(edge_program, input_data) @pytest.mark.parametrize( @@ -98,15 +122,22 @@ def test_constant_pad_nd_conversion__channels_first(input_shape, paddings): input_data = np.random.random(input_shape).astype(np.float32) - convert_run_compare( - edge_program, - input_data, - tflite_input_preprocess=ToNHWCPreprocess(), - tflite_output_preprocess=ToNCHWPreprocess(), - conversion_config=ConversionConfig( - {"use_neutron_for_format_conversion": False} - ), - ) + # Ignore the target requirement, as this test is target agnostic. + def supported_target(*_): + return True + + with OverrideTargetSupportCheck( + ConstantPadNDConverter, new_target_support_check=supported_target + ): + convert_run_compare( + edge_program, + input_data, + tflite_input_preprocess=ToNHWCPreprocess(), + tflite_output_preprocess=ToNCHWPreprocess(), + conversion_config=ConversionConfig( + {"use_neutron_for_format_conversion": False} + ), + ) @pytest.mark.parametrize( diff --git a/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py index 5953b9dcac3..45f5723c8f0 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025 NXP +# Copyright 2024-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -6,14 +6,19 @@ import numpy as np import pytest import torch - from executorch.backends.nxp.backend.edge_program_converter import ( EdgeProgramToIRConverter, ) from executorch.backends.nxp.backend.ir.conversion_config import ConversionConfig +from executorch.backends.nxp.backend.ir.converter.node_converters.ops_converters.softmax_converter import ( + SoftmaxConverter, +) from executorch.backends.nxp.backend.node_format_inference import NodeFormatInference from executorch.backends.nxp.tests.executorch_pipeline import to_edge_program -from executorch.backends.nxp.tests.executors import convert_run_compare +from executorch.backends.nxp.tests.executors import ( + convert_run_compare, + OverrideTargetSupportCheck, +) from executorch.backends.nxp.tests.models import SoftmaxConvModule, SoftmaxModule @@ -39,7 +44,14 @@ def test_softmax_conversion__formatless_input(input_shape, dim: int): input_data = np.random.random(input_shape).astype(np.float32) - convert_run_compare(edge_program, input_data=input_data) + # Ignore the target requirement, as this test is target agnostic. + def supported_target(*_): + return True + + with OverrideTargetSupportCheck( + SoftmaxConverter, new_target_support_check=supported_target + ): + convert_run_compare(edge_program, input_data=input_data) @pytest.mark.parametrize( @@ -60,9 +72,7 @@ def test_softmax_conversion__unknown_input_format(input_shape, dim: int): NodeFormatInference(edge_program).identify_node_formats() # Currently this test not pass because the convertibility checker doesn't use tensor formats. - with pytest.raises( - AssertionError, match="`aten__softmax_default` is not convertible" - ): + with pytest.raises(AssertionError, match="aten__softmax_default.*not convertible"): EdgeProgramToIRConverter().convert_program(edge_program, ConversionConfig()) # input_data = np.random.random(input_shape).astype(np.float32) @@ -83,9 +93,7 @@ def test_softmax_conversion_channel_last(input_shape, dim: int): NodeFormatInference(edge_program).identify_node_formats() # TODO (Robert Kalmar) Currently this test not pass because the convertibility checker doesn't use tensor formats. - with pytest.raises( - AssertionError, match="`aten__softmax_default` is not convertible" - ): + with pytest.raises(AssertionError, match="aten__softmax_default.*not convertible"): EdgeProgramToIRConverter().convert_program(edge_program, ConversionConfig()) # input_data = np.random.random(input_shape).astype(np.float32) @@ -109,7 +117,5 @@ def test_softmax_conversion_unsupported_dims(input_shape, dim: int): edge_program = to_edge_program(model, input_shape).exported_program() NodeFormatInference(edge_program).identify_node_formats() - with pytest.raises( - AssertionError, match="`aten__softmax_default` is not convertible" - ): + with pytest.raises(AssertionError, match="aten__softmax_default.*not convertible"): EdgeProgramToIRConverter().convert_program(edge_program, ConversionConfig()) From 769ae19d48ddcaefdb6696701aee1d47035034f1 Mon Sep 17 00:00:00 2001 From: Martin Pavella Date: Fri, 30 Jan 2026 13:10:41 +0100 Subject: [PATCH 2/2] NXP backend: Unify ExecuTorch and Neutron data format enums. --- backends/nxp/_passes/remove_getitem_pass.py | 6 +- backends/nxp/backend/data_format.py | 49 +++++++++++++ .../nxp/backend/edge_program_converter.py | 4 +- .../builder/aten_model_builder_director.py | 23 ++++--- .../ir/converter/builder/model_builder.py | 19 ++--- .../ops_converters/cat_converter.py | 4 +- .../constant_pad_nd_converter.py | 6 +- .../ops_converters/convolution_converter.py | 6 +- .../ops_converters/mean_dim_converter.py | 4 +- .../ops_converters/permute_copy_converter.py | 13 ++-- .../qdq_dequantize_converter.py | 6 +- .../ops_converters/slice_tensor_converter.py | 4 +- .../ops_converters/view_copy_converter.py | 4 +- .../node_converters/shared/recurrent_utils.py | 12 ++-- .../node_converters/shared/reduce_utils.py | 12 ++-- .../shared/reshape_transposition.py | 14 ++-- .../tensor_rules.py | 6 +- backends/nxp/backend/ir/tensor_formatting.py | 69 ------------------- .../ir/tflite_generator/tflite_model.py | 8 +-- backends/nxp/backend/node_format.py | 26 ------- backends/nxp/backend/node_format_inference.py | 49 ++++++------- backends/nxp/nxp_backend.py | 12 ++-- .../nxp/tests/test_node_format_inference.py | 26 +++---- 23 files changed, 164 insertions(+), 218 deletions(-) create mode 100644 backends/nxp/backend/data_format.py delete mode 100644 backends/nxp/backend/ir/tensor_formatting.py delete mode 100644 backends/nxp/backend/node_format.py diff --git a/backends/nxp/_passes/remove_getitem_pass.py b/backends/nxp/_passes/remove_getitem_pass.py index 316cc13f49c..6e5f2535746 100644 --- a/backends/nxp/_passes/remove_getitem_pass.py +++ b/backends/nxp/_passes/remove_getitem_pass.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# Copyright 2025 NXP +# Copyright 2025-2026 NXP # All rights reserved. # # This source code is licensed under the BSD-style license found in the @@ -7,7 +7,7 @@ import torch -from executorch.backends.nxp.backend.node_format import NodeFormat, NXP_NODE_FORMAT +from executorch.backends.nxp.backend.data_format import DataFormat, NXP_NODE_FORMAT from executorch.exir.dialects._ops import ops as exir_ops from executorch.exir.pass_base import ExportPass, PassResult @@ -89,7 +89,7 @@ def call(self, graph_module: torch.fx.GraphModule): # MODIFIED PART START # Make sure to preserve the inferred node format. new_max_wd.meta[NXP_NODE_FORMAT] = node.meta.get( - NXP_NODE_FORMAT, NodeFormat.NONE + NXP_NODE_FORMAT, DataFormat.NONE ) # MODIFIED PART END diff --git a/backends/nxp/backend/data_format.py b/backends/nxp/backend/data_format.py new file mode 100644 index 00000000000..c10c8ece1bb --- /dev/null +++ b/backends/nxp/backend/data_format.py @@ -0,0 +1,49 @@ +# Copyright 2023 Martin Pavella +# Copyright 2023-2026 NXP +# +# License: MIT +# See the LICENSE_MIT for more details. +# + +from enum import Enum + +# Key into the `meta` attribute of nodes, which is mapped to their inferred node format. +NXP_NODE_FORMAT = "nxp_node_format" + + +class DataFormat(Enum): + CHANNELS_FIRST = 0 + + CHANNELS_LAST = 10 + + # The format of TFLite Conv3D weights tensor: [output_channels, input_channels, D, H, W] + CONV_3D_WEIGHT_FORMAT = 11 + + # Intermediate format between 'Transpose' and 'Reshape' ops when single dimension with value 1 + # is added/removed via reshaping + RESHAPE_SINGLE_UNITARY_TRANSPOSITION = 12 + + # The format of TFLite TransposeConv 2D weights tensor: [M/group, kH, kW, C] + TRANSPOSE_CONV_2D_WEIGHT_FORMAT = 13 + + # No special format (matrices, vectors, shapes etc.). All tensors with the FORMATLESS format MUST have EXACTLY + # the same shape and data in the NeutronIR model and in the ExecuTorch model. + FORMATLESS = 20 + + NONE = 30 # Format has not been identified + + def is_channels_first(self) -> bool: + return self == DataFormat.CHANNELS_FIRST + + def is_channels_last(self) -> bool: + return self == DataFormat.CHANNELS_LAST + + @staticmethod + def convert_executorch_format_to_neutron( + executorch_format: "DataFormat", + ) -> "DataFormat": + if executorch_format == DataFormat.CHANNELS_FIRST: + return DataFormat.CHANNELS_LAST # Format is converted. + + else: + return executorch_format # Other formats remain unchanged. diff --git a/backends/nxp/backend/edge_program_converter.py b/backends/nxp/backend/edge_program_converter.py index 6922e4abba6..5575e636027 100644 --- a/backends/nxp/backend/edge_program_converter.py +++ b/backends/nxp/backend/edge_program_converter.py @@ -18,8 +18,8 @@ from torch.fx import Node from torch.nn.parameter import Parameter from executorch.backends.nxp.backend.ir.converter.node_converters.ops_converters import * # noqa F403 +from executorch.backends.nxp.backend.data_format import DataFormat, NXP_NODE_FORMAT from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec -from executorch.backends.nxp.backend.node_format import NodeFormat, NXP_NODE_FORMAT from executorch.exir.dialects._ops import ops as exir_ops # noinspection PyProtectedMember @@ -65,7 +65,7 @@ def convert_program( conversion_config: ConversionConfig = _default_conversion_config, neutron_target_spec: NeutronTargetSpec = _default_target_spec, custom_delegation_options: CustomDelegationOptions = _default_delegation_options, - ) -> (bytes, dict[str, NodeFormat]): + ) -> tuple[bytes, dict[str, DataFormat]]: """ Convert ExportedProgram in Edge dialect to IR (TFLite flatbuffers) as bytes. diff --git a/backends/nxp/backend/ir/converter/builder/aten_model_builder_director.py b/backends/nxp/backend/ir/converter/builder/aten_model_builder_director.py index 658b4fc93f7..d4c4d96a5c6 100644 --- a/backends/nxp/backend/ir/converter/builder/aten_model_builder_director.py +++ b/backends/nxp/backend/ir/converter/builder/aten_model_builder_director.py @@ -1,15 +1,14 @@ -# Copyright 2024 NXP +# Copyright 2024,2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +from executorch.backends.nxp.backend.data_format import DataFormat from executorch.backends.nxp.backend.ir.converter.builder.model_builder import ( ModelBuilder, ) from executorch.backends.nxp.backend.ir.converter.conversion import translator -from executorch.backends.nxp.backend.ir.tensor_formatting import TensorFormat from executorch.backends.nxp.backend.ir.tflite_generator import tflite_model -from executorch.backends.nxp.backend.node_format import NodeFormat from torch.fx import Node from torch.nn import Parameter @@ -20,13 +19,13 @@ class AtenModelBuilderDirector(ModelBuilder): contains methods related to Edge program nodes conversion. """ - def append_as_fake_tensor(self, node: Node, node_format: NodeFormat): + def append_as_fake_tensor(self, node: Node, node_format: DataFormat): """ Append node into ModelBuilder as tensor without data (FakeTensor). Can be used for activations and output tensors. :param node: Node instance. - :param node_format: NodeFormat definition. + :param node_format: DataFormat definition. """ if self.tensor_exists(node.name): return @@ -41,17 +40,19 @@ def append_as_fake_tensor(self, node: Node, node_format: NodeFormat): shape = translator.dims_to_channels_last(shape) tensor = self.create_empty_tensor(node.name, _type, shape) - tensor.tensor_format = TensorFormat.from_node_format(node_format) + tensor.tensor_format = DataFormat.convert_executorch_format_to_neutron( + node_format + ) def append_as_static_tensor( - self, node: Node, node_format: NodeFormat, tensor: Parameter + self, node: Node, node_format: DataFormat, tensor: Parameter ): """ Append node into ModelBuilder as tensor with data (static). Can be used for weights, permutations etc. :param node: Node instance. - :param node_format: NodeFormat definition. + :param node_format: DataFormat definition. :param tensor: Torch Tensor (Parameter) that holds tensor data. """ assert not self.tensor_exists(node.name), f"Tensor '{node.name}' already added!" @@ -65,7 +66,9 @@ def append_as_static_tensor( data = translator.convert_data_to_channels_last(data) tensor = self.create_tensor_for_data(data, node.name) - tensor.tensor_format = TensorFormat.from_node_format(node_format) + tensor.tensor_format = DataFormat.convert_executorch_format_to_neutron( + node_format + ) def append_operators(self, ops_to_add: list[tflite_model.Operator]): """ @@ -88,7 +91,7 @@ def append_operators(self, ops_to_add: list[tflite_model.Operator]): self.check_and_append_operator(op) - def get_io_formats(self, graph_signature) -> dict[str, dict[str, TensorFormat]]: + def get_io_formats(self, graph_signature) -> dict[str, dict[str, DataFormat]]: """Get a mapping from tensor names to their formats. :param graph_signature: Instance of GraphSignature. diff --git a/backends/nxp/backend/ir/converter/builder/model_builder.py b/backends/nxp/backend/ir/converter/builder/model_builder.py index 826b0038fae..d3547acb67f 100755 --- a/backends/nxp/backend/ir/converter/builder/model_builder.py +++ b/backends/nxp/backend/ir/converter/builder/model_builder.py @@ -7,13 +7,13 @@ # from copy import deepcopy -from itertools import chain from typing import List, Optional, Union import executorch.backends.nxp.backend.ir.converter.conversion.translator as translator import executorch.backends.nxp.backend.ir.logger as logger import executorch.backends.nxp.backend.ir.tflite_generator.tflite_model as tflite_model import numpy as np +from executorch.backends.nxp.backend.data_format import DataFormat from executorch.backends.nxp.backend.edge_helper import is_channels_last_dim_order from executorch.backends.nxp.backend.ir.conversion_config import ConversionConfig from executorch.backends.nxp.backend.ir.converter.builder import ( @@ -35,7 +35,6 @@ ) from executorch.backends.nxp.backend.ir.lib.tflite.TensorType import TensorType from executorch.backends.nxp.backend.ir.neutron_ir_post_processing import optimizer -from executorch.backends.nxp.backend.ir.tensor_formatting import TensorFormat from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options import ( cast_options, dequantize_options, @@ -227,7 +226,7 @@ def channels_first_version_of(self, t_tensor: tflite_model.Tensor): new_tensor.shape = translator.channels_last_shape_to_channels_first( t_tensor.shape ) - new_tensor.tensor_format = TensorFormat.CHANNELS_FIRST + new_tensor.tensor_format = DataFormat.CHANNELS_FIRST perm = translator.create_channels_last_to_channels_first_permutation( t_tensor.rank @@ -398,7 +397,7 @@ def _make_inputs_channels_first(self): input_tensor, input_tensor.name + "_channels_first" ) new_input.shape = new_input_shape - new_input.tensor_format = TensorFormat.CHANNELS_FIRST + new_input.tensor_format = DataFormat.CHANNELS_FIRST transpose = self._create_transpose_operator( new_input, input_tensor, perm @@ -484,14 +483,6 @@ def _keep_one_empty_buffer(self): # It's safe to replace the buffer. t.tmp_buffer = empty_buffer - def replace_io_tensor_format_with_node_format(self): - for t in chain( - self.get_sub_graph().inputs.tmp_inputs, - self.get_sub_graph().outputs.tmp_outputs, - ): - if isinstance(t.tensor_format, TensorFormat): - t.tensor_format = t.tensor_format.to_equal_node_format() - def finish(self) -> tflite_model.Model: """Finalize and optimize the converted TFLite model. Then return it. @@ -514,8 +505,6 @@ def finish(self) -> tflite_model.Model: self._keep_one_empty_buffer() - self.replace_io_tensor_format_with_node_format() - # Remove outputs, which are not produced by any node. Otherwise, there would be errors after inference. operator_outputs = [] for op in self.get_operators().vector: @@ -1584,7 +1573,7 @@ def prepare_dynamic_tensor_for_correct_broadcasting_with_channels_first_tensors( transpose_output.shape = tflite_model.Shape( translator.apply_permutation_to(transpose_output.shape.vector, perm) ) - transpose_output.tensor_format = TensorFormat.CHANNELS_LAST + transpose_output.tensor_format = DataFormat.CHANNELS_LAST transpose = self._create_transpose_operator( transpose_input, transpose_output, perm diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/cat_converter.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/cat_converter.py index bb8ab1048eb..cdbd086b6b4 100644 --- a/backends/nxp/backend/ir/converter/node_converters/ops_converters/cat_converter.py +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/cat_converter.py @@ -1,4 +1,4 @@ -# Copyright 2025 NXP +# Copyright 2025-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -8,6 +8,7 @@ from executorch.backends.nxp.backend.custom_delegation_options import ( CustomDelegationOptions, ) +from executorch.backends.nxp.backend.data_format import NXP_NODE_FORMAT from executorch.backends.nxp.backend.edge_helper import previous_non_qdq_node from executorch.backends.nxp.backend.ir.converter.conversion import translator from executorch.backends.nxp.backend.ir.converter.conversion.translator import ( @@ -23,7 +24,6 @@ Concatenation, ) from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec -from executorch.backends.nxp.backend.node_format import NXP_NODE_FORMAT from torch.fx import Node from torch.fx.passes.infra.partitioner import Partition from torch.nn import Parameter diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/constant_pad_nd_converter.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/constant_pad_nd_converter.py index 29a8f7d51bb..efab4fb95c7 100644 --- a/backends/nxp/backend/ir/converter/node_converters/ops_converters/constant_pad_nd_converter.py +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/constant_pad_nd_converter.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025 NXP +# Copyright 2024-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -8,6 +8,8 @@ import numpy as np +from executorch.backends.nxp.backend.data_format import NXP_NODE_FORMAT + from executorch.backends.nxp.backend.edge_helper import input_rank from executorch.backends.nxp.backend.ir.converter.conversion.translator import ( apply_permutation_to, @@ -27,8 +29,6 @@ pad_v2_options, ) from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec - -from executorch.backends.nxp.backend.node_format import NXP_NODE_FORMAT from torch.fx import Node from torch.nn import Parameter diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/convolution_converter.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/convolution_converter.py index 645274c7870..148b90a331e 100644 --- a/backends/nxp/backend/ir/converter/node_converters/ops_converters/convolution_converter.py +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/convolution_converter.py @@ -1,10 +1,11 @@ -# Copyright 2024-2025 NXP +# Copyright 2024-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import numpy as np import torch +from executorch.backends.nxp.backend.data_format import DataFormat from executorch.backends.nxp.backend.edge_helper import ( input_tensor, @@ -37,7 +38,6 @@ ) from executorch.backends.nxp.backend.ir.converter.tensor_utils import tensor_has_data from executorch.backends.nxp.backend.ir.lib.tflite.TensorType import TensorType -from executorch.backends.nxp.backend.ir.tensor_formatting import TensorFormat from executorch.backends.nxp.backend.ir.tflite_generator import tflite_model from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options import ( conv_2d_options, @@ -386,7 +386,7 @@ def _convert_transpose_conv( w.quantization.quantized_dimension = 0 else: raise NotImplementedError("Dynamic Transpose Conv weights.") - w.tensor_format = TensorFormat.TRANSPOSE_CONV_2D_WEIGHT_FORMAT + w.tensor_format = DataFormat.TRANSPOSE_CONV_2D_WEIGHT_FORMAT output_shape_tensor_data = np.asarray(y.shape.vector, dtype=np.int32) o = self.builder.create_tensor_for_data( diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/mean_dim_converter.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/mean_dim_converter.py index ac09e564eb8..c4b828df39f 100644 --- a/backends/nxp/backend/ir/converter/node_converters/ops_converters/mean_dim_converter.py +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/mean_dim_converter.py @@ -1,9 +1,10 @@ -# Copyright 2025 NXP +# Copyright 2025-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import torch +from executorch.backends.nxp.backend.data_format import NXP_NODE_FORMAT from executorch.backends.nxp.backend.ir.converter.conversion.translator import ( create_channels_last_to_channels_first_permutation, @@ -19,7 +20,6 @@ mean_options, ) from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec -from executorch.backends.nxp.backend.node_format import NXP_NODE_FORMAT from torch.fx import Node from torch.nn import Parameter diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/permute_copy_converter.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/permute_copy_converter.py index 35bef6c8035..1a3c5abe54e 100644 --- a/backends/nxp/backend/ir/converter/node_converters/ops_converters/permute_copy_converter.py +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/permute_copy_converter.py @@ -1,10 +1,11 @@ -# Copyright 2024-2025 NXP +# Copyright 2024-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import numpy as np import torch +from executorch.backends.nxp.backend.data_format import DataFormat, NXP_NODE_FORMAT from executorch.backends.nxp.backend.edge_helper import ( node_is_effectively_static_tensor, @@ -18,7 +19,6 @@ NeutronTargetSpec, NodeConverter, ) -from executorch.backends.nxp.backend.ir.tensor_formatting import TensorFormat from executorch.backends.nxp.backend.ir.tflite_generator import tflite_model from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options import ( transpose_options, @@ -27,7 +27,6 @@ is_tensor_invariant_permutation, transposition_is_supported_on_neutron, ) -from executorch.backends.nxp.backend.node_format import NXP_NODE_FORMAT from torch.fx import Node from torch.nn import Parameter @@ -181,7 +180,7 @@ def _handle_channels_first_input_and_formatless_output( "A `permute_copy` node was incorrectly selected for delegation. Please report this." ) - t_op.tmp_inputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + t_op.tmp_inputs[0].tensor_format = DataFormat.CHANNELS_FIRST return perm @@ -216,7 +215,7 @@ def _handle_formatless_input_and_channels_first_output( "A `permute_copy` node was incorrectly selected for delegation. Please report this." ) - t_op.tmp_outputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + t_op.tmp_outputs[0].tensor_format = DataFormat.CHANNELS_FIRST return perm @@ -281,8 +280,8 @@ def _handle_channels_first_input_and_output( "A `permute_copy` node was incorrectly selected for delegation. Please report this." ) - t_op.tmp_inputs[0].tensor_format = TensorFormat.CHANNELS_FIRST - t_op.tmp_outputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + t_op.tmp_inputs[0].tensor_format = DataFormat.CHANNELS_FIRST + t_op.tmp_outputs[0].tensor_format = DataFormat.CHANNELS_FIRST return perm diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/qdq_dequantize_converter.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/qdq_dequantize_converter.py index 3e20e504e8a..5415bdf21f5 100644 --- a/backends/nxp/backend/ir/converter/node_converters/ops_converters/qdq_dequantize_converter.py +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/qdq_dequantize_converter.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025 NXP +# Copyright 2024-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -6,6 +6,7 @@ from abc import ABC, abstractmethod import numpy as np +from executorch.backends.nxp.backend.data_format import DataFormat from executorch.backends.nxp.backend.ir.converter.conversion.translator import ( create_channels_last_to_channels_first_permutation, @@ -18,7 +19,6 @@ from executorch.backends.nxp.backend.ir.converter.quantization_utils import ( set_quantization_parameters_to_tensor, ) -from executorch.backends.nxp.backend.ir.tensor_formatting import TensorFormat from executorch.backends.nxp.backend.ir.tflite_generator.tflite_model import Tensor from torch.fx import Node from torch.nn import Parameter @@ -107,7 +107,7 @@ def get_quantization_dimension(self, from_tensor: Tensor, node: Node) -> int: quantization_dimension = node.args[3] # Quantization dimension is affected by tensor format - if from_tensor.tensor_format == TensorFormat.CHANNELS_LAST: + if from_tensor.tensor_format == DataFormat.CHANNELS_LAST: tensor_rank = len(from_tensor.shape.vector) perm = create_channels_last_to_channels_first_permutation( tensor_rank, return_list=True diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/slice_tensor_converter.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/slice_tensor_converter.py index fd2aec7b8a0..20943d830ee 100644 --- a/backends/nxp/backend/ir/converter/node_converters/ops_converters/slice_tensor_converter.py +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/slice_tensor_converter.py @@ -1,9 +1,10 @@ -# Copyright 2025 NXP +# Copyright 2025-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import numpy as np +from executorch.backends.nxp.backend.data_format import NXP_NODE_FORMAT from executorch.backends.nxp.backend.edge_helper import input_tensor from executorch.backends.nxp.backend.ir.converter.conversion import translator from executorch.backends.nxp.backend.ir.converter.conversion.common import OpsList @@ -18,7 +19,6 @@ transposition_is_supported_on_neutron, ) from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec -from executorch.backends.nxp.backend.node_format import NXP_NODE_FORMAT from torch.fx import Node from torch.nn import Parameter diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/view_copy_converter.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/view_copy_converter.py index 5e03f1e2b40..d273db1758b 100644 --- a/backends/nxp/backend/ir/converter/node_converters/ops_converters/view_copy_converter.py +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/view_copy_converter.py @@ -1,9 +1,10 @@ -# Copyright 2024-2025 NXP +# Copyright 2024-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import numpy as np +from executorch.backends.nxp.backend.data_format import NXP_NODE_FORMAT from executorch.backends.nxp.backend.edge_helper import ( get_non_qdq_users, @@ -33,7 +34,6 @@ transposition_is_supported_on_neutron, ) from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec -from executorch.backends.nxp.backend.node_format import NXP_NODE_FORMAT from executorch.exir.dialects._ops import ops as exir_ops from torch.fx import Node from torch.fx.passes.infra.partitioner import Partition diff --git a/backends/nxp/backend/ir/converter/node_converters/shared/recurrent_utils.py b/backends/nxp/backend/ir/converter/node_converters/shared/recurrent_utils.py index 52b895d60cd..fb150931bea 100755 --- a/backends/nxp/backend/ir/converter/node_converters/shared/recurrent_utils.py +++ b/backends/nxp/backend/ir/converter/node_converters/shared/recurrent_utils.py @@ -1,17 +1,17 @@ -# Copyright 2024-2025 NXP +# Copyright 2024-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +from executorch.backends.nxp.backend.data_format import DataFormat from executorch.backends.nxp.backend.ir.converter.builder import model_builder from executorch.backends.nxp.backend.ir.converter.conversion import translator from executorch.backends.nxp.backend.ir.converter.conversion.common import OpsList from executorch.backends.nxp.backend.ir.converter.tensor_utils import tensor_has_data -from executorch.backends.nxp.backend.ir.tensor_formatting import TensorFormat from executorch.backends.nxp.backend.ir.tflite_generator import tflite_model -def ensure_correct_tensor_formatting( +def ensure_correct_data_format( t_op: tflite_model.Operator, builder: model_builder.ModelBuilder, ops: OpsList ): """Make sure that all input and output tensors of 't_op' have the correct format. 't_op' is assumed to be an LSTM @@ -28,7 +28,7 @@ def ensure_correct_tensor_formatting( :param ops: OpsList object, with operators to add to the model. May already contain some operators. """ - if t_op.tmp_inputs[0].tensor_format == TensorFormat.FORMATLESS: + if t_op.tmp_inputs[0].tensor_format == DataFormat.FORMATLESS: # Nothing to be done. All tensors should be formatless. return @@ -48,7 +48,7 @@ def ensure_correct_tensor_formatting( ) ops.pre_ops.append(transpose) - t_op.tmp_inputs[idx].tensor_format = TensorFormat.FORMATLESS + t_op.tmp_inputs[idx].tensor_format = DataFormat.FORMATLESS # LSTM/RNN produces 'FORMATLESS' outputs. However, if the output tensors have the 'channels_last' format, Transpose # operators must be added, to actually make the inputs 'channels_last'. @@ -61,4 +61,4 @@ def ensure_correct_tensor_formatting( transpose = builder.create_transpose_operator_after(t_op, idx, revert_perm) ops.post_ops.append(transpose) - t_op.tmp_outputs[idx].tensor_format = TensorFormat.FORMATLESS + t_op.tmp_outputs[idx].tensor_format = DataFormat.FORMATLESS diff --git a/backends/nxp/backend/ir/converter/node_converters/shared/reduce_utils.py b/backends/nxp/backend/ir/converter/node_converters/shared/reduce_utils.py index da92e359f1e..65528bb30f8 100755 --- a/backends/nxp/backend/ir/converter/node_converters/shared/reduce_utils.py +++ b/backends/nxp/backend/ir/converter/node_converters/shared/reduce_utils.py @@ -1,16 +1,16 @@ -# Copyright 2024-2025 NXP +# Copyright 2024-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import numpy as np +from executorch.backends.nxp.backend.data_format import DataFormat from executorch.backends.nxp.backend.ir.converter.builder.model_builder import ( ModelBuilder, ) from executorch.backends.nxp.backend.ir.converter.conversion import translator from executorch.backends.nxp.backend.ir.converter.conversion.common import OpsList -from executorch.backends.nxp.backend.ir.tensor_formatting import TensorFormat from executorch.backends.nxp.backend.ir.tflite_generator import tflite_model @@ -63,13 +63,13 @@ def ensure_reduce_transposition(builder, ops: OpsList): transpose_before = builder.create_transpose_operator_before( t_op, 0, to_executorch_perm ) - transpose_before.tmp_outputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + transpose_before.tmp_outputs[0].tensor_format = DataFormat.CHANNELS_FIRST ops.add_pre(transpose_before) transpose_after = builder.create_transpose_operator_after( t_op, 0, to_tflite_perm ) - transpose_after.tmp_inputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + transpose_after.tmp_inputs[0].tensor_format = DataFormat.CHANNELS_FIRST ops.post_ops.insert(0, transpose_after) elif input_format.is_channels_last() and not output_format.is_channels_last(): @@ -79,7 +79,7 @@ def ensure_reduce_transposition(builder, ops: OpsList): translator.create_channels_last_to_channels_first_permutation(input_rank) ) transpose = builder.create_transpose_operator_before(t_op, 0, permutation) - transpose.tmp_outputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + transpose.tmp_outputs[0].tensor_format = DataFormat.CHANNELS_FIRST ops.add_pre(transpose) @@ -92,6 +92,6 @@ def ensure_reduce_transposition(builder, ops: OpsList): translator.create_channels_first_to_channels_last_permutation(output_rank) ) transpose = builder.create_transpose_operator_after(t_op, 0, permutation) - transpose.tmp_inputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + transpose.tmp_inputs[0].tensor_format = DataFormat.CHANNELS_FIRST ops.post_ops.insert(0, transpose) diff --git a/backends/nxp/backend/ir/converter/node_converters/shared/reshape_transposition.py b/backends/nxp/backend/ir/converter/node_converters/shared/reshape_transposition.py index 55056614684..3ebae9c3715 100755 --- a/backends/nxp/backend/ir/converter/node_converters/shared/reshape_transposition.py +++ b/backends/nxp/backend/ir/converter/node_converters/shared/reshape_transposition.py @@ -1,4 +1,4 @@ -# Copyright 2023-2025 NXP +# Copyright 2023-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -6,10 +6,10 @@ from enum import Enum import numpy as np +from executorch.backends.nxp.backend.data_format import DataFormat from executorch.backends.nxp.backend.ir.converter.conversion import translator from executorch.backends.nxp.backend.ir.converter.conversion.common import OpsList -from executorch.backends.nxp.backend.ir.tensor_formatting import TensorFormat class SingleUnitaryDimensionChangeType(Enum): @@ -164,7 +164,7 @@ def ensure_reshape_transposition(builder, ops: OpsList) -> list[int]: translator.create_channels_last_to_channels_first_permutation(input_rank) ) transpose = builder.create_transpose_operator_before(t_op, 0, permutation) - transpose.tmp_outputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + transpose.tmp_outputs[0].tensor_format = DataFormat.CHANNELS_FIRST ops.add_pre(transpose) @@ -177,7 +177,7 @@ def ensure_reshape_transposition(builder, ops: OpsList) -> list[int]: translator.create_channels_first_to_channels_last_permutation(output_rank) ) transpose = builder.create_transpose_operator_after(t_op, 0, permutation) - transpose.tmp_inputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + transpose.tmp_inputs[0].tensor_format = DataFormat.CHANNELS_FIRST new_shape = translator.dims_to_channels_first(new_shape) @@ -196,7 +196,7 @@ def ensure_reshape_transposition(builder, ops: OpsList) -> list[int]: # Single added/removed dimension with value 1 transpose = builder.create_transpose_operator_before(t_op, 0, permutation) transpose.tmp_outputs[0].tensor_format = ( - TensorFormat.RESHAPE_SINGLE_UNITARY_TRANSPOSITION + DataFormat.RESHAPE_SINGLE_UNITARY_TRANSPOSITION ) ops.add_pre(transpose) @@ -213,7 +213,7 @@ def ensure_reshape_transposition(builder, ops: OpsList) -> list[int]: t_op, 0, list(last_to_first_perm) ) ) - t_op.tmp_inputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + t_op.tmp_inputs[0].tensor_format = DataFormat.CHANNELS_FIRST new_shape = translator.dims_to_channels_first(new_shape) @@ -228,6 +228,6 @@ def ensure_reshape_transposition(builder, ops: OpsList) -> list[int]: t_op, 0, list(first_to_last_perm) ), ) - t_op.tmp_outputs[0].tensor_format = TensorFormat.CHANNELS_FIRST + t_op.tmp_outputs[0].tensor_format = DataFormat.CHANNELS_FIRST return new_shape diff --git a/backends/nxp/backend/ir/neutron_ir_post_processing/tensor_rules.py b/backends/nxp/backend/ir/neutron_ir_post_processing/tensor_rules.py index 16d3d674297..7626c6893fa 100755 --- a/backends/nxp/backend/ir/neutron_ir_post_processing/tensor_rules.py +++ b/backends/nxp/backend/ir/neutron_ir_post_processing/tensor_rules.py @@ -9,6 +9,7 @@ import executorch.backends.nxp.backend.ir.converter.builder.model_builder as model_builder import numpy as np +from executorch.backends.nxp.backend.data_format import DataFormat from executorch.backends.nxp.backend.ir.lib.tflite.TensorType import TensorType from executorch.backends.nxp.backend.ir.neutron_ir_post_processing.optimizations.base_optimization import ( InputTensorToOpsMap, @@ -18,7 +19,6 @@ NameToTensorMap, operator_is_type, ) -from executorch.backends.nxp.backend.ir.tensor_formatting import TensorFormat from executorch.backends.nxp.backend.ir.tflite_generator import tflite_model @@ -523,10 +523,10 @@ def __call__( ) -> bool: match tensor_map[self.tensor]: case tflite_model.Tensor(): - return tensor_map[self.tensor].tensor_format == TensorFormat.FORMATLESS + return tensor_map[self.tensor].tensor_format == DataFormat.FORMATLESS case list(): return all( - t.tensor_format == TensorFormat.FORMATLESS + t.tensor_format == DataFormat.FORMATLESS for t in tensor_map[self.tensor] ) case _: diff --git a/backends/nxp/backend/ir/tensor_formatting.py b/backends/nxp/backend/ir/tensor_formatting.py deleted file mode 100644 index 71b697a0eba..00000000000 --- a/backends/nxp/backend/ir/tensor_formatting.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2023 Martin Pavella -# Copyright 2023-2025 NXP -# -# License: MIT -# See the LICENSE_MIT for more details. -# -from enum import Enum - -from executorch.backends.nxp.backend.node_format import NodeFormat - - -class TensorFormat(Enum): - CHANNELS_FIRST = 0 - - CHANNELS_LAST = 10 - - # The format of TFLite Conv3D weights tensor: [output_channels, input_channels, D, H, W] - CONV_3D_WEIGHT_FORMAT = 11 - - # Intermediate format between 'Transpose' and 'Reshape' ops when single dimension with value 1 - # is added/removed via reshaping - RESHAPE_SINGLE_UNITARY_TRANSPOSITION = 12 - - # The format of TFLite TransposeConv 2D weights tensor: [M/group, kH, kW, C] - TRANSPOSE_CONV_2D_WEIGHT_FORMAT = 13 - - # No special format (matrices, vectors, shapes etc.). All tensors with the FORMATLESS format MUST have EXACTLY - # the same shape and data in the NeutronIR model and in the ExecuTorch model. - FORMATLESS = 20 - - NONE = 30 # Format has not been identified - - def is_channels_first(self) -> bool: - return self == TensorFormat.CHANNELS_FIRST - - def is_channels_last(self) -> bool: - return self == TensorFormat.CHANNELS_LAST - - @staticmethod - def from_node_format(node_format: NodeFormat): - if node_format == NodeFormat.CHANNELS_FIRST: - return TensorFormat.CHANNELS_LAST # Format is swapped. - elif node_format == NodeFormat.CHANNELS_LAST: - return TensorFormat.CHANNELS_FIRST # Format is swapped. - elif node_format == NodeFormat.FORMATLESS: - return TensorFormat.FORMATLESS - else: - return TensorFormat.NONE - - def to_node_format(self): - if self == TensorFormat.CHANNELS_LAST: - return NodeFormat.CHANNELS_FIRST # Format is swapped. - elif self == TensorFormat.FORMATLESS: - return NodeFormat.FORMATLESS - elif self == TensorFormat.CHANNELS_FIRST: - return NodeFormat.CHANNELS_LAST # Format is swapped. - else: - return NodeFormat.NONE - - def to_equal_node_format(self): - match self: - case TensorFormat.CHANNELS_FIRST: - return NodeFormat.CHANNELS_FIRST - case TensorFormat.CHANNELS_LAST: - return NodeFormat.CHANNELS_LAST - case TensorFormat.FORMATLESS: - return NodeFormat.FORMATLESS - case _: - return NodeFormat.NONE diff --git a/backends/nxp/backend/ir/tflite_generator/tflite_model.py b/backends/nxp/backend/ir/tflite_generator/tflite_model.py index 76a50a2e177..6e8e7b6c33b 100755 --- a/backends/nxp/backend/ir/tflite_generator/tflite_model.py +++ b/backends/nxp/backend/ir/tflite_generator/tflite_model.py @@ -1,5 +1,5 @@ # Copyright 2023 Martin Pavella -# Copyright 2023-2025 NXP +# Copyright 2023-2026 NXP # # License: MIT # See the LICENSE_MIT for more details. @@ -24,7 +24,7 @@ import flatbuffers as fb import numpy as np -from executorch.backends.nxp.backend.ir import tensor_formatting +from executorch.backends.nxp.backend import data_format from executorch.backends.nxp.backend.ir.tflite_generator.meta import types from executorch.backends.nxp.backend.ir.tflite_generator.meta.types import name_for_type @@ -380,7 +380,7 @@ class Tensor(meta.TFLiteObject): # TODO shapeSignature # TODO variantTensors - tensor_format: tensor_formatting.TensorFormat + tensor_format: data_format.DataFormat # TODO If 'hasRank' is false, "shape" must be []. @@ -426,7 +426,7 @@ def __init__( self.tmp_null_tensor = False - self.tensor_format = tensor_formatting.TensorFormat.NONE + self.tensor_format = data_format.DataFormat.NONE def gen_tflite(self, builder: fb.Builder): diff --git a/backends/nxp/backend/node_format.py b/backends/nxp/backend/node_format.py deleted file mode 100644 index fd54e2365ed..00000000000 --- a/backends/nxp/backend/node_format.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2025 NXP -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -from enum import Enum - -# Key into the `meta` attribute of nodes, which is mapped to their inferred node format. -NXP_NODE_FORMAT = "nxp_node_format" - - -class NodeFormat(Enum): - # Node's output in NCHW format - CHANNELS_FIRST = 0 - - # Node's output format has no meaning - FORMATLESS = 1 - - # Format has not been identified - NONE = 2 - - # NHWC - CHANNELS_LAST = 3 - - def is_channels_first(self) -> bool: - return self == NodeFormat.CHANNELS_FIRST diff --git a/backends/nxp/backend/node_format_inference.py b/backends/nxp/backend/node_format_inference.py index 989d25fe15a..9154434f44b 100644 --- a/backends/nxp/backend/node_format_inference.py +++ b/backends/nxp/backend/node_format_inference.py @@ -6,9 +6,10 @@ import logging import operator +from executorch.backends.nxp.backend.data_format import DataFormat, NXP_NODE_FORMAT + from executorch.backends.nxp.backend.edge_helper import is_channels_last_dim_order from executorch.backends.nxp.backend.edge_program_converter import functions_converters -from executorch.backends.nxp.backend.node_format import NodeFormat, NXP_NODE_FORMAT from executorch.exir.dialects._ops import ops as exir_ops from executorch.exir.dialects.edge._ops import EdgeOpOverload from torch.export import ExportedProgram @@ -84,7 +85,7 @@ def identify_node_formats(self): node.meta = {} if NXP_NODE_FORMAT not in node.meta: logging.warning(f"Node `{node}` does not have inferred format.") - node.meta[NXP_NODE_FORMAT] = NodeFormat.NONE + node.meta[NXP_NODE_FORMAT] = DataFormat.NONE def _infer_format_of_nodes(self, node: Node): op_type = self._get_node_op_type(node) @@ -102,10 +103,10 @@ def _infer_format_of_nodes(self, node: Node): # Note: If the format for the input/output has already been assigned as channels first, it will NOT be # overwritten. self._assign_format_to_node( - self._node_outputs[node][0], NodeFormat.FORMATLESS + self._node_outputs[node][0], DataFormat.FORMATLESS ) self._assign_format_to_node( - self._node_inputs[node][0], NodeFormat.FORMATLESS + self._node_inputs[node][0], DataFormat.FORMATLESS ) else: @@ -123,7 +124,7 @@ def _infer_format_of_nodes(self, node: Node): # delegated partitions. Propagating the format here could unnecessarily enforce the format in one of these # partitions, which would require extra transpositions. for processed_node in self._node_inputs[node] + [node]: - self._assign_format_to_node(processed_node, NodeFormat.NONE) + self._assign_format_to_node(processed_node, DataFormat.NONE) def _infer_format_based_on_io_ranks(self, node: Node): """Determine the format of the output tensor of given "reshape style operator" based on the ranks of its input @@ -141,12 +142,12 @@ def _infer_format_based_on_io_ranks(self, node: Node): else: # Either the op 'flattens' the tensor, so output is formatless, or it scales it up, in which case the # format is assumed to be 'FORMATLESS', and may be back propagated as channels first later. - self._assign_format_to_node(node, NodeFormat.FORMATLESS) + self._assign_format_to_node(node, DataFormat.FORMATLESS) except: # Some shape data is not known, so we cannot be extra clever. Just set the output to `FORMATLESS` and # everything will be alright. - self._assign_format_to_node(node, NodeFormat.FORMATLESS) + self._assign_format_to_node(node, DataFormat.FORMATLESS) def _match_formats_of_nodes(self, node_1, node_2): """If one of 'node_1' or 'node_2' is channels first, make the other channels first as well. @@ -159,25 +160,25 @@ def _match_formats_of_nodes(self, node_1, node_2): if format_1.is_channels_first() or format_2.is_channels_first(): # At least 1 is channels first if not format_1.is_channels_first(): - self._assign_format_to_node(node_1, NodeFormat.CHANNELS_FIRST) + self._assign_format_to_node(node_1, DataFormat.CHANNELS_FIRST) elif not format_2.is_channels_first(): - self._assign_format_to_node(node_2, NodeFormat.CHANNELS_FIRST) + self._assign_format_to_node(node_2, DataFormat.CHANNELS_FIRST) else: - self._assign_format_to_node(node_1, NodeFormat.FORMATLESS) - self._assign_format_to_node(node_2, NodeFormat.FORMATLESS) + self._assign_format_to_node(node_1, DataFormat.FORMATLESS) + self._assign_format_to_node(node_2, DataFormat.FORMATLESS) - def _assign_format_to_node(self, node: Node, node_format: NodeFormat): + def _assign_format_to_node(self, node: Node, node_format: DataFormat): """ Assign format to node, but only if it's not channels first. """ old_node_format = self._get_node_format(node) - if old_node_format is NodeFormat.CHANNELS_FIRST: + if old_node_format is DataFormat.CHANNELS_FIRST: # Once CHANNEL_FIRST was assigned, we don't want to reassign return - if node_format is NodeFormat.NONE and old_node_format is not NodeFormat.NONE: + if node_format is DataFormat.NONE and old_node_format is not DataFormat.NONE: # A format has already been assigned to the node before. Don't replace it with `NONE`. return @@ -204,16 +205,16 @@ def _handle_node_which_uses_channels_first_format(self, node: Node): for index, ancestor_node in enumerate(self._node_inputs[node]): # Go through input nodes and assign them correct format if index in self.ops_with_channels_first_nodes[op_type]["inputs"]: - self._assign_format_to_node(ancestor_node, NodeFormat.CHANNELS_FIRST) + self._assign_format_to_node(ancestor_node, DataFormat.CHANNELS_FIRST) # We need to propagate channels first format up to already visited nodes self._propagate_channels_first_format_up(ancestor_node) else: - self._assign_format_to_node(ancestor_node, NodeFormat.FORMATLESS) + self._assign_format_to_node(ancestor_node, DataFormat.FORMATLESS) # (TODO Lukas Sztefek): It is expected here, that CHANNELS_FIRST node always produces CHANNELS_FIRST output. # Validate the assumption. - self._assign_format_to_node(node, NodeFormat.CHANNELS_FIRST) + self._assign_format_to_node(node, DataFormat.CHANNELS_FIRST) def _handle_node_which_can_use_any_node_format(self, node: Node): """ @@ -226,12 +227,12 @@ def _handle_node_which_can_use_any_node_format(self, node: Node): if hasattr(val := node.meta["val"], "dim_order") and is_channels_last_dim_order( list(val.dim_order()) ): - self._assign_format_to_node(node, NodeFormat.CHANNELS_FIRST) + self._assign_format_to_node(node, DataFormat.CHANNELS_FIRST) if not self._node_produces_or_consumes_channels_first_format(node): # Neither inputs nor current node are channels first -> assign everything to formatless for processed_node in self._node_inputs[node] + [node]: - self._assign_format_to_node(processed_node, NodeFormat.FORMATLESS) + self._assign_format_to_node(processed_node, DataFormat.FORMATLESS) else: # Node produces or consumes channels first content @@ -243,18 +244,18 @@ def _handle_node_which_can_use_any_node_format(self, node: Node): continue elif is_0d_to_2d: # Node has less than 3 dimensions so it cannot be considered CHANNELS_FIRST - self._assign_format_to_node(processed_node, NodeFormat.FORMATLESS) + self._assign_format_to_node(processed_node, DataFormat.FORMATLESS) else: # Node has more than 2D output -> make it channels first self._assign_format_to_node( - processed_node, NodeFormat.CHANNELS_FIRST + processed_node, DataFormat.CHANNELS_FIRST ) self._propagate_channels_first_format_up(processed_node) def _propagate_channels_first_format_up(self, node: Node): if self._node_is_placeholder(node): # Input or buffer node -> there is no parent node so we can end propagation here - self._assign_format_to_node(node, NodeFormat.CHANNELS_FIRST) + self._assign_format_to_node(node, DataFormat.CHANNELS_FIRST) return if node in self.ops_that_can_change_tensor_format: @@ -293,10 +294,10 @@ def _node_produces_or_consumes_channels_first_format(self, node) -> bool: for ancestor_node in input_nodes ) - def _get_node_format(self, node) -> NodeFormat: + def _get_node_format(self, node) -> DataFormat: if not hasattr(node, "meta"): node.meta = {} - return node.meta.get(NXP_NODE_FORMAT, NodeFormat.NONE) + return node.meta.get(NXP_NODE_FORMAT, DataFormat.NONE) def _node_is_placeholder(self, node: Node) -> bool: return node.op == "placeholder" diff --git a/backends/nxp/nxp_backend.py b/backends/nxp/nxp_backend.py index 83c304566d5..fa48f3bbeab 100644 --- a/backends/nxp/nxp_backend.py +++ b/backends/nxp/nxp_backend.py @@ -16,6 +16,7 @@ import torch from executorch.backends.nxp._passes.remove_getitem_pass import RemoveGetItemPass +from executorch.backends.nxp.backend.data_format import DataFormat from executorch.backends.nxp.backend.edge_program_converter import ( EdgeProgramToIRConverter, ) @@ -24,7 +25,6 @@ NeutronConverterManager, ) from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec -from executorch.backends.nxp.backend.node_format import NodeFormat from executorch.backends.nxp.neutron_node_extraction import ( extract_artifacts_from_neutron_node, NeutronNodeArtifacts, @@ -265,7 +265,7 @@ def _format_string_for_array(self, array: np.ndarray) -> str: return f"{array.size}s{self._padding_format_string_for_array(array)}" def _create_payload_header( - self, io_formats: dict[str, list[NodeFormat]], neutron_artifacts + self, io_formats: dict[str, list[DataFormat]], neutron_artifacts ) -> np.ndarray: """ Create bytes header for returned payload. It contains information about @@ -306,7 +306,7 @@ def _create_payload_header( for input_name in neutron_artifacts.input_names: try: header_data.append( - 1 if inputs[input_name.decode()] == NodeFormat.CHANNELS_LAST else 0 + 1 if inputs[input_name.decode()] == DataFormat.CHANNELS_LAST else 0 ) except KeyError: raise AssertionError( @@ -317,7 +317,7 @@ def _create_payload_header( try: header_data.append( 1 - if outputs[output_name.decode()] == NodeFormat.CHANNELS_LAST + if outputs[output_name.decode()] == DataFormat.CHANNELS_LAST else 0 ) except KeyError: @@ -358,7 +358,7 @@ def _pack_with_alignment( ) def get_binary_payload( - self, io_formats: dict[str, list[NodeFormat]], neutron_model + self, io_formats: dict[str, list[DataFormat]], neutron_model ) -> bytes: """ Get binary payload for provided input/output tensor formats and neutron_model. Returned data have @@ -379,7 +379,7 @@ def get_binary_payload( Tensor format definition: '0x1' == CHANNELS_LAST, '0x0' == FORMATLESS (no format). :param io_formats: Dictionary with keys 'inputs' and 'outputs' that contains dictionaries - mapping tensor name to NodeFormat. + mapping tensor name to DataFormat. :param neutron_model: Neutron model with single NeutronGraph node. :return: 16 bytes aligned binary payload. """ diff --git a/backends/nxp/tests/test_node_format_inference.py b/backends/nxp/tests/test_node_format_inference.py index 412c422dc6d..21ad95c6b64 100644 --- a/backends/nxp/tests/test_node_format_inference.py +++ b/backends/nxp/tests/test_node_format_inference.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025 NXP +# Copyright 2024-2026 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -7,7 +7,7 @@ from executorch import exir from executorch.backends.nxp.backend.node_format_inference import ( - NodeFormat, + DataFormat, NodeFormatInference, NXP_NODE_FORMAT, ) @@ -31,11 +31,11 @@ def test_convolution(): NodeFormatInference(edge_program).identify_node_formats() expected_mapping = { - "p_conv_weight": NodeFormat.CHANNELS_FIRST, - "p_conv_bias": NodeFormat.FORMATLESS, - "x": NodeFormat.CHANNELS_FIRST, - "aten_convolution_default": NodeFormat.CHANNELS_FIRST, - "output": NodeFormat.CHANNELS_FIRST, + "p_conv_weight": DataFormat.CHANNELS_FIRST, + "p_conv_bias": DataFormat.FORMATLESS, + "x": DataFormat.CHANNELS_FIRST, + "aten_convolution_default": DataFormat.CHANNELS_FIRST, + "output": DataFormat.CHANNELS_FIRST, } for node in edge_program.graph.nodes: @@ -52,9 +52,9 @@ def test_softmax(): NodeFormatInference(edge_program).identify_node_formats() expected_mapping = { - "x": NodeFormat.FORMATLESS, - "aten__softmax_default": NodeFormat.FORMATLESS, - "output": NodeFormat.FORMATLESS, + "x": DataFormat.FORMATLESS, + "aten__softmax_default": DataFormat.FORMATLESS, + "output": DataFormat.FORMATLESS, } for node in edge_program.graph.nodes: @@ -82,9 +82,9 @@ def test_maxpool2d(): NodeFormatInference(edge_program).identify_node_formats() expected_mapping = { - "x": NodeFormat.CHANNELS_FIRST, - "aten_max_pool2d_default": NodeFormat.CHANNELS_FIRST, - "output": NodeFormat.CHANNELS_FIRST, + "x": DataFormat.CHANNELS_FIRST, + "aten_max_pool2d_default": DataFormat.CHANNELS_FIRST, + "output": DataFormat.CHANNELS_FIRST, } for node in edge_program.graph.nodes: