fix: make dataset robust to empty samples

Signed-off-by: Mehant Kammakomati <mehant.kammakomati2@ibm.com>

Author: kmehant

tuning/utils/data_loaders.py

@@ -109,6 +109,11 @@ def __iter__(self):
                             sample[self.tokens_field] = self.tokenizer.encode(
                                 sample[self.text_field]
                             )
+                            if not sample[self.tokens_field]:
+                                logger.warning(
+                                    f"skipping an empty sample : {sample[self.tokens_field]}"
+                                )
+                                continue
                         except Exception as e:  # pylint: disable=broad-exception-caught
                             logger.warning(
                                 "failed to tokenize the data {} of type {}.".format(