From 0f5ed18c96985887ab7413132eba38e2c16884af Mon Sep 17 00:00:00 2001 From: Toolkit User Date: Thu, 22 Jan 2026 21:41:13 +0000 Subject: [PATCH] remove sample configs from auto import --- fast_llm/data/auto.py | 4 ---- fast_llm/data/dataset/memmap.py | 6 ++++++ fast_llm/data/preparator/dataset_discovery/__init__.py | 4 ---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/fast_llm/data/auto.py b/fast_llm/data/auto.py index d39ce1e4a..f400978bf 100644 --- a/fast_llm/data/auto.py +++ b/fast_llm/data/auto.py @@ -17,7 +17,3 @@ from fast_llm.data.preparator.dataset_discovery.config import DatasetDiscoveryConfig # isort: skip from fast_llm.data.preparator.gpt_memmap.config import GPTMemmapDatasetPreparatorConfig # isort: skip from fast_llm.data.sample.abstract import NullReaderConfig # isort: skip -from fast_llm.data.sample.language_model import LanguageModelReaderConfig # isort: skip -from fast_llm.data.sample.patch import PatchReaderConfig # isort: skip -from fast_llm.data.sample.range import RangeReaderConfig # isort: skip -from fast_llm.data.sample.token import TokenReaderConfig # isort: skip diff --git a/fast_llm/data/dataset/memmap.py b/fast_llm/data/dataset/memmap.py index 718466ac3..e571fc433 100644 --- a/fast_llm/data/dataset/memmap.py +++ b/fast_llm/data/dataset/memmap.py @@ -28,6 +28,12 @@ def read_reader_config(path: pathlib.Path | str) -> MemmapIndexDatasetReaderConf """ Read the MemmapIndexDatasetReaderConfig from a memmap file. """ + # Import reader configs to register them in the dynamic class registry + from fast_llm.data.sample.language_model import LanguageModelReaderConfig # noqa: F401 + from fast_llm.data.sample.patch import PatchReaderConfig # noqa: F401 + from fast_llm.data.sample.range import RangeReaderConfig # noqa: F401 + from fast_llm.data.sample.token import TokenReaderConfig # noqa: F401 + path = pathlib.Path(path) if isinstance(path, str) else path with path.open("rb") as stream: # Verify file type. diff --git a/fast_llm/data/preparator/dataset_discovery/__init__.py b/fast_llm/data/preparator/dataset_discovery/__init__.py index a9d38880a..e69de29bb 100644 --- a/fast_llm/data/preparator/dataset_discovery/__init__.py +++ b/fast_llm/data/preparator/dataset_discovery/__init__.py @@ -1,4 +0,0 @@ -from fast_llm.data.preparator.dataset_discovery.config import DatasetDiscoveryConfig -from fast_llm.data.preparator.dataset_discovery.prepare import DatasetDiscoveryPreparator - -__all__ = ["DatasetDiscoveryConfig", "DatasetDiscoveryPreparator"]