Exclude repo_id from LoadDataFromFileSystem (#898)

* Exclude repo_id from LoadDataFromFileSystem generator class and update tests

* Update code to be compatible with python 3.9

src/distilabel/steps/generators/huggingface.py

@@ -17,13 +17,15 @@
 from pathlib import Path
 from typing import (
     TYPE_CHECKING,
+    Annotated,
     Any,
     Dict,
     List,
     Mapping,
     Optional,
     Sequence,
     Tuple,
+    TypeVar,
     Union,
 )
 
@@ -46,6 +48,13 @@
     from distilabel.steps.typing import GeneratorStepOutput
 
 
+T = TypeVar("T")
+
+# To avoid using repo_id in LoadDataFromFileSystem:
+# https://github.com/pydantic/pydantic/discussions/7076#discussioncomment-6699138
+ExcludedField = Annotated[T, Field(exclude=True)]
+
+
 class LoadDataFromHub(GeneratorStep):
     """Loads a dataset from the Hugging Face Hub.
 
@@ -334,6 +343,7 @@ class LoadDataFromFileSystem(LoadDataFromHub):
         default=None,
         description="The expected filetype. If not provided, it will be inferred from the file extension.",
     )
+    repo_id: ExcludedField[Union[str, None]] = None
 
     def load(self) -> None:
         """Load the dataset from the file/s in disk."""
@@ -416,9 +426,7 @@ def outputs(self) -> List[str]:
         """
         # We assume there are Dataset/IterableDataset, not it's ...Dict counterparts
         if self._dataset is None:
-            raise ValueError(
-                "Dataset not loaded yet, you must call `load` method first."
-            )
+            self.load()
 
         return self._dataset.column_names
 

tests/unit/steps/generators/test_huggingface.py

@@ -27,6 +27,8 @@
     LoadDataFromHub,
 )
 
+from tests.unit.pipeline.utils import DummyStep1
+
 DISTILABEL_RUN_SLOW_TESTS = os.getenv("DISTILABEL_RUN_SLOW_TESTS", False)
 
 
@@ -133,18 +135,23 @@ def test_read_from_jsonl_with_nested_folder(
             assert isinstance(generator_step_output[1], bool)
             assert len(generator_step_output[0]) == 22
 
-    @pytest.mark.parametrize("load", [True, False])
-    def test_outputs(self, load: bool) -> None:
+    def test_outputs(self) -> None:
         loader = LoadDataFromFileSystem(
             filetype="json",
             data_files=str(Path(__file__).parent / "sample_functions.jsonl"),
         )
-        if load:
-            loader.load()
-            assert loader.outputs == ["type", "function"]
-        else:
-            with pytest.raises(ValueError):
-                loader.outputs  # noqa: B018
+        loader.load()
+        assert loader.outputs == ["type", "function"]
+
+    def test_loading_in_pipeline(self):
+        with Pipeline():
+            loader = LoadDataFromFileSystem(
+                filetype="json",
+                data_files=str(Path(__file__).parent / "sample_functions.jsonl"),
+            )
+            dummy = DummyStep1(input_mappings={"instruction": "function"})
+            loader >> dummy
+        assert loader.outputs == ["type", "function"]
 
 
 class TestLoadDataFromDisk: