Fix loader to read from a glob pattern (#877)

* Fix loader to read from a glob pattern

* Fix to read from general UPath instead of Path

* Update tests to use glob patterns

* Refactor to simplify check for glob pattern

src/distilabel/steps/generators/huggingface.py

@@ -333,6 +333,23 @@ class LoadDataFromFileSystem(LoadDataFromHub):
         # >>> result
         # ([{'type': 'function', 'function':...', False)
         ```
+
+        Load data passing a glob pattern:
+
+        ```python
+        from distilabel.steps import LoadDataFromFileSystem
+
+        loader = LoadDataFromFileSystem(
+            data_files="path/to/dataset/*.jsonl",
+            streaming=True
+        )
+        loader.load()
+
+        # Just like we saw with LoadDataFromDicts, the `process` method will yield batches.
+        result = next(loader.process())
+        # >>> result
+        # ([{'type': 'function', 'function':...', False)
+        ```
     """
 
     data_files: RuntimeParameter[Union[str, Path]] = Field(
@@ -376,7 +393,7 @@ def load(self) -> None:
                 self.num_examples = len(self._dataset)
 
     @staticmethod
-    def _prepare_data_files(
+    def _prepare_data_files(  # noqa: C901
         data_path: UPath,
     ) -> Tuple[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]], str]:
         """Prepare the loading process by setting the `data_files` attribute.
@@ -394,9 +411,12 @@ def get_filetype(data_path: UPath) -> str:
                 filetype = "json"
             return filetype
 
-        if data_path.is_file():
+        if data_path.is_file() or (
+            len(str(data_path.parent.glob(data_path.name))) >= 1
+        ):
             filetype = get_filetype(data_path)
             data_files = str(data_path)
+
         elif data_path.is_dir():
             file_sequence = []
             file_map = defaultdict(list)

tests/unit/steps/generators/test_huggingface.py

@@ -104,7 +104,7 @@ def test_read_from_jsonl_with_folder(self, filetype: Union[str, None]) -> None:
 
             loader = LoadDataFromFileSystem(
                 filetype=filetype,
-                data_files=tmpdir,
+                data_files=str(Path(tmpdir) / "*.jsonl"),
             )
             loader.load()
             generator_step_output = next(loader.process())
@@ -127,7 +127,7 @@ def test_read_from_jsonl_with_nested_folder(
 
             loader = LoadDataFromFileSystem(
                 filetype=filetype,
-                data_files=tmpdir,
+                data_files=str(Path(tmpdir) / "**/*.jsonl"),
             )
             loader.load()
             generator_step_output = next(loader.process())