Update flavours (#285)

* update requirements

* fix xrootd integration

* update sources

dev/fsspec_inspector/generate_flavours.py

@@ -169,10 +169,11 @@ def _fix_oss_file_system(x: str) -> str:
 
 def _fix_xrootd_file_system(x: str) -> str:
     x = re.sub(
-        r"client.URL",
-        "urlsplit",
+        r"return client[.]URL\(path\)[.]path_with_params",
+        "x = urlsplit(path); return (x.path + f'?{x.query}' if x.query else '')",
         x,
     )
+    x = re.sub(r"client[.]URL\(u\)", "urlsplit(u)", x)
     return re.sub(
         "url.hostid",
         "url.netloc",

dev/requirements.txt

@@ -1,11 +1,11 @@
-fsspec[git,hdfs,dask,http,sftp,smb]==2024.6.1
+fsspec[git,hdfs,dask,http,sftp,smb]==2024.9.0
 
 # these dependencies define their own filesystems
 adlfs==2024.7.0
 boxfs==0.3.0
 dropboxdrivefs==1.4.1
-gcsfs==2024.6.1
-s3fs==2024.6.1
+gcsfs==2024.9.0.post1
+s3fs==2024.9.0
 ocifs==1.3.1
 webdav4[fsspec]==0.10.0
 # gfrivefs @ git+https://github.com/fsspec/gdrivefs@master broken ...
@@ -14,5 +14,5 @@ dvc==3.55.2
 huggingface_hub==0.24.6
 lakefs-spec==0.10.0
 ossfs==2023.12.0
-fsspec-xrootd==0.3.0
+fsspec-xrootd==0.4.0
 wandbfs==0.0.2

upath/_flavour_sources.py

@@ -87,7 +87,7 @@ def __init_subclass__(cls: Any, **kwargs):
 
 class AbstractFileSystemFlavour(FileSystemFlavourBase):
     __orig_class__ = 'fsspec.spec.AbstractFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol: str | tuple[str, ...] = 'abstract'
     root_marker: Literal['', '/'] = ''
     sep: Literal['/'] = '/'
@@ -312,7 +312,7 @@ def _strip_protocol(cls, path) -> str:
 
 class DaskWorkerFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.dask.DaskWorkerFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('dask',)
     root_marker = ''
     sep = '/'
@@ -328,15 +328,15 @@ def _get_kwargs_from_urls(path):
 
 class DataFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.data.DataFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('data',)
     root_marker = ''
     sep = '/'
 
 
 class DatabricksFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.dbfs.DatabricksFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('dbfs',)
     root_marker = ''
     sep = '/'
@@ -369,7 +369,7 @@ class DropboxDriveFileSystemFlavour(AbstractFileSystemFlavour):
 
 class FTPFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.ftp.FTPFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('ftp',)
     root_marker = '/'
     sep = '/'
@@ -388,7 +388,7 @@ def _get_kwargs_from_urls(urlpath):
 
 class GCSFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'gcsfs.core.GCSFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0post1'
     protocol = ('gs', 'gcs')
     root_marker = ''
     sep = '/'
@@ -465,7 +465,7 @@ def _split_path(cls, path, version_aware=False):
 
 class GitFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.git.GitFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('git',)
     root_marker = ''
     sep = '/'
@@ -493,7 +493,7 @@ def _get_kwargs_from_urls(path):
 
 class GithubFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.github.GithubFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('github',)
     root_marker = ''
     sep = '/'
@@ -518,7 +518,7 @@ def _get_kwargs_from_urls(path):
 
 class HTTPFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.http.HTTPFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('http', 'https')
     root_marker = ''
     sep = '/'
@@ -539,7 +539,7 @@ def _parent(cls, path):
 
 class HadoopFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.arrow.HadoopFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('hdfs', 'arrow_hdfs')
     root_marker = '/'
     sep = '/'
@@ -580,7 +580,7 @@ class HfFileSystemFlavour(AbstractFileSystemFlavour):
 
 class JupyterFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.jupyter.JupyterFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('jupyter', 'jlab')
     root_marker = ''
     sep = '/'
@@ -606,7 +606,7 @@ def _strip_protocol(cls, path):
 
 class LibArchiveFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.libarchive.LibArchiveFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('libarchive',)
     root_marker = ''
     sep = '/'
@@ -619,7 +619,7 @@ def _strip_protocol(cls, path):
 
 class LocalFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.local.LocalFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('file', 'local')
     root_marker = '/'
     sep = '/'
@@ -697,7 +697,7 @@ def _strip_protocol(cls, path):
 
 class MemoryFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.memory.MemoryFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('memory',)
     root_marker = '/'
     sep = '/'
@@ -796,15 +796,15 @@ class OverlayFileSystemFlavour(AbstractFileSystemFlavour):
 
 class ReferenceFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.reference.ReferenceFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('reference',)
     root_marker = ''
     sep = '/'
 
 
 class S3FileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 's3fs.core.S3FileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('s3', 's3a')
     root_marker = ''
     sep = '/'
@@ -831,7 +831,7 @@ def _get_kwargs_from_urls(urlpath):
 
 class SFTPFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.sftp.SFTPFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('sftp', 'ssh')
     root_marker = ''
     sep = '/'
@@ -850,7 +850,7 @@ def _get_kwargs_from_urls(urlpath):
 
 class SMBFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.smb.SMBFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('smb',)
     root_marker = ''
     sep = '/'
@@ -870,7 +870,7 @@ def _get_kwargs_from_urls(path):
 
 class TarFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.tar.TarFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('tar',)
     root_marker = ''
     sep = '/'
@@ -886,7 +886,7 @@ class WandbFSFlavour(AbstractFileSystemFlavour):
 
 class WebHDFSFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.webhdfs.WebHDFS'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('webhdfs', 'webHDFS')
     root_marker = ''
     sep = '/'
@@ -921,7 +921,7 @@ def _strip_protocol(cls, path: str) -> str:
 
 class XRootDFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec_xrootd.xrootd.XRootDFileSystem'
-    __orig_version__ = '0.3.0'
+    __orig_version__ = '0.4.0'
     protocol = ('root',)
     root_marker = '/'
     sep = '/'
@@ -930,7 +930,7 @@ class XRootDFileSystemFlavour(AbstractFileSystemFlavour):
     def _strip_protocol(cls, path: str | list[str]) -> Any:
         if isinstance(path, str):
             if path.startswith(cls.protocol):
-                return urlsplit(path).path.rstrip("/") or cls.root_marker
+                x = urlsplit(path); return (x.path + f'?{x.query}' if x.query else '').rstrip("/") or cls.root_marker
             # assume already stripped
             return path.rstrip("/") or cls.root_marker
         elif isinstance(path, list):
@@ -947,7 +947,7 @@ def _get_kwargs_from_urls(u: str) -> dict[Any, Any]:
 
 class ZipFileSystemFlavour(AbstractFileSystemFlavour):
     __orig_class__ = 'fsspec.implementations.zip.ZipFileSystem'
-    __orig_version__ = '2024.6.1'
+    __orig_version__ = '2024.9.0'
     protocol = ('zip',)
     root_marker = ''
     sep = '/'