Depend on audeer>=2.0.0 (#348)

* Depend on audeer>=2.0.0

* Use audformat with new audeer

* Ensure correct Windows path for attachments

* Use same syntax for load_media|attachments

* Use audeer.path() when loading attachment

* Use audeer.path() in load_to() for attachments

* Fix publishing of attachments

* Undo unneeded changes

* Depend on audeer>=2.0.0, audformat>=1.1.1

audb/core/api.py

@@ -355,10 +355,10 @@ def exists(
             default_cache_root(False),
         ]
         if cache_root is None
-        else [cache_root]
+        else [audeer.path(cache_root, follow_symlink=True)]
     )
     for cache_root in cache_roots:
-        db_root = audeer.path(cache_root, relative_flavor_path)
+        db_root = os.path.join(cache_root, relative_flavor_path)
         if os.path.exists(db_root):
             return True
 

audb/core/cache.py

@@ -30,19 +30,12 @@ def database_cache_root(
             default_cache_root(False),
         ]
     else:
-        cache_roots = [cache_root]
+        cache_roots = [audeer.path(cache_root, follow_symlink=True)]
     for cache_root in cache_roots:
         if flavor is None:
-            db_root = audeer.path(
-                cache_root,
-                name,
-                version,
-            )
+            db_root = os.path.join(cache_root, name, version)
         else:
-            db_root = audeer.path(
-                cache_root,
-                flavor.path(name, version),
-            )
+            db_root = os.path.join(cache_root, flavor.path(name, version))
         if os.path.exists(db_root):
             break
 
@@ -98,4 +91,4 @@ def default_cache_root(
         cache = os.environ.get("AUDB_SHARED_CACHE_ROOT") or config.SHARED_CACHE_ROOT
     else:
         cache = os.environ.get("AUDB_CACHE_ROOT") or config.CACHE_ROOT
-    return audeer.path(cache)
+    return audeer.path(cache, follow_symlink=True)

audb/core/load.py

@@ -1271,8 +1271,10 @@ def load_attachment(
         )
 
     attachment_files = db.attachments[attachment].files
-    attachment_files = [audeer.path(db_root, a) for a in attachment_files]
-
+    attachment_files = [
+        os.path.join(db_root, os.path.normpath(file))  # convert "/" to os.sep
+        for file in attachment_files
+    ]
     return attachment_files
 
 
@@ -1509,7 +1511,10 @@ def load_media(
 
             if format is not None:
                 media = [audeer.replace_file_extension(m, format) for m in media]
-            files = [os.path.join(db_root, os.path.normpath(m)) for m in media]
+            files = [
+                os.path.join(db_root, os.path.normpath(file))  # convert "/" to os.sep
+                for file in media
+            ]
 
     except filelock.Timeout:
         utils.timeout_warning()

audb/core/load_to.py

@@ -297,7 +297,7 @@ def load_to(
     if version is None:
         version = latest_version(name)
 
-    db_root = audeer.path(root)
+    db_root = audeer.path(root, follow_symlink=True)
     db_root_tmp = database_tmp_root(db_root)
 
     # remove files with a wrong checksum

audb/core/utils.py

@@ -58,7 +58,7 @@ def mkdir_tree(
     for file in files:
         folders.add(os.path.dirname(file))
     for folder in folders:
-        audeer.mkdir(os.path.join(root, folder))
+        audeer.mkdir(root, folder)
 
 
 def _lookup(

pyproject.toml

@@ -34,8 +34,8 @@ classifiers = [
 requires-python = '>=3.8'
 dependencies = [
     'audbackend[artifactory] >=1.0.0',
-    'audeer >=1.20.0',
-    'audformat >=0.16.1',
+    'audeer >=2.0.0',
+    'audformat >=1.1.1',
     'audiofile >=1.0.0',
     'audobject >=0.5.0',
     'audresample >=0.1.6',

tests/conftest.py

@@ -130,7 +130,7 @@ def repository(tmpdir_factory):
         host=host,
         backend="file-system",
     )
-    audeer.mkdir(audeer.path(host, name))
+    audeer.mkdir(host, name)
     current_repositories = audb.config.REPOSITORIES
     audb.config.REPOSITORIES = [repository]
 
@@ -166,7 +166,7 @@ def persistent_repository(tmpdir_factory):
         host=host,
         backend="file-system",
     )
-    audeer.mkdir(audeer.path(host, name))
+    audeer.mkdir(host, name)
     current_repositories = audb.config.REPOSITORIES
     audb.config.REPOSITORIES = [repository]
 

tests/test_info.py

@@ -76,7 +76,7 @@ def db(tmpdir_factory, persistent_repository):
 
     db_root = tmpdir_factory.mktemp(DB_VERSION)
     sampling_rate = 8000
-    audeer.touch(audeer.path(db_root, db.attachments["attachment"].path))
+    audeer.touch(db_root, db.attachments["attachment"].path)
     for table in list(db.tables):
         for file in db[table].files:
             audiofile.write(

tests/test_load.py

@@ -91,11 +91,11 @@ def dbs(tmpdir_factory, persistent_repository):
     db_root = tmpdir_factory.mktemp(version)
     paths[version] = str(db_root)
 
-    audeer.mkdir(audeer.path(db_root, "extra/folder/sub-folder"))
-    audeer.touch(audeer.path(db_root, "extra/file.txt"))
-    audeer.touch(audeer.path(db_root, "extra/folder/file1.txt"))
-    audeer.touch(audeer.path(db_root, "extra/folder/file2.txt"))
-    audeer.touch(audeer.path(db_root, "extra/folder/sub-folder/file3.txt"))
+    audeer.mkdir(db_root, "extra/folder/sub-folder")
+    audeer.touch(db_root, "extra/file.txt")
+    audeer.touch(db_root, "extra/folder/file1.txt")
+    audeer.touch(db_root, "extra/folder/file2.txt")
+    audeer.touch(db_root, "extra/folder/sub-folder/file3.txt")
     db.save(db_root)
     audformat.testing.create_audio_files(db)
     archives = db["files"]["speaker"].get().dropna().to_dict()
@@ -741,7 +741,7 @@ def test_load_to_update(tmpdir, dbs, only_metadata):
                 db.attachments[attachment_id].path,
             )
             if os.path.isdir(attachment):
-                audeer.touch(audeer.path(attachment, "other-file.txt"))
+                audeer.touch(attachment, "other-file.txt")
             else:
                 with open(attachment, "a") as fp:
                     fp.write("next")

tests/test_load_on_demand.py

@@ -60,10 +60,10 @@ def dbs(tmpdir_factory, persistent_repository):
     )
     db.attachments["file"] = audformat.Attachment("file.txt")
     db.attachments["folder"] = audformat.Attachment("folder/")
-    audeer.mkdir(audeer.path(db_root, "folder"))
-    audeer.touch(audeer.path(db_root, "file.txt"))
-    audeer.touch(audeer.path(db_root, "folder/file1.txt"))
-    audeer.touch(audeer.path(db_root, "folder/file2.txt"))
+    audeer.mkdir(db_root, "folder")
+    audeer.touch(db_root, "file.txt")
+    audeer.touch(db_root, "folder/file1.txt")
+    audeer.touch(db_root, "folder/file2.txt")
     db.save(db_root)
     audformat.testing.create_audio_files(db)
 

tests/test_lock.py

@@ -27,7 +27,7 @@ def job(lock, wait, sleep):
 def test_lock(tmpdir):
     # create two lock folders
 
-    lock_folders = [audeer.mkdir(audeer.path(tmpdir, str(idx))) for idx in range(2)]
+    lock_folders = [audeer.mkdir(tmpdir, str(idx)) for idx in range(2)]
 
     # lock 1 and 2
 

tests/test_lock_db.py

@@ -132,7 +132,7 @@ def dbs(tmpdir_factory, persistent_repository):
     )
     db.attachments["file"] = audformat.Attachment("extra/file.txt")
     db.attachments["folder"] = audformat.Attachment("extra/folder")
-    audeer.mkdir(audeer.path(db_root, "extra/folder/sub-folder"))
+    audeer.mkdir(db_root, "extra/folder/sub-folder")
     for file in [
         "extra/file.txt",
         "extra/folder/file1.txt",

tests/test_publish.py

@@ -102,12 +102,12 @@ def dbs(tmpdir_factory):
         [0, 1, 1, 2],
         index=audformat.filewise_index(db.files[:4]),
     )
-    audeer.mkdir(audeer.path(db_root, "extra/folder"))
-    audeer.touch(audeer.path(db_root, "extra/file.txt"))
-    audeer.touch(audeer.path(db_root, "extra/folder/file1.txt"))
-    audeer.touch(audeer.path(db_root, "extra/folder/file2.txt"))
-    audeer.mkdir(audeer.path(db_root, "extra/folder/sub-folder"))
-    audeer.touch(audeer.path(db_root, "extra/folder/sub-folder/file3.txt"))
+    audeer.mkdir(db_root, "extra/folder")
+    audeer.touch(db_root, "extra/file.txt")
+    audeer.touch(db_root, "extra/folder/file1.txt")
+    audeer.touch(db_root, "extra/folder/file2.txt")
+    audeer.mkdir(db_root, "extra/folder/sub-folder")
+    audeer.touch(db_root, "extra/folder/sub-folder/file3.txt")
     # Create one file with different content to force different checksum
     file_with_different_content = audeer.path(
         db_root,
@@ -157,7 +157,7 @@ def dbs(tmpdir_factory):
         audeer.path(db_root, "extra"),
     )
     os.remove(audeer.path(db_root, "extra/folder/file2.txt"))
-    audeer.touch(audeer.path(db_root, "extra/folder/file3.txt"))
+    audeer.touch(db_root, "extra/folder/file3.txt")
     db["files"].extend_index(audformat.filewise_index(LONG_PATH), inplace=True)
     db.save(db_root)
     audformat.testing.create_audio_files(db)
@@ -496,7 +496,7 @@ def test_publish_attachment(tmpdir, repository):
         audb.publish(db_path, "1.0.0", repository)
 
     # Publish database, path is not allowed to be a symlink
-    audeer.mkdir(audeer.path(db_path, folder_path))
+    audeer.mkdir(db_path, folder_path)
     os.symlink(
         audeer.path(db_path, folder_path),
         audeer.path(db_path, file_path),
@@ -510,7 +510,7 @@ def test_publish_attachment(tmpdir, repository):
         audb.publish(db_path, "1.0.0", repository)
 
     os.remove(os.path.join(db_path, file_path))
-    audeer.touch(audeer.path(db_path, file_path))
+    audeer.touch(db_path, file_path)
     db.save(db_path)
 
     # File exist now, folder is empty
@@ -527,7 +527,7 @@ def test_publish_attachment(tmpdir, repository):
 
     # Add empty sub-folder
     subfolder_path = f"{folder_path}/sub-folder"
-    audeer.mkdir(audeer.path(db_path, subfolder_path))
+    audeer.mkdir(db_path, subfolder_path)
     assert db.attachments["folder"].files == []
     error_msg = (
         "An attachment must not "
@@ -540,15 +540,15 @@ def test_publish_attachment(tmpdir, repository):
 
     # Add file to folder, sub-folder still empty
     file2_path = f"{folder_path}/file.txt"
-    audeer.touch(audeer.path(db_path, file2_path))
+    audeer.touch(db_path, file2_path)
     assert db.attachments["file"].files == [file_path]
     assert db.attachments["folder"].files == [file2_path]
     with pytest.raises(RuntimeError, match=error_msg):
         audb.publish(db_path, "1.0.0", repository)
 
     # Add file to sub-folder
     file3_path = f"{subfolder_path}/file.txt"
-    audeer.touch(audeer.path(db_path, file3_path))
+    audeer.touch(db_path, file3_path)
     assert db.attachments["file"].files == [file_path]
     assert db.attachments["folder"].files == [file2_path, file3_path]
 
@@ -774,7 +774,7 @@ def test_publish_error_allowed_chars(tmpdir, repository):
     # on the backends
 
     # Prepare database files
-    db_path = audeer.mkdir(audeer.path(tmpdir, "db"))
+    db_path = audeer.mkdir(tmpdir, "db")
     audio_file = audeer.path(db_path, "f1.wav")
     attachment_file = audeer.path(db_path, "attachment.txt")
     signal = np.zeros((2, 1000))
@@ -837,8 +837,8 @@ def test_publish_error_changed_deps_file_type(tmpdir, repository):
         "But attachment 'attachment' contains 'data/file.wav'."
     )
     db_name = "test_publish_error_changed_deps_file_type-1"
-    db_path = audeer.mkdir(audeer.path(tmpdir, "db"))
-    data_path = audeer.mkdir(audeer.path(db_path, "data"))
+    db_path = audeer.mkdir(tmpdir, "db")
+    data_path = audeer.mkdir(db_path, "data")
     signal = np.zeros((2, 1000))
     sampling_rate = 8000
     audiofile.write(audeer.path(data_path, "file.wav"), signal, sampling_rate)
@@ -856,8 +856,8 @@ def test_publish_error_changed_deps_file_type(tmpdir, repository):
         "But attachment 'attachment' contains 'db.table.csv'."
     )
     db_name = "test_publish_error_changed_deps_file_type-2"
-    db_path = audeer.mkdir(audeer.path(tmpdir, "db"))
-    data_path = audeer.mkdir(audeer.path(db_path, "data"))
+    db_path = audeer.mkdir(tmpdir, "db")
+    data_path = audeer.mkdir(db_path, "data")
     signal = np.zeros((2, 1000))
     sampling_rate = 8000
     audiofile.write(audeer.path(data_path, "file.wav"), signal, sampling_rate)
@@ -875,8 +875,8 @@ def test_publish_error_changed_deps_file_type(tmpdir, repository):
         "But attachment 'attachment' contains 'data/file2.wav'."
     )
     db_name = "test_publish_error_changed_deps_file_type-3"
-    db_path = audeer.mkdir(audeer.path(tmpdir, "db"))
-    data_path = audeer.mkdir(audeer.path(db_path, "data"))
+    db_path = audeer.mkdir(tmpdir, "db")
+    data_path = audeer.mkdir(db_path, "data")
     signal = np.zeros((2, 1000))
     sampling_rate = 8000
     audiofile.write(audeer.path(data_path, "file1.wav"), signal, sampling_rate)
@@ -902,15 +902,15 @@ def test_publish_error_changed_deps_file_type(tmpdir, repository):
         "But attachment 'attachment' contains 'db.table2.csv'."
     )
     db_name = "test_publish_error_changed_deps_file_type-4"
-    db_path = audeer.mkdir(audeer.path(tmpdir, "db"))
-    data_path = audeer.mkdir(audeer.path(db_path, "data"))
+    db_path = audeer.mkdir(tmpdir, "db")
+    data_path = audeer.mkdir(db_path, "data")
     signal = np.zeros((2, 1000))
     sampling_rate = 8000
     audiofile.write(audeer.path(data_path, "file.wav"), signal, sampling_rate)
     db = audformat.Database(db_name)
     db["table1"] = audformat.Table(audformat.filewise_index("data/file.wav"))
     db.attachments["attachment"] = audformat.Attachment("db.table2.csv")
-    audeer.touch(audeer.path(db_path, "db.table2.csv"))
+    audeer.touch(db_path, "db.table2.csv")
     db.save(db_path)
     audb.publish(db_path, "1.0.0", repository)
     audeer.rmdir(db_path)
@@ -942,8 +942,8 @@ def test_publish_error_repository_does_not_exist(tmpdir, repository):
 )
 def test_publish_error_uppercase_file_extension(tmpdir, repository, file):
     # Prepare files
-    db_path = audeer.mkdir(audeer.path(tmpdir, "db"))
-    audeer.touch(audeer.path(db_path, file))
+    db_path = audeer.mkdir(tmpdir, "db")
+    audeer.touch(db_path, file)
     # Prepare database
     db = audformat.Database("db")
     db["table"] = audformat.Table(audformat.filewise_index([file]))
@@ -964,7 +964,7 @@ def test_publish_error_version(tmpdir, repository):
     # are allowed
 
     # Create simple database
-    db_path = audeer.mkdir(audeer.path(tmpdir, "db"))
+    db_path = audeer.mkdir(tmpdir, "db")
     audio_file = audeer.path(db_path, "f1.wav")
     signal = np.zeros((2, 1000))
     sampling_rate = 8000
@@ -1225,8 +1225,8 @@ def test_update_database_without_media(tmpdir, persistent_repository):
         assert not os.path.exists(path)
 
     # add new attachment
-    audeer.mkdir(audeer.path(build_root, "extra"))
-    audeer.touch(audeer.path(build_root, "extra/new.txt"))
+    audeer.mkdir(build_root, "extra")
+    audeer.touch(build_root, "extra/new.txt")
     db.attachments[new_attachment] = audformat.Attachment("extra/new.txt")
 
     db.save(build_root)