From d151f101ec5d66c3036a995403ae847fdcf060fe Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Thu, 25 Jan 2024 10:09:26 +0100 Subject: [PATCH] Depend on audeer>=2.0.0 (#348) * Depend on audeer>=2.0.0 * Use audformat with new audeer * Ensure correct Windows path for attachments * Use same syntax for load_media|attachments * Use audeer.path() when loading attachment * Use audeer.path() in load_to() for attachments * Fix publishing of attachments * Undo unneeded changes * Depend on audeer>=2.0.0, audformat>=1.1.1 --- audb/core/api.py | 4 +-- audb/core/cache.py | 15 +++------- audb/core/load.py | 11 ++++++-- audb/core/load_to.py | 2 +- audb/core/utils.py | 2 +- pyproject.toml | 4 +-- tests/conftest.py | 4 +-- tests/test_info.py | 2 +- tests/test_load.py | 12 ++++---- tests/test_load_on_demand.py | 8 +++--- tests/test_lock.py | 2 +- tests/test_lock_db.py | 2 +- tests/test_publish.py | 54 ++++++++++++++++++------------------ 13 files changed, 60 insertions(+), 62 deletions(-) diff --git a/audb/core/api.py b/audb/core/api.py index b1a00979..3a5be6eb 100644 --- a/audb/core/api.py +++ b/audb/core/api.py @@ -355,10 +355,10 @@ def exists( default_cache_root(False), ] if cache_root is None - else [cache_root] + else [audeer.path(cache_root, follow_symlink=True)] ) for cache_root in cache_roots: - db_root = audeer.path(cache_root, relative_flavor_path) + db_root = os.path.join(cache_root, relative_flavor_path) if os.path.exists(db_root): return True diff --git a/audb/core/cache.py b/audb/core/cache.py index c117091e..754024ca 100644 --- a/audb/core/cache.py +++ b/audb/core/cache.py @@ -30,19 +30,12 @@ def database_cache_root( default_cache_root(False), ] else: - cache_roots = [cache_root] + cache_roots = [audeer.path(cache_root, follow_symlink=True)] for cache_root in cache_roots: if flavor is None: - db_root = audeer.path( - cache_root, - name, - version, - ) + db_root = os.path.join(cache_root, name, version) else: - db_root = audeer.path( - cache_root, - flavor.path(name, version), - ) + db_root = os.path.join(cache_root, flavor.path(name, version)) if os.path.exists(db_root): break @@ -98,4 +91,4 @@ def default_cache_root( cache = os.environ.get("AUDB_SHARED_CACHE_ROOT") or config.SHARED_CACHE_ROOT else: cache = os.environ.get("AUDB_CACHE_ROOT") or config.CACHE_ROOT - return audeer.path(cache) + return audeer.path(cache, follow_symlink=True) diff --git a/audb/core/load.py b/audb/core/load.py index 6694fc04..66b0e672 100644 --- a/audb/core/load.py +++ b/audb/core/load.py @@ -1271,8 +1271,10 @@ def load_attachment( ) attachment_files = db.attachments[attachment].files - attachment_files = [audeer.path(db_root, a) for a in attachment_files] - + attachment_files = [ + os.path.join(db_root, os.path.normpath(file)) # convert "/" to os.sep + for file in attachment_files + ] return attachment_files @@ -1509,7 +1511,10 @@ def load_media( if format is not None: media = [audeer.replace_file_extension(m, format) for m in media] - files = [os.path.join(db_root, os.path.normpath(m)) for m in media] + files = [ + os.path.join(db_root, os.path.normpath(file)) # convert "/" to os.sep + for file in media + ] except filelock.Timeout: utils.timeout_warning() diff --git a/audb/core/load_to.py b/audb/core/load_to.py index ffa49bf0..3ed14eb5 100644 --- a/audb/core/load_to.py +++ b/audb/core/load_to.py @@ -297,7 +297,7 @@ def load_to( if version is None: version = latest_version(name) - db_root = audeer.path(root) + db_root = audeer.path(root, follow_symlink=True) db_root_tmp = database_tmp_root(db_root) # remove files with a wrong checksum diff --git a/audb/core/utils.py b/audb/core/utils.py index f5317ea2..a66cd29d 100644 --- a/audb/core/utils.py +++ b/audb/core/utils.py @@ -58,7 +58,7 @@ def mkdir_tree( for file in files: folders.add(os.path.dirname(file)) for folder in folders: - audeer.mkdir(os.path.join(root, folder)) + audeer.mkdir(root, folder) def _lookup( diff --git a/pyproject.toml b/pyproject.toml index 4f26d936..1c2144da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,8 +34,8 @@ classifiers = [ requires-python = '>=3.8' dependencies = [ 'audbackend[artifactory] >=1.0.0', - 'audeer >=1.20.0', - 'audformat >=0.16.1', + 'audeer >=2.0.0', + 'audformat >=1.1.1', 'audiofile >=1.0.0', 'audobject >=0.5.0', 'audresample >=0.1.6', diff --git a/tests/conftest.py b/tests/conftest.py index db74dc3f..d8e5a19f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -130,7 +130,7 @@ def repository(tmpdir_factory): host=host, backend="file-system", ) - audeer.mkdir(audeer.path(host, name)) + audeer.mkdir(host, name) current_repositories = audb.config.REPOSITORIES audb.config.REPOSITORIES = [repository] @@ -166,7 +166,7 @@ def persistent_repository(tmpdir_factory): host=host, backend="file-system", ) - audeer.mkdir(audeer.path(host, name)) + audeer.mkdir(host, name) current_repositories = audb.config.REPOSITORIES audb.config.REPOSITORIES = [repository] diff --git a/tests/test_info.py b/tests/test_info.py index 24632bc1..af7059d9 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -76,7 +76,7 @@ def db(tmpdir_factory, persistent_repository): db_root = tmpdir_factory.mktemp(DB_VERSION) sampling_rate = 8000 - audeer.touch(audeer.path(db_root, db.attachments["attachment"].path)) + audeer.touch(db_root, db.attachments["attachment"].path) for table in list(db.tables): for file in db[table].files: audiofile.write( diff --git a/tests/test_load.py b/tests/test_load.py index 650d9f4a..4c64d5cc 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -91,11 +91,11 @@ def dbs(tmpdir_factory, persistent_repository): db_root = tmpdir_factory.mktemp(version) paths[version] = str(db_root) - audeer.mkdir(audeer.path(db_root, "extra/folder/sub-folder")) - audeer.touch(audeer.path(db_root, "extra/file.txt")) - audeer.touch(audeer.path(db_root, "extra/folder/file1.txt")) - audeer.touch(audeer.path(db_root, "extra/folder/file2.txt")) - audeer.touch(audeer.path(db_root, "extra/folder/sub-folder/file3.txt")) + audeer.mkdir(db_root, "extra/folder/sub-folder") + audeer.touch(db_root, "extra/file.txt") + audeer.touch(db_root, "extra/folder/file1.txt") + audeer.touch(db_root, "extra/folder/file2.txt") + audeer.touch(db_root, "extra/folder/sub-folder/file3.txt") db.save(db_root) audformat.testing.create_audio_files(db) archives = db["files"]["speaker"].get().dropna().to_dict() @@ -741,7 +741,7 @@ def test_load_to_update(tmpdir, dbs, only_metadata): db.attachments[attachment_id].path, ) if os.path.isdir(attachment): - audeer.touch(audeer.path(attachment, "other-file.txt")) + audeer.touch(attachment, "other-file.txt") else: with open(attachment, "a") as fp: fp.write("next") diff --git a/tests/test_load_on_demand.py b/tests/test_load_on_demand.py index 0a34fa9e..830bc777 100644 --- a/tests/test_load_on_demand.py +++ b/tests/test_load_on_demand.py @@ -60,10 +60,10 @@ def dbs(tmpdir_factory, persistent_repository): ) db.attachments["file"] = audformat.Attachment("file.txt") db.attachments["folder"] = audformat.Attachment("folder/") - audeer.mkdir(audeer.path(db_root, "folder")) - audeer.touch(audeer.path(db_root, "file.txt")) - audeer.touch(audeer.path(db_root, "folder/file1.txt")) - audeer.touch(audeer.path(db_root, "folder/file2.txt")) + audeer.mkdir(db_root, "folder") + audeer.touch(db_root, "file.txt") + audeer.touch(db_root, "folder/file1.txt") + audeer.touch(db_root, "folder/file2.txt") db.save(db_root) audformat.testing.create_audio_files(db) diff --git a/tests/test_lock.py b/tests/test_lock.py index ddff42ea..27569742 100644 --- a/tests/test_lock.py +++ b/tests/test_lock.py @@ -27,7 +27,7 @@ def job(lock, wait, sleep): def test_lock(tmpdir): # create two lock folders - lock_folders = [audeer.mkdir(audeer.path(tmpdir, str(idx))) for idx in range(2)] + lock_folders = [audeer.mkdir(tmpdir, str(idx)) for idx in range(2)] # lock 1 and 2 diff --git a/tests/test_lock_db.py b/tests/test_lock_db.py index c6a63f30..086da868 100644 --- a/tests/test_lock_db.py +++ b/tests/test_lock_db.py @@ -132,7 +132,7 @@ def dbs(tmpdir_factory, persistent_repository): ) db.attachments["file"] = audformat.Attachment("extra/file.txt") db.attachments["folder"] = audformat.Attachment("extra/folder") - audeer.mkdir(audeer.path(db_root, "extra/folder/sub-folder")) + audeer.mkdir(db_root, "extra/folder/sub-folder") for file in [ "extra/file.txt", "extra/folder/file1.txt", diff --git a/tests/test_publish.py b/tests/test_publish.py index 97aad797..292b6d98 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -102,12 +102,12 @@ def dbs(tmpdir_factory): [0, 1, 1, 2], index=audformat.filewise_index(db.files[:4]), ) - audeer.mkdir(audeer.path(db_root, "extra/folder")) - audeer.touch(audeer.path(db_root, "extra/file.txt")) - audeer.touch(audeer.path(db_root, "extra/folder/file1.txt")) - audeer.touch(audeer.path(db_root, "extra/folder/file2.txt")) - audeer.mkdir(audeer.path(db_root, "extra/folder/sub-folder")) - audeer.touch(audeer.path(db_root, "extra/folder/sub-folder/file3.txt")) + audeer.mkdir(db_root, "extra/folder") + audeer.touch(db_root, "extra/file.txt") + audeer.touch(db_root, "extra/folder/file1.txt") + audeer.touch(db_root, "extra/folder/file2.txt") + audeer.mkdir(db_root, "extra/folder/sub-folder") + audeer.touch(db_root, "extra/folder/sub-folder/file3.txt") # Create one file with different content to force different checksum file_with_different_content = audeer.path( db_root, @@ -157,7 +157,7 @@ def dbs(tmpdir_factory): audeer.path(db_root, "extra"), ) os.remove(audeer.path(db_root, "extra/folder/file2.txt")) - audeer.touch(audeer.path(db_root, "extra/folder/file3.txt")) + audeer.touch(db_root, "extra/folder/file3.txt") db["files"].extend_index(audformat.filewise_index(LONG_PATH), inplace=True) db.save(db_root) audformat.testing.create_audio_files(db) @@ -496,7 +496,7 @@ def test_publish_attachment(tmpdir, repository): audb.publish(db_path, "1.0.0", repository) # Publish database, path is not allowed to be a symlink - audeer.mkdir(audeer.path(db_path, folder_path)) + audeer.mkdir(db_path, folder_path) os.symlink( audeer.path(db_path, folder_path), audeer.path(db_path, file_path), @@ -510,7 +510,7 @@ def test_publish_attachment(tmpdir, repository): audb.publish(db_path, "1.0.0", repository) os.remove(os.path.join(db_path, file_path)) - audeer.touch(audeer.path(db_path, file_path)) + audeer.touch(db_path, file_path) db.save(db_path) # File exist now, folder is empty @@ -527,7 +527,7 @@ def test_publish_attachment(tmpdir, repository): # Add empty sub-folder subfolder_path = f"{folder_path}/sub-folder" - audeer.mkdir(audeer.path(db_path, subfolder_path)) + audeer.mkdir(db_path, subfolder_path) assert db.attachments["folder"].files == [] error_msg = ( "An attachment must not " @@ -540,7 +540,7 @@ def test_publish_attachment(tmpdir, repository): # Add file to folder, sub-folder still empty file2_path = f"{folder_path}/file.txt" - audeer.touch(audeer.path(db_path, file2_path)) + audeer.touch(db_path, file2_path) assert db.attachments["file"].files == [file_path] assert db.attachments["folder"].files == [file2_path] with pytest.raises(RuntimeError, match=error_msg): @@ -548,7 +548,7 @@ def test_publish_attachment(tmpdir, repository): # Add file to sub-folder file3_path = f"{subfolder_path}/file.txt" - audeer.touch(audeer.path(db_path, file3_path)) + audeer.touch(db_path, file3_path) assert db.attachments["file"].files == [file_path] assert db.attachments["folder"].files == [file2_path, file3_path] @@ -774,7 +774,7 @@ def test_publish_error_allowed_chars(tmpdir, repository): # on the backends # Prepare database files - db_path = audeer.mkdir(audeer.path(tmpdir, "db")) + db_path = audeer.mkdir(tmpdir, "db") audio_file = audeer.path(db_path, "f1.wav") attachment_file = audeer.path(db_path, "attachment.txt") signal = np.zeros((2, 1000)) @@ -837,8 +837,8 @@ def test_publish_error_changed_deps_file_type(tmpdir, repository): "But attachment 'attachment' contains 'data/file.wav'." ) db_name = "test_publish_error_changed_deps_file_type-1" - db_path = audeer.mkdir(audeer.path(tmpdir, "db")) - data_path = audeer.mkdir(audeer.path(db_path, "data")) + db_path = audeer.mkdir(tmpdir, "db") + data_path = audeer.mkdir(db_path, "data") signal = np.zeros((2, 1000)) sampling_rate = 8000 audiofile.write(audeer.path(data_path, "file.wav"), signal, sampling_rate) @@ -856,8 +856,8 @@ def test_publish_error_changed_deps_file_type(tmpdir, repository): "But attachment 'attachment' contains 'db.table.csv'." ) db_name = "test_publish_error_changed_deps_file_type-2" - db_path = audeer.mkdir(audeer.path(tmpdir, "db")) - data_path = audeer.mkdir(audeer.path(db_path, "data")) + db_path = audeer.mkdir(tmpdir, "db") + data_path = audeer.mkdir(db_path, "data") signal = np.zeros((2, 1000)) sampling_rate = 8000 audiofile.write(audeer.path(data_path, "file.wav"), signal, sampling_rate) @@ -875,8 +875,8 @@ def test_publish_error_changed_deps_file_type(tmpdir, repository): "But attachment 'attachment' contains 'data/file2.wav'." ) db_name = "test_publish_error_changed_deps_file_type-3" - db_path = audeer.mkdir(audeer.path(tmpdir, "db")) - data_path = audeer.mkdir(audeer.path(db_path, "data")) + db_path = audeer.mkdir(tmpdir, "db") + data_path = audeer.mkdir(db_path, "data") signal = np.zeros((2, 1000)) sampling_rate = 8000 audiofile.write(audeer.path(data_path, "file1.wav"), signal, sampling_rate) @@ -902,15 +902,15 @@ def test_publish_error_changed_deps_file_type(tmpdir, repository): "But attachment 'attachment' contains 'db.table2.csv'." ) db_name = "test_publish_error_changed_deps_file_type-4" - db_path = audeer.mkdir(audeer.path(tmpdir, "db")) - data_path = audeer.mkdir(audeer.path(db_path, "data")) + db_path = audeer.mkdir(tmpdir, "db") + data_path = audeer.mkdir(db_path, "data") signal = np.zeros((2, 1000)) sampling_rate = 8000 audiofile.write(audeer.path(data_path, "file.wav"), signal, sampling_rate) db = audformat.Database(db_name) db["table1"] = audformat.Table(audformat.filewise_index("data/file.wav")) db.attachments["attachment"] = audformat.Attachment("db.table2.csv") - audeer.touch(audeer.path(db_path, "db.table2.csv")) + audeer.touch(db_path, "db.table2.csv") db.save(db_path) audb.publish(db_path, "1.0.0", repository) audeer.rmdir(db_path) @@ -942,8 +942,8 @@ def test_publish_error_repository_does_not_exist(tmpdir, repository): ) def test_publish_error_uppercase_file_extension(tmpdir, repository, file): # Prepare files - db_path = audeer.mkdir(audeer.path(tmpdir, "db")) - audeer.touch(audeer.path(db_path, file)) + db_path = audeer.mkdir(tmpdir, "db") + audeer.touch(db_path, file) # Prepare database db = audformat.Database("db") db["table"] = audformat.Table(audformat.filewise_index([file])) @@ -964,7 +964,7 @@ def test_publish_error_version(tmpdir, repository): # are allowed # Create simple database - db_path = audeer.mkdir(audeer.path(tmpdir, "db")) + db_path = audeer.mkdir(tmpdir, "db") audio_file = audeer.path(db_path, "f1.wav") signal = np.zeros((2, 1000)) sampling_rate = 8000 @@ -1225,8 +1225,8 @@ def test_update_database_without_media(tmpdir, persistent_repository): assert not os.path.exists(path) # add new attachment - audeer.mkdir(audeer.path(build_root, "extra")) - audeer.touch(audeer.path(build_root, "extra/new.txt")) + audeer.mkdir(build_root, "extra") + audeer.touch(build_root, "extra/new.txt") db.attachments[new_attachment] = audformat.Attachment("extra/new.txt") db.save(build_root)