From f1c86268f5a2f275120fac88de6613e35a285b18 Mon Sep 17 00:00:00 2001 From: David Nicholson Date: Sat, 11 May 2024 08:51:04 -0400 Subject: [PATCH] CLN/ENH: Rename and refactor datapipes, add datasets; fix #574 #724 #754 (#755) * Rename vak/datasets -> vak/datapipes * Rename frame_classifcation.window_dataset.WindowDataset -> TrainDatapipe * Rename frame_classification/window_dataset.py -> train_datapipe.py * Fix WindowDataset -> TrainDatapipe in docstrings * Rename frame_classification.frames_dataset.FramesDataset -> infer_datapipe.InferDatapipe * Rename transforms.StandardizeSpect -> FramesStandarizer * Import FramesStandarizer in datapipes/frame_classification/infer_datapipe.py * Add module-level docstring in vak/datapipes/__init__.py * Rewrite transforms.defaults.frames_classification.EvalItemTransform and PredictItemTransform as a single class, InferItemTransform, and remname spect_standardizer -> frames_standardizer in that module * Fix bug in view_as_window_batch so it works on 1-D arrays, add type hinting in src/vak/transforms/functional.py * Change frame_labels_transform in InferItemTransform to be a torchvision.transforms.Compose, so we get back a windowed batch * Remove TODO in src/vak/models/frame_classification_model.py * Rewrite TrainDatapipe to always use TrainItemTransform, add parameters that get passed to TrainItemTransform when instatiating it inside TrainDatapipe.__init__ * Rewrite frames_classification.InferDatapipe to always use transforms.default.frame_classification.InferItemTransform, add parameters that get passed to InferItemTransform when instatiating it inside InferDatapipe.__init__ * Rewrite train.frame_classification to pass kwargs into datapipes that now use default transforms, and no longer call transforms.defaults.get * Rewrite predict.frame_classification to pass kwargs into datapipes that now use default transforms, and no longer call transforms.defaults.get * Rewrite eval.frame_classification to pass kwargs into datapipes that now use default transforms, and no longer call transforms.defaults.get * Rewrite predict.frame_classification to pass kwargs into datapipes that now use default transforms, and no longer call transforms.defaults.get * Rename 'spect_scaler_path' -> 'frames_standardizer_path' * Rename 'normalize_spectrogram' -> 'standardize_frames' * Fix 'SpectScaler' -> 'FramesStandardizer', 'normalize spectrogram' -> 'standardize (normalize) frames' * Fix 'SpectScaler' -> 'FramesStandardizer' in tests/ * Fix key names in doc/toml * Add missing comma in src/vak/train/frame_classification.py * Rename config/valid-version-1.1.toml -> valid-version-1.2.toml * Fix normalize spectrograms -> standardize frames more places in docs * Fix datapipes.frame_classification.InferDatapipe to have needed parameters for item transform * Fix datapipes.frame_classification.TrainDatapipe to have needed parameters for item transform * Fix arg name 'spect_standardizer -> frames_standardizer in src/vak/train/frame_classification.py * fixup fix TrainDatapipe parameters * Fix variable name in src/vak/datapipes/frame_classification/train_datapipe.py * Add missing arg return_padding_mask in src/vak/train/frame_classification.py * Fix transforms.default.frame_classification.InferItemTransform to not window frame labels, just convert them to LongTensor * Revise docstring in eval/frame_classification * Remove item_transform from docstring in datapipes/frame_classification/train_datapipe.py * Add return_padding_mask arg in vak/predict/frame_classification.py * Remove src/vak/transforms/defaults/parametric_umap.py * Rename/rewrite Datapipe class for ParametricUMAP, hard-code in transform * Remove transforms/defaults/get.py, remove related imports in transforms/defaults/__init__.py * Finish removing transform fetching for ParametricUMAP * Fix typo in src/vak/eval/frame_classification.py * Fix "StandardizeSpect" -> "FramesStandardizer" in src/vak/learncurve/frame_classification.py * Apply changes from nox lint session * Make flake8 fixes, remove unused function get_default_frame_classification_transform * Fix "StandardizeSpect" -> "FramesStandardizer" in tests/scripts/vaktestdata/configs.py" * WIP: Add datasets/ with biosoundsegbench * Renam tests/test_datasets -> test_datapipes, fix tests * Fix 'StandardizeSpect' -> 'FramesStandardizer' in two tests * Remove two uses of vak.transforms.defaults.get_default_transform from tests * Fix datapipe used in tests/test_models/test_parametric_umap_model.py * Use TYPE_CHECKING to avoid circular import in src/vak/datapipes/frame_classification/infer_datapipe.py * Add method 'fit_inputs_targets_csv_path' to FramesStandardizer, rewrite 'fit_dataset_path' method to just call this new method * fixup add method * Add unit test for FramesStandardizer.fit_inputs_targets_csv_path * Remove unused import from src/vak/transforms/transforms.py * Remove unused import in src/vak/transforms/defaults/frame_classification.py * Pep8 fix in src/vak/datasets/__init__.py * Apply linting to src/vak/transforms/transforms.py * Correct docstring in src/vak/transforms/defaults/frame_classification.py * Import datasets in src/vak/__init__.py * Rename datapipes/frame_classification/constants.FRAME_LABELS_EXT -> MULTI_FRAME_LABELS_EXT, and change value to 'multi-frame-labels.npy', and change value of FRAME_LABELS_NPY_PATH_COL_NAME to 'multi_frame_labels_npy_path' * Rename vak.datapipes.frame_classification.constants.FRAME_LABELS_NPY_PATH_COL_NAME -> MULTI_FRAME_LABELS_PATH_COL_NAME * Rename key in item returned by frame_classification.TrainItemTransform and InferItemTransform; 'frame_labels' -> 'multi_frame_labels' * WIP: Get BioSoundSegBench class working * Rewrite FrameClassificationModel to handle different target types * Add VALID_SPLITS to common.constants * In datasets/biosoundsegbench.py: change VALID_TARGET_TYPES to be the ones we're using for experiments right now, fix TrainItemTransform to handle target types, clean up __init__ method validation * Add initial unit tests for BioSoundSegBench dataset * Add helper function vak.datasets.get * Clean up how we validate target_type in datasets.BioSoundSegBench.__init__ * Add tests/test_datasets/__init__.py (to make a sub-package) * Add initial unit tests for vak.datasets.get * Modify BioSoundSegBench.__init__ so we can write splits_path as just the filename * Use expanded_user_path converter on path and splits_path attributes of DatasetConfig * Rename BOUNDARY_ONEHOT_PATH_COL_NAME -> BOUNDARY_FRAME_LABELS_PATH_COL_NAME in datasets/biosoundsegbench.py * Modify datasets.BioSoundSegBench to compute metadata from splits_json path * Fix mock_biosoundsegbench_dataset fixture so mocked files follow naming conventions of dataset * Modify mock_biosoundsegbench_dataset fixture to save labelmaps.json * Change BioSoundSegBench.__init__ so we have training_replicate_metadata attribute, frame_dur attribute, and labelmap attribute * Add DATASETS dict in dataset/__init__.py, used by vak.datasets.get to look up class (value) by name (key) * Use vak.datasets.DATASETS in vak.datasets.get to get class * Rewrite BioSoundSegBench.__init__ so we can either pass in a FramesStandardizer instance or tell it to fit a new one to the specified split, that then gets added to the transform * Import DATASETS inside vak.datasets.get to avoid circular import * Make fixes in datasets/biosoundsegbench.py: import FramesStandardizer inside TrainItemTransform.__init__, fix tmp_splits_path -> splits-jsons (plural), add needed __len__ method to class * Rename BioSoundSegBench property 'input_shape' -> 'shape' for consistency with frame_classification datapipes * Get vak/train/frame_classification.py to the point where it runs * Add missing self in BioSoundSegBench._getitemval * Rewrite src/vak/eval/frame_classification.py to work with built-in datasets, and remove 'split' parameter from eval_frame_classification_model function -- check if 'split' is in dataset_config and if not, default to 'test' * Remove split argument in call to eval_frame_classification_model inside src/vak/learncurve/frame_classification.py * Remove split parameter from eval._eval.eval -- it's not an attribute of EvalConfig and we can now pass in a 'split' through dataset_config * Remove 'split' parameter from eval_parametric_umap_model, check if 'split' in dataset_config and if not default to 'test' * Rewrite src/vak/predict/frame_classification.py to work with built-in datasets; check if 'split' is in dataset_config and if not, default to 'predict' * Add comments to structure src/vak/train/frame_classification.py * Fix how we check for key in src/vak/predict/frame_classification.py * Fix how we check for key in dict in src/vak/eval/parametric_umap.py * Fix how we check for key in dict in src/vak/eval/frame_classification.py * Fix unit tests in test_dataset.py: assert that path attributes are vak.converters.expanded_user_path(value from config), not pathlib.Path * Fix how we parametrize tests/test_dataset/test_get.py * In BioSoundSegBench.__init__, fix how we calculate frame_dur and how we set labelmap attribute for binary/boundary frame labels * In FrameClassificationModel.validation_step, convert Levenshtein distance to float to squelch warning from Lightning * Fix FrameClassificationModel so train/val with multi-class + boundary labels works * Fix vak.cli.predict to not assume that config has a prep attribute * Fix how we override default split with a split from dataset_config['params'] in predict/frame_classification and eval/frame_classification * Change BioSoundSegBench so __getitem__ can return 'frames_path' in 'item' for eval/predict * In predict.frame_classification, set 'return_frames_path' to True in dataset_config['params'] since we need this for predictions * Add constant DEFAULT_SPECT_FORMAT in common.constants * Fix SPECT_KEY -> TIMEBINS_KEY in cli.prep * Fix how we determine input_type and spect_format for built-in datasets in predict/frame_classification * Add nn/loss/crossentropy.py, wraps torch.nn.CrossEntropy, but converts weight arg as list to tensor * Fixup add loss * Use nn.loss.CrossEntropy with TweetyNet model * Clean up prediction_step in FrameClassificationModel * Get predict working for multi_frame_labels and boundary_frame_labels, still need to test binary_frame_labels and (boundary, multi) * Rename 'unlabeled_label' -> 'background_label' in transforms/frame_labels * Rename 'unlabeled_label' -> 'background_label' in tests/test_transforms/test_frame_labels * Rewrite transforms/frame_labels/functional.py to handle boundary labels - Add `boundary_labels_to_segment_inds_list' that finds segment indexing arrays from a list of boundary labels - Rename `to_segment_inds` -> `frame_labels_to_segment_inds_list - Have `preprocess` optionally take `boundary_labels` and use it to find segments, instead of frame labels - Fix type annotations to use npt.NDArray instead of np.ndarray * Change how FrameClassificationModel calls loss for multi-class + boundary targets -- assume we pass to an instance of a loss function, and get back either a scalar loss or a dict mapping loss names to scalar values * Change arg name 'unlabeled_label' -> 'background_label' in prep/frame_classification/make_splits.py * Fix predict.frame_classification for multi-class, and add logic for multi-class frame labels with boundary frame labels * Add DEFAULT_BACKGROUND_LABEL to common.constants * Use DEFAULT_BACKGROUND_LABEL in transforms.frame_labels.functional * Rename unlabeled -> background_label in common.labels * Add background_label in docstring in common/labels.py * Add 'background_label' to FrameClassificationModel, defaults to common.constants.DEFAULT_BACKGROUND_LABEL, used to validate length of string labels in labelmap * Fix 'unlabeled' -> common.constants.DEFAULT_BACKGROUND_LABEL in anohter place in common/labels.py * Fix unlabeled -> background label in docstrings in transforms * Use 'background_label' argument in place of magic string 'unlabeled' in prep/frame_classification/learncurve.py * Fix unlabeled -> background label in docstrings in transforms/frame_labels/functional.py * Add background_label to docstring in src/vak/prep/frame_classification/learncurve.py * Add background_label to function in src/vak/prep/frame_classification/make_splits.py * Add background_label parameter to src/vak/predict/frame_classification.py and add type annotations to function signature * Fix unlabeled -> background / vak.common.constants.DEFAULT_BACKGROUND_LABEL in tests * Fix 'map_unlabeled' -> 'map_background' in tests/ * Fix 'constants' -> 'common' in src/vak/models/frame_classification_model.py * Fix arg name map_unlabeled -> map_background * Fix arg name map_unlabeled -> map_background in prep/parametric_umap * Fix 'unlabeled' -> vak.common.constants.DEFAULT_BACKGROUND_LABEL in tests/ * Fix name `to_inds_list` -> segment_inds_list_from_class_labels` in test_transforms/test_frame_labels/test_functional.py --- doc/api/index.rst | 14 + doc/toml/gy6or6_eval.toml | 4 +- doc/toml/gy6or6_predict.toml | 4 +- doc/toml/gy6or6_train.toml | 4 +- src/vak/__init__.py | 2 + src/vak/cli/eval.py | 2 +- src/vak/cli/learncurve.py | 2 +- src/vak/cli/predict.py | 8 +- src/vak/cli/train.py | 4 +- src/vak/common/__init__.py | 2 +- src/vak/common/constants.py | 7 +- src/vak/common/labels.py | 43 +- src/vak/config/__init__.py | 1 - src/vak/config/dataset.py | 6 +- src/vak/config/eval.py | 6 +- src/vak/config/learncurve.py | 17 +- src/vak/config/predict.py | 7 +- src/vak/config/train.py | 15 +- src/vak/config/trainer.py | 27 +- ...ersion-1.1.toml => valid-version-1.2.toml} | 10 +- src/vak/config/validators.py | 2 +- src/vak/datapipes/__init__.py | 7 + .../frame_classification/__init__.py | 6 + .../frame_classification/constants.py | 4 +- .../frame_classification/helper.py | 0 .../frame_classification/infer_datapipe.py} | 107 ++- .../frame_classification/metadata.py | 0 .../frame_classification/train_datapipe.py} | 78 ++- src/vak/datapipes/parametric_umap/__init__.py | 8 + .../parametric_umap/metadata.py | 0 .../parametric_umap/parametric_umap.py | 118 +--- src/vak/datasets/__init__.py | 16 +- src/vak/datasets/biosoundsegbench.py | 646 ++++++++++++++++++ .../datasets/frame_classification/__init__.py | 6 - src/vak/datasets/get.py | 66 ++ src/vak/datasets/parametric_umap/__init__.py | 4 - src/vak/eval/eval_.py | 21 +- src/vak/eval/frame_classification.py | 135 ++-- src/vak/eval/parametric_umap.py | 27 +- src/vak/learncurve/frame_classification.py | 27 +- src/vak/learncurve/learncurve.py | 10 +- src/vak/models/__init__.py | 2 +- src/vak/models/decorator.py | 9 +- src/vak/models/definition.py | 2 +- src/vak/models/factory.py | 45 +- src/vak/models/frame_classification_model.py | 268 ++++++-- src/vak/models/parametric_umap_model.py | 22 +- src/vak/models/registry.py | 3 +- src/vak/models/tweetynet.py | 6 +- src/vak/nn/loss/__init__.py | 1 + src/vak/nn/loss/crossentropy.py | 17 + src/vak/predict/frame_classification.py | 365 +++++++--- src/vak/predict/parametric_umap.py | 35 +- src/vak/predict/predict_.py | 12 +- .../frame_classification.py | 6 +- .../prep/frame_classification/learncurve.py | 22 +- .../prep/frame_classification/make_splits.py | 24 +- .../prep/parametric_umap/parametric_umap.py | 6 +- src/vak/prep/spectrogram_dataset/prep.py | 4 +- src/vak/train/frame_classification.py | 251 ++++--- src/vak/train/parametric_umap.py | 23 +- src/vak/train/train_.py | 28 +- src/vak/transforms/defaults/__init__.py | 5 +- .../defaults/frame_classification.py | 241 ++----- src/vak/transforms/defaults/get.py | 48 -- .../transforms/defaults/parametric_umap.py | 31 - src/vak/transforms/frame_labels/functional.py | 208 ++++-- src/vak/transforms/frame_labels/transforms.py | 21 +- src/vak/transforms/functional.py | 48 +- src/vak/transforms/transforms.py | 100 ++- ...weetyNet_eval_audio_cbin_annot_notmat.toml | 2 +- ...et_learncurve_audio_cbin_annot_notmat.toml | 2 +- ...tyNet_predict_audio_cbin_annot_notmat.toml | 2 +- ...eetyNet_train_audio_cbin_annot_notmat.toml | 2 +- ...rain_continue_audio_cbin_annot_notmat.toml | 4 +- ...train_continue_spect_mat_annot_yarden.toml | 2 +- ...weetyNet_train_spect_mat_annot_yarden.toml | 2 +- .../configs/invalid_key_config.toml | 2 +- .../configs/invalid_table_config.toml | 2 +- .../invalid_train_and_learncurve_config.toml | 4 +- tests/fixtures/csv.py | 4 +- tests/scripts/vaktestdata/configs.py | 18 +- tests/test_common/test_labels.py | 23 +- tests/test_config/test_dataset.py | 10 +- tests/test_config/test_eval.py | 12 +- tests/test_config/test_learncurve.py | 10 +- tests/test_config/test_predict.py | 10 +- tests/test_config/test_train.py | 8 +- .../__init__.py | 0 .../test_frame_classification}/__init__.py | 0 .../test_frame_classification/test_helper.py | 12 +- .../test_infer_datapipe.py} | 17 +- .../test_metadata.py | 18 +- .../test_train_datapipe.py} | 13 +- .../test_parametric_umap}/__init__.py | 0 .../test_parametric_umap.py | 16 +- tests/test_datapipes/test_seq/__init__.py | 0 tests/test_datasets/conftest.py | 373 ++++++++++ tests/test_datasets/test_biosoundsegbench.py | 129 ++++ tests/test_datasets/test_get.py | 86 +++ tests/test_eval/test_eval.py | 2 +- tests/test_eval/test_frame_classification.py | 10 +- .../test_frame_classification.py | 8 +- tests/test_models/test_factory.py | 2 +- .../test_frame_classification_model.py | 10 +- .../test_models/test_parametric_umap_model.py | 8 +- tests/test_models/test_tweetynet.py | 2 +- .../test_predict/test_frame_classification.py | 10 +- tests/test_predict/test_predict.py | 2 +- .../test_frame_classification.py | 10 +- .../test_learncurve.py | 20 +- .../test_make_splits.py | 18 +- tests/test_prep/test_split/test_split.py | 9 +- tests/test_train/test_frame_classification.py | 26 +- tests/test_train/test_train.py | 4 +- .../test_frame_labels/test_functional.py | 53 +- .../test_frame_labels/test_transforms.py | 37 +- tests/test_transforms/test_transforms.py | 83 ++- 118 files changed, 3060 insertions(+), 1363 deletions(-) rename src/vak/config/{valid-version-1.1.toml => valid-version-1.2.toml} (92%) create mode 100644 src/vak/datapipes/__init__.py create mode 100644 src/vak/datapipes/frame_classification/__init__.py rename src/vak/{datasets => datapipes}/frame_classification/constants.py (70%) rename src/vak/{datasets => datapipes}/frame_classification/helper.py (100%) rename src/vak/{datasets/frame_classification/frames_dataset.py => datapipes/frame_classification/infer_datapipe.py} (65%) rename src/vak/{datasets => datapipes}/frame_classification/metadata.py (100%) rename src/vak/{datasets/frame_classification/window_dataset.py => datapipes/frame_classification/train_datapipe.py} (85%) create mode 100644 src/vak/datapipes/parametric_umap/__init__.py rename src/vak/{datasets => datapipes}/parametric_umap/metadata.py (100%) rename src/vak/{datasets => datapipes}/parametric_umap/parametric_umap.py (79%) create mode 100644 src/vak/datasets/biosoundsegbench.py delete mode 100644 src/vak/datasets/frame_classification/__init__.py create mode 100644 src/vak/datasets/get.py delete mode 100644 src/vak/datasets/parametric_umap/__init__.py create mode 100644 src/vak/nn/loss/crossentropy.py delete mode 100644 src/vak/transforms/defaults/get.py delete mode 100644 src/vak/transforms/defaults/parametric_umap.py rename tests/{test_datasets/test_frame_classification => test_datapipes}/__init__.py (100%) rename tests/{test_datasets/test_parametric_umap => test_datapipes/test_frame_classification}/__init__.py (100%) rename tests/{test_datasets => test_datapipes}/test_frame_classification/test_helper.py (59%) rename tests/{test_datasets/test_frame_classification/test_frames_dataset.py => test_datapipes/test_frame_classification/test_infer_datapipe.py} (66%) rename tests/{test_datasets => test_datapipes}/test_frame_classification/test_metadata.py (74%) rename tests/{test_datasets/test_frame_classification/test_window_dataset.py => test_datapipes/test_frame_classification/test_train_datapipe.py} (74%) rename tests/{test_datasets/test_seq => test_datapipes/test_parametric_umap}/__init__.py (100%) rename tests/{test_datasets => test_datapipes}/test_parametric_umap/test_parametric_umap.py (65%) create mode 100644 tests/test_datapipes/test_seq/__init__.py create mode 100644 tests/test_datasets/conftest.py create mode 100644 tests/test_datasets/test_biosoundsegbench.py create mode 100644 tests/test_datasets/test_get.py diff --git a/doc/api/index.rst b/doc/api/index.rst index 4b3d2c4d2..0951a5a84 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -154,6 +154,20 @@ The :mod:`vak.datasets` module contains datasets built into vak. datasets.frame_classification datasets.parametric_umap +Datapipes +--------- + +The :mod:`vak.datapipes` module contains datapipes for loading dataset +generated by :func:`vak.prep.prep`. + +.. autosummary:: + :toctree: generated + :template: module.rst + :recursive: + + datapipes.frame_classification + datapipes.parametric_umap + Metrics ------- The :mod:`vak.metrics` module contains metrics used diff --git a/doc/toml/gy6or6_eval.toml b/doc/toml/gy6or6_eval.toml index 41d25a1c7..434576d71 100644 --- a/doc/toml/gy6or6_eval.toml +++ b/doc/toml/gy6or6_eval.toml @@ -33,9 +33,9 @@ checkpoint_path = "/PATH/TO/FOLDER/results/train/RESULTS_TIMESTAMP/TweetyNet/che # labelmap_path: path to file that maps from outputs of model (integers) to text labels in annotations; # this is used when generating predictions labelmap_path = "/PATH/TO/FOLDER/results/train/RESULTS_TIMESTAMP/labelmap.json" -# spect_scaler_path: path to file containing SpectScaler that was fit to training set +# frames_standardizer_path: path to file containing SpectScaler that was fit to training set # We want to transform the data we predict on in the exact same way -spect_scaler_path = "/PATH/TO/FOLDER/results/train/RESULTS_TIMESTAMP/StandardizeSpect" +frames_standardizer_path = "/PATH/TO/FOLDER/results/train/RESULTS_TIMESTAMP/StandardizeSpect" # batch_size # for predictions with a frame classification model, this should always be 1 # and will be ignored if it's not diff --git a/doc/toml/gy6or6_predict.toml b/doc/toml/gy6or6_predict.toml index 4bbfa1dd2..5061488be 100644 --- a/doc/toml/gy6or6_predict.toml +++ b/doc/toml/gy6or6_predict.toml @@ -29,9 +29,9 @@ checkpoint_path = "/PATH/TO/FOLDER/results/train/RESULTS_TIMESTAMP/TweetyNet/che # labelmap_path: path to file that maps from outputs of model (integers) to text labels in annotations; # this is used when generating predictions labelmap_path = "/PATH/TO/FOLDER/results/train/RESULTS_TIMESTAMP/labelmap.json" -# spect_scaler_path: path to file containing SpectScaler that was fit to training set +# frames_standardizer_path: path to file containing SpectScaler that was fit to training set # We want to transform the data we predict on in the exact same way -spect_scaler_path = "/PATH/TO/FOLDER/results/train/RESULTS_TIMESTAMP/StandardizeSpect" +frames_standardizer_path = "/PATH/TO/FOLDER/results/train/RESULTS_TIMESTAMP/StandardizeSpect" # batch_size # for predictions with a frame classification model, this should always be 1 # and will be ignored if it's not diff --git a/doc/toml/gy6or6_train.toml b/doc/toml/gy6or6_train.toml index 6c802b2ec..3d3794f92 100644 --- a/doc/toml/gy6or6_train.toml +++ b/doc/toml/gy6or6_train.toml @@ -37,9 +37,9 @@ root_results_dir = "/PATH/TO/FOLDER/results/train" batch_size = 8 # num_epochs: number of training epochs, where an epoch is one iteration through all samples in training split num_epochs = 2 -# normalize_spectrograms: if true, normalize spectrograms per frequency bin, so mean of each is 0.0 and std is 1.0 +# standardize_frames: if true, standardize (normalize) frames (input to neural network) per frequency bin, so mean of each is 0.0 and std is 1.0 # across the entire training split -normalize_spectrograms = true +standardize_frames = true # val_step: step number on which to compute metrics with validation set, every time step % val_step == 0 # (a step is one batch fed through the network) # saves a checkpoint if the monitored evaluation metric improves (which is model specific) diff --git a/src/vak/__init__.py b/src/vak/__init__.py index f6bbb4eed..99c40a1ef 100644 --- a/src/vak/__init__.py +++ b/src/vak/__init__.py @@ -3,6 +3,7 @@ cli, common, config, + datapipes, datasets, eval, learncurve, @@ -42,6 +43,7 @@ "cli", "common", "config", + "datapipes", "datasets", "eval", "learncurve", diff --git a/src/vak/cli/eval.py b/src/vak/cli/eval.py index a80245464..503a2f33e 100644 --- a/src/vak/cli/eval.py +++ b/src/vak/cli/eval.py @@ -58,6 +58,6 @@ def eval(toml_path: str | pathlib.Path) -> None: output_dir=cfg.eval.output_dir, num_workers=cfg.eval.num_workers, batch_size=cfg.eval.batch_size, - spect_scaler_path=cfg.eval.spect_scaler_path, + frames_standardizer_path=cfg.eval.frames_standardizer_path, post_tfm_kwargs=cfg.eval.post_tfm_kwargs, ) diff --git a/src/vak/cli/learncurve.py b/src/vak/cli/learncurve.py index c8f407fe0..f57a1b97d 100644 --- a/src/vak/cli/learncurve.py +++ b/src/vak/cli/learncurve.py @@ -61,7 +61,7 @@ def learning_curve(toml_path): num_workers=cfg.learncurve.num_workers, results_path=results_path, post_tfm_kwargs=cfg.learncurve.post_tfm_kwargs, - normalize_spectrograms=cfg.learncurve.normalize_spectrograms, + standardize_frames=cfg.learncurve.standardize_frames, shuffle=cfg.learncurve.shuffle, val_step=cfg.learncurve.val_step, ckpt_step=cfg.learncurve.ckpt_step, diff --git a/src/vak/cli/predict.py b/src/vak/cli/predict.py index 474b2b8ca..c0079e360 100644 --- a/src/vak/cli/predict.py +++ b/src/vak/cli/predict.py @@ -1,7 +1,7 @@ import logging from pathlib import Path -from .. import config +from .. import common, config from .. import predict as predict_module from ..common.logging import config_logging_for_cli, log_version @@ -33,7 +33,7 @@ def predict(toml_path): force=True, ) log_version(logger) - logger.info("Logging results to {}".format(cfg.prep.output_dir)) + logger.info("Logging results to {}".format(cfg.predict.output_dir)) if cfg.predict.dataset.path is None: raise ValueError( @@ -49,8 +49,8 @@ def predict(toml_path): checkpoint_path=cfg.predict.checkpoint_path, labelmap_path=cfg.predict.labelmap_path, num_workers=cfg.predict.num_workers, - timebins_key=cfg.prep.spect_params.timebins_key, - spect_scaler_path=cfg.predict.spect_scaler_path, + timebins_key=cfg.prep.spect_params.timebins_key if cfg.prep else common.constants.TIMEBINS_KEY, + frames_standardizer_path=cfg.predict.frames_standardizer_path, annot_csv_filename=cfg.predict.annot_csv_filename, output_dir=cfg.predict.output_dir, min_segment_dur=cfg.predict.min_segment_dur, diff --git a/src/vak/cli/train.py b/src/vak/cli/train.py index 88c2fb6d9..fbdcaf916 100644 --- a/src/vak/cli/train.py +++ b/src/vak/cli/train.py @@ -60,9 +60,9 @@ def train(toml_path): num_epochs=cfg.train.num_epochs, num_workers=cfg.train.num_workers, checkpoint_path=cfg.train.checkpoint_path, - spect_scaler_path=cfg.train.spect_scaler_path, + frames_standardizer_path=cfg.train.frames_standardizer_path, results_path=results_path, - normalize_spectrograms=cfg.train.normalize_spectrograms, + standardize_frames=cfg.train.standardize_frames, shuffle=cfg.train.shuffle, val_step=cfg.train.val_step, ckpt_step=cfg.train.ckpt_step, diff --git a/src/vak/common/__init__.py b/src/vak/common/__init__.py index 11c4e330d..c5be9ccfd 100644 --- a/src/vak/common/__init__.py +++ b/src/vak/common/__init__.py @@ -5,7 +5,7 @@ If a helper/utility function is only used in one module, it should live either in that module or another at the same level. See for example :mod:`vak.prep.prep_helper` or -:mod:`vak.datsets.window_dataset._helper`. +:mod:`vak.datsets.train_datapipe._helper`. """ from . import ( diff --git a/src/vak/common/constants.py b/src/vak/common/constants.py index a3a34315f..044395d2b 100644 --- a/src/vak/common/constants.py +++ b/src/vak/common/constants.py @@ -1,4 +1,4 @@ -"""constants used by multiple modules. +"""Constants used by multiple modules. Defined here to avoid circular imports. """ @@ -26,6 +26,7 @@ "npz": np.load, } VALID_SPECT_FORMATS = list(SPECT_FORMAT_LOAD_FUNCTION_MAP.keys()) +DEFAULT_SPECT_FORMAT = "npz" # ---- valid types of training data, the $x$ that goes into a network VALID_X_SOURCES = {"audio", "spect"} @@ -57,3 +58,7 @@ "npz": SPECT_NPZ_EXTENSION, "mat": ".mat", } + +VALID_SPLITS = ("predict", "test", "train", "val") + +DEFAULT_BACKGROUND_LABEL = "background" \ No newline at end of file diff --git a/src/vak/common/labels.py b/src/vak/common/labels.py index 5f851cdec..e9c6d8296 100644 --- a/src/vak/common/labels.py +++ b/src/vak/common/labels.py @@ -5,10 +5,12 @@ import numpy as np import pandas as pd -from . import annotation +from . import annotation, constants -def to_map(labelset: set, map_unlabeled: bool = True) -> dict: +def to_map( + labelset: set, map_background: bool = True, background_label: str = constants.DEFAULT_BACKGROUND_LABEL +) -> dict: """Convert set of labels to `dict` mapping those labels to a series of consecutive integers from 0 to n inclusive, @@ -18,21 +20,31 @@ def to_map(labelset: set, map_unlabeled: bool = True) -> dict: from annotations of a vocalization into a label for every time bin in a spectrogram of that vocalization. - If ``map_unlabeled`` is True, then the label 'unlabeled' - will be added to labelset, and will map to 0, + If ``map_background`` is True, then a label + will be added to labelset representing a background class + (any segment that is not labeled). + The default for this label is + :const:`vak.common.constants.DEFAULT_BACKGROUND_LABEL`. + This string label will map to class index 0, so the total number of classes is n + 1. Parameters ---------- labelset : set Set of labels used to annotate a dataset. - map_unlabeled : bool - If True, include key 'unlabeled' in mapping. + map_background : bool + If True, include key specified by + ``background_label`` in mapping. Any time bins in a spectrogram that do not have a label associated with them, e.g. a silent gap between vocalizations, will be assigned the integer - that the 'unlabeled' key maps to. + that the background key maps to. + background_label: str, optional + The string label applied to segments belonging to the + background class. + Default is + :const:`vak.common.constants.DEFAULT_BACKGROUND_LABEL`. Returns ------- @@ -45,11 +57,12 @@ def to_map(labelset: set, map_unlabeled: bool = True) -> dict: ) labellist = [] - if map_unlabeled is True: - labellist.append("unlabeled") - + if map_background is True: + # NOTE we append background label *first* + labellist.append(background_label) + # **then** extend with the rest of the labels labellist.extend(sorted(list(labelset))) - + # so that background_label maps to class index 0 by default in next line labelmap = dict(zip(labellist, range(len(labellist)))) return labelmap @@ -124,7 +137,7 @@ def from_df( # added to fix https://github.com/NickleDave/vak/issues/373 def multi_char_labels_to_single_char( - labelmap: dict, skip: tuple[str] = ("unlabeled",) + labelmap: dict, skip: tuple[str] = (constants.DEFAULT_BACKGROUND_LABEL,) ) -> dict: """Return a copy of a ``labelmap`` where any labels that are strings with multiple characters @@ -146,9 +159,9 @@ def multi_char_labels_to_single_char( to integers. As returned by ``vak.labels.to_map``. skip : tuple - Of strings, labels to leave - as multiple characters. - Default is ('unlabeled',). + A tuple of labels to leave as multiple characters. + Default is a tuple containing just + :const:`vak.common.constants.DEFAULT_BACKGROUND_LABEL`. Returns ------- diff --git a/src/vak/config/__init__.py b/src/vak/config/__init__.py index e0184c522..c1828aff9 100644 --- a/src/vak/config/__init__.py +++ b/src/vak/config/__init__.py @@ -25,7 +25,6 @@ from .train import TrainConfig from .trainer import TrainerConfig - __all__ = [ "config", "dataset", diff --git a/src/vak/config/dataset.py b/src/vak/config/dataset.py index f75c34b73..68bab4c5e 100644 --- a/src/vak/config/dataset.py +++ b/src/vak/config/dataset.py @@ -7,6 +7,8 @@ import attr.validators from attr import asdict, define, field +from ..common.converters import expanded_user_path + @define class DatasetConfig: @@ -31,9 +33,9 @@ class DatasetConfig: Default is None. """ - path: pathlib.Path = field(converter=pathlib.Path) + path: pathlib.Path = field(converter=expanded_user_path) splits_path: pathlib.Path | None = field( - converter=attr.converters.optional(pathlib.Path), default=None + converter=attr.converters.optional(expanded_user_path), default=None ) name: str | None = field( converter=attr.converters.optional(str), default=None diff --git a/src/vak/config/eval.py b/src/vak/config/eval.py index e524efdca..ce59313a0 100644 --- a/src/vak/config/eval.py +++ b/src/vak/config/eval.py @@ -110,8 +110,8 @@ class EvalConfig: Argument to torch.DataLoader. Default is 2. labelmap_path : str path to 'labelmap.json' file. - spect_scaler_path : str - path to a saved SpectScaler object used to normalize spectrograms. + frames_standardizer_path : str + path to a saved :class:`vak.transforms.FramesStandardizer` object used to standardize (normalize) frames. If spectrograms were normalized and this is not provided, will give incorrect results. post_tfm_kwargs : dict @@ -152,7 +152,7 @@ class EvalConfig: converter=converters.optional(expanded_user_path), default=None ) # optional, transform - spect_scaler_path = field( + frames_standardizer_path = field( converter=converters.optional(expanded_user_path), default=None, ) diff --git a/src/vak/config/learncurve.py b/src/vak/config/learncurve.py index bb564db72..71f19ffe7 100644 --- a/src/vak/config/learncurve.py +++ b/src/vak/config/learncurve.py @@ -10,7 +10,12 @@ from .train import TrainConfig from .trainer import TrainerConfig -REQUIRED_KEYS = ("dataset", "model", "root_results_dir", "trainer",) +REQUIRED_KEYS = ( + "dataset", + "model", + "root_results_dir", + "trainer", +) @define @@ -45,9 +50,9 @@ class LearncurveConfig(TrainConfig): Argument to torch.DataLoader. shuffle: bool if True, shuffle training data before each epoch. Default is True. - normalize_spectrograms : bool - if True, use spect.utils.data.SpectScaler to normalize the spectrograms. - Normalization is done by subtracting off the mean for each frequency bin + standardize_frames : bool + if True, use :class:`vak.transforms.FramesStandardizer` to standardize the frames. + Normalization is done by subtracting off the mean for each row of the training set and then dividing by the std for that frequency bin. This same normalization is then applied to validation + test data. val_step : int @@ -75,6 +80,7 @@ class LearncurveConfig(TrainConfig): See the docstring of the transform for more details on these arguments and how they work. """ + post_tfm_kwargs = field( validator=validators.optional(are_valid_post_tfm_kwargs), converter=converters.optional(convert_post_tfm_kwargs), @@ -91,7 +97,8 @@ def from_config_dict(cls, config_dict: dict) -> LearncurveConfig: by loading a valid configuration toml file with :func:`vak.config.parse.from_toml_path`, and then using key ``learncurve``, - i.e., ``LearncurveConfig.from_config_dict(config_dict['learncurve'])``.""" + i.e., ``LearncurveConfig.from_config_dict(config_dict['learncurve'])``. + """ for required_key in REQUIRED_KEYS: if required_key not in config_dict: raise KeyError( diff --git a/src/vak/config/predict.py b/src/vak/config/predict.py index ee63ee093..e0712388d 100644 --- a/src/vak/config/predict.py +++ b/src/vak/config/predict.py @@ -14,7 +14,6 @@ from .model import ModelConfig from .trainer import TrainerConfig - REQUIRED_KEYS = ( "checkpoint_path", "dataset", @@ -50,8 +49,8 @@ class PredictConfig: num_workers : int Number of processes to use for parallel loading of data. Argument to torch.DataLoader. Default is 2. - spect_scaler_path : str - path to a saved SpectScaler object used to normalize spectrograms. + frames_standardizer_path : str + path to a saved :class:`vak.transforms.FramesStandardizer` object used to standardize (normalize) frames. If spectrograms were normalized and this is not provided, will give incorrect results. annot_csv_filename : str @@ -104,7 +103,7 @@ class PredictConfig: ) # optional, transform - spect_scaler_path = field( + frames_standardizer_path = field( converter=converters.optional(expanded_user_path), default=None, ) diff --git a/src/vak/config/train.py b/src/vak/config/train.py index 08b8a7d49..9466cd2e1 100644 --- a/src/vak/config/train.py +++ b/src/vak/config/train.py @@ -8,7 +8,6 @@ from .model import ModelConfig from .trainer import TrainerConfig - REQUIRED_KEYS = ( "dataset", "model", @@ -49,9 +48,9 @@ class TrainConfig: Argument to torch.DataLoader. shuffle: bool if True, shuffle training data before each epoch. Default is True. - normalize_spectrograms : bool - if True, use spect.utils.data.SpectScaler to normalize the spectrograms. - Normalization is done by subtracting off the mean for each frequency bin + standardize_frames : bool + if True, use :class:`vak.transforms.FramesStandardizer` to standardize the frames. + Normalization is done by subtracting off the mean for each row of the training set and then dividing by the std for that frequency bin. This same normalization is then applied to validation + test data. val_step : int @@ -71,8 +70,8 @@ class TrainConfig: checkpoint_path : str path to directory with checkpoint files saved by Torch, to reload model. Default is None, in which case a new model is initialized. - spect_scaler_path : str - path to a saved SpectScaler object used to normalize spectrograms. + frames_standardizer_path : str + path to a saved :class:`vak.transforms.FramesStandardizer` object used to standardize (normalize) frames. If spectrograms were normalized and this is not provided, will give incorrect results. Default is None. """ @@ -96,7 +95,7 @@ class TrainConfig: default=None, ) - normalize_spectrograms = field( + standardize_frames = field( converter=bool_from_str, validator=validators.optional(instance_of(bool)), default=False, @@ -126,7 +125,7 @@ class TrainConfig: converter=converters.optional(expanded_user_path), default=None, ) - spect_scaler_path = field( + frames_standardizer_path = field( converter=converters.optional(expanded_user_path), default=None, ) diff --git a/src/vak/config/trainer.py b/src/vak/config/trainer.py index b2e751905..3e29dc5e3 100644 --- a/src/vak/config/trainer.py +++ b/src/vak/config/trainer.py @@ -1,4 +1,3 @@ - from __future__ import annotations from attrs import asdict, define, field, validators @@ -20,15 +19,18 @@ def is_valid_accelerator(instance, attribute, value): else: raise ValueError( f"Invalid value for 'accelerator' key in 'trainer' table of configuration file: {value}. " - "Value must be one of: {\"cpu\", \"gpu\", \"tpu\", \"ipu\"}" + 'Value must be one of: {"cpu", "gpu", "tpu", "ipu"}' ) def is_valid_devices(instance, attribute, value): """Check if ``devices`` is valid""" if not ( - (isinstance(value, int)) or - (isinstance(value, list) and all([isinstance(el, int) for el in value])) + (isinstance(value, int)) + or ( + isinstance(value, list) + and all([isinstance(el, int) for el in value]) + ) ): raise ValueError( "Invalid value for 'devices' key in 'trainer' table of configuration file: {value}" @@ -63,9 +65,10 @@ class TrainerConfig: Please see this issue: https://github.com/vocalpy/vak/issues/691 If you need to use multiple GPUs, please use `vak` directly in a script. """ + accelerator: str = field( validator=is_valid_accelerator, - default=common.accelerator.get_default() + default=common.accelerator.get_default(), ) devices: int | list[int] = field( validator=validators.optional(is_valid_devices), @@ -87,12 +90,17 @@ def __attrs_post_init__(self): if self.accelerator in ("gpu", "tpu", "ipu"): if not ( - (isinstance(self.devices, int) and self.devices == 1) or - (isinstance(self.devices, list) and len(self.devices) == 1 and all([isinstance(el, int) for el in self.devices])) + (isinstance(self.devices, int) and self.devices == 1) + or ( + isinstance(self.devices, list) + and len(self.devices) == 1 + and all([isinstance(el, int) for el in self.devices]) + ) ): raise ValueError( "Setting a value for the `lightning.pytorch.Trainer` parameter `devices` that is not either 1 " - "(meaning \"use a single GPU\") or a list with a single number (meaning \"use this exact GPU\") currently " + '(meaning "use a single GPU") or a list with a single number ' + '(meaning "use this exact GPU") currently ' "breaks functionality for the command-line interface of `vak`. " "Please see this issue: https://github.com/vocalpy/vak/issues/691" "If you need to use multiple GPUs, please use `vak` directly in a script." @@ -105,7 +113,8 @@ def __attrs_post_init__(self): ) if self.devices < 1: raise ValueError( - f"When value for 'accelerator' is 'cpu', value for `devices` should be an int > 0, but was: {self.devices}" + "When value for 'accelerator' is 'cpu', value for `devices` " + f"should be an int > 0, but was: {self.devices}" ) def asdict(self): diff --git a/src/vak/config/valid-version-1.1.toml b/src/vak/config/valid-version-1.2.toml similarity index 92% rename from src/vak/config/valid-version-1.1.toml rename to src/vak/config/valid-version-1.2.toml index 95aa50372..2ecdda73a 100644 --- a/src/vak/config/valid-version-1.1.toml +++ b/src/vak/config/valid-version-1.2.toml @@ -38,14 +38,14 @@ root_results_dir = './tests/test_data/results/train' num_workers = 4 batch_size = 11 num_epochs = 2 -normalize_spectrograms = true +standardize_frames = true shuffle = true val_step = 1 ckpt_step = 1 patience = 4 results_dir_made_by_main_script = '/some/path/to/learncurve/' checkpoint_path = '/home/user/results_181014_194418/TweetyNet/checkpoints/' -spect_scaler_path = '/home/user/results_181014_194418/spect_scaler' +frames_standardizer_path = '/home/user/results_181014_194418/spect_scaler' [vak.train.dataset] name = 'IntlDistributedSongbirdConsortiumPack' @@ -65,7 +65,7 @@ labelmap_path = '/home/user/results_181014_194418/labelmap.json' output_dir = './tests/test_data/prep/learncurve' batch_size = 11 num_workers = 4 -spect_scaler_path = '/home/user/results_181014_194418/spect_scaler' +frames_standardizer_path = '/home/user/results_181014_194418/spect_scaler' post_tfm_kwargs = {'majority_vote' = true, 'min_segment_dur' = 0.01} [vak.eval.dataset] @@ -83,7 +83,7 @@ devices = [0] root_results_dir = './tests/test_data/results/learncurve' batch_size = 11 num_epochs = 2 -normalize_spectrograms = true +standardize_frames = true shuffle = true val_step = 1 ckpt_step = 1 @@ -111,7 +111,7 @@ annot_csv_filename = '032312_prep_191224_225910.annot.csv' output_dir = './tests/test_data/prep/learncurve' batch_size = 11 num_workers = 4 -spect_scaler_path = '/home/user/results_181014_194418/spect_scaler' +frames_standardizer_path = '/home/user/results_181014_194418/spect_scaler' min_segment_dur = 0.004 majority_vote = false save_net_outputs = false diff --git a/src/vak/config/validators.py b/src/vak/config/validators.py index da396b8d3..f349db746 100644 --- a/src/vak/config/validators.py +++ b/src/vak/config/validators.py @@ -59,7 +59,7 @@ def is_spect_format(instance, attribute, value): CONFIG_DIR = pathlib.Path(__file__).parent -VALID_TOML_PATH = CONFIG_DIR.joinpath("valid-version-1.1.toml") +VALID_TOML_PATH = CONFIG_DIR.joinpath("valid-version-1.2.toml") with VALID_TOML_PATH.open("r") as fp: VALID_DICT = tomlkit.load(fp)["vak"] VALID_TOP_LEVEL_TABLES = list(VALID_DICT.keys()) diff --git a/src/vak/datapipes/__init__.py b/src/vak/datapipes/__init__.py new file mode 100644 index 000000000..d6565ee46 --- /dev/null +++ b/src/vak/datapipes/__init__.py @@ -0,0 +1,7 @@ +"""Module that contains datapipe classes, +used to load inputs and targets for neural network models +from datasets prepared by :func:`vak.prep.prep`""" + +from . import frame_classification, parametric_umap + +__all__ = ["frame_classification", "parametric_umap"] diff --git a/src/vak/datapipes/frame_classification/__init__.py b/src/vak/datapipes/frame_classification/__init__.py new file mode 100644 index 000000000..f8a170d9b --- /dev/null +++ b/src/vak/datapipes/frame_classification/__init__.py @@ -0,0 +1,6 @@ +from . import constants, helper +from .infer_datapipe import InferDatapipe +from .metadata import Metadata +from .train_datapipe import TrainDatapipe + +__all__ = ["constants", "helper", "Metadata", "InferDatapipe", "TrainDatapipe"] diff --git a/src/vak/datasets/frame_classification/constants.py b/src/vak/datapipes/frame_classification/constants.py similarity index 70% rename from src/vak/datasets/frame_classification/constants.py rename to src/vak/datapipes/frame_classification/constants.py index 6867b9b67..101e9e7a3 100644 --- a/src/vak/datasets/frame_classification/constants.py +++ b/src/vak/datapipes/frame_classification/constants.py @@ -1,6 +1,6 @@ FRAMES_PATH_COL_NAME = "frames_path" -FRAME_LABELS_EXT = ".frame_labels.npy" -FRAME_LABELS_NPY_PATH_COL_NAME = "frame_labels_npy_path" +MULTI_FRAME_LABELS_EXT = ".multi-frame-labels.npy" +MULTI_FRAME_LABELS_PATH_COL_NAME = "multi_frame_labels_path" ANNOTATION_CSV_FILENAME = "y.csv" SAMPLE_IDS_ARRAY_FILENAME = "sample_ids.npy" INDS_IN_SAMPLE_ARRAY_FILENAME = "inds_in_sample.npy" diff --git a/src/vak/datasets/frame_classification/helper.py b/src/vak/datapipes/frame_classification/helper.py similarity index 100% rename from src/vak/datasets/frame_classification/helper.py rename to src/vak/datapipes/frame_classification/helper.py diff --git a/src/vak/datasets/frame_classification/frames_dataset.py b/src/vak/datapipes/frame_classification/infer_datapipe.py similarity index 65% rename from src/vak/datasets/frame_classification/frames_dataset.py rename to src/vak/datapipes/frame_classification/infer_datapipe.py index 94ea8169d..9c6e3e338 100644 --- a/src/vak/datasets/frame_classification/frames_dataset.py +++ b/src/vak/datapipes/frame_classification/infer_datapipe.py @@ -1,22 +1,26 @@ -"""A dataset class used for neural network models with the +"""A datapipe class used for neural network models with the frame classification task, where the source data consists of audio signals or spectrograms of varying lengths.""" from __future__ import annotations import pathlib -from typing import Callable +from typing import TYPE_CHECKING import numpy as np import numpy.typing as npt import pandas as pd +from ...transforms.defaults.frame_classification import InferItemTransform from . import constants, helper from .metadata import Metadata +if TYPE_CHECKING: + from ...transforms import FramesStandardizer -class FramesDataset: - """A dataset class used for + +class InferDatapipe: + """A datapipe class used for neural network models with the frame classification task, where the source data consists of audio signals @@ -65,9 +69,14 @@ class FramesDataset: frame_dur: float Duration of a frame, i.e., a single sample in audio or a single timebin in a spectrogram. - item_transform : callable, optional - Transform applied to each item :math:`(x, y)` - returned by :meth:`FramesDataset.__getitem__`. + window_size : int + Size of windows to return; + number of frames. + frames_standardizer : vak.transforms.FramesStandardizer, optional + Transform applied to frames, the input to the neural network model. + Optional, default is None. + If supplied, will be used with the transform applied to inputs and targets, + :class:`vak.transforms.defaults.frame_classification.TrainItemTransform`. """ def __init__( @@ -79,10 +88,14 @@ def __init__( sample_ids: npt.NDArray, inds_in_sample: npt.NDArray, frame_dur: float, - item_transform: Callable, + window_size: int, + frames_standardizer: FramesStandardizer | None = None, + frames_padval: float = 0.0, + frame_labels_padval: int = -1, + return_padding_mask: bool = False, subset: str | None = None, ): - """Initialize a new instance of a FramesDataset. + """Initialize a new instance of an :class:`InferDatapipe`. Parameters ---------- @@ -110,14 +123,33 @@ def __init__( frame_dur: float Duration of a frame, i.e., a single sample in audio or a single timebin in a spectrogram. + frames_standardizer : vak.transforms.FramesStandardizer, optional + Transform applied to frames, the input to the neural network model. + Optional, default is None. + If supplied, will be used with the transform applied to inputs and targets, + :class:`vak.transforms.defaults.frame_classification.InferItemTransform`. + window_size : int + Size of windows to return; + number of frames. + frames_padval : float + Value to pad frames with. Added to end of array, the "right side". + Argument to PadToWindow transform. Default is 0.0. + frame_labels_padval : int + Value to pad frame labels vector with. Added to the end of the array. + Argument to PadToWindow transform. Default is -1. + Used with ``ignore_index`` argument of :mod:`torch.nn.CrossEntropyLoss`. + return_padding_mask : bool + if True, the dictionary returned by ItemTransform classes will include + a boolean vector to use for cropping back down to size before padding. + padding_mask has size equal to width of padded array, i.e. original size + plus padding at the end, and has values of 1 where + columns in padded are from the original array, + and values of 0 where columns were added for padding. subset : str, optional Name of subset to use. If specified, this takes precedence over split. Subsets are typically taken from the training data for use when generating a learning curve. - item_transform : callable - The transform applied to each item :math:`(x, y)` - that is returned by :meth:`FramesDataset.__getitem__`. """ from ... import ( prep, @@ -144,14 +176,20 @@ def __init__( ].values if split != "predict": self.frame_labels_paths = self.dataset_df[ - constants.FRAME_LABELS_NPY_PATH_COL_NAME + constants.MULTI_FRAME_LABELS_PATH_COL_NAME ].values else: self.frame_labels_paths = None self.sample_ids = sample_ids self.inds_in_sample = inds_in_sample self.frame_dur = float(frame_dur) - self.item_transform = item_transform + self.item_transform = InferItemTransform( + window_size, + frames_standardizer, + frames_padval, + frame_labels_padval, + return_padding_mask, + ) @property def duration(self): @@ -196,11 +234,15 @@ def __len__(self): def from_dataset_path( cls, dataset_path: str | pathlib.Path, - item_transform: Callable, + window_size: int, + frames_standardizer: FramesStandardizer | None = None, + frames_padval: float = 0.0, + frame_labels_padval: int = -1, + return_padding_mask: bool = False, split: str = "val", subset: str | None = None, ): - """Make a :class:`FramesDataset` instance, + """Make a :class:`InferDatapipe` instance, given the path to a frame classification dataset. Parameters @@ -210,9 +252,28 @@ def from_dataset_path( frame classification dataset, as created by :func:`vak.prep.prep_frame_classification_dataset`. - item_transform : callable, optional - Transform applied to each item :math:`(x, y)` - returned by :meth:`FramesDataset.__getitem__`. + window_size : int + Size of windows to return; + number of frames. + frames_standardizer : vak.transforms.FramesStandardizer, optional + Transform applied to frames, the input to the neural network model. + Optional, default is None. + If supplied, will be used with the transform applied to inputs and targets, + :class:`vak.transforms.defaults.frame_classification.TrainItemTransform`. + frames_padval : float + Value to pad frames with. Added to end of array, the "right side". + Argument to PadToWindow transform. Default is 0.0. + frame_labels_padval : int + Value to pad frame labels vector with. Added to the end of the array. + Argument to PadToWindow transform. Default is -1. + Used with ``ignore_index`` argument of :mod:`torch.nn.CrossEntropyLoss`. + return_padding_mask : bool + if True, the dictionary returned by ItemTransform classes will include + a boolean vector to use for cropping back down to size before padding. + padding_mask has size equal to width of padded array, i.e. original size + plus padding at the end, and has values of 1 where + columns in padded are from the original array, + and values of 0 where columns were added for padding. split : str The name of a split from the dataset, one of {'train', 'val', 'test'}. @@ -225,7 +286,7 @@ def from_dataset_path( Returns ------- - frames_dataset : FramesDataset + infer_datapipe : InferDatapipe """ dataset_path = pathlib.Path(dataset_path) metadata = Metadata.from_dataset_path(dataset_path) @@ -264,6 +325,10 @@ def from_dataset_path( sample_ids, inds_in_sample, frame_dur, - item_transform, + window_size, + frames_standardizer, + frames_padval, + frame_labels_padval, + return_padding_mask, subset, ) diff --git a/src/vak/datasets/frame_classification/metadata.py b/src/vak/datapipes/frame_classification/metadata.py similarity index 100% rename from src/vak/datasets/frame_classification/metadata.py rename to src/vak/datapipes/frame_classification/metadata.py diff --git a/src/vak/datasets/frame_classification/window_dataset.py b/src/vak/datapipes/frame_classification/train_datapipe.py similarity index 85% rename from src/vak/datasets/frame_classification/window_dataset.py rename to src/vak/datapipes/frame_classification/train_datapipe.py index d916a6bcc..f160199f2 100644 --- a/src/vak/datasets/frame_classification/window_dataset.py +++ b/src/vak/datapipes/frame_classification/train_datapipe.py @@ -2,7 +2,7 @@ frame classification task, where the source data consists of audio signals or spectrograms of varying lengths. -Unlike :class:`vak.datasets.frame_classification.FramesDataset`, +Unlike :class:`vak.datasets.frame_classification.InferDatapipe`, this class does not return entire samples from the source dataset. Instead each paired samples :math:`(x_i, y_i)` @@ -19,22 +19,23 @@ from __future__ import annotations import pathlib -from typing import Callable import numpy as np import numpy.typing as npt import pandas as pd +from ...transforms import FramesStandardizer +from ...transforms.defaults.frame_classification import TrainItemTransform from . import constants, helper from .metadata import Metadata def get_window_inds(n_frames: int, window_size: int, stride: int = 1): - """Get indices of windows for a :class:`WindowDataset`, + """Get indices of windows for a :class:`TrainDatapipe`, given the number of frames in the dataset, the window size, and the stride. - This function is used by :class:`WindowDataset` + This function is used by :class:`TrainDatapipe` to compute the indices of windows in the dataset. The length of the vector of indices it returns is the number of windows in the dataset, @@ -59,14 +60,14 @@ def get_window_inds(n_frames: int, window_size: int, stride: int = 1): return np.arange(stop=n_frames - (window_size - 1), step=stride) -class WindowDataset: +class TrainDatapipe: """Dataset used for training neural network models on the frame classification task, where the source data consists of audio signals or spectrograms of varying lengths. Unlike - :class:`vak.datasets.frame_classification.FramesDataset`, + :class:`vak.datasets.frame_classification.InferDatapipe`, this class does not return entire samples from the source dataset. Instead each paired samples :math:`(x_i, y_i)` @@ -86,7 +87,7 @@ class WindowDataset: created by concatenating samples from the source data, e.g., audio files or spectrogram arrays. (This is true for - :class:`vak.datasets.frame_classification.FramesDataset` + :class:`vak.datasets.frame_classification.InferDatapipe` as well.) The dimensions of :math:`X` will be (channels, ..., frames), i.e., audio will have dimensions (channels, samples) @@ -144,6 +145,11 @@ class WindowDataset: window_size : int Size of windows to return; number of frames. + frames_standardizer : vak.transforms.FramesStandardizer, optional + Transform applied to frames, the input to the neural network model. + Optional, default is None. + If supplied, will be used with the transform applied to inputs and targets, + :class:`vak.transforms.defaults.frame_classification.TrainItemTransform`. frame_dur: float Duration of a frame, i.e., a single sample in audio or a single timebin in a spectrogram. @@ -152,16 +158,15 @@ class WindowDataset: are included in the dataset. The default is 1. Used to compute ``window_inds``, with the function - :func:`vak.datasets.frame_classification.window_dataset.get_window_inds`. + :func:`vak.datasets.frame_classification.train_datapipe.get_window_inds`. window_inds : numpy.ndarray, optional A vector of valid window indices for the dataset. If specified, this takes precedence over ``stride``. - transform : callable - The transform applied to the frames, - the input to the neural network :math:`x`. - target_transform : callable - The transform applied to the target for the output - of the neural network :math:`y`. + frames_standardizer : vak.transforms.FramesStandardizer, optional + Transform applied to frames, the input to the neural network model. + Optional, default is None. + If supplied, will be used with the transform applied to inputs and targets, + :class:`vak.transforms.defaults.frame_classification.TrainItemTransform`. """ def __init__( @@ -174,12 +179,12 @@ def __init__( inds_in_sample: npt.NDArray, window_size: int, frame_dur: float, - item_transform: Callable, stride: int = 1, subset: str | None = None, window_inds: npt.NDArray | None = None, + frames_standardizer: FramesStandardizer | None = None, ): - """Initialize a new instance of a WindowDataset. + """Initialize a new instance of a TrainDatapipe. Parameters ---------- @@ -210,15 +215,12 @@ def __init__( frame_dur: float Duration of a frame, i.e., a single sample in audio or a single timebin in a spectrogram. - item_transform : callable - The transform applied to each item :math:`(x, y)` - that is returned by :meth:`WindowDataset.__getitem__`. stride : int The size of the stride used to determine which windows are included in the dataset. The default is 1. Used to compute ``window_inds``, with the function - :func:`vak.datasets.frame_classification.window_dataset.get_window_inds`. + :func:`vak.datasets.frame_classification.train_datapipe.get_window_inds`. subset : str, optional Name of subset to use. If specified, this takes precedence over split. @@ -227,11 +229,11 @@ def __init__( window_inds : numpy.ndarray, optional A vector of valid window indices for the dataset. If specified, this takes precedence over ``stride``. - transform : callable - The transform applied to the input to the neural network :math:`x`. - target_transform : callable - The transform applied to the target for the output - of the neural network :math:`y`. + frames_standardizer : vak.transforms.FramesStandardizer, optional + Transform applied to frames, the input to the neural network model. + Optional, default is None. + If supplied, will be used with the transform applied to inputs and targets, + :class:`vak.transforms.defaults.frame_classification.TrainItemTransform`. """ from ... import ( prep, @@ -257,7 +259,7 @@ def __init__( constants.FRAMES_PATH_COL_NAME ].values self.frame_labels_paths = self.dataset_df[ - constants.FRAME_LABELS_NPY_PATH_COL_NAME + constants.MULTI_FRAME_LABELS_PATH_COL_NAME ].values self.sample_ids = sample_ids self.inds_in_sample = inds_in_sample @@ -269,7 +271,9 @@ def __init__( sample_ids.shape[-1], window_size, stride ) self.window_inds = window_inds - self.item_transform = item_transform + self.item_transform = TrainItemTransform( + frames_standardizer=frames_standardizer + ) @property def duration(self): @@ -351,12 +355,12 @@ def from_dataset_path( cls, dataset_path: str | pathlib.Path, window_size: int, - item_transform: Callable, stride: int = 1, split: str = "train", subset: str | None = None, + frames_standardizer: FramesStandardizer | None = None, ): - """Make a :class:`WindowDataset` instance, + """Make a :class:`TrainDatapipe` instance, given the path to a frame classification dataset. Parameters @@ -374,7 +378,7 @@ def from_dataset_path( are included in the dataset. The default is 1. Used to compute ``window_inds``, with the function - :func:`vak.datasets.frame_classification.window_dataset.get_window_inds`. + :func:`vak.datasets.frame_classification.train_datapipe.get_window_inds`. split : str The name of a split from the dataset, one of {'train', 'val', 'test'}. @@ -383,15 +387,15 @@ def from_dataset_path( If specified, this takes precedence over split. Subsets are typically taken from the training data for use when generating a learning curve. - transform : callable - The transform applied to the input to the neural network :math:`x`. - target_transform : callable - The transform applied to the target for the output - of the neural network :math:`y`. + frames_standardizer : vak.transforms.FramesStandardizer, optional + Transform applied to frames, the input to the neural network model. + Optional, default is None. + If supplied, will be used with the transform applied to inputs and targets, + :class:`vak.transforms.defaults.frame_classification.TrainItemTransform`. Returns ------- - dataset : vak.datasets.frame_classification.WindowDataset + dataset : vak.datasets.frame_classification.TrainDatapipe """ dataset_path = pathlib.Path(dataset_path) metadata = Metadata.from_dataset_path(dataset_path) @@ -437,8 +441,8 @@ def from_dataset_path( inds_in_sample, window_size, frame_dur, - item_transform, stride, subset, window_inds, + frames_standardizer, ) diff --git a/src/vak/datapipes/parametric_umap/__init__.py b/src/vak/datapipes/parametric_umap/__init__.py new file mode 100644 index 000000000..55cc4b924 --- /dev/null +++ b/src/vak/datapipes/parametric_umap/__init__.py @@ -0,0 +1,8 @@ +from .metadata import Metadata +from .parametric_umap import Datapipe + +__all__ = [ + "InferDatapipe", + "Metadata", + "Datapipe", +] diff --git a/src/vak/datasets/parametric_umap/metadata.py b/src/vak/datapipes/parametric_umap/metadata.py similarity index 100% rename from src/vak/datasets/parametric_umap/metadata.py rename to src/vak/datapipes/parametric_umap/metadata.py diff --git a/src/vak/datasets/parametric_umap/parametric_umap.py b/src/vak/datapipes/parametric_umap/parametric_umap.py similarity index 79% rename from src/vak/datasets/parametric_umap/parametric_umap.py rename to src/vak/datapipes/parametric_umap/parametric_umap.py index d95cb0150..f14d58d0e 100644 --- a/src/vak/datasets/parametric_umap/parametric_umap.py +++ b/src/vak/datapipes/parametric_umap/parametric_umap.py @@ -4,16 +4,18 @@ import pathlib import warnings -from typing import Callable import numpy as np import numpy.typing as npt import pandas as pd import scipy.sparse._coo +import torchvision.transforms from pynndescent import NNDescent from sklearn.utils import check_random_state from torch.utils.data import Dataset +from ... import transforms as vak_transforms + # isort: off # Ignore warnings from Numba deprecation: # https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit @@ -187,8 +189,8 @@ def get_graph_elements( return graph, epochs_per_sample, head, tail, weight, n_vertices -class ParametricUMAPDataset(Dataset): - """A dataset class used to train Parametric UMAP models.""" +class Datapipe(Dataset): + """A datapipe used with Parametric UMAP models.""" def __init__( self, @@ -200,7 +202,6 @@ def __init__( n_neighbors: int = 10, metric: str = "euclidean", random_state: int | None = None, - transform: Callable | None = None, ): """Initialize a :class:`ParametricUMAPDataset` instance. @@ -251,8 +252,8 @@ def __init__( epochs_per_sample, head, tail, - weight, - n_vertices, + _, + _, ) = get_graph_elements(graph, n_epochs) # we repeat each sample in (head, tail) a certain number of times depending on its probability @@ -269,7 +270,12 @@ def __init__( self.data = data self.dataset_df = dataset_df - self.transform = transform + self.transform = torchvision.transforms.Compose( + [ + vak_transforms.ToFloatTensor(), + vak_transforms.AddChannel(), + ] + ) @property def duration(self): @@ -287,9 +293,8 @@ def shape(self): def __getitem__(self, index): edges_to_exp = self.data[self.edges_to_exp[index]] edges_from_exp = self.data[self.edges_from_exp[index]] - if self.transform: - edges_to_exp = self.transform(edges_to_exp) - edges_from_exp = self.transform(edges_from_exp) + edges_to_exp = self.transform(edges_to_exp) + edges_from_exp = self.transform(edges_from_exp) return (edges_to_exp, edges_from_exp) @classmethod @@ -302,7 +307,6 @@ def from_dataset_path( metric: str = "euclidean", random_state: int | None = None, n_epochs: int = 200, - transform: Callable | None = None, ): """Make a :class:`ParametricUMAPDataset` instance, given the path to parametric UMAP dataset. @@ -334,17 +338,15 @@ def from_dataset_path( random_state : numpy.random.RandomState Either a numpy.random.RandomState instance, or None. - transform : callable - The transform applied to the input to the neural network :math:`x`. Returns ------- - dataset : vak.datasets.parametric_umap.ParametricUMAPDataset + dataset : vak.datasets.parametric_umap.TrainDatapipe """ - import vak.datasets # import here just to make classmethod more explicit + import vak.datapipes # import here just to make classmethod more explicit dataset_path = pathlib.Path(dataset_path) - metadata = vak.datasets.parametric_umap.Metadata.from_dataset_path( + metadata = vak.datapipes.parametric_umap.Metadata.from_dataset_path( dataset_path ) @@ -360,88 +362,4 @@ def from_dataset_path( n_neighbors, metric, random_state, - transform, - ) - - -class ParametricUMAPInferenceDataset(Dataset): - def __init__( - self, - data: npt.NDArray, - dataset_df: pd.DataFrame, - transform: Callable | None = None, - ): - self.data = data - self.dataset_df = dataset_df - self.transform = transform - - @property - def duration(self): - return self.dataset_df["duration"].sum() - - def __len__(self): - return self.data.shape[0] - - @property - def shape(self): - tmp_x_ind = 0 - tmp_item = self.__getitem__(tmp_x_ind) - return tmp_item[0].shape - - def __getitem__(self, index): - x = self.data[index] - df_index = self.dataset_df.index[index] - if self.transform: - x = self.transform(x) - return {"x": x, "df_index": df_index} - - @classmethod - def from_dataset_path( - cls, - dataset_path: str | pathlib.Path, - split: str, - n_neighbors: int = 10, - metric: str = "euclidean", - random_state: int | None = None, - n_epochs: int = 200, - transform: Callable | None = None, - ): - """ - - Parameters - ---------- - dataset_path : str, pathlib.Path - Path to a directory that represents a dataset. - split - n_neighbors - metric - random_state - n_epochs - transform - - Returns - ------- - - """ - import vak.datasets # import here just to make classmethod more explicit - - dataset_path = pathlib.Path(dataset_path) - metadata = vak.datasets.parametric_umap.Metadata.from_dataset_path( - dataset_path - ) - - dataset_csv_path = dataset_path / metadata.dataset_csv_filename - dataset_df = pd.read_csv(dataset_csv_path) - split_df = dataset_df[dataset_df.split == split] - - data = np.stack( - [ - np.load(dataset_path / spect_path) - for spect_path in split_df.spect_path.values - ] - ) - return cls( - data, - split_df, - transform=transform, ) diff --git a/src/vak/datasets/__init__.py b/src/vak/datasets/__init__.py index 0a8cc3764..cb7734cde 100644 --- a/src/vak/datasets/__init__.py +++ b/src/vak/datasets/__init__.py @@ -1,3 +1,15 @@ -from . import frame_classification, parametric_umap +from . import biosoundsegbench +from .biosoundsegbench import BioSoundSegBench, SplitsMetadata +from .get import get -__all__ = ["frame_classification", "parametric_umap"] +__all__ = [ + "biosoundsegbench", + "BioSoundSegBench", + "get", + "SplitsMetadata", +] + +# TODO: make this a proper registry +DATASETS = { + "BioSoundSegBench": BioSoundSegBench +} diff --git a/src/vak/datasets/biosoundsegbench.py b/src/vak/datasets/biosoundsegbench.py new file mode 100644 index 000000000..a610c6530 --- /dev/null +++ b/src/vak/datasets/biosoundsegbench.py @@ -0,0 +1,646 @@ +"""Class representing BioSoundSegBench dataset.""" +from __future__ import annotations + +import json +import pathlib +from typing import Callable, Literal, TYPE_CHECKING + +from attrs import define +import numpy as np +import pandas as pd + +import torch +import torchvision.transforms + +from .. import common, datapipes, transforms + +if TYPE_CHECKING: + from ..transforms import FramesStandardizer + + +VALID_TARGET_TYPES = ( + "boundary_frame_labels", + "multi_frame_labels", + "binary_frame_labels", + ("boundary_frame_labels", "multi_frame_labels"), + "None", +) + + +FRAMES_PATH_COL_NAME = "frames_path" +MULTI_FRAME_LABELS_PATH_COL_NAME = "multi_frame_labels_path" +BINARY_FRAME_LABELS_PATH_COL_NAME = "binary_frame_labels_path" +BOUNDARY_FRAME_LABELS_PATH_COL_NAME = "boundary_frame_labels_path" + + +@define +class SampleIDVectorPaths: + train: pathlib.Path + val: pathlib.Path + test: pathlib.Path + + +@define +class IndsInSampleVectorPaths: + train: pathlib.Path + val: pathlib.Path + test: pathlib.Path + + +@define +class SplitsMetadata: + """Class that represents metadata about dataset splits + in the BioSoundSegBench dataset, loaded from a json file""" + + splits_csv_path: pathlib.Path + sample_id_vector_paths: SampleIDVectorPaths + inds_in_sample_vector_paths: IndsInSampleVectorPaths + + @classmethod + def from_paths(cls, json_path, dataset_path): + json_path = pathlib.Path(json_path) + with json_path.open("r") as fp: + splits_json = json.load(fp) + + dataset_path = pathlib.Path(dataset_path) + if not dataset_path.exists() or not dataset_path.is_dir(): + raise NotADirectoryError( + f"`dataset_path` not found or not a directory: {dataset_path}" + ) + + splits_csv_path = pathlib.Path( + dataset_path / splits_json["splits_csv_path"] + ) + if not splits_csv_path.exists(): + raise FileNotFoundError( + f"`splits_csv_path` not found: {splits_csv_path}" + ) + + sample_id_vector_paths = { + split: dataset_path / path + for split, path in splits_json["sample_id_vec_path"].items() + } + for split, vec_path in sample_id_vector_paths.items(): + if not vec_path.exists(): + raise FileNotFoundError( + f"`sample_id_vector_path` for split '{split}' not found: {vec_path}" + ) + sample_id_vector_paths = SampleIDVectorPaths(**sample_id_vector_paths) + + inds_in_sample_vector_paths = { + split: dataset_path / path + for split, path in splits_json["inds_in_sample_vec_path"].items() + } + for split, vec_path in inds_in_sample_vector_paths.items(): + if not vec_path.exists(): + raise FileNotFoundError( + f"`inds_in_sample_vec_path` for split '{split}' not found: {vec_path}" + ) + inds_in_sample_vector_paths = IndsInSampleVectorPaths( + **inds_in_sample_vector_paths + ) + + return cls( + splits_csv_path, + sample_id_vector_paths, + inds_in_sample_vector_paths, + ) + + +@define +class TrainingReplicateMetadata: + """Class representing metadata for a + pre-defined training replicate + in the BioSoundSegBench dataset. + """ + biosound_group: str + id: str | None + frame_dur: float + unit: str + data_source: str | None + train_dur: float + replicate_num: int + + +def metadata_from_splits_json_path( + splits_json_path: pathlib.Path, datset_path: pathlib.Path + ) -> TrainingReplicateMetadata: + try: + # Human-Speech doesn't have ID or data source in filename + # so it will raise a ValueError + name = splits_json_path.name + (biosound_group, + id_, + timebin_dur_1st_half, + timebin_dur_2nd_half, + unit, + data_source, + train_dur_1st_half, + train_dur_2nd_half, + replicate_num, + _, _ + ) = name.split('.') + except ValueError: + name = splits_json_path.name + (biosound_group, + timebin_dur_1st_half, + timebin_dur_2nd_half, + unit, + train_dur_1st_half, + train_dur_2nd_half, + replicate_num, + _, _ + ) = name.split('.') + id_ = None + data_source = None + if id_ is not None: + id_ = id_.split('-')[-1] + timebin_dur = float( + timebin_dur_1st_half.split('-')[-1] + '.' + timebin_dur_2nd_half.split('-')[0] + ) + train_dur = float( + train_dur_1st_half.split('-')[-1] + '.' + train_dur_2nd_half.split('-')[0] + ) + replicate_num = int( + replicate_num.split('-')[-1] + ) + return TrainingReplicateMetadata( + biosound_group, + id_, + timebin_dur, + unit, + data_source, + train_dur, + replicate_num, + ) + + +class TrainItemTransform: + """Default transform used when training frame classification models + with :class:`BioSoundSegBench` dataset.""" + + def __init__( + self, + frames_standardizer: FramesStandardizer | None = None, + ): + from ..transforms import FramesStandardizer # avoid circular import + if frames_standardizer is not None: + if isinstance( + frames_standardizer, FramesStandardizer + ): + frames_transform = [frames_standardizer] + else: + raise TypeError( + f"invalid type for frames_standardizer: {type(frames_standardizer)}. " + "Should be an instance of vak.transforms.StandardizeSpect" + ) + else: + frames_transform = [] + # add as an attribute on self so that high-level functions can save this class as needed + self.frames_standardizer = frames_standardizer + + frames_transform.extend( + [ + transforms.ToFloatTensor(), + transforms.AddChannel(), + ] + ) + self.frames_transform = torchvision.transforms.Compose( + frames_transform + ) + self.frame_labels_transform = transforms.ToLongTensor() + + def __call__( + self, + frames: torch.Tensor, + multi_frame_labels: torch.Tensor | None = None, + binary_frame_labels: torch.Tensor | None = None, + boundary_frame_labels: torch.Tensor | None = None, + ) -> dict: + frames = self.frames_transform(frames) + item = { + "frames": frames, + } + if multi_frame_labels is not None: + item["multi_frame_labels"] = self.frame_labels_transform(multi_frame_labels) + + if binary_frame_labels is not None: + item["binary_frame_labels"] = self.frame_labels_transform(binary_frame_labels) + + if boundary_frame_labels is not None: + item["boundary_frame_labels"] = self.frame_labels_transform(boundary_frame_labels) + + return item + + +class InferItemTransform: + """Default transform used when running inference on classification models + with :class:`BioSoundSegBench` dataset, for evaluation or to generate new predictions. + + Returned item includes frames reshaped into a stack of windows, + with padded added to make reshaping possible. + Any `frame_labels` are not padded and reshaped, + but are converted to :class:`torch.LongTensor`. + If return_padding_mask is True, item includes 'padding_mask' that + can be used to crop off any predictions made on the padding. + + Attributes + ---------- + frames_standardizer : vak.transforms.FramesStandardizer + instance that has already been fit to dataset, using fit_df method. + Default is None, in which case no standardization transform is applied. + window_size : int + width of window in number of elements. Argument to PadToWindow transform. + frames_padval : float + Value to pad frames with. Added to end of array, the "right side". + Argument to PadToWindow transform. Default is 0.0. + frame_labels_padval : int + Value to pad frame labels vector with. Added to the end of the array. + Argument to PadToWindow transform. Default is -1. + Used with ``ignore_index`` argument of :mod:`torch.nn.CrossEntropyLoss`. + return_padding_mask : bool + if True, the dictionary returned by ItemTransform classes will include + a boolean vector to use for cropping back down to size before padding. + padding_mask has size equal to width of padded array, i.e. original size + plus padding at the end, and has values of 1 where + columns in padded are from the original array, + and values of 0 where columns were added for padding. + """ + + def __init__( + self, + window_size, + frames_standardizer=None, + frames_padval=0.0, + frame_labels_padval=-1, + return_padding_mask=True, + channel_dim=1, + ): + from ..transforms import FramesStandardizer # avoid circular import + + self.window_size = window_size + self.frames_padval = frames_padval + self.frame_labels_padval = frame_labels_padval + self.return_padding_mask = return_padding_mask + self.channel_dim = channel_dim + + if frames_standardizer is not None: + if not isinstance( + frames_standardizer, FramesStandardizer + ): + raise TypeError( + f"Invalid type for frames_standardizer: {type(frames_standardizer)}. " + "Should be an instance of vak.transforms.FramesStandardizer" + ) + # add as an attribute on self to use inside __call__ + # *and* so that high-level functions can save this class as needed + self.frames_standardizer = frames_standardizer + + self.pad_to_window = transforms.PadToWindow( + window_size, frames_padval, return_padding_mask=return_padding_mask + ) + + self.frames_transform_after_pad = torchvision.transforms.Compose( + [ + transforms.ViewAsWindowBatch(window_size), + transforms.ToFloatTensor(), + # below, add channel at first dimension because windows become batch + transforms.AddChannel(channel_dim=channel_dim), + ] + ) + + self.frame_labels_padval = frame_labels_padval + self.frame_labels_transform = transforms.ToLongTensor() + + def __call__( + self, + frames: torch.Tensor, + multi_frame_labels: torch.Tensor | None = None, + binary_frame_labels: torch.Tensor | None = None, + boundary_frame_labels: torch.Tensor | None = None, + frames_path=None, + ) -> dict: + if self.frames_standardizer: + frames = self.frames_standardizer(frames) + + if self.pad_to_window.return_padding_mask: + frames, padding_mask = self.pad_to_window(frames) + else: + frames = self.pad_to_window(frames) + padding_mask = None + frames = self.frames_transform_after_pad(frames) + + item = { + "frames": frames, + } + + if multi_frame_labels is not None: + item["multi_frame_labels"] = self.frame_labels_transform(multi_frame_labels) + + if binary_frame_labels is not None: + item["binary_frame_labels"] = self.frame_labels_transform(binary_frame_labels) + + if boundary_frame_labels is not None: + item["boundary_frame_labels"] = self.frame_labels_transform(boundary_frame_labels) + + if padding_mask is not None: + item["padding_mask"] = padding_mask + + if frames_path is not None: + # make sure frames_path is a str, not a pathlib.Path + item["frames_path"] = str(frames_path) + + return item + + +class BioSoundSegBench: + """Class representing BioSoundSegBench dataset.""" + def __init__( + self, + dataset_path: str | pathlib.Path, + splits_path: str | pathlib.Path, + split: Literal["train", "val", "test"], + window_size: int, + target_type: str | list[str] | tuple[str] | None = None, + stride: int = 1, + standardize_frames: bool = False, + frames_standardizer: transforms.FramesStandardizer | None = None, + frames_padval: float = 0.0, + frame_labels_padval: int = -1, + return_padding_mask: bool = False, + return_frames_path: bool = False, + item_transform: Callable | None = None + ): + """BioSoundSegBench dataset.""" + # ---- validate args, roughly in order + dataset_path = pathlib.Path(dataset_path) + if not dataset_path.exists() or not dataset_path.is_dir(): + raise NotADirectoryError( + f"`dataset_path` for dataset not found, or not a directory: {dataset_path}" + ) + self.dataset_path = dataset_path + if split not in common.constants.VALID_SPLITS: + raise ValueError( + f"Invalid split name: {split}\n" + f"Valid splits are: {common.constants.VALID_SPLITS}" + ) + + splits_path = pathlib.Path(splits_path) + if not splits_path.exists(): + tmp_splits_path = dataset_path / "splits" / "splits-jsons" / splits_path + if not tmp_splits_path.exists(): + raise FileNotFoundError( + f"Did not find `splits_path` using either absolute path ({splits_path})" + f"or relative to `dataset_path` ({tmp_splits_path})" + ) + # if tmp_splits_path *does* exist, replace splits_path with it + splits_path = tmp_splits_path + self.splits_path = splits_path + self.splits_metadata = SplitsMetadata.from_paths( + json_path=splits_path, dataset_path=dataset_path + ) + + if target_type is None and split != "predict": + raise ValueError( + f"Must specify `target_type` if split is '{split}', but " + "`target_type` is None. `target_type` can only be None if split is 'predict'." + ) + if target_type is None: + target_type = "None" + if not isinstance(target_type, (str, list, tuple)): + raise TypeError( + f"`target_type` must be string or sequence of strings but type was: {type(target_type)}\n" + f"Valid `target_type` arguments are: {VALID_TARGET_TYPES}" + ) + if isinstance(target_type, (list, tuple)): + if not all([ + isinstance(target_type_, str) + for target_type_ in target_type + ]): + types_in_target_types = set( + [type(target_type_) for target_type_ in target_type] + ) + raise TypeError( + "A list or tuple of `target_type` must be all strings, " + f"but found the following types: {types_in_target_types}\n" + f"`target_type` was: {target_type}\n" + f"Valid `target_type` arguments are: {VALID_TARGET_TYPES}" + ) + # alphabetically sort list or tuple, and make sure it's a tuple + target_type = tuple(sorted(target_type)) + if target_type not in VALID_TARGET_TYPES: + raise ValueError( + f"Invalid `target_type`: {target_type}. " + f"Valid target types are: {VALID_TARGET_TYPES}" + ) + if isinstance(target_type, str): + # make single str a tuple so we can do ``if 'some target' in self.target_type`` + target_type = (target_type,) + self.target_type = target_type + + # this is a bit convoluted: we are setting metadata, to set frame dur, + # to be able to compute duration in property below + self.training_replicate_metadata = metadata_from_splits_json_path( + self.splits_path, self.dataset_path + ) + self.frame_dur = self.training_replicate_metadata.frame_dur * 1e-3 # convert from ms to s! + + if "multi_frame_labels" in target_type: + labelmaps_json_path = self.dataset_path / "labelmaps.json" + if not labelmaps_json_path.exists(): + raise FileNotFoundError( + "`target_type` includes \"multi_frame_labels\" but " + "'labelmaps.json' was not found in root of dataset path:\n" + f"{labelmaps_json_path}" + ) + with labelmaps_json_path.open("r") as fp: + labelmaps = json.load(fp) + group = self.training_replicate_metadata.biosound_group + unit = self.training_replicate_metadata.unit + id_ = self.training_replicate_metadata.id + if id_ is not None: + if group == "Mouse-Pup-Call": + self.labelmap = labelmaps[group][unit]["all"] + else: + self.labelmap = labelmaps[group][unit][id_] + else: + if group == "Human-Speech": + self.labelmap = labelmaps[group][unit]["all"] + else: + raise ValueError( + "Unable to determine labelmap to use for " + f"group '{group}', unit '{unit}', and id '{id}'. " + "Please check that splits_json path is correct." + ) + elif target_type == ('binary_frame_labels',): + self.labelmap = {'no segment': 0, 'segment': 1} + elif target_type == ('boundary_frame_labels',): + self.labelmap = {'no boundary': 0, 'boundary': 1} + + self.split = split + split_df = pd.read_csv(self.splits_metadata.splits_csv_path) + split_df = split_df[split_df.split == split].copy() + self.split_df = split_df + + self.frames_paths = self.split_df[FRAMES_PATH_COL_NAME].values + self.target_paths = {} + if "multi_frame_labels" in self.target_type: + self.target_paths["multi_frame_labels"] = self.split_df[ + MULTI_FRAME_LABELS_PATH_COL_NAME + ].values + if "binary_frame_labels" in self.target_type: + self.target_paths["binary_frame_labels"] = self.split_df[ + BINARY_FRAME_LABELS_PATH_COL_NAME + ].values + if "boundary_frame_labels" in self.target_type: + self.target_paths["boundary_frame_labels"] = self.split_df[ + BOUNDARY_FRAME_LABELS_PATH_COL_NAME + ].values + + self.window_size = window_size + self.stride = stride + + # we need all these vectors for getting batches of windows during training + # for other splits, we use these to determine the duration of the dataset + self.sample_ids = np.load( + getattr(self.splits_metadata.sample_id_vector_paths, split) + ) + self.inds_in_sample = np.load( + getattr(self.splits_metadata.inds_in_sample_vector_paths, split) + ) + self.window_inds = datapipes.frame_classification.train_datapipe.get_window_inds( + self.sample_ids.shape[-1], window_size, stride + ) + + if item_transform is None: + if standardize_frames and frames_standardizer is None: + from ..transforms import FramesStandardizer + frames_standardizer = FramesStandardizer.fit_inputs_targets_csv_path( + self.splits_metadata.splits_csv_path, self.dataset_path + ) + if split == "train": + self.item_transform = TrainItemTransform( + frames_standardizer=frames_standardizer + ) + elif split in ("val", "test", "predict"): + self.item_transform = InferItemTransform( + window_size, + frames_standardizer, + frames_padval, + frame_labels_padval, + return_padding_mask, + ) + else: + if not callable(item_transform): + raise ValueError( + "`item_transform` should be `callable` " + f"but value for `item_transform` was not: {item_transform}" + ) + self.item_transform = item_transform + + self.return_frames_path = return_frames_path + + @property + def shape(self): + tmp_x_ind = 0 + tmp_item = self.__getitem__(tmp_x_ind) + input_shape = tmp_item["frames"].shape + if self.split == "train" and len(input_shape) == 3: + return input_shape + elif ( + self.split in ("val", "test", "predict") and len(input_shape) == 4 + ): + # discard windows dimension from shape -- + # it's sample dependent and not what we want + return input_shape[1:] + + @property + def duration(self): + return self.sample_ids.shape[-1] * self.frame_dur + + def __len__(self): + """number of batches""" + if self.split == "train": + return len(self.window_inds) + else: + return len(np.unique(self.sample_ids)) + + def _getitem_train(self, idx): + window_idx = self.window_inds[idx] + sample_ids = self.sample_ids[ + window_idx : window_idx + self.window_size # noqa: E203 + ] + uniq_sample_ids = np.unique(sample_ids) + item = {} + if len(uniq_sample_ids) == 1: + # repeat myself to avoid running a loop on one item + sample_id = uniq_sample_ids[0] + frames_path = self.dataset_path / self.frames_paths[sample_id] + spect_dict = common.files.spect.load(frames_path) + item["frames"] = spect_dict[common.constants.SPECT_KEY] + for target_type in self.target_type: + item[target_type] = np.load( + self.dataset_path / self.target_paths[target_type][sample_id] + ) + + elif len(uniq_sample_ids) > 1: + item["frames"] = [] + for target_type in self.target_type: + # do this to append instead of using defaultdict + # so that when we do `'target_type' in item` we don't get empty list + item[target_type] = [] + for sample_id in sorted(uniq_sample_ids): + frames_path = self.dataset_path / self.frames_paths[sample_id] + spect_dict = common.files.spect.load(frames_path) + item["frames"].append( + spect_dict[common.constants.SPECT_KEY] + ) + for target_type in self.target_type: + item[target_type].append( + np.load( + self.dataset_path + / self.target_paths[target_type][sample_id] + ) + ) + + item["frames"] = np.concatenate(item["frames"], axis=1) + for target_type in self.target_type: + item[target_type] = np.concatenate(item[target_type]) + else: + raise ValueError( + f"Unexpected number of ``uniq_sample_ids``: {uniq_sample_ids}" + ) + + ind_in_sample = self.inds_in_sample[window_idx] + item["frames"] = item["frames"][ + ..., + ind_in_sample : ind_in_sample + self.window_size, # noqa: E203 + ] + for target_type in self.target_type: + item[target_type] = item[target_type][ + ind_in_sample : ind_in_sample + self.window_size # noqa: E203 + ] + item = self.item_transform(**item) + return item + + def _getitem_infer(self, idx): + item = {} + frames_path = self.dataset_path / self.frames_paths[idx] + if self.return_frames_path: + item["frames_path"] = frames_path + spect_dict = common.files.spect.load(frames_path) + item["frames"] = spect_dict[common.constants.SPECT_KEY] + if self.target_type != "None": # target_type can be None for predict + for target_type in self.target_type: + item[target_type] = np.load( + self.dataset_path / self.target_paths[target_type][idx] + ) + item = self.item_transform(**item) + return item + + def __getitem__(self, idx): + if self.split == "train": + item = self._getitem_train(idx) + elif self.split in ("val", "test", "predict"): + item = self._getitem_infer(idx) + return item diff --git a/src/vak/datasets/frame_classification/__init__.py b/src/vak/datasets/frame_classification/__init__.py deleted file mode 100644 index 29bc8b68c..000000000 --- a/src/vak/datasets/frame_classification/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from . import constants, helper -from .frames_dataset import FramesDataset -from .metadata import Metadata -from .window_dataset import WindowDataset - -__all__ = ["constants", "helper", "Metadata", "FramesDataset", "WindowDataset"] diff --git a/src/vak/datasets/get.py b/src/vak/datasets/get.py new file mode 100644 index 000000000..3b38182a4 --- /dev/null +++ b/src/vak/datasets/get.py @@ -0,0 +1,66 @@ +"""Helper function that gets instances of classes representing datasets built into :mod:`vak`.""" +from __future__ import annotations + +from typing import Literal, Mapping, TYPE_CHECKING + +from .. import common + +Dataset = Mapping + +if TYPE_CHECKING: + from ..transforms import FramesStandardizer + +def get( + dataset_config: dict, + split: Literal["predict", "test", "train", "val"], + frames_standardizer: FramesStandardizer | None = None, + ) -> Dataset: + """Get an instance of a dataset class from :mod:`vak.datasets`. + + Parameters + ---------- + dataset_config: dict + Dataset configuration in a :class:`dict`. + Can be obtained by calling :meth:`vak.config.DatasetConfig.asdict`. + split : str + Name of split to use. + One of {"predict", "test", "train", "val"}. + + Returns + ------- + dataset : class + An instance of a class from :mod:`vak.datasets`, + e.g. :class:`vak.datasets.BioSoundSegBench`. + """ + from . import DATASETS # avoid circular import + + if "name" not in dataset_config: + raise KeyError( + "A name is required to get a dataset, but " + "`vak.datasets.get` received a `dataset_config` " + f"without a \"name\":\n{dataset_config}" + ) + if split not in common.constants.VALID_SPLITS: + raise ValueError( + f"Invalid value for `split`: {split}.\n" + f"Valid splits are: {common.constants.VALID_SPLITS}" + ) + + from .. import datasets + dataset_name = dataset_config["name"] + try: + dataset_class = DATASETS[dataset_name] + except KeyError as e: + raise ValueError( + f"Invalid dataset name: {dataset_name}\n." + f"Built-in dataset names are: {DATASETS.keys()}" + ) + if frames_standardizer is not None: + dataset_config["params"]["frames_standardizer"] = frames_standardizer + dataset = dataset_class( + dataset_path=dataset_config["path"], + splits_path=dataset_config["splits_path"], + split=split, + **dataset_config["params"] + ) + return dataset diff --git a/src/vak/datasets/parametric_umap/__init__.py b/src/vak/datasets/parametric_umap/__init__.py deleted file mode 100644 index 42ef1f109..000000000 --- a/src/vak/datasets/parametric_umap/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .metadata import Metadata -from .parametric_umap import ParametricUMAPDataset - -__all__ = ["Metadata", "ParametricUMAPDataset"] diff --git a/src/vak/eval/eval_.py b/src/vak/eval/eval_.py index 665ec7d0d..29ffdc858 100644 --- a/src/vak/eval/eval_.py +++ b/src/vak/eval/eval_.py @@ -22,8 +22,7 @@ def eval( num_workers: int, labelmap_path: str | pathlib.Path | None = None, batch_size: int | None = None, - split: str = "test", - spect_scaler_path: str | pathlib.Path = None, + frames_standardizer_path: str | pathlib.Path = None, post_tfm_kwargs: dict | None = None, device: str | None = None, ) -> None: @@ -56,10 +55,10 @@ def eval( split : str split of dataset on which model should be evaluated. One of {'train', 'val', 'test'}. Default is 'test'. - spect_scaler_path : str, pathlib.Path - path to a saved SpectScaler object used to normalize spectrograms. - If spectrograms were normalized and this is not provided, will give - incorrect results. + frames_standardizer_path : str, pathlib.Path + path to a saved FramesStandardizer object used to standardize frames. + If frames were standardized during training, and this is not provided, + then evaluation will give incorrect results. Default is None. post_tfm_kwargs : dict Keyword arguments to post-processing transform. @@ -88,10 +87,10 @@ def eval( """ # ---- pre-conditions ---------------------------------------------------------------------------------------------- for path, path_name in zip( - (checkpoint_path, labelmap_path, spect_scaler_path), - ("checkpoint_path", "labelmap_path", "spect_scaler_path"), + (checkpoint_path, labelmap_path, frames_standardizer_path), + ("checkpoint_path", "labelmap_path", "frames_standardizer_path"), ): - if path is not None: # because `spect_scaler_path` is optional + if path is not None: # because `frames_standardizer_path` is optional if not validators.is_a_file(path): raise FileNotFoundError( f"value for ``{path_name}`` not recognized as a file: {path}" @@ -120,8 +119,7 @@ def eval( labelmap_path=labelmap_path, output_dir=output_dir, num_workers=num_workers, - split=split, - spect_scaler_path=spect_scaler_path, + frames_standardizer_path=frames_standardizer_path, post_tfm_kwargs=post_tfm_kwargs, ) elif model_family == "ParametricUMAPModel": @@ -133,7 +131,6 @@ def eval( output_dir=output_dir, batch_size=batch_size, num_workers=num_workers, - split=split, ) else: raise ValueError(f"Model family not recognized: {model_family}") diff --git a/src/vak/eval/frame_classification.py b/src/vak/eval/frame_classification.py index 60be44b3e..ea20b72e6 100644 --- a/src/vak/eval/frame_classification.py +++ b/src/vak/eval/frame_classification.py @@ -9,13 +9,13 @@ from datetime import datetime import joblib -import pandas as pd import lightning +import pandas as pd import torch.utils.data -from .. import datasets, models, transforms +from .. import datapipes, datasets, models, transforms from ..common import validators -from ..datasets.frame_classification import FramesDataset +from ..datapipes.frame_classification import InferDatapipe logger = logging.getLogger(__name__) @@ -28,8 +28,7 @@ def eval_frame_classification_model( labelmap_path: str | pathlib.Path, output_dir: str | pathlib.Path, num_workers: int, - split: str = "test", - spect_scaler_path: str | pathlib.Path = None, + frames_standardizer_path: str | pathlib.Path = None, post_tfm_kwargs: dict | None = None, ) -> None: """Evaluate a trained model. @@ -54,14 +53,11 @@ def eval_frame_classification_model( num_workers : int Number of processes to use for parallel loading of data. Argument to torch.DataLoader. Default is 2. - split : str - Split of dataset on which model should be evaluated. - One of {'train', 'val', 'test'}. Default is 'test'. - spect_scaler_path : str, pathlib.Path - Path to a saved SpectScaler object used to normalize spectrograms. - If spectrograms were normalized and this is not provided, will give - incorrect results. - Default is None. + frames_standardizer_path : str, pathlib.Path + Path to a saved :class:`vak.transforms.FramesStandardizer` + object used to standardize (normalize) frames. + If frames were standardized during training and this is not provided, + will give incorrect results. Default is None. post_tfm_kwargs : dict Keyword arguments to post-processing transform. If None, then no additional clean-up is applied @@ -77,7 +73,7 @@ def eval_frame_classification_model( Notes ----- - Note that unlike :func:`core.predict`, this function + Note that unlike :func:`~vak.predict.predict`, this function can modify ``labelmap`` so that metrics like edit distance are correctly computed, by converting any string labels in ``labelmap`` with multiple characters @@ -86,30 +82,22 @@ def eval_frame_classification_model( """ # ---- pre-conditions ---------------------------------------------------------------------------------------------- for path, path_name in zip( - (checkpoint_path, labelmap_path, spect_scaler_path), - ("checkpoint_path", "labelmap_path", "spect_scaler_path"), + (checkpoint_path, labelmap_path, frames_standardizer_path), + ("checkpoint_path", "labelmap_path", "frames_standardizer_path"), ): - if path is not None: # because `spect_scaler_path` is optional + if path is not None: # because `frames_standardizer_path` is optional if not validators.is_a_file(path): raise FileNotFoundError( f"value for ``{path_name}`` not recognized as a file: {path}" ) - dataset_path = pathlib.Path(dataset_config["path"]) - if not dataset_path.exists() or not dataset_path.is_dir(): - raise NotADirectoryError( - f"`dataset_path` not found or not recognized as a directory: {dataset_path}" + model_name = model_config["name"] # we use this var again below + if "window_size" not in dataset_config["params"]: + raise KeyError( + f"The `dataset_config` for frame classification model '{model_name}' must include a 'params' sub-table " + f"that sets a value for 'window_size', but received a `dataset_config` that did not:\n{dataset_config}" ) - # we unpack `frame_dur` to log it, regardless of whether we use it with post_tfm below - metadata = datasets.frame_classification.Metadata.from_dataset_path( - dataset_path - ) - frame_dur = metadata.frame_dur - logger.info( - f"Duration of a frame in dataset, in seconds: {frame_dur}", - ) - if not validators.is_a_directory(output_dir): raise NotADirectoryError( f"value for ``output_dir`` not recognized as a directory: {output_dir}" @@ -118,42 +106,59 @@ def eval_frame_classification_model( # ---- get time for .csv file -------------------------------------------------------------------------------------- timenow = datetime.now().strftime("%y%m%d_%H%M%S") - # ---------------- load data for evaluation ------------------------------------------------------------------------ - if spect_scaler_path: - logger.info(f"loading spect scaler from path: {spect_scaler_path}") - spect_standardizer = joblib.load(spect_scaler_path) + # ---- load what we need to transform data ------------------------------------------------------------------------- + if frames_standardizer_path: + logger.info( + f"loading frames standardizer from path: {frames_standardizer_path}" + ) + frames_standardizer = joblib.load(frames_standardizer_path) else: - logger.info("not using a spect scaler") - spect_standardizer = None + logger.info("No `frames_standardizer_path` provided, not standardizing frames.") + frames_standardizer = None logger.info(f"loading labelmap from path: {labelmap_path}") with labelmap_path.open("r") as f: labelmap = json.load(f) - model_name = model_config["name"] - # TODO: move this into datapipe once each datapipe uses a fixed set of transforms - # that will require adding `spect_standardizer`` as a parameter to the datapipe, - # maybe rename to `frames_standardizer`? - try: - window_size = dataset_config["params"]["window_size"] - except KeyError as e: - raise KeyError( - f"The `dataset_config` for frame classification model '{model_name}' must include a 'params' sub-table " - f"that sets a value for 'window_size', but received a `dataset_config` that did not:\n{dataset_config}" - ) from e - transform_params = { - "spect_standardizer": spect_standardizer, - "window_size": window_size, - } - - item_transform = transforms.defaults.get_default_transform( - model_name, "eval", transform_params - ) - val_dataset = FramesDataset.from_dataset_path( - dataset_path=dataset_path, - split=split, - item_transform=item_transform, - ) + # ---------------- load data for evaluation ------------------------------------------------------------------------ + if "split" in dataset_config["params"]: + split = dataset_config["params"]["split"] + # we do this convoluted thing to avoid 'TypeError: Dataset got multiple values for split` + del dataset_config["params"]["split"] + else: + split = "test" + # ---- *not* using a built-in dataset ------------------------------------------------------------------------------ + if dataset_config["name"] is None: + dataset_path = pathlib.Path(dataset_config["path"]) + if not dataset_path.exists() or not dataset_path.is_dir(): + raise NotADirectoryError( + f"`dataset_path` not found or not recognized as a directory: {dataset_path}" + ) + + # we unpack `frame_dur` to log it, regardless of whether we use it with post_tfm below + metadata = datapipes.frame_classification.Metadata.from_dataset_path( + dataset_path + ) + frame_dur = metadata.frame_dur + logger.info( + f"Duration of a frame in dataset, in seconds: {frame_dur}", + ) + val_dataset = InferDatapipe.from_dataset_path( + dataset_path=dataset_path, + split=split, + window_size=dataset_config["params"]["window_size"], + frames_standardizer=frames_standardizer, + return_padding_mask=True, + ) + # ---- *yes* using a built-in dataset ------------------------------------------------------------------------------# ---- *yes* using a built-in dataset ------------------------------------------------------------------------------ + else: + dataset_config["params"]["return_padding_mask"] = True + val_dataset = datasets.get( + dataset_config, + split=split, + frames_standardizer=frames_standardizer, + ) + val_loader = torch.utils.data.DataLoader( dataset=val_dataset, shuffle=False, @@ -190,12 +195,14 @@ def eval_frame_classification_model( model.load_state_dict_from_path(checkpoint_path) - trainer_logger = lightning.pytorch.loggers.TensorBoardLogger(save_dir=output_dir) + trainer_logger = lightning.pytorch.loggers.TensorBoardLogger( + save_dir=output_dir + ) trainer = lightning.pytorch.Trainer( accelerator=trainer_config["accelerator"], devices=trainer_config["devices"], - logger=trainer_logger - ) + logger=trainer_logger, + ) # TODO: check for hasattr(model, test_step) and if so run test # below, [0] because validate returns list of dicts, length of no. of val loaders metric_vals = trainer.validate(model, dataloaders=val_loader)[0] @@ -211,7 +218,7 @@ def eval_frame_classification_model( ("model_name", model_name), ("checkpoint_path", checkpoint_path), ("labelmap_path", labelmap_path), - ("spect_scaler_path", spect_scaler_path), + ("frames_standardizer_path", frames_standardizer_path), ("dataset_path", dataset_path), ] ) diff --git a/src/vak/eval/parametric_umap.py b/src/vak/eval/parametric_umap.py index cf4b13f41..215821d8b 100644 --- a/src/vak/eval/parametric_umap.py +++ b/src/vak/eval/parametric_umap.py @@ -7,13 +7,13 @@ from collections import OrderedDict from datetime import datetime -import pandas as pd import lightning +import pandas as pd import torch.utils.data -from .. import models, transforms +from .. import models from ..common import validators -from ..datasets.parametric_umap import ParametricUMAPDataset +from ..datapipes.parametric_umap import Datapipe logger = logging.getLogger(__name__) @@ -26,7 +26,6 @@ def eval_parametric_umap_model( batch_size: int, num_workers: int, trainer_config: dict, - split: str = "test", ) -> None: """Evaluate a trained model. @@ -59,7 +58,7 @@ def eval_parametric_umap_model( (checkpoint_path,), ("checkpoint_path",), ): - if path is not None: # because `spect_scaler_path` is optional + if path is not None: # because `frames_standardizer_path` is optional if not validators.is_a_file(path): raise FileNotFoundError( f"value for ``{path_name}`` not recognized as a file: {path}" @@ -83,14 +82,14 @@ def eval_parametric_umap_model( timenow = datetime.now().strftime("%y%m%d_%H%M%S") # ---------------- load data for evaluation ------------------------------------------------------------------------ + if "split" in dataset_config["params"]: + split = dataset_config["params"]["split"] + else: + split = "test" model_name = model_config["name"] - item_transform = transforms.defaults.get_default_transform( - model_name, "eval" - ) - val_dataset = ParametricUMAPDataset.from_dataset_path( + val_dataset = Datapipe.from_dataset_path( dataset_path=dataset_path, split=split, - transform=item_transform, **dataset_config["params"], ) val_loader = torch.utils.data.DataLoader( @@ -111,12 +110,14 @@ def eval_parametric_umap_model( model.load_state_dict_from_path(checkpoint_path) - trainer_logger = lightning.pytorch.loggers.TensorBoardLogger(save_dir=output_dir) + trainer_logger = lightning.pytorch.loggers.TensorBoardLogger( + save_dir=output_dir + ) trainer = lightning.pytorch.Trainer( accelerator=trainer_config["accelerator"], devices=trainer_config["devices"], - logger=trainer_logger - ) + logger=trainer_logger, + ) # TODO: check for hasattr(model, test_step) and if so run test # below, [0] because validate returns list of dicts, length of no. of val loaders metric_vals = trainer.validate(model, dataloaders=val_loader)[0] diff --git a/src/vak/learncurve/frame_classification.py b/src/vak/learncurve/frame_classification.py index fc95d59bb..e48671950 100644 --- a/src/vak/learncurve/frame_classification.py +++ b/src/vak/learncurve/frame_classification.py @@ -7,7 +7,7 @@ import pandas as pd -from .. import common, datasets +from .. import common, datapipes from ..common.converters import expanded_user_path from ..eval.frame_classification import eval_frame_classification_model from ..train.frame_classification import train_frame_classification_model @@ -25,7 +25,7 @@ def learning_curve_for_frame_classification_model( num_workers: int, results_path: str | pathlib.Path, post_tfm_kwargs: dict | None = None, - normalize_spectrograms: bool = True, + standardize_frames: bool = True, shuffle: bool = True, val_step: int | None = None, ckpt_step: int | None = None, @@ -89,9 +89,9 @@ def learning_curve_for_frame_classification_model( That function defaults to 'cuda' if torch.cuda.is_available is True. shuffle: bool if True, shuffle training data before each epoch. Default is True. - normalize_spectrograms : bool - if True, use spect.utils.data.SpectScaler to normalize the spectrograms. - Normalization is done by subtracting off the mean for each frequency bin + standardize_frames : bool + if True, use :class:`vak.transforms.FramesStandardizer` to standardize the frames. + Normalization is done by subtracting off the mean for each row of the training set and then dividing by the std for that frequency bin. This same normalization is then applied to validation + test data. val_step : int @@ -119,7 +119,7 @@ def learning_curve_for_frame_classification_model( logger.info( f"Loading dataset from path: {dataset_path}", ) - metadata = datasets.frame_classification.Metadata.from_dataset_path( + metadata = datapipes.frame_classification.Metadata.from_dataset_path( dataset_path ) dataset_csv_path = dataset_path / metadata.dataset_csv_filename @@ -196,7 +196,7 @@ def learning_curve_for_frame_classification_model( num_epochs, num_workers, results_path=results_path_this_replicate, - normalize_spectrograms=normalize_spectrograms, + standardize_frames=standardize_frames, shuffle=shuffle, val_step=val_step, ckpt_step=ckpt_step, @@ -228,15 +228,15 @@ def learning_curve_for_frame_classification_model( logger.info(f"Using checkpoint: {ckpt_path}") labelmap_path = results_path_this_replicate.joinpath("labelmap.json") logger.info(f"Using labelmap: {labelmap_path}") - if normalize_spectrograms: - spect_scaler_path = results_path_this_replicate.joinpath( - "StandardizeSpect" + if standardize_frames: + frames_standardizer_path = results_path_this_replicate.joinpath( + "FramesStandardizer" ) logger.info( - f"Using spect scaler to normalize: {spect_scaler_path}", + f"Using FramesStandardizer to standardize frames, from path: {frames_standardizer_path}", ) else: - spect_scaler_path = None + frames_standardizer_path = None eval_frame_classification_model( model_config, @@ -246,8 +246,7 @@ def learning_curve_for_frame_classification_model( labelmap_path, results_path_this_replicate, num_workers, - "test", - spect_scaler_path, + frames_standardizer_path, post_tfm_kwargs, ) diff --git a/src/vak/learncurve/learncurve.py b/src/vak/learncurve/learncurve.py index def4b722f..3b41f7e7f 100644 --- a/src/vak/learncurve/learncurve.py +++ b/src/vak/learncurve/learncurve.py @@ -21,7 +21,7 @@ def learning_curve( num_workers: int, results_path: str | pathlib.Path = None, post_tfm_kwargs: dict | None = None, - normalize_spectrograms: bool = True, + standardize_frames: bool = True, shuffle: bool = True, val_step: int | None = None, ckpt_step: int | None = None, @@ -79,9 +79,9 @@ def learning_curve( these arguments and how they work. shuffle: bool if True, shuffle training data before each epoch. Default is True. - normalize_spectrograms : bool - if True, use spect.utils.data.SpectScaler to normalize the spectrograms. - Normalization is done by subtracting off the mean for each frequency bin + standardize_frames : bool + if True, use :class:`vak.transforms.FramesStandardizer` to standardize the frames. + Normalization is done by subtracting off the mean for each row of the training set and then dividing by the std for that frequency bin. This same normalization is then applied to validation + test data. val_step : int @@ -123,7 +123,7 @@ def learning_curve( num_workers=num_workers, results_path=results_path, post_tfm_kwargs=post_tfm_kwargs, - normalize_spectrograms=normalize_spectrograms, + standardize_frames=standardize_frames, shuffle=shuffle, val_step=val_step, ckpt_step=ckpt_step, diff --git a/src/vak/models/__init__.py b/src/vak/models/__init__.py index 50404ccc1..294f6ac84 100644 --- a/src/vak/models/__init__.py +++ b/src/vak/models/__init__.py @@ -1,8 +1,8 @@ from . import decorator, definition, factory, registry -from .factory import ModelFactory from .convencoder_umap import ConvEncoderUMAP from .decorator import model from .ed_tcn import ED_TCN +from .factory import ModelFactory from .frame_classification_model import FrameClassificationModel from .get import get from .parametric_umap_model import ParametricUMAPModel diff --git a/src/vak/models/decorator.py b/src/vak/models/decorator.py index 1b3911f1f..11401a11d 100644 --- a/src/vak/models/decorator.py +++ b/src/vak/models/decorator.py @@ -11,16 +11,16 @@ from __future__ import annotations -from typing import Type, TYPE_CHECKING +from typing import TYPE_CHECKING, Type import lightning -from .definition import validate as validate_definition from .registry import register_model if TYPE_CHECKING: from .factory import ModelFactory + class ModelDefinitionValidationError(Exception): """Exception raised when validating a model definition fails. @@ -75,10 +75,7 @@ def model(family: lightning.pytorch.LightningModule): def _model(definition: Type) -> ModelFactory: from .factory import ModelFactory # avoid circular import - model_factory = ModelFactory( - definition, - family - ) + model_factory = ModelFactory(definition, family) model_factory.__name__ = definition.__name__ model_factory.__doc__ = definition.__doc__ model_factory.__module__ = definition.__module__ diff --git a/src/vak/models/definition.py b/src/vak/models/definition.py index a44db3108..3adb0c1c2 100644 --- a/src/vak/models/definition.py +++ b/src/vak/models/definition.py @@ -128,7 +128,7 @@ def validate(definition: Type) -> Type: ``vak.models.Model``. It's also used by :class:`vak.models.ModelFactory`, - to validate a definition before building + to validate a definition before building a new model instance from the definition. """ # need to set this default first diff --git a/src/vak/models/factory.py b/src/vak/models/factory.py index d5ff274ce..4852a725d 100644 --- a/src/vak/models/factory.py +++ b/src/vak/models/factory.py @@ -8,8 +8,8 @@ import lightning import torch -from .definition import validate as validate_definition from .decorator import ModelDefinitionValidationError +from .definition import validate as validate_definition class ModelFactory: @@ -30,10 +30,11 @@ class ModelFactory: for more detail. """ - def __init__(self, - definition: Type, - family: lightning.pytorch.LightningModule, - ) -> None: + def __init__( + self, + definition: Type, + family: lightning.pytorch.LightningModule, + ) -> None: if not issubclass(family, lightning.pytorch.LightningModule): raise TypeError( "The ``family`` argument to the ``vak.models.model`` decorator" @@ -112,7 +113,9 @@ def attributes_from_config(self, config: dict): optimizer_kwargs = config.get( "optimizer", self.definition.default_config["optimizer"] ) - optimizer = self.definition.optimizer(params=params, **optimizer_kwargs) + optimizer = self.definition.optimizer( + params=params, **optimizer_kwargs + ) if inspect.isclass(self.definition.loss): loss_kwargs = config.get( @@ -386,11 +389,20 @@ def from_config(self, config: dict, **kwargs): that are initialized using parameters from ``config``. """ network, loss, optimizer, metrics = self.attributes_from_config(config) - network, loss, optimizer, metrics = self.validate_instances_or_get_default( - network, loss, optimizer, metrics, + network, loss, optimizer, metrics = ( + self.validate_instances_or_get_default( + network, + loss, + optimizer, + metrics, + ) ) return self.family( - network=network, loss=loss, optimizer=optimizer, metrics=metrics, **kwargs + network=network, + loss=loss, + optimizer=optimizer, + metrics=metrics, + **kwargs, ) def from_instances( @@ -424,9 +436,18 @@ def from_instances( to ``Callable`` functions, used to measure performance of the model. """ - network, loss, optimizer, metrics = self.validate_instances_or_get_default( - network, loss, optimizer, metrics, + network, loss, optimizer, metrics = ( + self.validate_instances_or_get_default( + network, + loss, + optimizer, + metrics, + ) ) return self.family( - network=network, loss=loss, optimizer=optimizer, metrics=metrics, **kwargs + network=network, + loss=loss, + optimizer=optimizer, + metrics=metrics, + **kwargs, ) diff --git a/src/vak/models/frame_classification_model.py b/src/vak/models/frame_classification_model.py index 8c2e29d20..edd691685 100644 --- a/src/vak/models/frame_classification_model.py +++ b/src/vak/models/frame_classification_model.py @@ -11,7 +11,7 @@ import lightning import torch -from .. import transforms +from .. import common, transforms from ..common import labels from .registry import model_family @@ -51,7 +51,7 @@ class FrameClassificationModel(lightning.LightningModule): or a ``dict`` that maps human-readable string names to a set of such instances. loss : torch.nn.Module, callable - An instance of a ``torch.nn.Module`` + An instance of a ``torch.nn.Module` ` that implements a loss function, or a callable Python function that computes a scalar loss. @@ -74,10 +74,10 @@ class FrameClassificationModel(lightning.LightningModule): to compute metrics that require strings, such as edit distance. If ``labelmap`` contains keys with multiple characters, this will be ``labelmap`` re-mapped so that all labels have - single characters (except "unlabeled"), to avoid artificially - changing the edit distance. + single characters (except the background label, if specified), + to avoid artificially changing the edit distance. See https://github.com/vocalpy/vak/issues/373 for more detail. - If all keys (except "unlabeled") are single-character, + If all keys (except background label) are single-character, then ``eval_labelmap`` will just be ``labelmap``. to_labels_eval : vak.transforms.frame_labels.ToLabels Instance of :class:`~vak.transforms.frame_labels.ToLabels` @@ -85,6 +85,7 @@ class FrameClassificationModel(lightning.LightningModule): to string labels inside of ``validation_step``, for computing edit distance. """ + def __init__( self, labelmap: Mapping, @@ -93,6 +94,7 @@ def __init__( optimizer: torch.optim.Optimizer | None = None, metrics: dict | None = None, post_tfm: Callable | None = None, + background_label = common.constants.DEFAULT_BACKGROUND_LABEL, ): """Initialize a new instance of a :class:`~vak.models.frame_classification_model.FrameClassificationModel`. @@ -121,6 +123,11 @@ def __init__( performance of the model. post_tfm : callable Post-processing transform applied to predictions. + background_label: str, optional + The string label applied to segments belonging to the + background class. + Default is + :const:`vak.common.constants.DEFAULT_BACKGROUND_LABEL`. """ super().__init__() @@ -134,7 +141,7 @@ def __init__( # with single-character labels # so that we do not affect edit distance computation # see https://github.com/NickleDave/vak/issues/373 - labelmap_keys = [lbl for lbl in labelmap.keys() if lbl != "unlabeled"] + labelmap_keys = [lbl for lbl in labelmap.keys() if lbl != background_label] if any( [len(label) > 1 for label in labelmap_keys] ): # only re-map if necessary @@ -199,10 +206,51 @@ def training_step(self, batch: tuple, batch_idx: int): Scalar loss value computed by the loss function, ``self.loss``. """ - frames, frame_labels = batch["frames"], batch["frame_labels"] - out = self.network(frames) - loss = self.loss(out, frame_labels) - self.log("train_loss", loss, on_step=True) + frames = batch["frames"] + + # we repeat this code in validation step + # because I'm assuming it's faster than a call to a staticmethod that factors it out + if ( # multi-class frame classificaton + "multi_frame_labels" in batch and + "binary_frame_labels" not in batch and + "boundary_frame_labels" not in batch + ): + target_types = ("multi_frame_labels",) + elif ( # binary frame classification + "binary_frame_labels" in batch and + "multi_frame_labels" not in batch and + "boundary_frame_labels" not in batch + ): + target_types = ("binary_frame_labels",) + elif ( # boundary "detection" -- i.e. different kind of binary frame classification + "boundary_frame_labels" in batch and + "multi_frame_labels" not in batch and + "binary_frame_labels" not in batch + ): + target_types = ("boundary_frame_labels",) + elif ( # multi-class frame classification *and* boundary detection + "multi_frame_labels" in batch and + "boundary_frame_labels" in batch and + "binary_frame_labels" not in batch + ): + target_types = ("multi_frame_labels", "boundary_frame_labels") + + if len(target_types) == 1: + class_logits = self.network(frames) + loss = self.loss(class_logits, batch[target_types[0]]) + self.log("train_loss", loss, on_step=True) + else: + multi_logits, boundary_logits = self.network(frames) + loss = self.loss( + multi_logits, boundary_logits, batch["multi_frame_labels"], batch["boundary_frame_labels"] + ) + if isinstance(loss, torch.Tensor): + self.log("train_loss", loss, on_step=True) + elif isinstance(loss, dict): + # this provides a mechanism to values for all terms of a loss function with multiple terms + for loss_name, loss_val in loss.items(): + self.log(f"train_{loss_name}", loss_val, on_step=True) + return loss def validation_step(self, batch: tuple, batch_idx: int): @@ -222,28 +270,68 @@ def validation_step(self, batch: tuple, batch_idx: int): ------- None """ - x, y = batch["frames"], batch["frame_labels"] - # remove "batch" dimension added by collate_fn to x - # we keep for y because loss still expects the first dimension to be batch + frames = batch["frames"] + # remove "batch" dimension added by collate_fn to frames # TODO: fix this weirdness. Diff't collate_fn? - if x.ndim in (5, 4): - if x.shape[0] == 1: - x = torch.squeeze(x, dim=0) + if frames.ndim in (5, 4): + if frames.shape[0] == 1: + frames = torch.squeeze(frames, dim=0) + else: + raise ValueError(f"invalid shape for frames: {frames.shape}") + + # we repeat this code in training step + # because I'm assuming it's faster than a call to a staticmethod that factors it out + if ( # multi-class frame classificaton + "multi_frame_labels" in batch and + "binary_frame_labels" not in batch and + "boundary_frame_labels" not in batch + ): + target_types = ("multi_frame_labels",) + elif ( # binary frame classification + "binary_frame_labels" in batch and + "multi_frame_labels" not in batch and + "boundary_frame_labels" not in batch + ): + target_types = ("binary_frame_labels",) + elif ( # boundary "detection" -- i.e. different kind of binary frame classification + "boundary_frame_labels" in batch and + "multi_frame_labels" not in batch and + "binary_frame_labels" not in batch + ): + target_types = ("boundary_frame_labels",) + elif ( # multi-class frame classification *and* boundary detection + "multi_frame_labels" in batch and + "boundary_frame_labels" in batch and + "binary_frame_labels" not in batch + ): + target_types = ("multi_frame_labels", "boundary_frame_labels") + + if len(target_types) == 1: + class_logits = self.network(frames) + boundary_logits = None else: - raise ValueError(f"invalid shape for x: {x.shape}") + class_logits, boundary_logits = self.network(frames) - out = self.network(x) # permute and flatten out # so that it has shape (1, number classes, number of time bins) # ** NOTICE ** just calling out.reshape(1, out.shape(1), -1) does not work, it will change the data - out = out.permute(1, 0, 2) - out = torch.flatten(out, start_dim=1) - out = torch.unsqueeze(out, dim=0) + class_logits = class_logits.permute(1, 0, 2) + class_logits = torch.flatten(class_logits, start_dim=1) + class_logits = torch.unsqueeze(class_logits, dim=0) # reduce to predictions, assuming class dimension is 1 - y_pred = torch.argmax( - out, dim=1 + class_preds = torch.argmax( + class_logits, dim=1 ) # y_pred has dims (batch size 1, predicted label per time bin) + if boundary_logits is not None: + boundary_logits = boundary_logits.permute(1, 0, 2) + boundary_logits = torch.flatten(boundary_logits, start_dim=1) + boundary_logits = torch.unsqueeze(boundary_logits, dim=0) + # reduce to predictions, assuming class dimension is 1 + boundary_preds = torch.argmax( + boundary_logits, dim=1 + ) # y_pred has dims (batch size 1, predicted label per time bin) + if "padding_mask" in batch: padding_mask = batch[ "padding_mask" @@ -258,53 +346,110 @@ def validation_step(self, batch: tuple, batch_idx: int): f"invalid shape for padding mask: {padding_mask.shape}" ) - out = out[:, :, padding_mask] - y_pred = y_pred[:, padding_mask] + class_logits = class_logits[:, :, padding_mask] + class_preds = class_preds[:, padding_mask] - y_labels = self.to_labels_eval(y.cpu().numpy()) - y_pred_labels = self.to_labels_eval(y_pred.cpu().numpy()) + if boundary_logits is not None: + boundary_logits = boundary_logits[:, :, padding_mask] + boundary_preds = boundary_preds[:, padding_mask] - if self.post_tfm: - y_pred_tfm = self.post_tfm( - y_pred.cpu().numpy(), - ) - y_pred_tfm_labels = self.to_labels_eval(y_pred_tfm) - # convert back to tensor so we can compute accuracy - y_pred_tfm = torch.from_numpy(y_pred_tfm).to(self.device) + if "multi_frame_labels" in target_types: + multi_frame_labels_str = self.to_labels_eval(batch["multi_frame_labels"].cpu().numpy()) + class_preds_str = self.to_labels_eval(class_preds.cpu().numpy()) + + if self.post_tfm: + class_preds_tfm = self.post_tfm( + class_preds.cpu().numpy(), + ) + class_preds_tfm_str = self.to_labels_eval(class_preds_tfm) + # convert back to tensor so we can compute accuracy + class_preds_tfm = torch.from_numpy(class_preds_tfm).to(self.device) + + if len(target_types) == 1: + target = batch[target_types[0]] + else: + target = {target_type: batch[target_type] for target_type in target_types} - # TODO: figure out smarter way to do this for metric_name, metric_callable in self.metrics.items(): if metric_name == "loss": - self.log( - f"val_{metric_name}", - metric_callable(out, y), - batch_size=1, - on_step=True, - sync_dist=True, - ) + if len(target_types) == 1: + self.log( + f"val_{metric_name}", + metric_callable(class_logits, target), + batch_size=1, + on_step=True, + sync_dist=True, + ) + else: + loss = self.loss( + class_logits, boundary_logits, batch["multi_frame_labels"], batch["boundary_frame_labels"] + ) + if isinstance(loss, torch.Tensor): + self.log( + f"val_{metric_name}", + loss, + batch_size=1, + on_step=True, + sync_dist=True, + ) + elif isinstance(loss, dict): + # this provides a mechanism to values for all terms of a loss function with multiple terms + for loss_name, loss_val in loss.items(): + self.log( + f"val_{loss_name}", + loss_val, + batch_size=1, + on_step=True, + sync_dist=True, + ) elif metric_name == "acc": - self.log( - f"val_{metric_name}", - metric_callable(y_pred, y), - batch_size=1, - on_step=True, - sync_dist=True, - ) - if self.post_tfm: + if len(target_types) == 1: self.log( - f"val_{metric_name}_tfm", - metric_callable(y_pred_tfm, y), + f"val_{metric_name}", + metric_callable(class_preds, target), + batch_size=1, + on_step=True, + sync_dist=True, + ) + if self.post_tfm and "multi_frame_labels" in target_types: + self.log( + f"val_{metric_name}_tfm", + metric_callable(class_preds_tfm, target), + batch_size=1, + on_step=True, + sync_dist=True, + ) + else: + self.log( + f"val_{metric_name}", + metric_callable(class_preds, target["multi_frame_labels"]), batch_size=1, on_step=True, sync_dist=True, ) + self.log( + f"val_boundary_{metric_name}", + metric_callable(boundary_preds, target["boundary_frame_labels"]), + batch_size=1, + on_step=True, + sync_dist=True, + ) + if self.post_tfm and "multi_frame_labels" in target_types: + self.log( + f"val_multi_{metric_name}_tfm", + metric_callable(class_preds_tfm, target["multi_frame_labels"]), + batch_size=1, + on_step=True, + sync_dist=True, + ) elif ( metric_name == "levenshtein" or metric_name == "character_error_rate" - ): + ) and "multi_frame_labels" in target_types: self.log( f"val_{metric_name}", - metric_callable(y_pred_labels, y_labels), + # next line: convert to float to squelch warning from lightning + float(metric_callable(class_preds_str, multi_frame_labels_str)), batch_size=1, on_step=True, sync_dist=True, @@ -312,7 +457,8 @@ def validation_step(self, batch: tuple, batch_idx: int): if self.post_tfm: self.log( f"val_{metric_name}_tfm", - metric_callable(y_pred_tfm_labels, y_labels), + # next line: convert to float to squelch warning from lightning + float(metric_callable(class_preds_tfm_str, multi_frame_labels_str)), batch_size=1, on_step=True, sync_dist=True, @@ -339,16 +485,16 @@ def predict_step(self, batch: tuple, batch_idx: int): containing the spectrogram for which a prediction was generated. """ - x, frames_path = batch["frames"].to(self.device), batch["frames_path"] + frames, frames_path = batch["frames"].to(self.device), batch["frames_path"] if isinstance(frames_path, list) and len(frames_path) == 1: frames_path = frames_path[0] # TODO: fix this weirdness. Diff't collate_fn? - if x.ndim in (5, 4): - if x.shape[0] == 1: - x = torch.squeeze(x, dim=0) + if frames.ndim in (5, 4): + if frames.shape[0] == 1: + frames = torch.squeeze(frames, dim=0) else: - raise ValueError(f"invalid shape for x: {x.shape}") - y_pred = self.network(x) + raise ValueError(f"invalid shape for `frames`: {frames.shape}") + y_pred = self.network(frames) return {frames_path: y_pred} def load_state_dict_from_path(self, ckpt_path): diff --git a/src/vak/models/parametric_umap_model.py b/src/vak/models/parametric_umap_model.py index b9a31474a..a597b9d38 100644 --- a/src/vak/models/parametric_umap_model.py +++ b/src/vak/models/parametric_umap_model.py @@ -48,9 +48,7 @@ def __init__( metrics: dict[str:Type], ): super().__init__() - self.network = torch.nn.ModuleDict( - network - ) + self.network = torch.nn.ModuleDict(network) self.loss = loss self.optimizer = optimizer self.metrics = metrics @@ -60,11 +58,11 @@ def configure_optimizers(self): def training_step(self, batch, batch_idx): (edges_to_exp, edges_from_exp) = batch - embedding_to = self.network['encoder'](edges_to_exp) - embedding_from = self.network['encoder'](edges_from_exp) + embedding_to = self.network["encoder"](edges_to_exp) + embedding_from = self.network["encoder"](edges_from_exp) - if 'decoder' in self.network: - reconstruction = self.network['decoder'](embedding_to) + if "decoder" in self.network: + reconstruction = self.network["decoder"](embedding_to) before_encoding = edges_to_exp else: reconstruction = None @@ -83,11 +81,11 @@ def training_step(self, batch, batch_idx): def validation_step(self, batch, batch_idx): (edges_to_exp, edges_from_exp) = batch - embedding_to = self.network['encoder'](edges_to_exp) - embedding_from = self.network['encoder'](edges_from_exp) + embedding_to = self.network["encoder"](edges_to_exp) + embedding_from = self.network["encoder"](edges_from_exp) - if 'decoder' in self.network is not None: - reconstruction = self.network['decoder'](embedding_to) + if "decoder" in self.network is not None: + reconstruction = self.network["decoder"](embedding_to) before_encoding = edges_to_exp else: reconstruction = None @@ -191,7 +189,7 @@ def fit( dataset_path: str | pathlib.Path, transform=None, ): - from vak.datasets.parametric_umap import ParametricUMAPDataset + from vak.datapipes.parametric_umap import ParametricUMAPDataset dataset = ParametricUMAPDataset.from_dataset_path( dataset_path, diff --git a/src/vak/models/registry.py b/src/vak/models/registry.py index f4fd0472b..3d6717820 100644 --- a/src/vak/models/registry.py +++ b/src/vak/models/registry.py @@ -6,8 +6,7 @@ from __future__ import annotations -import inspect -from typing import Any, Type, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Type import lightning diff --git a/src/vak/models/tweetynet.py b/src/vak/models/tweetynet.py index b9631be59..97b060312 100644 --- a/src/vak/models/tweetynet.py +++ b/src/vak/models/tweetynet.py @@ -11,7 +11,7 @@ import torch -from .. import metrics, nets +from .. import metrics, nets, nn from .decorator import model from .frame_classification_model import FrameClassificationModel @@ -56,12 +56,12 @@ class TweetyNet: """ network = nets.TweetyNet - loss = torch.nn.CrossEntropyLoss + loss = nn.loss.CrossEntropyLoss optimizer = torch.optim.Adam metrics = { "acc": metrics.Accuracy, "levenshtein": metrics.Levenshtein, "character_error_rate": metrics.CharacterErrorRate, - "loss": torch.nn.CrossEntropyLoss, + "loss": nn.loss.CrossEntropyLoss, } default_config = {"optimizer": {"lr": 0.003}} diff --git a/src/vak/nn/loss/__init__.py b/src/vak/nn/loss/__init__.py index 18f4e6d2f..59e0194ce 100644 --- a/src/vak/nn/loss/__init__.py +++ b/src/vak/nn/loss/__init__.py @@ -1,3 +1,4 @@ +from .crossentropy import CrossEntropyLoss from .dice import DiceLoss, dice_loss from .umap import UmapLoss, umap_loss diff --git a/src/vak/nn/loss/crossentropy.py b/src/vak/nn/loss/crossentropy.py new file mode 100644 index 000000000..eb0882629 --- /dev/null +++ b/src/vak/nn/loss/crossentropy.py @@ -0,0 +1,17 @@ +import torch + +class CrossEntropyLoss(torch.nn.CrossEntropyLoss): + """Wrapper around :class:`torch.nn.CrossEntropyLoss` + + Converts the argument ``weight`` to a :class:`torch.Tensor` + if it is a :class:`list`. + """ + def __init__(self, weight=None, size_average=None, ignore_index=-100, reduce=None, + reduction='mean', label_smoothing=0.0): + + if weight is not None: + if isinstance(weight, torch.Tensor): + pass + elif isinstance(weight, list): + weight = torch.Tensor(weight) + super().__init__(weight, size_average, ignore_index, reduce, reduction, label_smoothing) diff --git a/src/vak/predict/frame_classification.py b/src/vak/predict/frame_classification.py index e5a3cb6f7..57d06d852 100644 --- a/src/vak/predict/frame_classification.py +++ b/src/vak/predict/frame_classification.py @@ -7,35 +7,51 @@ import os import pathlib +from attrs import define import crowsetta import joblib -import numpy as np import lightning +import pandas as pd +import numpy as np import torch.utils.data from tqdm import tqdm -from .. import datasets, models, transforms +from .. import common, datapipes, datasets, models, transforms from ..common import constants, files, validators -from ..datasets.frame_classification import FramesDataset +from ..datapipes.frame_classification import InferDatapipe logger = logging.getLogger(__name__) + +@define +class AnnotationDataFrame: + """Data class that represents annotations + for an audio file, in a :class:`pandas.DataFrame`. + + Used to save annotations that currently can't + be saved with :mod:`crowsetta`, e.g. boundary times. + """ + df: pd.DataFrame + audio_path : str | pathlib.Path + + def predict_with_frame_classification_model( model_config: dict, dataset_config: dict, trainer_config: dict, - checkpoint_path, - labelmap_path, - num_workers=2, - timebins_key="t", - spect_scaler_path=None, - annot_csv_filename=None, - output_dir=None, - min_segment_dur=None, - majority_vote=False, - save_net_outputs=False, -): + checkpoint_path: str | pathlib.Path, + labelmap_path: str | pathlib.Path, + num_workers: int = 2, + timebins_key:str = "t", + frames_standardizer_path: str | pathlib.Path | None = None, + annot_csv_filename: str | None = None, + output_dir: str | pathlib.Path | None = None, + min_segment_dur: float | None = None, + majority_vote: bool = False, + save_net_outputs: bool = False, + background_label: str = common.constants.DEFAULT_BACKGROUND_LABEL, +) -> None: """Make predictions on a dataset with a trained :class:`~vak.models.FrameClassificationModel`. @@ -61,8 +77,8 @@ def predict_with_frame_classification_model( key for accessing spectrogram in files. Default is 's'. timebins_key : str key for accessing vector of time bins in files. Default is 't'. - spect_scaler_path : str - path to a saved SpectScaler object used to normalize spectrograms. + frames_standardizer_path : str + path to a saved :class:`vak.transforms.FramesStandardizer` object used to standardize (normalize) frames. If spectrograms were normalized and this is not provided, will give incorrect results. annot_csv_filename : str @@ -97,9 +113,10 @@ def predict_with_frame_classification_model( and the network is `TweetyNet`, then the net output file will be `gy6or6_032312_081416.tweetynet.output.npz`. """ + # ---- pre-conditions ---------------------------------------------------------------------------------------------- for path, path_name in zip( - (checkpoint_path, labelmap_path, spect_scaler_path), - ("checkpoint_path", "labelmap_path", "spect_scaler_path"), + (checkpoint_path, labelmap_path, frames_standardizer_path), + ("checkpoint_path", "labelmap_path", "frames_standardizer_path"), ): if path is not None: if not validators.is_a_file(path): @@ -107,12 +124,21 @@ def predict_with_frame_classification_model( f"value for ``{path_name}`` not recognized as a file: {path}" ) + model_name = model_config["name"] # we use this var again below + if "window_size" not in dataset_config["params"]: + raise KeyError( + f"The `dataset_config` for frame classification model '{model_name}' must include a 'params' sub-table " + f"that sets a value for 'window_size', but received a `dataset_config` that did not:\n{dataset_config}" + ) + dataset_path = pathlib.Path(dataset_config["path"]) if not dataset_path.exists() or not dataset_path.is_dir(): raise NotADirectoryError( f"`dataset_path` not found or not recognized as a directory: {dataset_path}" ) + # ---- set up directory to save output ----------------------------------------------------------------------------- + # we do this first to make sure we can save things if output_dir is None: output_dir = pathlib.Path(os.getcwd()) else: @@ -123,52 +149,74 @@ def predict_with_frame_classification_model( f"value specified for output_dir is not recognized as a directory: {output_dir}" ) - # ---------------- load data for prediction ------------------------------------------------------------------------ - if spect_scaler_path: - logger.info(f"loading SpectScaler from path: {spect_scaler_path}") - spect_standardizer = joblib.load(spect_scaler_path) + # ---- load what we need to transform data ------------------------------------------------------------------------- + if frames_standardizer_path: + logger.info( + f"loading FramesStandardizer from path: {frames_standardizer_path}" + ) + frames_standardizer = joblib.load(frames_standardizer_path) else: - logger.info("Not loading SpectScaler, no path was specified") - spect_standardizer = None - - model_name = model_config["name"] - # TODO: move this into datapipe once each datapipe uses a fixed set of transforms - # that will require adding `spect_standardizer`` as a parameter to the datapipe, - # maybe rename to `frames_standardizer`? - try: - window_size = dataset_config["params"]["window_size"] - except KeyError as e: - raise KeyError( - f"The `dataset_config` for frame classification model '{model_name}' must include a 'params' sub-table " - f"that sets a value for 'window_size', but received a `dataset_config` that did not:\n{dataset_config}" - ) from e - transform_params = { - "spect_standardizer": spect_standardizer, - "window_size": window_size, - } - item_transform = transforms.defaults.get_default_transform( - model_name, "predict", transform_params - ) + logger.info("Not loading FramesStandardizer, no path was specified") + frames_standardizer = None logger.info(f"loading labelmap from path: {labelmap_path}") with labelmap_path.open("r") as f: labelmap = json.load(f) - metadata = datasets.frame_classification.Metadata.from_dataset_path( - dataset_path - ) - dataset_csv_path = dataset_path / metadata.dataset_csv_filename + # ---------------- load data for prediction ------------------------------------------------------------------------ + if "split" in dataset_config["params"]: + split = dataset_config["params"]["split"] + # we do this convoluted thing to avoid 'TypeError: Dataset got multiple values for split` + del dataset_config["params"]["split"] + else: + split = "predict" + # ---- *not* using a built-in dataset ------------------------------------------------------------------------------ + if dataset_config["name"] is None: + metadata = datapipes.frame_classification.Metadata.from_dataset_path( + dataset_path + ) + dataset_csv_path = dataset_path / metadata.dataset_csv_filename + metadata = ( + datapipes.frame_classification.metadata.Metadata.from_dataset_path( + dataset_path + ) + ) + # we use this below to convert annotations from frames to seconds + frame_dur = metadata.frame_dur - logger.info( - f"loading dataset to predict from csv path: {dataset_csv_path}" - ) + logger.info( + f"loading dataset to predict from csv path: {dataset_csv_path}" + ) - # TODO: fix this when we build transforms into datasets; pass in `window_size` here - pred_dataset = FramesDataset.from_dataset_path( - dataset_path=dataset_path, - split="predict", - item_transform=item_transform, - ) + pred_dataset = InferDatapipe.from_dataset_path( + dataset_path=dataset_path, + split=split, + window_size=dataset_config["params"]["window_size"], + frames_standardizer=frames_standardizer, + return_padding_mask=True, + ) + # ---- *yes* using a built-in dataset ------------------------------------------------------------------------------ + else: + # we need "target_type" below when converting predictions to annotations, + # but fail early here if we don't have it + if "target_type" not in dataset_config["params"]: + from ..datasets.biosoundsegbench import VALID_TARGET_TYPES + raise ValueError( + "The dataset table in the configuration file requires a 'target_type' " + "when running predictions on built-in datasets. " + "Please add a key to the table whose value is a valid target type: " + f"{VALID_TARGET_TYPES}" + ) + dataset_config["params"]["return_padding_mask"] = True + # next line, required to be true regardless of split so we set it here + dataset_config["params"]["return_frames_path"] = True + pred_dataset = datasets.get( + dataset_config, + split=split, + frames_standardizer=frames_standardizer, + ) + # we use this below to convert annotations from frames to seconds + frame_dur = pred_dataset.frame_dur pred_loader = torch.utils.data.DataLoader( dataset=pred_dataset, @@ -178,20 +226,6 @@ def predict_with_frame_classification_model( num_workers=num_workers, ) - # ---------------- set up to convert predictions to annotation files ----------------------------------------------- - if annot_csv_filename is None: - annot_csv_filename = ( - pathlib.Path(dataset_path).stem + constants.ANNOT_CSV_SUFFIX - ) - annot_csv_path = pathlib.Path(output_dir).joinpath(annot_csv_filename) - logger.info(f"will save annotations in .csv file: {annot_csv_path}") - - metadata = ( - datasets.frame_classification.metadata.Metadata.from_dataset_path( - dataset_path - ) - ) - frame_dur = metadata.frame_dur logger.info( f"Duration of a frame in dataset, in seconds: {frame_dur}", ) @@ -222,11 +256,13 @@ def predict_with_frame_classification_model( ) model.load_state_dict_from_path(checkpoint_path) - trainer_logger = lightning.pytorch.loggers.TensorBoardLogger(save_dir=output_dir) + trainer_logger = lightning.pytorch.loggers.TensorBoardLogger( + save_dir=output_dir + ) trainer = lightning.pytorch.Trainer( accelerator=trainer_config["accelerator"], devices=trainer_config["devices"], - logger=trainer_logger + logger=trainer_logger, ) logger.info(f"running predict method of {model_name}") @@ -237,16 +273,43 @@ def predict_with_frame_classification_model( for result in results for frames_path, y_pred in result.items() } + + # ---------------- set up to convert predictions to annotation files ----------------------------------------------- + if dataset_config["name"] is None: + # we assume this default for now -- prep'd datasets are always multi-class frame label + target_type = "multi_frame_labels" + else: + # we made sure we have this above when determining the kind of dataset + target_type = dataset_config["params"]["target_type"] + if isinstance(target_type, str): + pass + elif isinstance(target_type, (list, tuple)): + target_type = tuple(sorted(target_type)) + + if annot_csv_filename is None: + annot_csv_filename = ( + pathlib.Path(dataset_path).stem + constants.ANNOT_CSV_SUFFIX + ) + annot_csv_path = pathlib.Path(output_dir).joinpath(annot_csv_filename) + logger.info(f"will save annotations in .csv file: {annot_csv_path}") + # ---------------- converting to annotations ------------------------------------------------------------------ progress_bar = tqdm(pred_loader) - input_type = ( - metadata.input_type - ) # we use this to get frame_times inside loop - if input_type == "audio": - audio_format = metadata.audio_format - elif input_type == "spect": - spect_format = metadata.spect_format + if dataset_config["name"] is None: + # we're using a user-prepped dataset, not a built-in dataset + # so assume we have metadata from above + input_type = ( + metadata.input_type + ) # we use this to get frame_times inside loop + if input_type == "audio": + audio_format = metadata.audio_format + elif input_type == "spect": + spect_format = metadata.spect_format + else: + input_type = "spect" # assume this for now + spect_format = common.constants.DEFAULT_SPECT_FORMAT + annots = [] logger.info("converting predictions to annotations") for ind, batch in enumerate(progress_bar): @@ -254,14 +317,22 @@ def predict_with_frame_classification_model( padding_mask = np.squeeze(padding_mask) if isinstance(frames_path, list) and len(frames_path) == 1: frames_path = frames_path[0] - y_pred = pred_dict[frames_path] + # we do all this basically to have clear naming below + if target_type == "multi_frame_labels" or target_type == "binary_frame_labels": + class_logits = pred_dict[frames_path] + boundary_logits = None + elif target_type == "boundary_frame_labels": + boundary_logits = pred_dict[frames_path] + class_logits = None + elif target_type == ("boundary_frame_labels", "multi_frame_labels"): + class_logits, boundary_logits = pred_dict[frames_path] if save_net_outputs: # not sure if there's a better way to get outputs into right shape; # can't just call y_pred.reshape() because that basically flattens the whole array first # meaning we end up with elements in the wrong order # so instead we convert to sequence then stack horizontally, on column axis - net_output = torch.hstack(y_pred.unbind()) + net_output = torch.hstack(class_logits.unbind()) net_output = net_output[:, padding_mask] net_output = net_output.cpu().numpy() net_output_path = output_dir.joinpath( @@ -270,8 +341,12 @@ def predict_with_frame_classification_model( ) np.savez(net_output_path, net_output) - y_pred = torch.argmax(y_pred, dim=1) # assumes class dimension is 1 - y_pred = torch.flatten(y_pred).cpu().numpy()[padding_mask] + if class_logits is not None: + class_preds = torch.argmax(class_logits, dim=1) # assumes class dimension is 1 + class_preds = torch.flatten(class_preds).cpu().numpy()[padding_mask] + if boundary_logits is not None: + boundary_preds = torch.argmax(boundary_logits, dim=1) # assumes class dimension is 1 + boundary_preds = torch.flatten(boundary_preds).cpu().numpy()[padding_mask] if input_type == "audio": frames, samplefreq = constants.AUDIO_FORMAT_FUNC_MAP[audio_format]( @@ -284,32 +359,106 @@ def predict_with_frame_classification_model( ) frame_times = spect_dict[timebins_key] - if majority_vote or min_segment_dur: - y_pred = transforms.frame_labels.postprocess( - y_pred, - timebin_dur=frame_dur, - min_segment_dur=min_segment_dur, - majority_vote=majority_vote, + # audio_fname is used for audio_path attribute of crowsetta.Annotation below + audio_fname = files.spect.find_audio_fname(frames_path) + if target_type == "multi_frame_labels" or target_type == "binary_frame_labels": + if majority_vote or min_segment_dur: + if background_label in labelmap: + background_label = labelmap[background_label] + elif "unlabeled" in labelmap: # some backward compatibility here + background_label = labelmap["unlabeled"] + else: + background_label = 0 # set a default value anyway just to not throw an error + class_preds = transforms.frame_labels.postprocess( + class_preds, + timebin_dur=frame_dur, + min_segment_dur=min_segment_dur, + majority_vote=majority_vote, + background_label=background_label, + ) + labels, onsets_s, offsets_s = transforms.frame_labels.to_segments( + class_preds, + labelmap=labelmap, + frame_times=frame_times, + ) + if labels is None and onsets_s is None and offsets_s is None: + # handle the case when all time bins are predicted to be unlabeled + # see https://github.com/NickleDave/vak/issues/383 + continue + seq = crowsetta.Sequence.from_keyword( + labels=labels, onsets_s=onsets_s, offsets_s=offsets_s ) - labels, onsets_s, offsets_s = transforms.frame_labels.to_segments( - y_pred, - labelmap=labelmap, - frame_times=frame_times, - ) - if labels is None and onsets_s is None and offsets_s is None: - # handle the case when all time bins are predicted to be unlabeled - # see https://github.com/NickleDave/vak/issues/383 - continue - seq = crowsetta.Sequence.from_keyword( - labels=labels, onsets_s=onsets_s, offsets_s=offsets_s - ) + annot = crowsetta.Annotation( + seq=seq, notated_path=audio_fname, annot_path=annot_csv_path.name + ) + annots.append(annot) - audio_fname = files.spect.find_audio_fname(frames_path) - annot = crowsetta.Annotation( - seq=seq, notated_path=audio_fname, annot_path=annot_csv_path.name - ) - annots.append(annot) + elif target_type == "boundary_frame_labels": + boundary_inds = transforms.frame_labels.boundary_inds_from_boundary_labels( + boundary_preds, + force_boundary_first_ind=True, + ) + boundary_times = frame_times[boundary_inds] # fancy indexing + df = pd.DataFrame.from_records({'boundary_time': boundary_times}) + annots.append( + AnnotationDataFrame(df=df, audio_path=audio_fname) + ) + elif target_type == ("boundary_frame_labels", "multi_frame_labels"): + if majority_vote is False: + logger.warn( + "`majority_vote` was set to False but `vak.predict.predict_with_frame_classification_model` " + "determined that this model predicts both multi-class labels and boundary labels, " + "so `majority_vote` will be set to True (to assign a single label to each segment determined by " + "a boundary)" + ) + if background_label in labelmap: + background_label = labelmap[background_label] + elif "unlabeled" in labelmap: # some backward compatibility here + background_label = labelmap["unlabeled"] + else: + background_label = 0 # set a default value anyway just to not throw an error + # Notice here we *always* call post-process, with majority_vote=True + # because we are using boundary labels + class_preds = transforms.frame_labels.postprocess( + frame_labels=class_preds, + timebin_dur=frame_dur, + min_segment_dur=min_segment_dur, + majority_vote=True, + background_label=background_label, + boundary_labels=boundary_preds, + ) + labels, onsets_s, offsets_s = transforms.frame_labels.to_segments( + class_preds, + labelmap=labelmap, + frame_times=frame_times, + ) + if labels is None and onsets_s is None and offsets_s is None: + # handle the case when all time bins are predicted to be unlabeled + # see https://github.com/NickleDave/vak/issues/383 + continue + if labels is None and onsets_s is None and offsets_s is None: + # handle the case when all time bins are predicted to be unlabeled + # see https://github.com/NickleDave/vak/issues/383 + continue + seq = crowsetta.Sequence.from_keyword( + labels=labels, onsets_s=onsets_s, offsets_s=offsets_s + ) - generic_seq = crowsetta.formats.seq.GenericSeq(annots=annots) - generic_seq.to_file(annot_path=annot_csv_path) + annot = crowsetta.Annotation( + seq=seq, notated_path=audio_fname, annot_path=annot_csv_path.name + ) + annots.append(annot) + + if all([isinstance(annot, crowsetta.Annotation) for annot in annots]): + generic_seq = crowsetta.formats.seq.GenericSeq(annots=annots) + generic_seq.to_file(annot_path=annot_csv_path) + elif all([isinstance(annot, AnnotationDataFrame) for annot in annots]): + df_out = [] + for sample_num, annot_df in enumerate(annots): + df = annot_df.df + df['audio_path'] = str(annot_df.audio_path) + df['sample_num'] = sample_num + df_out.append(df) + df_out = pd.concat(df_out) + df_out.to_csv(annot_csv_path, index=False) diff --git a/src/vak/predict/parametric_umap.py b/src/vak/predict/parametric_umap.py index 331b571b2..6e2694bc0 100644 --- a/src/vak/predict/parametric_umap.py +++ b/src/vak/predict/parametric_umap.py @@ -9,9 +9,9 @@ import lightning import torch.utils.data -from .. import datasets, models, transforms +from .. import datapipes, models from ..common import validators -from ..datasets.parametric_umap import ParametricUMAPDataset +from ..datapipes.parametric_umap import Datapipe logger = logging.getLogger(__name__) @@ -23,8 +23,6 @@ def predict_with_parametric_umap_model( checkpoint_path, num_workers=2, transform_params: dict | None = None, - dataset_params: dict | None = None, - timebins_key="t", output_dir=None, ): """Make predictions on a dataset with a trained @@ -46,14 +44,6 @@ def predict_with_parametric_umap_model( num_workers : int Number of processes to use for parallel loading of data. Argument to torch.DataLoader. Default is 2. - transform_params: dict, optional - Parameters for data transform. - Passed as keyword arguments. - Optional, default is None. - dataset_params: dict, optional - Parameters for dataset. - Passed as keyword arguments. - Optional, default is None. timebins_key : str key for accessing vector of time bins in files. Default is 't'. annot_csv_filename : str @@ -82,7 +72,7 @@ def predict_with_parametric_umap_model( logger.info( f"Loading metadata from dataset path: {dataset_path}", ) - metadata = datasets.frame_classification.Metadata.from_dataset_path( + metadata = datapipes.frame_classification.Metadata.from_dataset_path( dataset_path ) @@ -98,26 +88,15 @@ def predict_with_parametric_umap_model( # ---------------- load data for prediction ------------------------------------------------------------------------ model_name = model_config["name"] - # TODO: fix this when we build transforms into datasets - transform_params = { - "padding": dataset_config["params"].get( - "padding", - models.convencoder_umap.get_default_padding(metadata.shape), - ) - } - item_transform = transforms.defaults.get_default_transform( - model_name, "predict", transform_params - ) dataset_csv_path = dataset_path / metadata.dataset_csv_filename logger.info( f"loading dataset to predict from csv path: {dataset_csv_path}" ) - pred_dataset = ParametricUMAPDataset.from_dataset_path( + pred_dataset = Datapipe.from_dataset_path( dataset_path=dataset_path, split="predict", - transform=item_transform, **dataset_config["params"], ) @@ -153,11 +132,13 @@ def predict_with_parametric_umap_model( ) model.load_state_dict_from_path(checkpoint_path) - trainer_logger = lightning.pytorch.loggers.TensorBoardLogger(save_dir=output_dir) + trainer_logger = lightning.pytorch.loggers.TensorBoardLogger( + save_dir=output_dir + ) trainer = lightning.pytorch.Trainer( accelerator=trainer_config["accelerator"], devices=trainer_config["devices"], - logger=trainer_logger + logger=trainer_logger, ) logger.info(f"running predict method of {model_name}") diff --git a/src/vak/predict/predict_.py b/src/vak/predict/predict_.py index 2cc60ea61..432500097 100644 --- a/src/vak/predict/predict_.py +++ b/src/vak/predict/predict_.py @@ -22,7 +22,7 @@ def predict( labelmap_path: str | pathlib.Path, num_workers: int = 2, timebins_key: str = "t", - spect_scaler_path: str | pathlib.Path | None = None, + frames_standardizer_path: str | pathlib.Path | None = None, device: str | None = None, annot_csv_filename: str | None = None, output_dir: str | pathlib.Path | None = None, @@ -56,8 +56,8 @@ def predict( Argument to torch.DataLoader. Default is 2. timebins_key : str key for accessing vector of time bins in files. Default is 't'. - spect_scaler_path : str - path to a saved SpectScaler object used to normalize spectrograms. + frames_standardizer_path : str + path to a saved :class:`vak.transforms.FramesStandardizer` object used to standardize (normalize) frames. If spectrograms were normalized and this is not provided, will give incorrect results. annot_csv_filename : str @@ -93,8 +93,8 @@ def predict( will be `gy6or6_032312_081416.tweetynet.output.npz`. """ for path, path_name in zip( - (checkpoint_path, labelmap_path, spect_scaler_path), - ("checkpoint_path", "labelmap_path", "spect_scaler_path"), + (checkpoint_path, labelmap_path, frames_standardizer_path), + ("checkpoint_path", "labelmap_path", "frames_standardizer_path"), ): if path is not None: if not validators.is_a_file(path): @@ -137,7 +137,7 @@ def predict( labelmap_path=labelmap_path, num_workers=num_workers, timebins_key=timebins_key, - spect_scaler_path=spect_scaler_path, + frames_standardizer_path=frames_standardizer_path, annot_csv_filename=annot_csv_filename, output_dir=output_dir, min_segment_dur=min_segment_dur, diff --git a/src/vak/prep/frame_classification/frame_classification.py b/src/vak/prep/frame_classification/frame_classification.py index 8ce6d29fe..4b833a9c0 100644 --- a/src/vak/prep/frame_classification/frame_classification.py +++ b/src/vak/prep/frame_classification/frame_classification.py @@ -11,7 +11,7 @@ import crowsetta.formats.seq import pandas as pd -from ... import datasets +from ... import datapipes from ...common import labels from ...common.converters import expanded_user_path, labelset_to_set from ...common.logging import config_logging_for_cli, log_version @@ -310,7 +310,7 @@ def prep_frame_classification_dataset( dataset_df ) labelmap = labels.to_map( - labelset, map_unlabeled=map_unlabeled_segments + labelset, map_background=map_unlabeled_segments ) logger.info( f"Number of classes in labelmap: {len(labelmap)}", @@ -360,7 +360,7 @@ def prep_frame_classification_dataset( # We need this to be correct for other functions, e.g. predict when it loads spectrogram files spect_format = "npz" - metadata = datasets.frame_classification.Metadata( + metadata = datapipes.frame_classification.Metadata( dataset_csv_filename=str(dataset_csv_path.name), frame_dur=frame_dur, input_type=input_type, diff --git a/src/vak/prep/frame_classification/learncurve.py b/src/vak/prep/frame_classification/learncurve.py index bae335c5d..884e37469 100644 --- a/src/vak/prep/frame_classification/learncurve.py +++ b/src/vak/prep/frame_classification/learncurve.py @@ -13,7 +13,7 @@ import pandas as pd from dask.diagnostics import ProgressBar -from ... import common, datasets +from ... import common, datapipes from .. import split logger = logging.getLogger(__name__) @@ -56,8 +56,8 @@ def make_index_vectors_for_each_subset( in the "train" directory split inside ``dataset_path``. The indexing vectors are used by - :class:`vak.datasets.frame_classification.WindowDataset` - and :class:`vak.datasets.frame_classification.FramesDataset`. + :class:`vak.datasets.frame_classification.TrainDatapipe` + and :class:`vak.datasets.frame_classification.InferDatapipe`. These vectors make it possible to work with files, to avoid loading the entire dataset into memory, and to avoid working with memory-mapped arrays. @@ -103,7 +103,7 @@ def make_index_vectors_for_each_subset( logger.info(f"Making indexing vectors for subset: {subset}") subset_df = subsets_df[subsets_df.subset == subset].copy() frames_paths = subset_df[ - datasets.frame_classification.constants.FRAMES_PATH_COL_NAME + datapipes.frame_classification.constants.FRAMES_PATH_COL_NAME ].values def _return_index_arrays( @@ -116,7 +116,7 @@ def _return_index_arrays( frames_path = dataset_path / pathlib.Path(frames_path) - frames = datasets.frame_classification.helper.load_frames( + frames = datapipes.frame_classification.helper.load_frames( frames_path, input_type ) @@ -151,7 +151,7 @@ def _return_index_arrays( np.save( dataset_path / "train" - / datasets.frame_classification.helper.sample_ids_array_filename_for_subset( + / datapipes.frame_classification.helper.sample_ids_array_filename_for_subset( subset ), sample_id_vec, @@ -162,7 +162,7 @@ def _return_index_arrays( np.save( dataset_path / "train" - / datasets.frame_classification.helper.inds_in_sample_array_filename_for_subset( + / datapipes.frame_classification.helper.inds_in_sample_array_filename_for_subset( subset ), inds_in_sample_vec, @@ -176,6 +176,7 @@ def make_subsets_from_dataset_df( num_replicates: int, dataset_path: pathlib.Path, labelmap: dict, + background_label : str = common.constants.DEFAULT_BACKGROUND_LABEL, ) -> pd.DataFrame: """Make subsets of the training data split for a learning curve. @@ -246,6 +247,11 @@ def make_subsets_from_dataset_df( input_type : str The type of input to the neural network model. One of {'audio', 'spect'}. + background_label: str, optional + The string label applied to segments belonging to the + background class. + Default is + :const:`vak.common.constants.DEFAULT_BACKGROUND_LABEL`. Returns ------- @@ -267,7 +273,7 @@ def make_subsets_from_dataset_df( # get just train split, to pass to split.dataframe # so we don't end up with other splits in the training set train_split_df = dataset_df[dataset_df["split"] == "train"].copy() - labelset = set([k for k in labelmap.keys() if k != "unlabeled"]) + labelset = set([k for k in labelmap.keys() if k != background_label]) # will concat after loop, then use ``csv_path`` to replace # original dataset df with this one diff --git a/src/vak/prep/frame_classification/make_splits.py b/src/vak/prep/frame_classification/make_splits.py index 41d521013..e9faf6abc 100644 --- a/src/vak/prep/frame_classification/make_splits.py +++ b/src/vak/prep/frame_classification/make_splits.py @@ -15,7 +15,7 @@ import pandas as pd from dask.diagnostics import ProgressBar -from ... import common, datasets, transforms +from ... import common, datapipes, transforms from .. import constants as prep_constants logger = logging.getLogger(__name__) @@ -128,6 +128,7 @@ def make_splits( spect_key: str = "s", timebins_key: str = "t", freqbins_key: str = "f", + background_label: str = common.constants.DEFAULT_BACKGROUND_LABEL, ) -> pd.DataFrame: r"""Make each split of a frame classification dataset. @@ -183,8 +184,8 @@ def make_splits( This function also creates two additional npy files for each split. These npy files are "indexing" vectors that - are used by :class:`vak.datasets.frame_classification.WindowDataset` - and :class:`vak.datasets.frame_classification.FramesDataset`. + are used by :class:`vak.datasets.frame_classification.TrainDatapipe` + and :class:`vak.datasets.frame_classification.InferDatapipe`. These vectors make it possible to work with files, to avoid loading the entire dataset into memory, and to avoid working with memory-mapped arrays. @@ -235,6 +236,11 @@ def make_splits( Key for accessing vector of time bins in files. Default is 't'. freqbins_key : str key for accessing vector of frequency bins in files. Default is 'f'. + background_label: str, optional + The string label applied to segments belonging to the + background class. + Default is + :const:`vak.common.constants.DEFAULT_BACKGROUND_LABEL`. Returns ------- @@ -361,11 +367,11 @@ def _save_dataset_arrays_and_return_index_arrays( annot.seq.onsets_s, annot.seq.offsets_s, frame_times, - unlabeled_label=labelmap["unlabeled"], + background_label=labelmap[background_label], ) frame_labels_npy_path = split_subdir / ( source_path.stem - + datasets.frame_classification.constants.FRAME_LABELS_EXT + + datapipes.frame_classification.constants.MULTI_FRAME_LABELS_EXT ) np.save(frame_labels_npy_path, frame_labels) frame_labels_npy_path = str( @@ -417,7 +423,7 @@ def _save_dataset_arrays_and_return_index_arrays( ) np.save( split_subdir - / datasets.frame_classification.constants.SAMPLE_IDS_ARRAY_FILENAME, + / datapipes.frame_classification.constants.SAMPLE_IDS_ARRAY_FILENAME, sample_id_vec, ) inds_in_sample_vec = np.concatenate( @@ -425,7 +431,7 @@ def _save_dataset_arrays_and_return_index_arrays( ) np.save( split_subdir - / datasets.frame_classification.constants.INDS_IN_SAMPLE_ARRAY_FILENAME, + / datapipes.frame_classification.constants.INDS_IN_SAMPLE_ARRAY_FILENAME, inds_in_sample_vec, ) @@ -434,7 +440,7 @@ def _save_dataset_arrays_and_return_index_arrays( # Note that these are all in split dirs, written relative to ``dataset_path``. frames_paths = [str(sample.source_path) for sample in samples] split_df[ - datasets.frame_classification.constants.FRAMES_PATH_COL_NAME + datapipes.frame_classification.constants.FRAMES_PATH_COL_NAME ] = frames_paths frame_labels_npy_paths = [ @@ -446,7 +452,7 @@ def _save_dataset_arrays_and_return_index_arrays( for sample in samples ] split_df[ - datasets.frame_classification.constants.FRAME_LABELS_NPY_PATH_COL_NAME + datapipes.frame_classification.constants.MULTI_FRAME_LABELS_PATH_COL_NAME ] = frame_labels_npy_paths dataset_df_out.append(split_df) diff --git a/src/vak/prep/parametric_umap/parametric_umap.py b/src/vak/prep/parametric_umap/parametric_umap.py index 560b5a699..31ede7e76 100644 --- a/src/vak/prep/parametric_umap/parametric_umap.py +++ b/src/vak/prep/parametric_umap/parametric_umap.py @@ -7,7 +7,7 @@ import crowsetta -from ... import datasets +from ... import datapipes from ...common import labels from ...common.converters import expanded_user_path, labelset_to_set from ...common.logging import config_logging_for_cli, log_version @@ -296,7 +296,7 @@ def prep_parametric_umap_dataset( # we do this before creating array files since we need to load the labelmap to make frame label vectors if purpose != "predict": # TODO: add option to generate predict using existing dataset, so we can get labelmap from it - labelmap = labels.to_map(labelset, map_unlabeled=False) + labelmap = labels.to_map(labelset, map_background=False) logger.info( f"Number of classes in labelmap: {len(labelmap)}", ) @@ -333,7 +333,7 @@ def prep_parametric_umap_dataset( ) # index is False to avoid having "Unnamed: 0" column when loading # ---- save metadata ----------------------------------------------------------------------------------------------- - metadata = datasets.parametric_umap.Metadata( + metadata = datapipes.parametric_umap.Metadata( dataset_csv_filename=str(dataset_csv_path.name), audio_format=audio_format, shape=shape, diff --git a/src/vak/prep/spectrogram_dataset/prep.py b/src/vak/prep/spectrogram_dataset/prep.py index d36266403..17936dd84 100644 --- a/src/vak/prep/spectrogram_dataset/prep.py +++ b/src/vak/prep/spectrogram_dataset/prep.py @@ -54,8 +54,8 @@ def prep_spectrogram_dataset( ``labelset`` is converted to a Python ``set`` using ``vak.converters.labelset_to_set``. See help for that function for details on how to specify labelset. load_spects : bool - if True, load spectrograms. If False, return a FramesDataset without spectograms loaded. - Default is True. Set to False when you want to create a FramesDataset for use + if True, load spectrograms. If False, return a InferDatapipe without spectograms loaded. + Default is True. Set to False when you want to create a InferDatapipe for use later, but don't want to load all the spectrograms into memory yet. audio_format : str format of audio files. One of {'wav', 'cbin'}. diff --git a/src/vak/train/frame_classification.py b/src/vak/train/frame_classification.py index e2d9074f6..ed658c6ed 100644 --- a/src/vak/train/frame_classification.py +++ b/src/vak/train/frame_classification.py @@ -12,10 +12,10 @@ import pandas as pd import torch.utils.data -from .. import datasets, models, transforms +from .. import datapipes, datasets, models, transforms from ..common import validators from ..common.trainer import get_default_trainer -from ..datasets.frame_classification import FramesDataset, WindowDataset +from ..datapipes.frame_classification import InferDatapipe, TrainDatapipe logger = logging.getLogger(__name__) @@ -33,9 +33,9 @@ def train_frame_classification_model( num_epochs: int, num_workers: int, checkpoint_path: str | pathlib.Path | None = None, - spect_scaler_path: str | pathlib.Path | None = None, + frames_standardizer_path: str | pathlib.Path | None = None, results_path: str | pathlib.Path | None = None, - normalize_spectrograms: bool = True, + standardize_frames: bool = True, shuffle: bool = True, val_step: int | None = None, ckpt_step: int | None = None, @@ -77,9 +77,11 @@ def train_frame_classification_model( If specified, this checkpoint will be loaded into model. Used when continuing training. Default is None, in which case a new model is initialized. - spect_scaler_path : str, pathlib.Path - path to a ``SpectScaler`` used to normalize spectrograms, - e.g., one generated by a previous run of ``vak.core.train``. + frames_standardizer_path : str, pathlib.Path + path to a saved :class:`~vak.transforms.FramesStandardizer` + used to standardize (normalize) frames, the input to a + frame classification model. + e.g., one generated by a previous run of :func:`vak.core.train`. Used when continuing training, for example on the same dataset. Default is None. root_results_dir : str, pathlib.Path @@ -90,9 +92,9 @@ def train_frame_classification_model( If specified, this parameter overrides ``root_results_dir``. shuffle: bool if True, shuffle training data before each epoch. Default is True. - normalize_spectrograms : bool - if True, use spect.utils.data.SpectScaler to normalize the spectrograms. - Normalization is done by subtracting off the mean for each frequency bin + standardize_frames : bool + if True, use :class:`vak.transforms.FramesStandardizer` to standardize the frames. + Normalization is done by subtracting off the mean for each row of the training set and then dividing by the std for that frequency bin. This same normalization is then applied to validation + test data. val_step : int @@ -116,8 +118,8 @@ def train_frame_classification_model( when training models for a learning curve. """ for path, path_name in zip( - (checkpoint_path, spect_scaler_path), - ("checkpoint_path", "spect_scaler_path"), + (checkpoint_path, frames_standardizer_path), + ("checkpoint_path", "frames_standardizer_path"), ): if path is not None: if not validators.is_a_file(path): @@ -125,116 +127,139 @@ def train_frame_classification_model( f"value for ``{path_name}`` not recognized as a file: {path}" ) + model_name = model_config["name"] # we use this var again below + if "window_size" not in dataset_config["params"]: + raise KeyError( + f"The `dataset_config` for frame classification model '{model_name}' must include a 'params' sub-table " + f"that sets a value for 'window_size', but received a `dataset_config` that did not:\n{dataset_config}" + ) + dataset_path = pathlib.Path(dataset_config["path"]) if not dataset_path.exists() or not dataset_path.is_dir(): raise NotADirectoryError( f"`dataset_path` not found or not recognized as a directory: {dataset_path}" ) - logger.info( - f"Loading dataset from `dataset_path`: {dataset_path}", - ) - metadata = datasets.frame_classification.Metadata.from_dataset_path( - dataset_path - ) - dataset_csv_path = dataset_path / metadata.dataset_csv_filename - dataset_df = pd.read_csv(dataset_csv_path) - # ---------------- pre-conditions ---------------------------------------------------------------------------------- - if val_step and not dataset_df["split"].str.contains("val").any(): - raise ValueError( - f"val_step set to {val_step} but dataset does not contain a validation set; " - f"please run `vak prep` with a config.toml file that specifies a duration for the validation set." - ) - # ---- set up directory to save output ----------------------------------------------------------------------------- + # we do this first to make sure we can save things in `results_path`: copy of toml config file, labelset.json, etc results_path = pathlib.Path(results_path).expanduser().resolve() if not results_path.is_dir(): raise NotADirectoryError( - f"results_path not recognized as a directory: {results_path}" + f"`results_path` not recognized as a directory: {results_path}" ) - - frame_dur = metadata.frame_dur logger.info( - f"Duration of a frame in dataset, in seconds: {frame_dur}", + f"Will save results in `results_path`: {results_path}", ) - # ---------------- load training data ----------------------------------------------------------------------------- - logger.info(f"Using training split from dataset: {dataset_path}") - # below, if we're going to train network to predict unlabeled segments, then - # we need to include a class for those unlabeled segments in labelmap, - # the mapping from labelset provided by user to a set of consecutive - # integers that the network learns to predict - train_dur = get_split_dur(dataset_df, "train") logger.info( - f"Total duration of training split from dataset (in s): {train_dur}", + f"Loading dataset from `dataset_path`: {dataset_path}\nUsing dataset config: {dataset_config}" ) + # ---------------- load training data ----------------------------------------------------------------------------- + # ---- *not* using a built-in dataset ------------------------------------------------------------------------------ + if dataset_config["name"] is None: + metadata = datapipes.frame_classification.Metadata.from_dataset_path( + dataset_path + ) + dataset_csv_path = dataset_path / metadata.dataset_csv_filename + dataset_df = pd.read_csv(dataset_csv_path) + # we have to check this pre-condition here since we need `dataset_df` to check + if val_step and not dataset_df["split"].str.contains("val").any(): + raise ValueError( + f"val_step set to {val_step} but dataset does not contain a validation set; " + f"please run `vak prep` with a config.toml file that specifies a duration for the validation set." + ) - labelmap_path = dataset_path / "labelmap.json" - logger.info(f"loading labelmap from path: {labelmap_path}") - with labelmap_path.open("r") as f: - labelmap = json.load(f) - # copy to new results_path - with open(results_path.joinpath("labelmap.json"), "w") as f: - json.dump(labelmap, f) + frame_dur = metadata.frame_dur + logger.info( + f"Duration of a frame in dataset, in seconds: {frame_dur}", + ) - if spect_scaler_path is not None and normalize_spectrograms: - logger.info(f"loading spect scaler from path: {spect_scaler_path}") - spect_standardizer = joblib.load(spect_scaler_path) - shutil.copy(spect_scaler_path, results_path) - # get transforms just before creating datasets with them - elif normalize_spectrograms and spect_scaler_path is None: + logger.info(f"Using training split from dataset: {dataset_path}") + train_dur = get_split_dur(dataset_df, "train") logger.info( - "no spect_scaler_path provided, not loading", + f"Total duration of training split from dataset (in s): {train_dur}", ) - logger.info("will normalize spectrograms") - spect_standardizer = transforms.StandardizeSpect.fit_dataset_path( - dataset_path, + + labelmap_path = dataset_path / "labelmap.json" + logger.info(f"loading labelmap from path: {labelmap_path}") + with labelmap_path.open("r") as f: + labelmap = json.load(f) + # copy to new results_path + with open(results_path.joinpath("labelmap.json"), "w") as f: + json.dump(labelmap, f) + + if frames_standardizer_path is not None and standardize_frames: + logger.info( + f"Loading frames standardizer from path: {frames_standardizer_path}" + ) + frames_standardizer = joblib.load(frames_standardizer_path) + shutil.copy(frames_standardizer_path, results_path) + # get transforms just before creating datasets with them + elif standardize_frames and frames_standardizer_path is None: + logger.info( + "No `frames_standardizer_path` provided, not loading", + ) + logger.info("Will standardize (normalize) frames") + frames_standardizer = transforms.FramesStandardizer.fit_dataset_path( + dataset_path, + split="train", + subset=subset, + ) + joblib.dump( + frames_standardizer, results_path.joinpath("FramesStandardizer") + ) + elif frames_standardizer_path is not None and not standardize_frames: + raise ValueError( + "`frames_standardizer_path` provided but `standardize_frames` was False, these options conflict" + ) + # ---- *yes* using a built-in dataset ------------------------------------------------------------------------------ + else: + # not standardize_frames and frames_standardizer_path is None: + logger.info( + "`standardize_frames` is False and no `frames_standardizer_path` was provided, " + "will not standardize spectrograms", + ) + frames_standardizer = None + + train_dataset = TrainDatapipe.from_dataset_path( + dataset_path=dataset_path, split="train", subset=subset, - ) - joblib.dump( - spect_standardizer, results_path.joinpath("StandardizeSpect") - ) - elif spect_scaler_path is not None and not normalize_spectrograms: - raise ValueError( - "spect_scaler_path provided but normalize_spectrograms was False, these options conflict" + window_size=dataset_config["params"]["window_size"], + frames_standardizer=frames_standardizer, ) else: - # not normalize_spectrograms and spect_scaler_path is None: + # ---- we are using a built-in dataset ----------------------------------------- + # TODO: fix this hack + # (by doing the same thing with the built-in datapipes, making this a Boolean parameter + # while still accepting a transform but defaulting to None) + if "standardize_frames" not in dataset_config: + logger.info( + f"Adding `standardize_frames` argument to dataset_config[\"params\"]: {standardize_frames}" + ) + dataset_config["params"]["standardize_frames"] = standardize_frames + train_dataset = datasets.get( + dataset_config, + split="train", + ) logger.info( - "normalize_spectrograms is False and no spect_scaler_path was provided, " - "will not standardize spectrograms", + f"Duration of a frame in dataset, in seconds: {train_dataset.frame_dur}", ) - spect_standardizer = None - - model_name = model_config["name"] - # TODO: move this into datapipe once each datapipe uses a fixed set of transforms - # that will require adding `spect_standardizer`` as a parameter to the datapipe, - # maybe rename to `frames_standardizer`? - try: - window_size = dataset_config["params"]["window_size"] - except KeyError as e: - raise KeyError( - f"The `dataset_config` for frame classification model '{model_name}' must include a 'params' sub-table " - f"that sets a value for 'window_size', but received a `dataset_config` that did not:\n{dataset_config}" - ) from e - transform_kwargs = { - "spect_standardizer": spect_standardizer, - "window_size": window_size, - } - train_transform = transforms.defaults.get_default_transform( - model_name, "train", transform_kwargs=transform_kwargs - ) + # copy labelmap from dataset to new results_path + labelmap = train_dataset.labelmap + with open(results_path.joinpath("labelmap.json"), "w") as fp: + json.dump(labelmap, fp) + frames_standardizer = getattr(train_dataset.item_transform, 'frames_standardizer') + if frames_standardizer is not None: + logger.info( + f"Saving `frames_standardizer` from item transform on training dataset" + ) + joblib.dump( + frames_standardizer, results_path.joinpath("FramesStandardizer") + ) - train_dataset = WindowDataset.from_dataset_path( - dataset_path=dataset_path, - split="train", - subset=subset, - item_transform=train_transform, - **dataset_config["params"], - ) logger.info( - f"Duration of WindowDataset used for training, in seconds: {train_dataset.duration}", + f"Duration of {train_dataset.__class__.__name__} used for training, in seconds: {train_dataset.duration}", ) train_loader = torch.utils.data.DataLoader( dataset=train_dataset, @@ -248,24 +273,28 @@ def train_frame_classification_model( logger.info( f"Will measure error on validation set every {val_step} steps of training", ) - logger.info(f"Using validation split from dataset:\n{dataset_path}") - val_dur = get_split_dur(dataset_df, "val") - logger.info( - f"Total duration of validation split from dataset (in s): {val_dur}", - ) - - # NOTE: we use same `transform_kwargs` here; will need to change to a `dataset_param` - # when we factor transform *into* fixed DataPipes as above - val_transform = transforms.defaults.get_default_transform( - model_name, "eval", transform_kwargs - ) - val_dataset = FramesDataset.from_dataset_path( - dataset_path=dataset_path, - split="val", - item_transform=val_transform, - ) + if dataset_config["name"] is None: + logger.info(f"Using validation split from dataset:\n{dataset_path}") + val_dur = get_split_dur(dataset_df, "val") + logger.info( + f"Total duration of validation split from dataset (in s): {val_dur}", + ) + val_dataset = InferDatapipe.from_dataset_path( + dataset_path=dataset_path, + split="val", + **dataset_config["params"], + frames_standardizer=frames_standardizer, + return_padding_mask=True, + ) + else: + dataset_config["params"]["return_padding_mask"] = True + val_dataset = datasets.get( + dataset_config, + split="val", + frames_standardizer=frames_standardizer, + ) logger.info( - f"Duration of FramesDataset used for evaluation, in seconds: {val_dataset.duration}", + f"Duration of {val_dataset.__class__.__name__} used for evaluation, in seconds: {val_dataset.duration}", ) val_loader = torch.utils.data.DataLoader( dataset=val_dataset, diff --git a/src/vak/train/parametric_umap.py b/src/vak/train/parametric_umap.py index 0fd72ca33..7c836d94e 100644 --- a/src/vak/train/parametric_umap.py +++ b/src/vak/train/parametric_umap.py @@ -6,14 +6,14 @@ import logging import pathlib -import pandas as pd import lightning +import pandas as pd import torch.utils.data -from .. import datasets, models, transforms +from .. import datapipes, models from ..common import validators from ..common.paths import generate_results_dir_name_as_path -from ..datasets.parametric_umap import ParametricUMAPDataset +from ..datapipes.parametric_umap import Datapipe logger = logging.getLogger(__name__) @@ -162,7 +162,7 @@ def train_parametric_umap_model( logger.info( f"Loading dataset from path: {dataset_path}", ) - metadata = datasets.parametric_umap.Metadata.from_dataset_path( + metadata = datapipes.parametric_umap.Metadata.from_dataset_path( dataset_path ) dataset_csv_path = dataset_path / metadata.dataset_csv_filename @@ -196,17 +196,11 @@ def train_parametric_umap_model( f"Total duration of training split from dataset (in s): {train_dur}", ) - model_name = model_config["name"] - train_transform = transforms.defaults.get_default_transform( - model_name, "train" - ) - dataset_params = dataset_config["params"] - train_dataset = ParametricUMAPDataset.from_dataset_path( + train_dataset = Datapipe.from_dataset_path( dataset_path=dataset_path, split="train", subset=subset, - transform=train_transform, **dataset_params, ) logger.info( @@ -221,13 +215,9 @@ def train_parametric_umap_model( # ---------------- load validation set (if there is one) ----------------------------------------------------------- if val_step: - transform = transforms.defaults.get_default_transform( - model_name, "eval" - ) - val_dataset = ParametricUMAPDataset.from_dataset_path( + val_dataset = Datapipe.from_dataset_path( dataset_path=dataset_path, split="val", - transform=transform, **dataset_params, ) logger.info( @@ -242,6 +232,7 @@ def train_parametric_umap_model( else: val_loader = None + model_name = model_config["name"] model = models.get( model_name, model_config, diff --git a/src/vak/train/train_.py b/src/vak/train/train_.py index 1a5527c88..b1da96eac 100644 --- a/src/vak/train/train_.py +++ b/src/vak/train/train_.py @@ -21,9 +21,9 @@ def train( num_epochs: int, num_workers: int, checkpoint_path: str | pathlib.Path | None = None, - spect_scaler_path: str | pathlib.Path | None = None, + frames_standardizer_path: str | pathlib.Path | None = None, results_path: str | pathlib.Path | None = None, - normalize_spectrograms: bool = True, + standardize_frames: bool = True, shuffle: bool = True, val_step: int | None = None, ckpt_step: int | None = None, @@ -64,8 +64,8 @@ def train( If specified, this checkpoint will be loaded into model. Used when continuing training. Default is None, in which case a new model is initialized. - spect_scaler_path : str, pathlib.Path - path to a ``SpectScaler`` used to normalize spectrograms, + frames_standardizer_path : str, pathlib.Path + path to a ``:class:`vak.transforms.FramesStandardizer``` used to standardize (normalize) frames, e.g., one generated by a previous run of ``vak.core.train``. Used when continuing training, for example on the same dataset. Default is None. @@ -81,21 +81,21 @@ def train( That function defaults to 'cuda' if torch.cuda.is_available is True. shuffle: bool if True, shuffle training data before each epoch. Default is True. - normalize_spectrograms : bool - if True, use spect.utils.data.SpectScaler to normalize the spectrograms. - Normalization is done by subtracting off the mean for each frequency bin + standardize_frames : bool + if True, use :class:`vak.transforms.FramesStandardizer` to standardize the frames. + Normalization is done by subtracting off the mean for each row of the training set and then dividing by the std for that frequency bin. This same normalization is then applied to validation + test data. source_ids : numpy.ndarray - Parameter for WindowDataset. Represents the 'id' of any spectrogram, + Parameter for TrainDatapipe. Represents the 'id' of any spectrogram, i.e., the index into spect_paths that will let us load it. Default is None. source_inds : numpy.ndarray - Parameter for WindowDataset. Same length as source_ids + Parameter for TrainDatapipe. Same length as source_ids but values represent indices within each spectrogram. Default is None. window_inds : numpy.ndarray - Parameter for WindowDataset. + Parameter for TrainDatapipe. Indices of each window in the dataset. The value at x[0] represents the start index of the first window; using that value, we can index into source_ids to get the path @@ -124,8 +124,8 @@ def train( training set to use when training models for a learning curve. """ for path, path_name in zip( - (checkpoint_path, spect_scaler_path), - ("checkpoint_path", "spect_scaler_path"), + (checkpoint_path, frames_standardizer_path), + ("checkpoint_path", "frames_standardizer_path"), ): if path is not None: if not validators.is_a_file(path): @@ -155,9 +155,9 @@ def train( num_epochs=num_epochs, num_workers=num_workers, checkpoint_path=checkpoint_path, - spect_scaler_path=spect_scaler_path, + frames_standardizer_path=frames_standardizer_path, results_path=results_path, - normalize_spectrograms=normalize_spectrograms, + standardize_frames=standardize_frames, shuffle=shuffle, val_step=val_step, ckpt_step=ckpt_step, diff --git a/src/vak/transforms/defaults/__init__.py b/src/vak/transforms/defaults/__init__.py index d82644b60..81545eb79 100644 --- a/src/vak/transforms/defaults/__init__.py +++ b/src/vak/transforms/defaults/__init__.py @@ -1,4 +1,3 @@ -from . import frame_classification, parametric_umap -from .get import get_default_transform +from . import frame_classification -__all__ = ["get_default_transform", "frame_classification", "parametric_umap"] +__all__ = ["frame_classification"] diff --git a/src/vak/transforms/defaults/frame_classification.py b/src/vak/transforms/defaults/frame_classification.py index 9d20cfe50..ff47b7451 100644 --- a/src/vak/transforms/defaults/frame_classification.py +++ b/src/vak/transforms/defaults/frame_classification.py @@ -2,20 +2,20 @@ These are "item" transforms because they apply transforms to input parameters and then return them in an "item" (dictionary) -that is turn returned by the __getitem__ method of a vak.FramesDataset. +that is turn returned by the __getitem__ method of a vak.InferDatapipe. Having the transform return a dictionary makes it possible to avoid -coupling the FramesDataset __getitem__ implementation to the transforms +coupling the InferDatapipe __getitem__ implementation to the transforms needed for specific neural network models, e.g., whether the returned output includes a mask to crop off padding that was added. """ from __future__ import annotations -from typing import Callable - +import torch import torchvision.transforms from .. import transforms as vak_transforms +from ..transforms import FramesStandardizer class TrainItemTransform: @@ -23,14 +23,16 @@ class TrainItemTransform: def __init__( self, - spect_standardizer=None, + frames_standardizer: FramesStandardizer | None = None, ): - if spect_standardizer is not None: - if isinstance(spect_standardizer, vak_transforms.StandardizeSpect): - frames_transform = [spect_standardizer] + if frames_standardizer is not None: + if isinstance( + frames_standardizer, vak_transforms.FramesStandardizer + ): + frames_transform = [frames_standardizer] else: raise TypeError( - f"invalid type for spect_standardizer: {type(spect_standardizer)}. " + f"invalid type for frames_standardizer: {type(frames_standardizer)}. " "Should be an instance of vak.transforms.StandardizeSpect" ) else: @@ -52,7 +54,7 @@ def __call__(self, frames, frame_labels, spect_path=None): frame_labels = self.frame_labels_transform(frame_labels) item = { "frames": frames, - "frame_labels": frame_labels, + "multi_frame_labels": frame_labels, } if spect_path is not None: @@ -61,39 +63,70 @@ def __call__(self, frames, frame_labels, spect_path=None): return item -class EvalItemTransform: - """Default transform used when evaluating frame classification models. +class InferItemTransform: + """Default transform used when running inference on frame classification models, + for evaluation or to generate new predictions. - Returned item includes "source" spectrogram reshaped into a stack of windows, - with padded added to make reshaping possible, and annotation also padded and - reshaped. + Returned item includes frames reshaped into a stack of windows, + with padded added to make reshaping possible. + Any `frame_labels` are not padded and reshaped, + but are converted to :class:`torch.LongTensor`. If return_padding_mask is True, item includes 'padding_mask' that can be used to crop off any predictions made on the padding. + + Attributes + ---------- + frames_standardizer : vak.transforms.FramesStandardizer + instance that has already been fit to dataset, using fit_df method. + Default is None, in which case no standardization transform is applied. + window_size : int + width of window in number of elements. Argument to PadToWindow transform. + frames_padval : float + Value to pad frames with. Added to end of array, the "right side". + Argument to PadToWindow transform. Default is 0.0. + frame_labels_padval : int + Value to pad frame labels vector with. Added to the end of the array. + Argument to PadToWindow transform. Default is -1. + Used with ``ignore_index`` argument of :mod:`torch.nn.CrossEntropyLoss`. + return_padding_mask : bool + if True, the dictionary returned by ItemTransform classes will include + a boolean vector to use for cropping back down to size before padding. + padding_mask has size equal to width of padded array, i.e. original size + plus padding at the end, and has values of 1 where + columns in padded are from the original array, + and values of 0 where columns were added for padding. """ def __init__( self, window_size, - spect_standardizer=None, - padval=0.0, + frames_standardizer=None, + frames_padval=0.0, + frame_labels_padval=-1, return_padding_mask=True, channel_dim=1, ): - if spect_standardizer is not None: + self.window_size = window_size + self.frames_padval = frames_padval + self.frame_labels_padval = frame_labels_padval + self.return_padding_mask = return_padding_mask + self.channel_dim = channel_dim + + if frames_standardizer is not None: if not isinstance( - spect_standardizer, vak_transforms.StandardizeSpect + frames_standardizer, vak_transforms.FramesStandardizer ): raise TypeError( - f"invalid type for spect_standardizer: {type(spect_standardizer)}. " - "Should be an instance of vak.transforms.StandardizeSpect" + f"Invalid type for frames_standardizer: {type(frames_standardizer)}. " + "Should be an instance of vak.transforms.FramesStandardizer" ) - self.spect_standardizer = spect_standardizer + self.frames_standardizer = frames_standardizer self.pad_to_window = vak_transforms.PadToWindow( - window_size, padval, return_padding_mask=return_padding_mask + window_size, frames_padval, return_padding_mask=return_padding_mask ) - self.source_transform_after_pad = torchvision.transforms.Compose( + self.frames_transform_after_pad = torchvision.transforms.Compose( [ vak_transforms.ViewAsWindowBatch(window_size), vak_transforms.ToFloatTensor(), @@ -102,93 +135,33 @@ def __init__( ] ) - self.annot_transform = vak_transforms.ToLongTensor() - - def __call__(self, frames, frame_labels, frames_path=None): - if self.spect_standardizer: - frames = self.spect_standardizer(frames) - - if self.pad_to_window.return_padding_mask: - frames, padding_mask = self.pad_to_window(frames) - else: - frames = self.pad_to_window(frames) - padding_mask = None - frames = self.source_transform_after_pad(frames) - - frame_labels = self.annot_transform(frame_labels) - - item = { - "frames": frames, - "frame_labels": frame_labels, - } - - if padding_mask is not None: - item["padding_mask"] = padding_mask - - if frames_path is not None: - # make sure frames_path is a str, not a pathlib.Path - item["frames_path"] = str(frames_path) - - return item - - -class PredictItemTransform: - """Default transform used when using trained frame classification models - to make predictions. - - Returned item includes "source" spectrogram reshaped into a stack of windows, - with padded added to make reshaping possible. - If return_padding_mask is True, item includes 'padding_mask' that - can be used to crop off any predictions made on the padding. - """ + self.frame_labels_padval = frame_labels_padval + self.frame_labels_transform = vak_transforms.ToLongTensor() - def __init__( + def __call__( self, - window_size, - spect_standardizer=None, - padval=0.0, - return_padding_mask=True, - channel_dim=1, - ): - if spect_standardizer is not None: - if not isinstance( - spect_standardizer, vak_transforms.StandardizeSpect - ): - raise TypeError( - f"invalid type for spect_standardizer: {type(spect_standardizer)}. " - "Should be an instance of vak.transforms.StandardizeSpect" - ) - self.spect_standardizer = spect_standardizer - - self.pad_to_window = vak_transforms.PadToWindow( - window_size, padval, return_padding_mask=return_padding_mask - ) - - self.source_transform_after_pad = torchvision.transforms.Compose( - [ - vak_transforms.ViewAsWindowBatch(window_size), - vak_transforms.ToFloatTensor(), - # below, add channel at first dimension because windows become batch - vak_transforms.AddChannel(channel_dim=channel_dim), - ] - ) - - def __call__(self, frames, frames_path=None): - if self.spect_standardizer: - frames = self.spect_standardizer(frames) + frames: torch.Tensor, + frame_labels: torch.Tensor | None = None, + frames_path=None, + ) -> dict: + if self.frames_standardizer: + frames = self.frames_standardizer(frames) if self.pad_to_window.return_padding_mask: frames, padding_mask = self.pad_to_window(frames) else: frames = self.pad_to_window(frames) padding_mask = None - - frames = self.source_transform_after_pad(frames) + frames = self.frames_transform_after_pad(frames) item = { "frames": frames, } + if frame_labels is not None: + frame_labels = self.frame_labels_transform(frame_labels) + item["multi_frame_labels"] = frame_labels + if padding_mask is not None: item["padding_mask"] = padding_mask @@ -197,75 +170,3 @@ def __call__(self, frames, frames_path=None): item["frames_path"] = str(frames_path) return item - - -def get_default_frame_classification_transform( - mode: str, transform_kwargs: dict | None = None -) -> tuple[Callable, Callable] | Callable: - """Get default transform for frame classification model. - - Parameters - ---------- - mode : str - transform_kwargs : dict, optional - Keyword arguments for transform class. - Default is None. - If supplied, should be a :class:`dict`, - that can include the following key-value pairs: - spect_standardizer : vak.transforms.StandardizeSpect - instance that has already been fit to dataset, using fit_df method. - Default is None, in which case no standardization transform is applied. - window_size : int - width of window in number of elements. Argument to PadToWindow transform. - padval : float - value to pad with. Added to end of array, the "right side" if 2-dimensional. - Argument to PadToWindow transform. Default is 0. - return_padding_mask : bool - if True, the dictionary returned by ItemTransform classes will include - a boolean vector to use for cropping back down to size before padding. - padding_mask has size equal to width of padded array, i.e. original size - plus padding at the end, and has values of 1 where - columns in padded are from the original array, - and values of 0 where columns were added for padding. - - Returns - ------- - transform: TrainItemTransform, EvalItemTransform, or PredictItemTransform - """ - if transform_kwargs is None: - transform_kwargs = {} - spect_standardizer = transform_kwargs.get("spect_standardizer", None) - # regardless of mode, transform always starts with StandardizeSpect, if used - if spect_standardizer is not None: - if not isinstance(spect_standardizer, vak_transforms.StandardizeSpect): - raise TypeError( - f"invalid type for spect_standardizer: {type(spect_standardizer)}. " - "Should be an instance of vak.transforms.StandardizeSpect" - ) - - if mode == "train": - return TrainItemTransform(spect_standardizer) - - elif mode == "predict": - item_transform = PredictItemTransform( - spect_standardizer=spect_standardizer, - window_size=transform_kwargs["window_size"], - padval=transform_kwargs.get("padval", 0.0), - return_padding_mask=transform_kwargs.get( - "return_padding_mask", True - ), - ) - return item_transform - - elif mode == "eval": - item_transform = EvalItemTransform( - spect_standardizer=spect_standardizer, - window_size=transform_kwargs["window_size"], - padval=transform_kwargs.get("padval", 0.0), - return_padding_mask=transform_kwargs.get( - "return_padding_mask", True - ), - ) - return item_transform - else: - raise ValueError(f"invalid mode: {mode}") diff --git a/src/vak/transforms/defaults/get.py b/src/vak/transforms/defaults/get.py deleted file mode 100644 index 3d567bde7..000000000 --- a/src/vak/transforms/defaults/get.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Helper function that gets default transforms for a model.""" - -from __future__ import annotations - -from typing import Callable, Literal - -from ... import models -from . import frame_classification, parametric_umap - - -def get_default_transform( - model_name: str, - mode: Literal["eval", "predict", "train"], - transform_kwargs: dict | None = None, -) -> Callable: - """Get default transform for a model, - according to its family and what mode - the model is being used in. - - Parameters - ---------- - model_name : str - Name of model. - mode : str - One of {'eval', 'predict', 'train'}. - - Returns - ------- - item_transform : callable - Transform to be applied to input :math:`x` to a model and, - during training, the target :math:`y`. - """ - try: - model_family = models.registry.MODEL_FAMILY_FROM_NAME[model_name] - except KeyError as e: - raise ValueError( - f"No model family found for the model name specified: {model_name}" - ) from e - - if model_family == "FrameClassificationModel": - return frame_classification.get_default_frame_classification_transform( - mode, transform_kwargs - ) - - elif model_family == "ParametricUMAPModel": - return parametric_umap.get_default_parametric_umap_transform( - transform_kwargs - ) diff --git a/src/vak/transforms/defaults/parametric_umap.py b/src/vak/transforms/defaults/parametric_umap.py deleted file mode 100644 index a62a7c29b..000000000 --- a/src/vak/transforms/defaults/parametric_umap.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Default transforms for Parametric UMAP models.""" - -from __future__ import annotations - -import torchvision.transforms - -from .. import transforms as vak_transforms - - -def get_default_parametric_umap_transform( - transform_kwargs: dict | None = None, -) -> torchvision.transforms.Compose: - """Get default transform for frame classification model. - - Parameters - ---------- - transform_kwargs : dict, optional - Keyword arguments for transform class. - Default is None. - - Returns - ------- - transform : Callable - """ - if transform_kwargs is None: - transform_kwargs = {} - transforms = [ - vak_transforms.ToFloatTensor(), - vak_transforms.AddChannel(), - ] - return torchvision.transforms.Compose(transforms) diff --git a/src/vak/transforms/frame_labels/functional.py b/src/vak/transforms/frame_labels/functional.py index 7fd73ff30..85f435eea 100644 --- a/src/vak/transforms/frame_labels/functional.py +++ b/src/vak/transforms/frame_labels/functional.py @@ -11,9 +11,9 @@ - to_segments: transform to get back segment onsets, offsets, and labels from frame labels. Inverse of ``from_segments``. - post-processing transforms that can be used to "clean up" a vector of frame labels - - to_inds_list: helper function used to find segments in a vector of frame labels + - segment_inds_list_from_class_labels: helper function used to find segments in a vector of frame labels - remove_short_segments: remove any segment less than a minimum duration - - take_majority_vote: take a "majority vote" within each segment bounded by the "unlabeled" label, + - take_majority_vote: take a "majority vote" within each segment bounded by the background label, and apply the most "popular" label within each segment to all timebins in that segment - postprocess: combines remove_short_segments and take_majority_vote in one transform """ @@ -21,30 +21,33 @@ from __future__ import annotations import numpy as np +import numpy.typing as npt import scipy.stats +from ... import common from ...common.timebins import timebin_dur_from_vec from ...common.validators import column_or_1d, row_or_1d __all__ = [ # keep alphabetized + "boundary_inds_from_boundary_labels", "from_segments", "postprocess", "remove_short_segments", "take_majority_vote", - "to_inds_list", + "segment_inds_list_from_class_labels", "to_labels", "to_segments", ] def from_segments( - labels_int: np.ndarray, - onsets_s: np.ndarray, - offsets_s: np.ndarray, - time_bins: np.ndarray, - unlabeled_label: int = 0, -) -> np.ndarray: + labels_int: npt.NDArray, + onsets_s: npt.NDArray, + offsets_s: npt.NDArray, + time_bins: npt.NDArray, + background_label: int = 0, +) -> npt.NDArray: """Make a vector of labels for a vector of frames, given labeled segments in the form of onset times, offset times, and segment labels. @@ -60,7 +63,7 @@ def from_segments( 1-d vector of floats, segment offsets in seconds. time_bins : numpy.ndarray 1-d vector of floats, time in seconds for center of each time bin of a spectrogram. - unlabeled_label : int + background_label : int Label assigned to frames that do not have labels associated with them. Default is 0. @@ -80,7 +83,7 @@ def from_segments( "labels_int must be a list or numpy.ndarray of integers" ) - label_vec = np.ones((time_bins.shape[-1],), dtype="int8") * unlabeled_label + label_vec = np.ones((time_bins.shape[-1],), dtype="int8") * background_label onset_inds = [np.argmin(np.abs(time_bins - onset)) for onset in onsets_s] offset_inds = [ np.argmin(np.abs(time_bins - offset)) for offset in offsets_s @@ -92,7 +95,10 @@ def from_segments( return label_vec -def to_labels(frame_labels: np.ndarray, labelmap: dict) -> str: +def to_labels( + frame_labels: npt.NDArray, labelmap: dict, + background_label: str = common.constants.DEFAULT_BACKGROUND_LABEL +) -> str: """Convert vector of frame labels to a string, one character for each continuous segment. @@ -111,6 +117,11 @@ def to_labels(frame_labels: np.ndarray, labelmap: dict) -> str: labelmap : dict That maps string labels to integers. The mapping is inverted to convert back to string labels. + background_label: str, optional + The string label applied to segments belonging to the + background class. + Default is + :const:`vak.common.constants.DEFAULT_BACKGROUND_LABEL`. Returns ------- @@ -125,9 +136,9 @@ def to_labels(frame_labels: np.ndarray, labelmap: dict) -> str: labels = frame_labels[onset_inds] - # remove 'unlabeled' label - if "unlabeled" in labelmap: - labels = labels[labels != labelmap["unlabeled"]] + # remove background label + if background_label in labelmap: + labels = labels[labels != labelmap[background_label]] if len(labels) < 1: # if removing all the 'unlabeled' leaves nothing return "" @@ -141,11 +152,12 @@ def to_labels(frame_labels: np.ndarray, labelmap: dict) -> str: def to_segments( - frame_labels: np.ndarray, + frame_labels: npt.NDArray, labelmap: dict, - frame_times: np.ndarray, + frame_times: npt.NDArray, n_decimals_trunc: int = 5, -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + background_label: str = common.constants.DEFAULT_BACKGROUND_LABEL +) -> tuple[npt.NDArray, npt.NDArray, npt.NDArray]: """Convert a vector of frame labels into segments in the form of onset indices, offset indices, and labels. @@ -191,13 +203,13 @@ def to_segments( """ frame_labels = column_or_1d(frame_labels) - if "unlabeled" in labelmap: + if background_label in labelmap: # handle the case when all time bins are predicted to be unlabeled # see https://github.com/NickleDave/vak/issues/383 uniq_frame_labels = np.unique(frame_labels) if ( len(uniq_frame_labels) == 1 - and uniq_frame_labels[0] == labelmap["unlabeled"] + and uniq_frame_labels[0] == labelmap[background_label] ): return None, None, None @@ -214,9 +226,9 @@ def to_segments( onset_inds = np.concatenate((np.asarray([0]), onset_inds)) labels = frame_labels[onset_inds] - # remove 'unlabeled' label - if "unlabeled" in labelmap: - keep = np.where(labels != labelmap["unlabeled"])[0] + # remove background label + if background_label in labelmap: + keep = np.where(labels != labelmap[background_label])[0] labels = labels[keep] onset_inds = onset_inds[keep] offset_inds = offset_inds[keep] @@ -252,12 +264,13 @@ def to_segments( return labels, onsets_s, offsets_s -def to_inds_list( - frame_labels: np.ndarray, unlabeled_label: int = 0 -) -> list[np.ndarray]: +def segment_inds_list_from_class_labels( + frame_labels: npt.NDArray, background_label: int = 0 +) -> list[npt.NDArray]: """Given a vector of frame labels, returns a list of indexing vectors, - one for each labeled segment in the vector. + one for each segment in the vector + that is not labeled with the background label. Parameters ---------- @@ -265,29 +278,27 @@ def to_inds_list( A vector where each element represents a label for a frame, either a single sample in audio or a single time bin from a spectrogram. - unlabeled_label : int + background_label : int Label that was given to segments that were not labeled in annotation, e.g. silent periods between annotated segments. Default is 0. - return_inds : bool - If True, return list of indices for segments in frame_labels, in addition to the segments themselves. - If False, just return list of numpy.ndarrays that are the segments from frame_labels. Returns ------- segment_inds_list : list - of numpy.ndarray, indices that will recover segments list from frame_labels. + Of fancy indexing arrays. Each array can be used to index + one segment in ``frame_labels``. """ - segment_inds = np.nonzero(frame_labels != unlabeled_label)[0] + segment_inds = np.nonzero(frame_labels != background_label)[0] return np.split(segment_inds, np.where(np.diff(segment_inds) != 1)[0] + 1) def remove_short_segments( - frame_labels: np.ndarray, - segment_inds_list: list[np.ndarray], + frame_labels: npt.NDArray, + segment_inds_list: list[npt.NDArray], timebin_dur: float, min_segment_dur: float | int, - unlabeled_label: int = 0, -) -> tuple[np.ndarray, list[np.ndarray]]: + background_label: int = 0, +) -> tuple[npt.NDArray, list[npt.NDArray]]: """Remove segments from vector of frame labels that are shorter than a specified duration. @@ -309,7 +320,7 @@ def remove_short_segments( any segment with a duration less than min_segment_dur is removed from frame_labels. Default is None, in which case no segments are removed. - unlabeled_label : int + background_label : int Label that was given to segments that were not labeled in annotation, e.g. silent periods between annotated segments. Default is 0. @@ -320,7 +331,7 @@ def remove_short_segments( a label for a frame, either a single sample in audio or a single time bin from a spectrogram. With segments whose duration is shorter than ``min_segment_dur`` - set to ``unlabeled_label`` + set to ``background_label`` segment_inds_list : list Of numpy.ndarray, with arrays removed that represented segments in ``frame_labels`` that were shorter than ``min_segment_dur``. @@ -329,7 +340,7 @@ def remove_short_segments( for segment_inds in segment_inds_list: if segment_inds.shape[-1] * timebin_dur < min_segment_dur: - frame_labels[segment_inds] = unlabeled_label + frame_labels[segment_inds] = background_label # DO NOT keep segment_inds array else: # do keep segment_inds array, don't change frame_labels @@ -339,8 +350,8 @@ def remove_short_segments( def take_majority_vote( - frame_labels: np.ndarray, segment_inds_list: list[np.ndarray] -) -> np.ndarray: + frame_labels: npt.NDArray, segment_inds_list: list[npt.NDArray] +) -> npt.NDArray: """Transform segments containing multiple labels into segments with a single label by taking a "majority vote", i.e. assign all frames in the segment the most frequently @@ -370,25 +381,90 @@ def take_majority_vote( return frame_labels +def boundary_inds_from_boundary_labels( + boundary_labels: npt.NDArray, + force_boundary_first_ind: bool = True +) -> npt.NDArray: + """Return a :class:`numpy.ndarray` with the indices + of boundaries, given a 1-D vector of boundary labels. + + Parameters + ---------- + boundary_labels : numpy.ndarray + Vector of integers ``{0, 1}``, where ``1`` indicates a boundary, + and ``0`` indicates no boundary. + Output of a frame classification model trained to classify each + frame as "boundary" or "no boundary". + force_boundary_first_ind : bool + If ``True``, and the first index of ``boundary_labels`` is not classified as a boundary, + force it to be a boundary. + """ + boundary_inds = np.nonzero(boundary_labels)[0] + + if boundary_inds[0] != 0 and force_boundary_first_ind: + # force there to be a boundary at index 0 + np.insert(boundary_inds, 0, 0) + + return boundary_inds + + +def segment_inds_list_from_boundary_labels( + boundary_labels: npt.NDArray, + force_boundary_first_ind: bool = True +) -> list[npt.NDArray]: + """Given an array of boundary labels, + return a list of :class:`numpy.ndarray` vectors, + each of which can be used to index one segment + in a vector of frame labels. + + Parameters + ---------- + boundary_labels : numpy.ndarray + Vector of integers ``{0, 1}``, where ``1`` indicates a boundary, + and ``0`` indicates no boundary. + Output of a frame classification model trained to classify each + frame as "boundary" or "no boundary". + force_boundary_first_ind : bool + If ``True``, and the first index of ``boundary_labels`` is not classified as a boundary, + force it to be a boundary. + + Returns + ------- + segment_inds_list : list + Of fancy indexing arrays. Each array can be used to index + one segment in ``frame_labels``. + """ + boundary_inds = boundary_inds_from_boundary_labels(boundary_labels, force_boundary_first_ind) + + # at the end of `boundary_inds``, insert an imaginary "last" boundary we use just with ``np.arange`` below + np.insert(boundary_inds, boundary_inds.shape[0], boundary_labels.shape[0]) + + segment_inds_list = [] + for start, stop in zip(boundary_inds[:-1], boundary_inds[1:]): + segment_inds_list.append(np.arange(start, stop)) + + return segment_inds_list + + def postprocess( - frame_labels: np.ndarray, + frame_labels: npt.NDArray, timebin_dur: float, - unlabeled_label: int = 0, + background_label: int = 0, min_segment_dur: float | None = None, majority_vote: bool = False, -) -> np.ndarray: + boundary_labels: npt.NDArray | None = None +) -> npt.NDArray: """Apply post-processing transformations to a vector of frame labels. Optional post-processing consist of two transforms, that both rely on there being a label - that corresponds to the "unlabeled" - (or "background") class. + that corresponds to the background class. The first removes any segments that are shorter than a specified duration, by converting labels in those segments to the - "background" / "unlabeled" class label. + background class label. The second performs a "majority vote" transform within run of labels that is bordered on both sides by the "background" label. @@ -418,7 +494,7 @@ def postprocess( Duration of a time bin in a spectrogram, e.g., as estimated from vector of times using ``vak.timebins.timebin_dur_from_vec``. - unlabeled_label : int + background_label : int Label that was given to segments that were not labeled in annotation, e.g. silent periods between annotated segments. Default is 0. min_segment_dur : float @@ -430,10 +506,21 @@ def postprocess( If True, transform segments containing multiple labels into segments with a single label by taking a "majority vote", i.e. assign all time bins in the segment the most frequently - occurring label in the segment. This transform can only be - applied if the labelmap contains an 'unlabeled' label, - because unlabeled segments makes it possible to identify - the labeled segments. Default is False. + occurring label in the segment. + This transform requires either a background label + or a vector of boundary labels. + Default is False. + boundary_labels : numpy.ndarray, optional. + Vector of integers ``{0, 1}``, where ``1`` indicates a boundary, + and ``0`` indicates no boundary. + Output of one head of a frame classification model, + that has been trained to classify each frame as either + "boundary" or "no boundary". + Optional, default is None. + If supplied, this vector is used to find segments + before applying post-processing, instead + of recovering them from ``frame_labels`` using the + ``background_label`` Returns ------- @@ -445,12 +532,17 @@ def postprocess( # handle the case when all time bins are predicted to be unlabeled # see https://github.com/NickleDave/vak/issues/383 uniq_frame_labels = np.unique(frame_labels) - if len(uniq_frame_labels) == 1 and uniq_frame_labels[0] == unlabeled_label: + if len(uniq_frame_labels) == 1 and uniq_frame_labels[0] == background_label: return frame_labels # -> no need to do any of the post-processing - segment_inds_list = to_inds_list( - frame_labels, unlabeled_label=unlabeled_label - ) + if boundary_labels is not None: + segment_inds_list = segment_inds_list_from_boundary_labels( + boundary_labels + ) + else: + segment_inds_list = segment_inds_list_from_class_labels( + frame_labels, background_label=background_label + ) if min_segment_dur is not None: frame_labels, segment_inds_list = remove_short_segments( @@ -458,7 +550,7 @@ def postprocess( segment_inds_list, timebin_dur, min_segment_dur, - unlabeled_label, + background_label, ) if len(segment_inds_list) == 0: # no segments left after removing return frame_labels # -> no need to do any of the post-processing diff --git a/src/vak/transforms/frame_labels/transforms.py b/src/vak/transforms/frame_labels/transforms.py index bcb81bc48..0dab0c504 100644 --- a/src/vak/transforms/frame_labels/transforms.py +++ b/src/vak/transforms/frame_labels/transforms.py @@ -35,13 +35,13 @@ class FromSegments: Attributes ---------- - unlabeled_label : int + background_label : int Label assigned to time bins that do not have labels associated with them. Default is 0. """ - def __init__(self, unlabeled_label: int = 0): - self.unlabeled_label = unlabeled_label + def __init__(self, background_label: int = 0): + self.background_label = background_label def __call__( self, @@ -76,7 +76,7 @@ def __call__( onsets_s, offsets_s, time_bins, - unlabeled_label=self.unlabeled_label, + background_label=self.background_label, ) @@ -199,12 +199,11 @@ class PostProcess: Optional post-processing consist of two transforms, that both rely on there being a label - that corresponds to the "unlabeled" - (or "background") class. + that corresponds to the background class. The first removes any segments that are shorter than a specified duration, by converting labels in those segments to the - "background" / "unlabeled" class label. + background class label. The second performs a "majority vote" transform within run of labels that is bordered on both sides by the "background" label. @@ -229,7 +228,7 @@ class PostProcess: Duration of a time bin in a spectrogram, e.g., as estimated from vector of times using ``vak.timebins.timebin_dur_from_vec``. - unlabeled_label : int + background_label : int Label that was given to segments that were not labeled in annotation, e.g. silent periods between annotated segments. Default is 0. min_segment_dur : float @@ -250,12 +249,12 @@ class PostProcess: def __init__( self, timebin_dur: float, - unlabeled_label: int = 0, + background_label: int = 0, min_segment_dur: float | None = None, majority_vote: bool = False, ): self.timebin_dur = timebin_dur - self.unlabeled_label = unlabeled_label + self.background_label = background_label self.min_segment_dur = min_segment_dur self.majority_vote = majority_vote @@ -278,7 +277,7 @@ def __call__(self, frame_labels: np.ndarray) -> np.ndarray: return F.postprocess( frame_labels, self.timebin_dur, - self.unlabeled_label, + self.background_label, self.min_segment_dur, self.majority_vote, ) diff --git a/src/vak/transforms/functional.py b/src/vak/transforms/functional.py index c65597c7a..6a882d839 100644 --- a/src/vak/transforms/functional.py +++ b/src/vak/transforms/functional.py @@ -1,4 +1,9 @@ +"""Functional forms of input transforms.""" + +from __future__ import annotations + import numpy as np +import numpy.typing as npt import torch __all__ = [ @@ -18,9 +23,9 @@ def standardize_spect(spect, mean_freqs, std_freqs, non_zero_std): spect : numpy.ndarray with shape (frequencies, time bins) mean_freqs : numpy.ndarray - vector of mean values for each frequency bin across the fit set of spectrograms + vector of mean values for each row across the fit set of spectrograms std_freqs : numpy.ndarray - vector of standard deviations for each frequency bin across the fit set of spectrograms + vector of standard deviations for each row across the fit set of spectrograms non_zero_std : numpy.ndarray boolean, indicates where std_freqs has non-zero values. Used to avoid divide-by-zero errors. @@ -38,9 +43,14 @@ def standardize_spect(spect, mean_freqs, std_freqs, non_zero_std): return tfm -def pad_to_window(arr, window_size, padval=0.0, return_padding_mask=True): - """pad a 1d or 2d array so that it can be reshaped - into consecutive windows of specified size +def pad_to_window( + arr: npt.NDArray, + window_size: int, + padval: float = 0.0, + return_padding_mask: bool = True, +) -> npt.NDArray: + """Pad a 1d or 2d array so that it can be reshaped + into consecutive windows of specified size. Parameters ---------- @@ -98,7 +108,7 @@ def pad_to_window(arr, window_size, padval=0.0, return_padding_mask=True): return padded -def view_as_window_batch(arr, window_width): +def view_as_window_batch(arr: npt.NDArray, window_size: int) -> npt.NDArray: """return view of a 1d or 2d array as a batch of non-overlapping windows Parameters @@ -108,7 +118,7 @@ def view_as_window_batch(arr, window_width): or a 2-d array representing a spectrogram. If the array has 2-d dimensions, the returned array will have dimensions (batch, height of array, window width) - window_width : int + window_size : int width of window in number of elements. Returns @@ -116,7 +126,7 @@ def view_as_window_batch(arr, window_width): batch_windows : numpy.ndarray with shape (batch size, window_size) if array is 1d, or with shape (batch size, height, window_size) if array is 2d. - Batch size will be arr.shape[-1] // window_width. + Batch size will be arr.shape[-1] // window_size. Window width must divide arr.shape[-1] evenly. To pad the array so it can be divided into windows of the specified width, use the `pad_to_window` transform @@ -126,16 +136,16 @@ def view_as_window_batch(arr, window_width): adapted from skimage.util.view_as_blocks https://github.com/scikit-image/scikit-image/blob/f1b7cf60fb80822849129cb76269b75b8ef18db1/skimage/util/shape.py#L9 """ - if not isinstance(window_width, int) or window_width < 1: + if not isinstance(window_size, int) or window_size < 1: raise ValueError( - f"`window_width` must be a positive integer, but was: {window_width}" + f"`window_size` must be a positive integer, but was: {window_size}" ) if arr.ndim == 1: - window_shape = (window_width,) + window_shape = (window_size,) elif arr.ndim == 2: height, _ = arr.shape - window_shape = (height, window_width) + window_shape = (height, window_size) else: raise ValueError( f"input array must be 1d or 2d but number of dimensions was: {arr.ndim}" @@ -145,7 +155,7 @@ def view_as_window_batch(arr, window_width): arr_shape = np.array(arr.shape) if (arr_shape % window_shape).sum() != 0: raise ValueError( - "'window_width' does not divide evenly into with 'arr' shape. " + "'window_size' does not divide evenly into with 'arr' shape. " "Use 'pad_to_window' transform to pad array so it can be windowed." ) @@ -154,10 +164,14 @@ def view_as_window_batch(arr, window_width): batch_windows = np.lib.stride_tricks.as_strided( arr, shape=new_shape, strides=new_strides ) - # TODO: figure out if there's a better way to do this where we don't need to squeeze - # The current version always add an initial dim of size 1 - batch_windows = np.squeeze(batch_windows, axis=0) - # By squeezing just that first axis, we always end up with (batch, freq. bins, time bins) for a spectrogram + + if ( + arr.ndim == 2 + ): # avoids bug in vak version of transform, by not doing this to 1-D arrays + # TODO: figure out if there's a better way to do this where we don't need to squeeze + # The current version always add an initial dim of size 1 + batch_windows = np.squeeze(batch_windows, axis=0) + # By squeezing just that first axis, we always end up with (batch, freq. bins, time bins) for a spectrogram return batch_windows diff --git a/src/vak/transforms/transforms.py b/src/vak/transforms/transforms.py index 23d900b86..7fab8a0f8 100644 --- a/src/vak/transforms/transforms.py +++ b/src/vak/transforms/transforms.py @@ -12,7 +12,7 @@ __all__ = [ "AddChannel", "PadToWindow", - "StandardizeSpect", + "FramesStandardizer", "ToFloatTensor", "ToLongTensor", "ViewAsWindowBatch", @@ -21,7 +21,7 @@ # adapted from: # https://github.com/NickleDave/hybrid-vocal-classifier/blob/master/hvc/neuralnet/utils.py -class StandardizeSpect: +class FramesStandardizer: """transform that standardizes spectrograms so they are all on the same scale, by subtracting off the mean and dividing by the standard deviation from a 'fit' set of spectrograms. @@ -29,9 +29,9 @@ class StandardizeSpect: Attributes ---------- mean_freqs : numpy.ndarray - mean values for each frequency bin across the fit set of spectrograms + mean values for each row across the fit set of spectrograms std_freqs : numpy.ndarray - standard deviation for each frequency bin across the fit set of spectrograms + standard deviation for each row across the fit set of spectrograms non_zero_std : numpy.ndarray boolean, indicates where std_freqs has non-zero values. Used to avoid divide-by-zero errors. """ @@ -42,9 +42,9 @@ def __init__(self, mean_freqs=None, std_freqs=None, non_zero_std=None): Parameters ---------- mean_freqs : numpy.ndarray - vector of mean values for each frequency bin across the fit set of spectrograms + vector of mean values for each row across the fit set of spectrograms std_freqs : numpy.ndarray - vector of standard deviations for each frequency bin across the fit set of spectrograms + vector of standard deviations for each row across the fit set of spectrograms non_zero_std : numpy.ndarray boolean, indicates where std_freqs has non-zero values. Used to avoid divide-by-zero errors. """ @@ -76,6 +76,59 @@ def __init__(self, mean_freqs=None, std_freqs=None, non_zero_std=None): self.std_freqs = std_freqs self.non_zero_std = non_zero_std + @classmethod + def fit_inputs_targets_csv_path( + cls, + inputs_targets_csv_path: str | pathlib.Path, + dataset_path: str | pathlib.Path, + split: str = "train", + subset: str | None = None, + frames_path_col_name: str | None = None, + frames_key: str | None = None, + ): + if frames_path_col_name is None: + from .. import datapipes + + frames_path_col_name = ( + datapipes.frame_classification.constants.FRAMES_PATH_COL_NAME + ) + if frames_key is None: + frames_key = constants.SPECT_KEY + + inputs_targets_csv_path = pathlib.Path(inputs_targets_csv_path) + if not inputs_targets_csv_path.exists(): + raise FileNotFoundError( + f"`inputs_targets_csv_path` for dataset not found: {inputs_targets_csv_path}" + ) + + dataset_path = pathlib.Path(dataset_path) + if not dataset_path.exists() or not dataset_path.is_dir(): + raise NotADirectoryError( + f"`dataset_path` not found, or not a directory: {dataset_path}" + ) + + df = pd.read_csv(inputs_targets_csv_path) + if subset: + df = df[df.split == split].copy() + else: + df = df[df.split == split].copy() + frames_paths = df[frames_path_col_name].values + frames = np.load(dataset_path / frames_paths[0])[frames_key] + + # in spectrograms files, spectrograms are in orientation (freq bins, time bins) + # so we take mean and std across columns, i.e. time bins, i.e. axis 1 + mean_freqs = np.mean(frames, axis=1) + std_freqs = np.std(frames, axis=1) + + for frames_path in frames_paths[1:]: + frames = np.load(dataset_path / frames_path)[frames_key] + mean_freqs += np.mean(frames, axis=1) + std_freqs += np.std(frames, axis=1) + mean_freqs = mean_freqs / len(frames_paths) + std_freqs = std_freqs / len(frames_paths) + non_zero_std = np.argwhere(std_freqs != 0) + return cls(mean_freqs, std_freqs, non_zero_std) + @classmethod def fit_dataset_path( cls, dataset_path, split="train", subset: str | None = None @@ -97,36 +150,15 @@ def fit_dataset_path( standardize_spect : StandardizeSpect Instance that has been fit to input data from split. """ - from vak.datasets import frame_classification - from vak.datasets.frame_classification import Metadata + from vak.datapipes.frame_classification import Metadata dataset_path = pathlib.Path(dataset_path) metadata = Metadata.from_dataset_path(dataset_path) dataset_csv_path = dataset_path / metadata.dataset_csv_filename dataset_path = dataset_csv_path.parent - dataset_df = pd.read_csv(dataset_csv_path) - if subset: - dataset_df = dataset_df[dataset_df.split == split].copy() - else: - dataset_df = dataset_df[dataset_df.split == split].copy() - frames_paths = dataset_df[ - frame_classification.constants.FRAMES_PATH_COL_NAME - ].values - frames = np.load(dataset_path / frames_paths[0])[constants.SPECT_KEY] - - # in files, spectrograms are in orientation (freq bins, time bins) - # so we take mean and std across columns, i.e. time bins, i.e. axis 1 - mean_freqs = np.mean(frames, axis=1) - std_freqs = np.std(frames, axis=1) - - for frames_path in frames_paths[1:]: - frames = np.load(dataset_path / frames_path)[constants.SPECT_KEY] - mean_freqs += np.mean(frames, axis=1) - std_freqs += np.std(frames, axis=1) - mean_freqs = mean_freqs / len(frames_paths) - std_freqs = std_freqs / len(frames_paths) - non_zero_std = np.argwhere(std_freqs != 0) - return cls(mean_freqs, std_freqs, non_zero_std) + return cls.fit_inputs_targets_csv_path( + dataset_csv_path, dataset_path, split, subset + ) @classmethod def fit(cls, spect): @@ -140,7 +172,7 @@ def fit(cls, spect): Notes ----- Input should be spectrogram. - Fit function finds the mean and standard deviation of each frequency bin, + Fit function finds the mean and standard deviation of each row, which are used by `transform` method to scale other spectrograms. """ # TODO: make this function accept list and/or ndarray with batch dimension @@ -164,13 +196,13 @@ def __call__(self, spect): ------- z_norm_spect : numpy.ndarray array standardized to same scale as set of spectrograms that - SpectScaler was fit with + :class:`vak.transforms.FramesStandardizer` was fit with """ if any( [not hasattr(self, attr) for attr in ["mean_freqs", "std_freqs"]] ): raise AttributeError( - "SpectScaler properties are set to None," + "FramesStandardizer properties are set to None," "must call fit method first to set the" "value of these properties before calling" "transform" diff --git a/tests/data_for_tests/configs/TweetyNet_eval_audio_cbin_annot_notmat.toml b/tests/data_for_tests/configs/TweetyNet_eval_audio_cbin_annot_notmat.toml index 98e51a7b0..22926bddf 100644 --- a/tests/data_for_tests/configs/TweetyNet_eval_audio_cbin_annot_notmat.toml +++ b/tests/data_for_tests/configs/TweetyNet_eval_audio_cbin_annot_notmat.toml @@ -20,7 +20,7 @@ labelmap_path = "~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepos batch_size = 11 num_workers = 16 -spect_scaler_path = "~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect" +frames_standardizer_path = "~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect" output_dir = "./tests/data_for_tests/generated/results/eval/audio_cbin_annot_notmat/TweetyNet" [vak.eval.post_tfm_kwargs] diff --git a/tests/data_for_tests/configs/TweetyNet_learncurve_audio_cbin_annot_notmat.toml b/tests/data_for_tests/configs/TweetyNet_learncurve_audio_cbin_annot_notmat.toml index 744cec82e..f2780a6fc 100644 --- a/tests/data_for_tests/configs/TweetyNet_learncurve_audio_cbin_annot_notmat.toml +++ b/tests/data_for_tests/configs/TweetyNet_learncurve_audio_cbin_annot_notmat.toml @@ -20,7 +20,7 @@ thresh = 6.25 transform_type = "log_spect" [vak.learncurve] -normalize_spectrograms = true +standardize_frames = true batch_size = 11 num_epochs = 2 val_step = 50 diff --git a/tests/data_for_tests/configs/TweetyNet_predict_audio_cbin_annot_notmat.toml b/tests/data_for_tests/configs/TweetyNet_predict_audio_cbin_annot_notmat.toml index 0d1cd8f13..8ad0875b8 100644 --- a/tests/data_for_tests/configs/TweetyNet_predict_audio_cbin_annot_notmat.toml +++ b/tests/data_for_tests/configs/TweetyNet_predict_audio_cbin_annot_notmat.toml @@ -13,7 +13,7 @@ thresh = 6.25 transform_type = "log_spect" [vak.predict] -spect_scaler_path = "/home/user/results_181014_194418/spect_scaler" +frames_standardizer_path = "/home/user/results_181014_194418/spect_scaler" checkpoint_path = "~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/TweetyNet/checkpoints/max-val-acc-checkpoint.pt" labelmap_path = "~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/labelmap.json" batch_size = 11 diff --git a/tests/data_for_tests/configs/TweetyNet_train_audio_cbin_annot_notmat.toml b/tests/data_for_tests/configs/TweetyNet_train_audio_cbin_annot_notmat.toml index c5d26bf88..99333b895 100644 --- a/tests/data_for_tests/configs/TweetyNet_train_audio_cbin_annot_notmat.toml +++ b/tests/data_for_tests/configs/TweetyNet_train_audio_cbin_annot_notmat.toml @@ -18,7 +18,7 @@ thresh = 6.25 transform_type = "log_spect" [vak.train] -normalize_spectrograms = true +standardize_frames = true batch_size = 11 num_epochs = 2 val_step = 50 diff --git a/tests/data_for_tests/configs/TweetyNet_train_continue_audio_cbin_annot_notmat.toml b/tests/data_for_tests/configs/TweetyNet_train_continue_audio_cbin_annot_notmat.toml index 5af7f78d2..4bb3b6da9 100644 --- a/tests/data_for_tests/configs/TweetyNet_train_continue_audio_cbin_annot_notmat.toml +++ b/tests/data_for_tests/configs/TweetyNet_train_continue_audio_cbin_annot_notmat.toml @@ -18,7 +18,7 @@ thresh = 6.25 transform_type = "log_spect" [vak.train] -normalize_spectrograms = true +standardize_frames = true batch_size = 11 num_epochs = 2 val_step = 50 @@ -28,7 +28,7 @@ num_workers = 16 root_results_dir = "./tests/data_for_tests/generated/results/train_continue/audio_cbin_annot_notmat/TweetyNet" checkpoint_path = "~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/TweetyNet/checkpoints/max-val-acc-checkpoint.pt" -spect_scaler_path = "~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect" +frames_standardizer_path = "~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect" [vak.train.dataset] params = { window_size = 88 } diff --git a/tests/data_for_tests/configs/TweetyNet_train_continue_spect_mat_annot_yarden.toml b/tests/data_for_tests/configs/TweetyNet_train_continue_spect_mat_annot_yarden.toml index 05d897eba..ad0213101 100644 --- a/tests/data_for_tests/configs/TweetyNet_train_continue_spect_mat_annot_yarden.toml +++ b/tests/data_for_tests/configs/TweetyNet_train_continue_spect_mat_annot_yarden.toml @@ -18,7 +18,7 @@ thresh = 6.25 transform_type = "log_spect" [vak.train] -normalize_spectrograms = false +standardize_frames = false batch_size = 11 num_epochs = 2 val_step = 50 diff --git a/tests/data_for_tests/configs/TweetyNet_train_spect_mat_annot_yarden.toml b/tests/data_for_tests/configs/TweetyNet_train_spect_mat_annot_yarden.toml index 4796edb60..93b9fe96d 100644 --- a/tests/data_for_tests/configs/TweetyNet_train_spect_mat_annot_yarden.toml +++ b/tests/data_for_tests/configs/TweetyNet_train_spect_mat_annot_yarden.toml @@ -18,7 +18,7 @@ thresh = 6.25 transform_type = "log_spect" [vak.train] -normalize_spectrograms = false +standardize_frames = false batch_size = 11 num_epochs = 2 val_step = 50 diff --git a/tests/data_for_tests/configs/invalid_key_config.toml b/tests/data_for_tests/configs/invalid_key_config.toml index 76770862c..efc3d606f 100644 --- a/tests/data_for_tests/configs/invalid_key_config.toml +++ b/tests/data_for_tests/configs/invalid_key_config.toml @@ -23,7 +23,7 @@ transform_type = 'log_spect' [vak.train] root_results_dir = '/home/user/data/subdir/' -normalize_spectrograms = true +standardize_frames = true num_epochs = 2 batch_size = 11 val_error_step = 1 diff --git a/tests/data_for_tests/configs/invalid_table_config.toml b/tests/data_for_tests/configs/invalid_table_config.toml index 09898375f..8ed12f712 100644 --- a/tests/data_for_tests/configs/invalid_table_config.toml +++ b/tests/data_for_tests/configs/invalid_table_config.toml @@ -23,7 +23,7 @@ transform_type = 'log_spect' [vak.trian] # <-- invalid section 'trian' (instead of 'vak.train') model = 'TweetyNet' root_results_dir = '/home/user/data/subdir/' -normalize_spectrograms = true +standardize_frames = true num_epochs = 2 batch_size = 11 val_error_step = 1 diff --git a/tests/data_for_tests/configs/invalid_train_and_learncurve_config.toml b/tests/data_for_tests/configs/invalid_train_and_learncurve_config.toml index 3107b538c..c3708b838 100644 --- a/tests/data_for_tests/configs/invalid_train_and_learncurve_config.toml +++ b/tests/data_for_tests/configs/invalid_train_and_learncurve_config.toml @@ -20,7 +20,7 @@ transform_type = "log_spect" # this .toml file should cause 'vak.config.parse.from_toml' to raise a ValueError # because it defines both a vak.train and a vak.learncurve section [vak.train] -normalize_spectrograms = true +standardize_frames = true batch_size = 11 num_epochs = 2 val_step = 50 @@ -35,7 +35,7 @@ accelerator = "gpu" devices = [0] [vak.learncurve] -normalize_spectrograms = true +standardize_frames = true batch_size = 11 num_epochs = 2 val_step = 50 diff --git a/tests/fixtures/csv.py b/tests/fixtures/csv.py index 0e21cc196..6a34c244b 100644 --- a/tests/fixtures/csv.py +++ b/tests/fixtures/csv.py @@ -28,9 +28,9 @@ def _specific_csv_path( dataset_path = Path(config_toml[config_type]["dataset"]["path"]) # TODO: make this more general -- dataset registry? if config_toml['prep']['dataset_type'] == 'frame classification': - metadata = vak.datasets.frame_classification.Metadata.from_dataset_path(dataset_path) + metadata = vak.datapipes.frame_classification.Metadata.from_dataset_path(dataset_path) elif config_toml['prep']['dataset_type'] == 'parametric umap': - metadata = vak.datasets.parametric_umap.Metadata.from_dataset_path(dataset_path) + metadata = vak.datapipes.parametric_umap.Metadata.from_dataset_path(dataset_path) dataset_csv_path = dataset_path / metadata.dataset_csv_filename return dataset_csv_path diff --git a/tests/scripts/vaktestdata/configs.py b/tests/scripts/vaktestdata/configs.py index 3134f7f57..ee883bb7a 100644 --- a/tests/scripts/vaktestdata/configs.py +++ b/tests/scripts/vaktestdata/configs.py @@ -103,7 +103,7 @@ def fix_options_in_configs(config_metadata_list, command, single_train_result=Tr config_to_use_result_from = constants.GENERATED_TEST_CONFIGS_ROOT / config_metadata.use_result_from_config # now use the config to find the results dir and get the values for the options we need to set - # which are checkpoint_path, spect_scaler_path, and labelmap_path + # which are checkpoint_path, frames_standardizer_path, and labelmap_path with config_to_use_result_from.open("r") as fp: config_toml = tomlkit.load(fp) root_results_dir = pathlib.Path(config_toml["vak"]["train"]["root_results_dir"]) @@ -131,10 +131,10 @@ def fix_options_in_configs(config_metadata_list, command, single_train_result=Tr # these are the only options whose values we need to change # and they are the same for both predict and eval checkpoint_path = sorted(results_dir.glob("**/checkpoints/checkpoint.pt"))[0] - if 'normalize_spectrograms' in config_toml["vak"]['train'] and config_toml["vak"]['train']['normalize_spectrograms']: - spect_scaler_path = sorted(results_dir.glob("StandardizeSpect"))[0] + if 'standardize_frames' in config_toml["vak"]['train'] and config_toml["vak"]['train']['standardize_frames']: + frames_standardizer_path = sorted(results_dir.glob("FramesStandardizer"))[0] else: - spect_scaler_path = None + frames_standardizer_path = None labelmap_path = sorted(results_dir.glob("labelmap.json")) if len(labelmap_path) == 1: @@ -159,12 +159,12 @@ def fix_options_in_configs(config_metadata_list, command, single_train_result=Tr table = command config_toml["vak"][table]["checkpoint_path"] = str(checkpoint_path) - if spect_scaler_path: - config_toml["vak"][table]["spect_scaler_path"] = str(spect_scaler_path) + if frames_standardizer_path: + config_toml["vak"][table]["frames_standardizer_path"] = str(frames_standardizer_path) else: - if 'spect_scaler_path' in config_toml["vak"][table]: - # remove any existing 'spect_scaler_path' option - del config_toml["vak"][table]["spect_scaler_path"] + if 'frames_standardizer_path' in config_toml["vak"][table]: + # remove any existing 'frames_standardizer_path' option + del config_toml["vak"][table]["frames_standardizer_path"] if command != 'train_continue': # train always gets labelmap from dataset dir, not from a config option if labelmap_path is not None: config_toml["vak"][table]["labelmap_path"] = str(labelmap_path) diff --git a/tests/test_common/test_labels.py b/tests/test_common/test_labels.py index 3a3f794e2..398d53684 100644 --- a/tests/test_common/test_labels.py +++ b/tests/test_common/test_labels.py @@ -4,12 +4,13 @@ import pandas as pd import pytest +import vak.common # for constants import vak.common.files.spect import vak.common.labels @pytest.mark.parametrize( - 'labelset, map_unlabeled', + 'labelset, map_background', [ ( set(list("abcde")), @@ -29,14 +30,14 @@ ) ] ) -def test_to_map(labelset, map_unlabeled): - labelmap = vak.common.labels.to_map(labelset, map_unlabeled=map_unlabeled) +def test_to_map(labelset, map_background): + labelmap = vak.common.labels.to_map(labelset, map_background=map_background) assert isinstance(labelmap, dict) - if map_unlabeled: - # because map_unlabeled=True + if map_background: + # because map_background=True assert len(labelmap) == len(labelset) + 1 else: - # because map_unlabeled=False + # because map_background=False assert len(labelmap) == len(labelset) @@ -77,19 +78,19 @@ def test_from_df(config_type, model_name, audio_format, spect_format, annot_form INTS_LABELMAP = {str(val): val for val in range(1, 20)} INTS_LABELMAP_WITH_UNLABELED = copy.deepcopy(INTS_LABELMAP) -INTS_LABELMAP_WITH_UNLABELED['unlabeled'] = 0 +INTS_LABELMAP_WITH_UNLABELED[vak.common.constants.DEFAULT_BACKGROUND_LABEL] = 0 -DEFAULT_SKIP = ('unlabeled',) +DEFAULT_SKIP = (vak.common.constants.DEFAULT_BACKGROUND_LABEL,) @pytest.mark.parametrize( 'labelmap, skip', [ ({'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5}, None), - ({'unlabeled': 0, 'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5}, None), - ({'unlabeled': 0, 'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5}, ('unlabeled',)), + ({vak.common.constants.DEFAULT_BACKGROUND_LABEL: 0, 'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5}, None), + ({vak.common.constants.DEFAULT_BACKGROUND_LABEL: 0, 'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5}, (vak.common.constants.DEFAULT_BACKGROUND_LABEL,)), (INTS_LABELMAP, None), - (INTS_LABELMAP_WITH_UNLABELED, ('unlabeled',)) + (INTS_LABELMAP_WITH_UNLABELED, (vak.common.constants.DEFAULT_BACKGROUND_LABEL,)) ] ) def test_multi_char_labels_to_single_char(labelmap, skip): diff --git a/tests/test_config/test_dataset.py b/tests/test_config/test_dataset.py index ecc9fed0d..bd83f724c 100644 --- a/tests/test_config/test_dataset.py +++ b/tests/test_config/test_dataset.py @@ -35,9 +35,9 @@ def test_init(self, path, splits_path, name): splits_path=splits_path, ) assert isinstance(dataset_config, vak.config.dataset.DatasetConfig) - assert dataset_config.path == pathlib.Path(path) + assert dataset_config.path == vak.common.converters.expanded_user_path(path) if splits_path is not None: - assert dataset_config.splits_path == pathlib.Path(splits_path) + assert dataset_config.splits_path == vak.common.converters.expanded_user_path(splits_path) else: assert dataset_config.splits_path is None if name is not None: @@ -74,9 +74,9 @@ def test_init(self, path, splits_path, name): def test_from_config_dict(self, config_dict): dataset_config = vak.config.dataset.DatasetConfig.from_config_dict(config_dict) assert isinstance(dataset_config, vak.config.dataset.DatasetConfig) - assert dataset_config.path == pathlib.Path(config_dict['path']) + assert dataset_config.path == vak.common.converters.expanded_user_path(config_dict['path']) if 'splits_path' in config_dict: - assert dataset_config.splits_path == pathlib.Path(config_dict['splits_path']) + assert dataset_config.splits_path == vak.common.converters.expanded_user_path(config_dict['splits_path']) else: assert dataset_config.splits_path is None if 'name' in config_dict: @@ -123,7 +123,7 @@ def test_asdict(self, config_dict): for key in ('name', 'path', 'splits_path', 'params'): if key in config_dict: if 'path' in key: - assert dataset_config_as_dict[key] == pathlib.Path(config_dict[key]) + assert dataset_config_as_dict[key] == vak.common.converters.expanded_user_path(config_dict[key]) else: assert dataset_config_as_dict[key] == config_dict[key] else: diff --git a/tests/test_config/test_eval.py b/tests/test_config/test_eval.py index 7917e4a6b..89516a82d 100644 --- a/tests/test_config/test_eval.py +++ b/tests/test_config/test_eval.py @@ -15,7 +15,7 @@ class TestEval: 'batch_size': 11, 'num_workers': 16, 'trainer': {'accelerator': 'gpu', 'devices': [0]}, - 'spect_scaler_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', + 'frames_standardizer_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', 'output_dir': './tests/data_for_tests/generated/results/eval/audio_cbin_annot_notmat/TweetyNet', 'post_tfm_kwargs': { 'majority_vote': True, 'min_segment_dur': 0.02 @@ -60,7 +60,7 @@ def test_init(self, config_dict): 'batch_size': 11, 'num_workers': 16, 'trainer': {'accelerator': 'gpu', 'devices': [0]}, - 'spect_scaler_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', + 'frames_standardizer_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', 'output_dir': './tests/data_for_tests/generated/results/eval/audio_cbin_annot_notmat/TweetyNet', 'post_tfm_kwargs': { 'majority_vote': True, 'min_segment_dur': 0.02 @@ -110,7 +110,7 @@ def test_from_config_dict_with_real_config(self, a_generated_eval_config_dict): 'batch_size': 11, 'num_workers': 16, 'trainer': {'accelerator': 'gpu', 'devices': [0]}, - 'spect_scaler_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', + 'frames_standardizer_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', 'output_dir': './tests/data_for_tests/generated/results/eval/audio_cbin_annot_notmat/TweetyNet', 'post_tfm_kwargs': { 'majority_vote': True, 'min_segment_dur': 0.02 @@ -129,7 +129,7 @@ def test_from_config_dict_with_real_config(self, a_generated_eval_config_dict): 'batch_size': 11, 'num_workers': 16, 'trainer': {'accelerator': 'gpu', 'devices': [0]}, - 'spect_scaler_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', + 'frames_standardizer_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', 'output_dir': './tests/data_for_tests/generated/results/eval/audio_cbin_annot_notmat/TweetyNet', 'post_tfm_kwargs': { 'majority_vote': True, 'min_segment_dur': 0.02 @@ -160,7 +160,7 @@ def test_from_config_dict_with_real_config(self, a_generated_eval_config_dict): 'batch_size': 11, 'num_workers': 16, 'trainer': {'accelerator': 'gpu', 'devices': [0]}, - 'spect_scaler_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', + 'frames_standardizer_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', 'output_dir': './tests/data_for_tests/generated/results/eval/audio_cbin_annot_notmat/TweetyNet', 'post_tfm_kwargs': { 'majority_vote': True, 'min_segment_dur': 0.02 @@ -195,7 +195,7 @@ def test_from_config_dict_with_real_config(self, a_generated_eval_config_dict): 'batch_size': 11, 'num_workers': 16, 'trainer': {'accelerator': 'gpu', 'devices': [0]}, - 'spect_scaler_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', + 'frames_standardizer_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/gy6or6/results_200620_165308/StandardizeSpect', 'post_tfm_kwargs': { 'majority_vote': True, 'min_segment_dur': 0.02 }, diff --git a/tests/test_config/test_learncurve.py b/tests/test_config/test_learncurve.py index fa21c152b..2b07e9f17 100644 --- a/tests/test_config/test_learncurve.py +++ b/tests/test_config/test_learncurve.py @@ -10,7 +10,7 @@ class TestLearncurveConfig: 'config_dict', [ { - 'normalize_spectrograms': True, + 'standardize_frames': True, 'batch_size': 11, 'num_epochs': 2, 'val_step': 50, @@ -57,7 +57,7 @@ def test_init(self, config_dict): 'config_dict', [ { - 'normalize_spectrograms': True, + 'standardize_frames': True, 'batch_size': 11, 'num_epochs': 2, 'val_step': 50, @@ -112,7 +112,7 @@ def test_from_config_dict_with_real_config(self, a_generated_learncurve_config_d # missing 'model', should raise KeyError ( { - 'normalize_spectrograms': True, + 'standardize_frames': True, 'batch_size': 11, 'num_epochs': 2, 'val_step': 50, @@ -131,7 +131,7 @@ def test_from_config_dict_with_real_config(self, a_generated_learncurve_config_d # missing 'dataset', should raise KeyError ( { - 'normalize_spectrograms': True, + 'standardize_frames': True, 'batch_size': 11, 'num_epochs': 2, 'val_step': 50, @@ -165,7 +165,7 @@ def test_from_config_dict_with_real_config(self, a_generated_learncurve_config_d # missing 'root_results_dir', should raise KeyError ( { - 'normalize_spectrograms': True, + 'standardize_frames': True, 'batch_size': 11, 'num_epochs': 2, 'val_step': 50, diff --git a/tests/test_config/test_predict.py b/tests/test_config/test_predict.py index 8dc77e7d8..f52c46bf8 100644 --- a/tests/test_config/test_predict.py +++ b/tests/test_config/test_predict.py @@ -10,7 +10,7 @@ class TestPredictConfig: 'config_dict', [ { - 'spect_scaler_path': '/home/user/results_181014_194418/spect_scaler', + 'frames_standardizer_path': '/home/user/results_181014_194418/spect_scaler', 'checkpoint_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/TweetyNet/checkpoints/max-val-acc-checkpoint.pt', 'labelmap_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/labelmap.json', 'batch_size': 11, @@ -53,7 +53,7 @@ def test_init(self, config_dict): 'config_dict', [ { - 'spect_scaler_path': '/home/user/results_181014_194418/spect_scaler', + 'frames_standardizer_path': '/home/user/results_181014_194418/spect_scaler', 'checkpoint_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/TweetyNet/checkpoints/max-val-acc-checkpoint.pt', 'labelmap_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/labelmap.json', 'batch_size': 11, @@ -101,7 +101,7 @@ def test_from_config_dict_with_real_config(self, a_generated_predict_config_dict # missing 'checkpoint_path', should raise KeyError ( { - 'spect_scaler_path': '/home/user/results_181014_194418/spect_scaler', + 'frames_standardizer_path': '/home/user/results_181014_194418/spect_scaler', 'labelmap_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/labelmap.json', 'batch_size': 11, 'num_workers': 16, @@ -133,7 +133,7 @@ def test_from_config_dict_with_real_config(self, a_generated_predict_config_dict # missing 'dataset', should raise KeyError ( { - 'spect_scaler_path': '/home/user/results_181014_194418/spect_scaler', + 'frames_standardizer_path': '/home/user/results_181014_194418/spect_scaler', 'checkpoint_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/TweetyNet/checkpoints/max-val-acc-checkpoint.pt', 'labelmap_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/labelmap.json', 'batch_size': 11, @@ -163,7 +163,7 @@ def test_from_config_dict_with_real_config(self, a_generated_predict_config_dict # missing 'model', should raise KeyError ( { - 'spect_scaler_path': '/home/user/results_181014_194418/spect_scaler', + 'frames_standardizer_path': '/home/user/results_181014_194418/spect_scaler', 'checkpoint_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/TweetyNet/checkpoints/max-val-acc-checkpoint.pt', 'labelmap_path': '~/Documents/repos/coding/birdsong/TweetyNet/results/BFSongRepository/bl26lb16/results_200620_164245/labelmap.json', 'batch_size': 11, diff --git a/tests/test_config/test_train.py b/tests/test_config/test_train.py index 970b1abb3..ef8d57276 100644 --- a/tests/test_config/test_train.py +++ b/tests/test_config/test_train.py @@ -10,7 +10,7 @@ class TestTrainConfig: 'config_dict', [ { - 'normalize_spectrograms': True, + 'standardize_frames': True, 'batch_size': 11, 'num_epochs': 2, 'val_step': 50, @@ -56,7 +56,7 @@ def test_init(self, config_dict): 'config_dict', [ { - 'normalize_spectrograms': True, + 'standardize_frames': True, 'batch_size': 11, 'num_epochs': 2, 'val_step': 50, @@ -106,7 +106,7 @@ def test_from_config_dict_with_real_config(self, a_generated_train_config_dict): [ ( { - 'normalize_spectrograms': True, + 'standardize_frames': True, 'batch_size': 11, 'num_epochs': 2, 'val_step': 50, @@ -123,7 +123,7 @@ def test_from_config_dict_with_real_config(self, a_generated_train_config_dict): ), ( { - 'normalize_spectrograms': True, + 'standardize_frames': True, 'batch_size': 11, 'num_epochs': 2, 'val_step': 50, diff --git a/tests/test_datasets/test_frame_classification/__init__.py b/tests/test_datapipes/__init__.py similarity index 100% rename from tests/test_datasets/test_frame_classification/__init__.py rename to tests/test_datapipes/__init__.py diff --git a/tests/test_datasets/test_parametric_umap/__init__.py b/tests/test_datapipes/test_frame_classification/__init__.py similarity index 100% rename from tests/test_datasets/test_parametric_umap/__init__.py rename to tests/test_datapipes/test_frame_classification/__init__.py diff --git a/tests/test_datasets/test_frame_classification/test_helper.py b/tests/test_datapipes/test_frame_classification/test_helper.py similarity index 59% rename from tests/test_datasets/test_frame_classification/test_helper.py rename to tests/test_datapipes/test_frame_classification/test_helper.py index 2be8e4fdc..288775c46 100644 --- a/tests/test_datasets/test_frame_classification/test_helper.py +++ b/tests/test_datapipes/test_frame_classification/test_helper.py @@ -1,7 +1,7 @@ import numpy as np import pytest -import vak.datasets.frame_classification.helper +import vak.datapipes.frame_classification.helper from ... import fixtures @@ -14,9 +14,9 @@ ] ) def test_sample_ids_array_filename_for_subset(subset): - out = vak.datasets.frame_classification.helper.sample_ids_array_filename_for_subset(subset) + out = vak.datapipes.frame_classification.helper.sample_ids_array_filename_for_subset(subset) assert isinstance(out, str) - assert out == vak.datasets.frame_classification.constants.SAMPLE_IDS_ARRAY_FILENAME.replace( + assert out == vak.datapipes.frame_classification.constants.SAMPLE_IDS_ARRAY_FILENAME.replace( '.npy', f'-{subset}.npy' ) @@ -29,9 +29,9 @@ def test_sample_ids_array_filename_for_subset(subset): ] ) def test_inds_in_sample_array_filename_for_subset(subset): - out = vak.datasets.frame_classification.helper.inds_in_sample_array_filename_for_subset(subset) + out = vak.datapipes.frame_classification.helper.inds_in_sample_array_filename_for_subset(subset) assert isinstance(out, str) - assert out == vak.datasets.frame_classification.constants.INDS_IN_SAMPLE_ARRAY_FILENAME.replace( + assert out == vak.datapipes.frame_classification.constants.INDS_IN_SAMPLE_ARRAY_FILENAME.replace( '.npy', f'-{subset}.npy' ) @@ -42,5 +42,5 @@ def frames_path(request): def test_load_frames(frames_path): - out = vak.datasets.frame_classification.helper.load_frames(frames_path, input_type="spect") + out = vak.datapipes.frame_classification.helper.load_frames(frames_path, input_type="spect") assert isinstance(out, np.ndarray) diff --git a/tests/test_datasets/test_frame_classification/test_frames_dataset.py b/tests/test_datapipes/test_frame_classification/test_infer_datapipe.py similarity index 66% rename from tests/test_datasets/test_frame_classification/test_frames_dataset.py rename to tests/test_datapipes/test_frame_classification/test_infer_datapipe.py index f71c7f9fb..7ad211c8e 100644 --- a/tests/test_datasets/test_frame_classification/test_frames_dataset.py +++ b/tests/test_datapipes/test_frame_classification/test_infer_datapipe.py @@ -1,10 +1,10 @@ import pytest import vak -import vak.datasets.frame_classification +import vak.datapipes.frame_classification -class TestWindowDataset: +class TestInferDatapipe: @pytest.mark.parametrize( 'config_type, model_name, audio_format, spect_format, annot_format, split', [ @@ -22,16 +22,9 @@ def test_from_dataset_path(self, config_type, model_name, audio_format, spect_fo cfg = vak.config.Config.from_toml_path(toml_path) cfg_command = getattr(cfg, config_type) - transform_kwargs = { - "window_size": cfg.eval.dataset.params["window_size"] - } - item_transform = vak.transforms.defaults.get_default_transform( - model_name, config_type, transform_kwargs - ) - - dataset = vak.datasets.frame_classification.FramesDataset.from_dataset_path( + datapipe = vak.datapipes.frame_classification.InferDatapipe.from_dataset_path( dataset_path=cfg_command.dataset.path, split=split, - item_transform=item_transform, + window_size=cfg.eval.dataset.params["window_size"] ) - assert isinstance(dataset, vak.datasets.frame_classification.FramesDataset) + assert isinstance(datapipe, vak.datapipes.frame_classification.InferDatapipe) diff --git a/tests/test_datasets/test_frame_classification/test_metadata.py b/tests/test_datapipes/test_frame_classification/test_metadata.py similarity index 74% rename from tests/test_datasets/test_frame_classification/test_metadata.py rename to tests/test_datapipes/test_frame_classification/test_metadata.py index d5d2f5145..9e1189fd4 100644 --- a/tests/test_datasets/test_frame_classification/test_metadata.py +++ b/tests/test_datapipes/test_frame_classification/test_metadata.py @@ -3,7 +3,7 @@ import pytest -import vak.datasets.frame_classification +import vak.datapipes.frame_classification ARGNAMES = 'dataset_csv_filename, input_type, frame_dur' @@ -23,8 +23,8 @@ class TestMetadata: ARGVALS ) def test_metadata_init(self, dataset_csv_filename, input_type, frame_dur): - metadata = vak.datasets.frame_classification.Metadata(dataset_csv_filename, input_type, frame_dur) - assert isinstance(metadata, vak.datasets.frame_classification.Metadata) + metadata = vak.datapipes.frame_classification.Metadata(dataset_csv_filename, input_type, frame_dur) + assert isinstance(metadata, vak.datapipes.frame_classification.Metadata) for attr_name, attr_val in zip( ('dataset_csv_filename', 'input_type', 'frame_dur'), (dataset_csv_filename, input_type, frame_dur), @@ -46,12 +46,12 @@ def test_metadata_from_path(self, dataset_csv_filename, input_type, frame_dur, t 'input_type': input_type, 'frame_dur': frame_dur, } - metadata_json_path = tmp_path / vak.datasets.frame_classification.Metadata.METADATA_JSON_FILENAME + metadata_json_path = tmp_path / vak.datapipes.frame_classification.Metadata.METADATA_JSON_FILENAME with metadata_json_path.open('w') as fp: json.dump(metadata_dict, fp, indent=4) - metadata = vak.datasets.frame_classification.Metadata.from_path(metadata_json_path) - assert isinstance(metadata, vak.datasets.frame_classification.Metadata) + metadata = vak.datapipes.frame_classification.Metadata.from_path(metadata_json_path) + assert isinstance(metadata, vak.datapipes.frame_classification.Metadata) for attr_name, attr_val in zip( ('dataset_csv_filename', 'input_type', 'frame_dur'), (dataset_csv_filename, input_type, frame_dur), @@ -67,13 +67,13 @@ def test_metadata_from_path(self, dataset_csv_filename, input_type, frame_dur, t ARGVALS ) def test_metadata_to_json(self, dataset_csv_filename, input_type, frame_dur, tmp_path): - metadata_to_json = vak.datasets.frame_classification.Metadata(dataset_csv_filename, input_type, frame_dur) + metadata_to_json = vak.datapipes.frame_classification.Metadata(dataset_csv_filename, input_type, frame_dur) mock_dataset_path = tmp_path / 'mock_dataset' mock_dataset_path.mkdir() metadata_to_json.to_json(dataset_path=mock_dataset_path) - expected_json_path = mock_dataset_path / vak.datasets.frame_classification.Metadata.METADATA_JSON_FILENAME + expected_json_path = mock_dataset_path / vak.datapipes.frame_classification.Metadata.METADATA_JSON_FILENAME assert expected_json_path.exists() - metadata_from_json = vak.datasets.frame_classification.Metadata.from_path(expected_json_path) + metadata_from_json = vak.datapipes.frame_classification.Metadata.from_path(expected_json_path) assert metadata_from_json == metadata_to_json diff --git a/tests/test_datasets/test_frame_classification/test_window_dataset.py b/tests/test_datapipes/test_frame_classification/test_train_datapipe.py similarity index 74% rename from tests/test_datasets/test_frame_classification/test_window_dataset.py rename to tests/test_datapipes/test_frame_classification/test_train_datapipe.py index 430917d2a..00462cc38 100644 --- a/tests/test_datasets/test_frame_classification/test_window_dataset.py +++ b/tests/test_datapipes/test_frame_classification/test_train_datapipe.py @@ -1,10 +1,10 @@ import pytest import vak -import vak.datasets.frame_classification +import vak.datapipes.frame_classification -class TestWindowDataset: +class TestTrainDatapipe: @pytest.mark.parametrize( 'config_type, model_name, audio_format, spect_format, annot_format, split, transform_kwargs', [ @@ -23,14 +23,9 @@ def test_from_dataset_path(self, config_type, model_name, audio_format, spect_fo cfg = vak.config.Config.from_toml_path(toml_path) cfg_command = getattr(cfg, config_type) - transform = vak.transforms.defaults.get_default_transform( - model_name, config_type, transform_kwargs - ) - - dataset = vak.datasets.frame_classification.WindowDataset.from_dataset_path( + dataset = vak.datapipes.frame_classification.TrainDatapipe.from_dataset_path( dataset_path=cfg_command.dataset.path, split=split, window_size=cfg_command.dataset.params['window_size'], - item_transform=transform, ) - assert isinstance(dataset, vak.datasets.frame_classification.WindowDataset) + assert isinstance(dataset, vak.datapipes.frame_classification.TrainDatapipe) diff --git a/tests/test_datasets/test_seq/__init__.py b/tests/test_datapipes/test_parametric_umap/__init__.py similarity index 100% rename from tests/test_datasets/test_seq/__init__.py rename to tests/test_datapipes/test_parametric_umap/__init__.py diff --git a/tests/test_datasets/test_parametric_umap/test_parametric_umap.py b/tests/test_datapipes/test_parametric_umap/test_parametric_umap.py similarity index 65% rename from tests/test_datasets/test_parametric_umap/test_parametric_umap.py rename to tests/test_datapipes/test_parametric_umap/test_parametric_umap.py index 7f2c0bb38..e35fe4199 100644 --- a/tests/test_datasets/test_parametric_umap/test_parametric_umap.py +++ b/tests/test_datapipes/test_parametric_umap/test_parametric_umap.py @@ -1,10 +1,10 @@ import pytest import vak -import vak.datasets.parametric_umap +import vak.datapipes.parametric_umap -class TestParametricUMAPDataset: +class TestDatapipe: @pytest.mark.parametrize( 'config_type, model_name, audio_format, spect_format, annot_format, split, transform_kwargs', [ @@ -13,7 +13,8 @@ class TestParametricUMAPDataset: ) def test_from_dataset_path(self, config_type, model_name, audio_format, spect_format, annot_format, split, transform_kwargs, specific_config_toml_path): - """Test we can get a WindowDataset instance from the classmethod ``from_dataset_path``""" + """Test we can get a :class:`vak.datapipes.parametric_umap.Datapipe` instance + from the classmethod ``from_dataset_path``""" toml_path = specific_config_toml_path(config_type, model_name, audio_format=audio_format, @@ -22,13 +23,8 @@ def test_from_dataset_path(self, config_type, model_name, audio_format, spect_fo cfg = vak.config.Config.from_toml_path(toml_path) cfg_command = getattr(cfg, config_type) - transform = vak.transforms.defaults.get_default_transform( - model_name, config_type, transform_kwargs - ) - - dataset = vak.datasets.parametric_umap.ParametricUMAPDataset.from_dataset_path( + dataset = vak.datapipes.parametric_umap.Datapipe.from_dataset_path( dataset_path=cfg_command.dataset.path, split=split, - transform=transform, ) - assert isinstance(dataset, vak.datasets.parametric_umap.ParametricUMAPDataset) + assert isinstance(dataset, vak.datapipes.parametric_umap.Datapipe) diff --git a/tests/test_datapipes/test_seq/__init__.py b/tests/test_datapipes/test_seq/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_datasets/conftest.py b/tests/test_datasets/conftest.py new file mode 100644 index 000000000..16699079d --- /dev/null +++ b/tests/test_datasets/conftest.py @@ -0,0 +1,373 @@ +"""Fixtures used just by test_datasets""" +import json + +import numpy as np +import pandas as pd +import pytest + + +SPLITS_JSON = { + "splits_csv_path": "splits/inputs-targets-paths-csvs/Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.csv", + "sample_id_vec_path": { + "test": "splits/sample-id-vectors/Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.test.sample_ids.npy", + "train": "splits/sample-id-vectors/Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.train.sample_ids.npy", + "val": "splits/sample-id-vectors/Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.val.sample_ids.npy" + }, + "inds_in_sample_vec_path": { + "test": "splits/inds-in-sample-vectors/Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.test.inds_in_sample.npy", + "train": "splits/inds-in-sample-vectors/Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.train.inds_in_sample.npy", + "val": "splits/inds-in-sample-vectors/Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.val.inds_in_sample.npy" + } +} + +INPUTS_TARGETS_CSV_RECORDS = [ + { + 'frames_path': '0.wav.spect.npz', + 'multi_frame_labels_path': '0.wav.multi-frame-labels.npy', + 'binary_frame_labels_path': '0.wav.binary-frame-labels.npy', + 'boundary_frame_labels_path': '0.wav.boundary-frame-labels.npy', + 'split': 'train' + }, + { + 'frames_path': '1.wav.spect.npz', + 'multi_frame_labels_path': '1.wav.multi-frame-labels.npy', + 'binary_frame_labels_path': '1.wav.binary-frame-labels.npy', + 'boundary_frame_labels_path': '1.wav.boundary-frame-labels.npy', + 'split': 'val' + }, + { + 'frames_path': '2.wav.spect.npz', + 'multi_frame_labels_path': '2.wav.multi-frame-labels.npy', + 'binary_frame_labels_path': '2.wav.binary-frame-labels.npy', + 'boundary_frame_labels_path': '2.wav.boundary-frame-labels.npy', + 'split': 'test' + }, + { + 'frames_path': '3.wav.spect.npz', + 'multi_frame_labels_path': '3.wav.multi-frame-labels.npy', + 'binary_frame_labels_path': '3.wav.binary-frame-labels.npy', + 'boundary_frame_labels_path': '3.wav.boundary-frame-labels.npy', + 'split': 'predict' + }, +] + +LABELMAPS_JSON = { + "Bengalese-Finch-Song": { + "syllable": { + "bl26lb16": { + "background": 0, + "a": 1, + "b": 2, + "c": 3, + "d": 4, + "e": 5, + "f": 6, + "i": 7 + }, + "gr41rd51": { + "background": 0, + "a": 1, + "b": 2, + "c": 3, + "d": 4, + "e": 5, + "f": 6, + "g": 7, + "i": 8, + "j": 9, + "k": 10, + "m": 11 + }, + "gy6or6": { + "background": 0, + "a": 1, + "b": 2, + "c": 3, + "d": 4, + "e": 5, + "f": 6, + "g": 7, + "h": 8, + "i": 9, + "j": 10, + "k": 11 + }, + "or60yw70": { + "background": 0, + "a": 1, + "b": 2, + "c": 3, + "d": 4, + "e": 5, + "f": 6, + "g": 7, + "i": 8 + }, + "Bird0": { + "background": 0, + "0": 1, + "1": 2, + "2": 3, + "3": 4, + "4": 5, + "5": 6, + "6": 7, + "7": 8, + "8": 9, + "9": 10 + }, + "Bird4": { + "background": 0, + "0": 1, + "1": 2, + "2": 3, + "3": 4, + "4": 5, + "5": 6, + "6": 7, + "7": 8 + }, + "Bird7": { + "background": 0, + "0": 1, + "1": 2, + "2": 3, + "3": 4, + "4": 5, + "5": 6, + "6": 7 + }, + "Bird9": { + "background": 0, + "0": 1, + "1": 2, + "2": 3, + "3": 4, + "4": 5, + "5": 6 + } + } + }, + "Canary-Song": { + "syllable": { + "llb3": { + "background": 0, + "1": 1, + "10": 2, + "11": 3, + "12": 4, + "13": 5, + "14": 6, + "15": 7, + "16": 8, + "17": 9, + "18": 10, + "19": 11, + "2": 12, + "20": 13, + "3": 14, + "4": 15, + "5": 16, + "6": 17, + "7": 18, + "8": 19, + "9": 20 + }, + "llb11": { + "background": 0, + "1": 1, + "10": 2, + "11": 3, + "12": 4, + "13": 5, + "14": 6, + "15": 7, + "16": 8, + "17": 9, + "18": 10, + "19": 11, + "2": 12, + "20": 13, + "21": 14, + "22": 15, + "23": 16, + "24": 17, + "25": 18, + "26": 19, + "27": 20, + "3": 21, + "4": 22, + "5": 23, + "6": 24, + "7": 25, + "8": 26, + "9": 27 + }, + "llb16": { + "background": 0, + "1": 1, + "10": 2, + "11": 3, + "12": 4, + "13": 5, + "14": 6, + "15": 7, + "16": 8, + "17": 9, + "18": 10, + "19": 11, + "2": 12, + "20": 13, + "21": 14, + "22": 15, + "23": 16, + "24": 17, + "25": 18, + "26": 19, + "27": 20, + "28": 21, + "29": 22, + "3": 23, + "30": 24, + "4": 25, + "5": 26, + "6": 27, + "7": 28, + "8": 29, + "9": 30 + } + } + }, + "Human-Speech": { + "phoneme": { + "all": { + "background": 0, + "aa": 1, + "ae": 2, + "ah": 3, + "ao": 4, + "aw": 5, + "ax": 6, + "ax-h": 7, + "axr": 8, + "ay": 9, + "b": 10, + "bcl": 11, + "ch": 12, + "d": 13, + "dcl": 14, + "dh": 15, + "dx": 16, + "eh": 17, + "el": 18, + "em": 19, + "en": 20, + "eng": 21, + "epi": 22, + "er": 23, + "ey": 24, + "f": 25, + "g": 26, + "gcl": 27, + "h#": 28, + "hh": 29, + "hv": 30, + "ih": 31, + "ix": 32, + "iy": 33, + "jh": 34, + "k": 35, + "kcl": 36, + "l": 37, + "m": 38, + "n": 39, + "ng": 40, + "nx": 41, + "ow": 42, + "oy": 43, + "p": 44, + "pau": 45, + "pcl": 46, + "q": 47, + "r": 48, + "s": 49, + "sh": 50, + "t": 51, + "tcl": 52, + "th": 53, + "uh": 54, + "uw": 55, + "ux": 56, + "v": 57, + "w": 58, + "y": 59, + "z": 60, + "zh": 61 + } + } + }, + "Mouse-Pup-Call": { + "call": { + "all": { + "background": 0, + "BK": 1, + "BW": 2, + "GO": 3, + "LL": 4, + "LO": 5, + "MU": 6, + "MZ": 7, + "NB": 8, + "PO": 9, + "SW": 10 + } + } + }, + "Zebra-Finch-Song": { + "syllable": { + "blu285": { + "background": 0, + "syll_0": 1, + "syll_1": 2, + "syll_2": 3, + "syll_3": 4, + "syll_4": 5, + "syll_5": 6 + } + } + } +} + + +@pytest.fixture +def mock_biosoundsegbench_dataset(tmp_path): + dataset_path = tmp_path / "BioSoundSegBench" + dataset_path.mkdir() + splits_dir = dataset_path / "splits" + splits_dir.mkdir() + + inputs_targets_csv_dir = splits_dir / "inputs-targets-paths-csvs" + inputs_targets_csv_dir.mkdir() + df = pd.DataFrame.from_records(INPUTS_TARGETS_CSV_RECORDS) + splits_csv = df.to_csv(inputs_targets_csv_dir / "Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.csv") + df.to_csv(splits_csv) + + sample_id_vecs_dir = splits_dir / "sample-id-vectors" + sample_id_vecs_dir.mkdir() + inds_in_sample_vecs_dir = splits_dir / "inds-in-sample-vectors" + inds_in_sample_vecs_dir.mkdir() + + for split in "train", "val", "test": + sample_id_vec = np.zeros(10) + np.save(sample_id_vecs_dir / f"Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.{split}.sample_ids.npy", sample_id_vec) + inds_in_sample_vec = np.arange(10) + np.save(inds_in_sample_vecs_dir / f"Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.{split}.inds_in_sample.npy", inds_in_sample_vec) + + splits_path = dataset_path / "Mouse-Pup-Call.id-SW.timebin-1.5-ms.call.id-data-only.train-dur-1500.0.replicate-1.splits.json" + with splits_path.open('w') as fp: + json.dump(SPLITS_JSON, fp) + + with (dataset_path / 'labelmaps.json').open('w') as fp: + json.dump(LABELMAPS_JSON, fp) + + return dataset_path, splits_path diff --git a/tests/test_datasets/test_biosoundsegbench.py b/tests/test_datasets/test_biosoundsegbench.py new file mode 100644 index 000000000..fdadaff43 --- /dev/null +++ b/tests/test_datasets/test_biosoundsegbench.py @@ -0,0 +1,129 @@ + +import pytest + +import vak.datasets + + +class TestBioSoundSegBench: + @pytest.mark.parametrize( + 'split, window_size, target_type', + [ + ( + "train", + 2000, + "multi_frame_labels", + ), + ( + "train", + 2000, + "binary_frame_labels", + ), + ( + "train", + 2000, + "boundary_frame_labels", + ), + ( + "train", + 2000, + ("multi_frame_labels", "boundary_frame_labels"), + ), + ] + ) + def test_init(self, split, window_size, target_type, mock_biosoundsegbench_dataset): + dataset_path, splits_path = mock_biosoundsegbench_dataset + dataset = vak.datasets.BioSoundSegBench( + dataset_path, + splits_path, + split, + window_size, + target_type, + ) + assert isinstance(dataset, vak.datasets.BioSoundSegBench) + + @pytest.mark.parametrize( + 'dataset_path, splits_path, split, window_size, target_type, expected_exception', + [ + # invalid dataset path -> NotADirectoryError + ( + 'path/to/dataset/that/doesnt/exist', + None, + "train", + 2000, + "multi_frame_labels", + NotADirectoryError, + ), + # invalid splits path -> FileNotFoundError + ( + None, + 'path/to/splits/that/doesnt/exist', + "train", + 2000, + "binary_frame_labels", + FileNotFoundError, + ), + # invalid split -> ValueError + ( + None, + None, + "evaluate", + 2000, + "boundary_frame_labels", + ValueError, + ), + # no target type when split != "predict" -> ValueError + ( + None, + None, + "train", + 2000, + None, + ValueError, + ), + # wrong type for target type -> TypeError + ( + None, + None, + "train", + 2000, + 1, + TypeError, + ), + # wrong type for target type -> TypeError + ( + None, + None, + "train", + 2000, + ("boundary_frame_labels", 1), + TypeError, + ), + # invalid target type -> ValueError + ( + None, + None, + "train", + 2000, + "frame_labels", + ValueError, + ), + ] + ) + def test_init_raises( + self, dataset_path, splits_path, split, window_size, target_type, expected_exception, mock_biosoundsegbench_dataset + ): + if dataset_path is None and splits_path is not None: + dataset_path, _ = mock_biosoundsegbench_dataset + elif dataset_path is not None and splits_path is None: + _, splits_path = mock_biosoundsegbench_dataset + elif dataset_path is None and splits_path is None: + dataset_path, splits_path = mock_biosoundsegbench_dataset + + with pytest.raises(expected_exception): + dataset = vak.datasets.BioSoundSegBench( + dataset_path, + splits_path, + split, + window_size, + target_type, + ) diff --git a/tests/test_datasets/test_get.py b/tests/test_datasets/test_get.py new file mode 100644 index 000000000..d45d97232 --- /dev/null +++ b/tests/test_datasets/test_get.py @@ -0,0 +1,86 @@ +import pathlib + +import pytest + +import vak.datasets + + +@pytest.mark.parametrize( + 'dataset_config, split', + [ + ( + { + 'name': 'BioSoundSegBench', + 'params': { + 'window_size': 2000, + 'target_type': 'multi_frame_labels', + }, + }, + 'train', + ), + ( + { + 'name': 'BioSoundSegBench', + 'params': { + 'window_size': 2000, + 'target_type': 'binary_frame_labels', + }, + }, + 'train', + ), + ( + { + 'name': 'BioSoundSegBench', + 'params': { + 'window_size': 2000, + 'target_type': 'boundary_frame_labels', + }, + }, + 'train', + ), + ( + { + 'name': 'BioSoundSegBench', + 'params': { + 'window_size': 2000, + 'target_type': 'boundary_frame_labels', + }, + }, + 'train', + ), + ( + { + 'name': 'BioSoundSegBench', + 'params': { + 'window_size': 2000, + 'target_type': 'boundary_frame_labels', + }, + }, + 'val', + ), + ( + { + 'name': 'BioSoundSegBench', + 'params': { + 'window_size': 2000, + 'target_type': 'boundary_frame_labels', + }, + }, + 'test', + ), + + ] +) +def test_get_biosoundsegbench(dataset_config, split, mock_biosoundsegbench_dataset): + dataset_path, splits_path = mock_biosoundsegbench_dataset + dataset_config["path"] = dataset_path + dataset_config["splits_path"] = splits_path + + dataset = vak.datasets.get(dataset_config, split) + + assert isinstance(dataset, vak.datasets.BioSoundSegBench) + assert dataset.dataset_path == pathlib.Path(dataset_config["path"]) + assert dataset.splits_path == pathlib.Path(dataset_config["splits_path"]) + assert dataset.window_size == dataset_config["params"]["window_size"] + + diff --git a/tests/test_eval/test_eval.py b/tests/test_eval/test_eval.py index 21d543828..4fd0b00f5 100644 --- a/tests/test_eval/test_eval.py +++ b/tests/test_eval/test_eval.py @@ -57,7 +57,7 @@ def test_eval( output_dir=cfg.eval.output_dir, num_workers=cfg.eval.num_workers, batch_size=cfg.eval.batch_size, - spect_scaler_path=cfg.eval.spect_scaler_path, + frames_standardizer_path=cfg.eval.frames_standardizer_path, post_tfm_kwargs=cfg.eval.post_tfm_kwargs, ) diff --git a/tests/test_eval/test_frame_classification.py b/tests/test_eval/test_frame_classification.py index 40eb04c07..8ac65539a 100644 --- a/tests/test_eval/test_frame_classification.py +++ b/tests/test_eval/test_frame_classification.py @@ -77,7 +77,7 @@ def test_eval_frame_classification_model( labelmap_path=cfg.eval.labelmap_path, output_dir=cfg.eval.output_dir, num_workers=cfg.eval.num_workers, - spect_scaler_path=cfg.eval.spect_scaler_path, + frames_standardizer_path=cfg.eval.frames_standardizer_path, post_tfm_kwargs=post_tfm_kwargs, ) @@ -89,7 +89,7 @@ def test_eval_frame_classification_model( [ {"table": "eval", "key": "checkpoint_path", "value": '/obviously/doesnt/exist/ckpt.pt'}, {"table": "eval", "key": "labelmap_path", "value": '/obviously/doesnt/exist/labelmap.json'}, - {"table": "eval", "key": "spect_scaler_path", "value": '/obviously/doesnt/exist/SpectScaler'}, + {"table": "eval", "key": "frames_standardizer_path", "value": '/obviously/doesnt/exist/FramesStandardizer'}, ] ) def test_eval_frame_classification_model_raises_file_not_found( @@ -100,7 +100,7 @@ def test_eval_frame_classification_model_raises_file_not_found( ): """Test that core.eval raises FileNotFoundError when one of the following does not exist: - checkpoint_path, labelmap_path, dataset_path, spect_scaler_path + checkpoint_path, labelmap_path, dataset_path, frames_standardizer_path """ output_dir = tmp_path.joinpath( f"test_eval_cbin_notmat_invalid_dataset_path" @@ -131,7 +131,7 @@ def test_eval_frame_classification_model_raises_file_not_found( labelmap_path=cfg.eval.labelmap_path, output_dir=cfg.eval.output_dir, num_workers=cfg.eval.num_workers, - spect_scaler_path=cfg.eval.spect_scaler_path, + frames_standardizer_path=cfg.eval.frames_standardizer_path, ) @@ -185,5 +185,5 @@ def test_eval_frame_classification_model_raises_not_a_directory( labelmap_path=cfg.eval.labelmap_path, output_dir=cfg.eval.output_dir, num_workers=cfg.eval.num_workers, - spect_scaler_path=cfg.eval.spect_scaler_path, + frames_standardizer_path=cfg.eval.frames_standardizer_path, ) diff --git a/tests/test_learncurve/test_frame_classification.py b/tests/test_learncurve/test_frame_classification.py index ddaa99f6a..51a8d1a69 100644 --- a/tests/test_learncurve/test_frame_classification.py +++ b/tests/test_learncurve/test_frame_classification.py @@ -20,8 +20,8 @@ def assert_learncurve_output_matches_expected(cfg, model_name, results_path): assert replicate_path.joinpath("labelmap.json").exists() - if cfg.learncurve.normalize_spectrograms: - assert replicate_path.joinpath("StandardizeSpect").exists() + if cfg.learncurve.standardize_frames: + assert replicate_path.joinpath("FramesStandardizer").exists() eval_csv = sorted(replicate_path.glob(f"eval_{model_name}*csv")) assert len(eval_csv) == 1 @@ -75,7 +75,7 @@ def test_learning_curve_for_frame_classification_model( num_workers=cfg.learncurve.num_workers, results_path=results_path, post_tfm_kwargs=cfg.learncurve.post_tfm_kwargs, - normalize_spectrograms=cfg.learncurve.normalize_spectrograms, + standardize_frames=cfg.learncurve.standardize_frames, shuffle=cfg.learncurve.shuffle, val_step=cfg.learncurve.val_step, ckpt_step=cfg.learncurve.ckpt_step, @@ -124,7 +124,7 @@ def test_learncurve_raises_not_a_directory(dir_option_to_change, num_workers=cfg.learncurve.num_workers, results_path=results_path, post_tfm_kwargs=cfg.learncurve.post_tfm_kwargs, - normalize_spectrograms=cfg.learncurve.normalize_spectrograms, + standardize_frames=cfg.learncurve.standardize_frames, shuffle=cfg.learncurve.shuffle, val_step=cfg.learncurve.val_step, ckpt_step=cfg.learncurve.ckpt_step, diff --git a/tests/test_models/test_factory.py b/tests/test_models/test_factory.py index 169b3ed8b..cb191f0ab 100644 --- a/tests/test_models/test_factory.py +++ b/tests/test_models/test_factory.py @@ -303,7 +303,7 @@ def test_from_config_with_frame_classification(self, definition, specific_config cfg = vak.config.Config.from_toml_path(toml_path) # stuff we need just to be able to instantiate network - labelmap = vak.common.labels.to_map(cfg.prep.labelset, map_unlabeled=True) + labelmap = vak.common.labels.to_map(cfg.prep.labelset, map_background=True) model_factory = vak.models.factory.ModelFactory( definition, diff --git a/tests/test_models/test_frame_classification_model.py b/tests/test_models/test_frame_classification_model.py index 3c1363496..b4c559b09 100644 --- a/tests/test_models/test_frame_classification_model.py +++ b/tests/test_models/test_frame_classification_model.py @@ -33,17 +33,11 @@ def test_load_state_dict_from_path(self, train_cfg = vak.config.Config.from_toml_path(train_toml_path) # stuff we need just to be able to instantiate network - labelmap = vak.common.labels.to_map(train_cfg.prep.labelset, map_unlabeled=True) - item_transform = vak.transforms.defaults.get_default_transform( - model_name, - "train", - transform_kwargs={}, - ) - train_dataset = vak.datasets.frame_classification.WindowDataset.from_dataset_path( + labelmap = vak.common.labels.to_map(train_cfg.prep.labelset, map_background=True) + train_dataset = vak.datapipes.frame_classification.TrainDatapipe.from_dataset_path( dataset_path=train_cfg.train.dataset.path, split="train", window_size=train_cfg.train.dataset.params['window_size'], - item_transform=item_transform, ) input_shape = train_dataset.shape num_input_channels = input_shape[-3] diff --git a/tests/test_models/test_parametric_umap_model.py b/tests/test_models/test_parametric_umap_model.py index 74bada3f6..b0b8777d8 100644 --- a/tests/test_models/test_parametric_umap_model.py +++ b/tests/test_models/test_parametric_umap_model.py @@ -35,15 +35,9 @@ def test_load_state_dict_from_path(self, train_cfg = vak.config.Config.from_toml_path(train_toml_path) # stuff we need just to be able to instantiate network - item_transform = vak.transforms.defaults.get_default_transform( - model_name, - "train", - transform_kwargs={}, - ) - train_dataset = vak.datasets.parametric_umap.ParametricUMAPDataset.from_dataset_path( + train_dataset = vak.datapipes.parametric_umap.Datapipe.from_dataset_path( dataset_path=train_cfg.train.dataset.path, split="train", - transform=item_transform, ) # network is the one thing that has required args diff --git a/tests/test_models/test_tweetynet.py b/tests/test_models/test_tweetynet.py index c0e8f8378..cd35b70b5 100644 --- a/tests/test_models/test_tweetynet.py +++ b/tests/test_models/test_tweetynet.py @@ -18,7 +18,7 @@ LABELMAPS = [] for labelset in LABELSETS: LABELMAPS.append( - vak.common.labels.to_map(labelset, map_unlabeled=True) + vak.common.labels.to_map(labelset, map_background=True) ) INPUT_SHAPES = ( diff --git a/tests/test_predict/test_frame_classification.py b/tests/test_predict/test_frame_classification.py index f0507a016..e1bd9df4e 100644 --- a/tests/test_predict/test_frame_classification.py +++ b/tests/test_predict/test_frame_classification.py @@ -59,7 +59,7 @@ def test_predict_with_frame_classification_model( labelmap_path=cfg.predict.labelmap_path, num_workers=cfg.predict.num_workers, timebins_key=cfg.prep.spect_params.timebins_key, - spect_scaler_path=cfg.predict.spect_scaler_path, + frames_standardizer_path=cfg.predict.frames_standardizer_path, annot_csv_filename=cfg.predict.annot_csv_filename, output_dir=cfg.predict.output_dir, min_segment_dur=cfg.predict.min_segment_dur, @@ -73,7 +73,7 @@ def test_predict_with_frame_classification_model( Path(output_dir).glob(f"*{vak.common.constants.NET_OUTPUT_SUFFIX}") ) - metadata = vak.datasets.frame_classification.Metadata.from_dataset_path(cfg.predict.dataset.path) + metadata = vak.datapipes.frame_classification.Metadata.from_dataset_path(cfg.predict.dataset.path) dataset_csv_path = cfg.predict.dataset.path / metadata.dataset_csv_filename dataset_df = pd.read_csv(dataset_csv_path) @@ -91,7 +91,7 @@ def test_predict_with_frame_classification_model( [ {"table": "predict", "key": "checkpoint_path", "value": '/obviously/doesnt/exist/ckpt.pt'}, {"table": "predict", "key": "labelmap_path", "value": '/obviously/doesnt/exist/labelmap.json'}, - {"table": "predict", "key": "spect_scaler_path", "value": '/obviously/doesnt/exist/SpectScaler'}, + {"table": "predict", "key": "frames_standardizer_path", "value": '/obviously/doesnt/exist/FramesStandardizer'}, ] ) def test_predict_with_frame_classification_model_raises_file_not_found( @@ -130,7 +130,7 @@ def test_predict_with_frame_classification_model_raises_file_not_found( labelmap_path=cfg.predict.labelmap_path, num_workers=cfg.predict.num_workers, timebins_key=cfg.prep.spect_params.timebins_key, - spect_scaler_path=cfg.predict.spect_scaler_path, + frames_standardizer_path=cfg.predict.frames_standardizer_path, annot_csv_filename=cfg.predict.annot_csv_filename, output_dir=cfg.predict.output_dir, min_segment_dur=cfg.predict.min_segment_dur, @@ -189,7 +189,7 @@ def test_predict_with_frame_classification_model_raises_not_a_directory( labelmap_path=cfg.predict.labelmap_path, num_workers=cfg.predict.num_workers, timebins_key=cfg.prep.spect_params.timebins_key, - spect_scaler_path=cfg.predict.spect_scaler_path, + frames_standardizer_path=cfg.predict.frames_standardizer_path, annot_csv_filename=cfg.predict.annot_csv_filename, output_dir=cfg.predict.output_dir, min_segment_dur=cfg.predict.min_segment_dur, diff --git a/tests/test_predict/test_predict.py b/tests/test_predict/test_predict.py index f7a7f0f7a..a31780ea1 100644 --- a/tests/test_predict/test_predict.py +++ b/tests/test_predict/test_predict.py @@ -52,7 +52,7 @@ def test_predict( labelmap_path=cfg.predict.labelmap_path, num_workers=cfg.predict.num_workers, timebins_key=cfg.prep.spect_params.timebins_key, - spect_scaler_path=cfg.predict.spect_scaler_path, + frames_standardizer_path=cfg.predict.frames_standardizer_path, annot_csv_filename=cfg.predict.annot_csv_filename, output_dir=cfg.predict.output_dir, min_segment_dur=cfg.predict.min_segment_dur, diff --git a/tests/test_prep/test_frame_classification/test_frame_classification.py b/tests/test_prep/test_frame_classification/test_frame_classification.py index 968f4d19b..541b6b5ff 100644 --- a/tests/test_prep/test_frame_classification/test_frame_classification.py +++ b/tests/test_prep/test_frame_classification/test_frame_classification.py @@ -18,7 +18,7 @@ def assert_prep_output_matches_expected(dataset_path, df_returned_by_prep): log_path = sorted(dataset_path.glob('*log')) assert len(log_path) == 1 - meta_json_path = dataset_path / vak.datasets.frame_classification.Metadata.METADATA_JSON_FILENAME + meta_json_path = dataset_path / vak.datapipes.frame_classification.Metadata.METADATA_JSON_FILENAME assert meta_json_path.exists() with meta_json_path.open('r') as fp: @@ -40,16 +40,16 @@ def assert_prep_output_matches_expected(dataset_path, df_returned_by_prep): check_exact=check_exact, ) - if vak.datasets.frame_classification.constants.FRAMES_PATH_COL_NAME in df_returned_by_prep.columns: + if vak.datapipes.frame_classification.constants.FRAMES_PATH_COL_NAME in df_returned_by_prep.columns: frames_paths = df_returned_by_prep[ - vak.datasets.frame_classification.constants.FRAMES_PATH_COL_NAME + vak.datapipes.frame_classification.constants.FRAMES_PATH_COL_NAME ].values for frames_path in frames_paths: assert (dataset_path / frames_path).exists() - if vak.datasets.frame_classification.constants.FRAME_LABELS_NPY_PATH_COL_NAME in df_returned_by_prep.columns: + if vak.datapipes.frame_classification.constants.MULTI_FRAME_LABELS_PATH_COL_NAME in df_returned_by_prep.columns: frame_labels_paths = df_returned_by_prep[ - vak.datasets.frame_classification.constants.FRAME_LABELS_NPY_PATH_COL_NAME + vak.datapipes.frame_classification.constants.MULTI_FRAME_LABELS_PATH_COL_NAME ].values if not all([frame_labels_path is None for frame_labels_path in frame_labels_paths]): for frame_labels_path in frame_labels_paths: diff --git a/tests/test_prep/test_frame_classification/test_learncurve.py b/tests/test_prep/test_frame_classification/test_learncurve.py index 0030e01d0..be628cc43 100644 --- a/tests/test_prep/test_frame_classification/test_learncurve.py +++ b/tests/test_prep/test_frame_classification/test_learncurve.py @@ -39,7 +39,7 @@ def test_make_index_vectors_for_each_subsets( cfg = vak.config.Config.from_toml_path(toml_path) dataset_path = cfg.learncurve.dataset.path - metadata = vak.datasets.frame_classification.Metadata.from_dataset_path(dataset_path) + metadata = vak.datapipes.frame_classification.Metadata.from_dataset_path(dataset_path) dataset_csv_path = dataset_path / metadata.dataset_csv_filename dataset_df = pd.read_csv(dataset_csv_path) @@ -56,12 +56,12 @@ def test_make_index_vectors_for_each_subsets( train_dur, replicate_num ) sample_id_vec_path = (tmp_dataset_path / "train" / - vak.datasets.frame_classification.helper.sample_ids_array_filename_for_subset( + vak.datapipes.frame_classification.helper.sample_ids_array_filename_for_subset( train_dur_replicate_subset_name) ) sample_id_vec_path.unlink() inds_in_sample_vec_path = (tmp_dataset_path / "train" / - vak.datasets.frame_classification.helper.inds_in_sample_array_filename_for_subset( + vak.datapipes.frame_classification.helper.inds_in_sample_array_filename_for_subset( train_dur_replicate_subset_name) ) inds_in_sample_vec_path.unlink() @@ -92,25 +92,25 @@ def test_make_index_vectors_for_each_subsets( # test that indexing vectors got made sample_id_vec_path = (tmp_dataset_path / "train" / - vak.datasets.frame_classification.helper.sample_ids_array_filename_for_subset( + vak.datapipes.frame_classification.helper.sample_ids_array_filename_for_subset( subset_name) ) assert sample_id_vec_path.exists() inds_in_sample_vec_path = (tmp_dataset_path / "train" / - vak.datasets.frame_classification.helper.inds_in_sample_array_filename_for_subset( + vak.datapipes.frame_classification.helper.inds_in_sample_array_filename_for_subset( subset_name) ) assert inds_in_sample_vec_path.exists() this_subset_df = subsets_df[subsets_df['subset'] == subset_name] frames_paths = this_subset_df[ - vak.datasets.frame_classification.constants.FRAMES_PATH_COL_NAME + vak.datapipes.frame_classification.constants.FRAMES_PATH_COL_NAME ].values sample_id_vec, inds_in_sample_vec = [], [] for sample_id, frames_path in enumerate(frames_paths): # make indexing vectors that we use to test - frames = vak.datasets.frame_classification.helper.load_frames(tmp_dataset_path / frames_path, + frames = vak.datapipes.frame_classification.helper.load_frames(tmp_dataset_path / frames_path, input_type) n_frames = frames.shape[-1] sample_id_vec.append(np.ones((n_frames,)).astype(np.int32) * sample_id) @@ -151,7 +151,7 @@ def test_make_subsets_from_dataset_df( cfg = vak.config.Config.from_toml_path(toml_path) dataset_path = cfg.learncurve.dataset.path - metadata = vak.datasets.frame_classification.Metadata.from_dataset_path(dataset_path) + metadata = vak.datapipes.frame_classification.Metadata.from_dataset_path(dataset_path) dataset_csv_path = dataset_path / metadata.dataset_csv_filename dataset_df = pd.read_csv(dataset_csv_path) @@ -168,12 +168,12 @@ def test_make_subsets_from_dataset_df( train_dur, replicate_num ) sample_id_vec_path = (tmp_dataset_path / "train" / - vak.datasets.frame_classification.helper.sample_ids_array_filename_for_subset( + vak.datapipes.frame_classification.helper.sample_ids_array_filename_for_subset( train_dur_replicate_subset_name) ) sample_id_vec_path.unlink() inds_in_sample_vec_path = (tmp_dataset_path / "train" / - vak.datasets.frame_classification.helper.inds_in_sample_array_filename_for_subset( + vak.datapipes.frame_classification.helper.inds_in_sample_array_filename_for_subset( train_dur_replicate_subset_name) ) inds_in_sample_vec_path.unlink() diff --git a/tests/test_prep/test_frame_classification/test_make_splits.py b/tests/test_prep/test_frame_classification/test_make_splits.py index 11037a5c3..7b469f20c 100644 --- a/tests/test_prep/test_frame_classification/test_make_splits.py +++ b/tests/test_prep/test_frame_classification/test_make_splits.py @@ -109,7 +109,7 @@ def test_make_splits(config_type, model_name, audio_format, spect_format, annot_ dataset_df ) labelmap = vak.common.labels.to_map( - cfg.prep.labelset, map_unlabeled=map_unlabeled_segments + cfg.prep.labelset, map_background=map_unlabeled_segments ) else: labelmap = None @@ -141,17 +141,17 @@ def test_make_splits(config_type, model_name, audio_format, spect_format, annot_ dataset_df_with_splits.split == split ].copy() - assert vak.datasets.frame_classification.constants.FRAMES_PATH_COL_NAME in split_df.columns + assert vak.datapipes.frame_classification.constants.FRAMES_PATH_COL_NAME in split_df.columns frames_paths = split_df[ - vak.datasets.frame_classification.constants.FRAMES_PATH_COL_NAME + vak.datapipes.frame_classification.constants.FRAMES_PATH_COL_NAME ].values if purpose != "predict": - assert vak.datasets.frame_classification.constants.FRAME_LABELS_NPY_PATH_COL_NAME in split_df.columns + assert vak.datapipes.frame_classification.constants.MULTI_FRAME_LABELS_PATH_COL_NAME in split_df.columns frame_labels_paths = split_df[ - vak.datasets.frame_classification.constants.FRAME_LABELS_NPY_PATH_COL_NAME + vak.datapipes.frame_classification.constants.MULTI_FRAME_LABELS_PATH_COL_NAME ].values annots = vak.common.annotation.from_df(split_df) @@ -176,7 +176,7 @@ def test_make_splits(config_type, model_name, audio_format, spect_format, annot_ # NOTE we load frames to confirm we can and also to make indexing vectors we use to test, # see next code block - frames = vak.datasets.frame_classification.helper.load_frames(tmp_dataset_path / frames_path, input_type) + frames = vak.datapipes.frame_classification.helper.load_frames(tmp_dataset_path / frames_path, input_type) assert isinstance(frames, np.ndarray) # make indexing vectors that we use to test @@ -203,7 +203,7 @@ def test_make_splits(config_type, model_name, audio_format, spect_format, annot_ annot.seq.onsets_s, annot.seq.offsets_s, frame_times, - unlabeled_label=labelmap["unlabeled"], + background_label=labelmap[vak.common.constants.DEFAULT_BACKGROUND_LABEL], ) frame_labels = np.load(frame_labels_file_that_should_exist) assert np.array_equal(frame_labels, expected_frame_labels) @@ -217,7 +217,7 @@ def test_make_splits(config_type, model_name, audio_format, spect_format, annot_ sample_id_vec_path = ( split_subdir / - vak.datasets.frame_classification.constants.SAMPLE_IDS_ARRAY_FILENAME + vak.datapipes.frame_classification.constants.SAMPLE_IDS_ARRAY_FILENAME ) assert sample_id_vec_path.exists() @@ -227,7 +227,7 @@ def test_make_splits(config_type, model_name, audio_format, spect_format, annot_ inds_in_sample_vec_path = ( split_subdir / - vak.datasets.frame_classification.constants.INDS_IN_SAMPLE_ARRAY_FILENAME + vak.datapipes.frame_classification.constants.INDS_IN_SAMPLE_ARRAY_FILENAME ) assert inds_in_sample_vec_path.exists() diff --git a/tests/test_prep/test_split/test_split.py b/tests/test_prep/test_split/test_split.py index 76b3822c6..844ade2a8 100644 --- a/tests/test_prep/test_split/test_split.py +++ b/tests/test_prep/test_split/test_split.py @@ -5,6 +5,7 @@ import pandas as pd import pytest +import vak.common # for constants import vak.common.annotation import vak.prep.spectrogram_dataset.spect_helper import vak.prep.split.split @@ -269,14 +270,14 @@ def test_split_frame_classification_dataframe( config_type=config_type, model=model_name, ) - metadata = vak.datasets.frame_classification.Metadata.from_dataset_path(dataset_path) + metadata = vak.datapipes.frame_classification.Metadata.from_dataset_path(dataset_path) dataset_csv_path = dataset_path / metadata.dataset_csv_filename dataset_df = pd.read_csv(dataset_csv_path) dataset_df = dataset_df.drop(columns=('split')) labelmap_path = dataset_path / "labelmap.json" with labelmap_path.open("r") as f: labelmap = json.load(f) - labelset = set(key for key in labelmap.keys() if key != 'unlabeled') + labelset = set(key for key in labelmap.keys() if key != vak.common.constants.DEFAULT_BACKGROUND_LABEL) dataset_df_split = vak.prep.split.split.frame_classification_dataframe( dataset_df, dataset_path, labelset=labelset, train_dur=train_dur, val_dur=val_dur, test_dur=test_dur @@ -314,14 +315,14 @@ def test_split_unit_dataframe( config_type=config_type, model=model_name, ) - metadata = vak.datasets.parametric_umap.Metadata.from_dataset_path(dataset_path) + metadata = vak.datapipes.parametric_umap.Metadata.from_dataset_path(dataset_path) dataset_csv_path = dataset_path / metadata.dataset_csv_filename dataset_df = pd.read_csv(dataset_csv_path) dataset_df = dataset_df.drop(columns=('split')) labelmap_path = dataset_path / "labelmap.json" with labelmap_path.open("r") as f: labelmap = json.load(f) - labelset = set(key for key in labelmap.keys() if key != 'unlabeled') + labelset = set(key for key in labelmap.keys() if key != vak.common.constants.DEFAULT_BACKGROUND_LABEL) dataset_df_split = vak.prep.split.split.unit_dataframe( dataset_df, dataset_path, labelset=labelset, train_dur=train_dur, val_dur=val_dur, test_dur=test_dur diff --git a/tests/test_train/test_frame_classification.py b/tests/test_train/test_frame_classification.py index e9d06aa98..c3661fd53 100644 --- a/tests/test_train/test_frame_classification.py +++ b/tests/test_train/test_frame_classification.py @@ -13,10 +13,10 @@ def assert_train_output_matches_expected(cfg: vak.config.config.Config, model_na results_path: pathlib.Path): assert results_path.joinpath("labelmap.json").exists() - if cfg.train.normalize_spectrograms or cfg.train.spect_scaler_path: - assert results_path.joinpath("StandardizeSpect").exists() + if cfg.train.standardize_frames or cfg.train.frames_standardizer_path: + assert results_path.joinpath("FramesStandardizer").exists() else: - assert not results_path.joinpath("StandardizeSpect").exists() + assert not results_path.joinpath("FramesStandardizer").exists() model_path = results_path.joinpath(model_name) assert model_path.exists() @@ -68,9 +68,9 @@ def test_train_frame_classification_model( num_epochs=cfg.train.num_epochs, num_workers=cfg.train.num_workers, checkpoint_path=cfg.train.checkpoint_path, - spect_scaler_path=cfg.train.spect_scaler_path, + frames_standardizer_path=cfg.train.frames_standardizer_path, results_path=results_path, - normalize_spectrograms=cfg.train.normalize_spectrograms, + standardize_frames=cfg.train.standardize_frames, shuffle=cfg.train.shuffle, val_step=cfg.train.val_step, ckpt_step=cfg.train.ckpt_step, @@ -115,9 +115,9 @@ def test_continue_training( num_epochs=cfg.train.num_epochs, num_workers=cfg.train.num_workers, checkpoint_path=cfg.train.checkpoint_path, - spect_scaler_path=cfg.train.spect_scaler_path, + frames_standardizer_path=cfg.train.frames_standardizer_path, results_path=results_path, - normalize_spectrograms=cfg.train.normalize_spectrograms, + standardize_frames=cfg.train.standardize_frames, shuffle=cfg.train.shuffle, val_step=cfg.train.val_step, ckpt_step=cfg.train.ckpt_step, @@ -131,7 +131,7 @@ def test_continue_training( 'path_option_to_change', [ {"table": "train", "key": "checkpoint_path", "value": '/obviously/doesnt/exist/ckpt.pt'}, - {"table": "train", "key": "spect_scaler_path", "value": '/obviously/doesnt/exist/SpectScaler'}, + {"table": "train", "key": "frames_standardizer_path", "value": '/obviously/doesnt/exist/FramesStandardizer'}, ] ) def test_train_raises_file_not_found( @@ -139,7 +139,7 @@ def test_train_raises_file_not_found( ): """Test that pre-conditions in `vak.train` raise FileNotFoundError when one of the following does not exist: - checkpoint_path, dataset_path, spect_scaler_path + checkpoint_path, dataset_path, frames_standardizer_path """ keys_to_change = [ {"table": "train", "key": "trainer", "value": trainer_table}, @@ -166,9 +166,9 @@ def test_train_raises_file_not_found( num_epochs=cfg.train.num_epochs, num_workers=cfg.train.num_workers, checkpoint_path=cfg.train.checkpoint_path, - spect_scaler_path=cfg.train.spect_scaler_path, + frames_standardizer_path=cfg.train.frames_standardizer_path, results_path=results_path, - normalize_spectrograms=cfg.train.normalize_spectrograms, + standardize_frames=cfg.train.standardize_frames, shuffle=cfg.train.shuffle, val_step=cfg.train.val_step, ckpt_step=cfg.train.ckpt_step, @@ -216,9 +216,9 @@ def test_train_raises_not_a_directory( num_epochs=cfg.train.num_epochs, num_workers=cfg.train.num_workers, checkpoint_path=cfg.train.checkpoint_path, - spect_scaler_path=cfg.train.spect_scaler_path, + frames_standardizer_path=cfg.train.frames_standardizer_path, results_path=results_path, - normalize_spectrograms=cfg.train.normalize_spectrograms, + standardize_frames=cfg.train.standardize_frames, shuffle=cfg.train.shuffle, val_step=cfg.train.val_step, ckpt_step=cfg.train.ckpt_step, diff --git a/tests/test_train/test_train.py b/tests/test_train/test_train.py index 11886db68..18823d570 100644 --- a/tests/test_train/test_train.py +++ b/tests/test_train/test_train.py @@ -60,9 +60,9 @@ def test_train( num_epochs=cfg.train.num_epochs, num_workers=cfg.train.num_workers, checkpoint_path=cfg.train.checkpoint_path, - spect_scaler_path=cfg.train.spect_scaler_path, + frames_standardizer_path=cfg.train.frames_standardizer_path, results_path=results_path, - normalize_spectrograms=cfg.train.normalize_spectrograms, + standardize_frames=cfg.train.standardize_frames, shuffle=cfg.train.shuffle, val_step=cfg.train.val_step, ckpt_step=cfg.train.ckpt_step, diff --git a/tests/test_transforms/test_frame_labels/test_functional.py b/tests/test_transforms/test_frame_labels/test_functional.py index 861835052..9e9013eb6 100644 --- a/tests/test_transforms/test_frame_labels/test_functional.py +++ b/tests/test_transforms/test_frame_labels/test_functional.py @@ -8,9 +8,9 @@ - to_segments: transform to get back segment onsets, offsets, and labels from labeled timebins. Inverse of ``from_segments``. - post-processing transforms that can be used to "clean up" a vector of labeled timebins - - to_inds_list: helper function used to find segments in a vector of labeled timebins + - segment_inds_list_from_class_labels: helper function used to find segments in a vector of labeled timebins - remove_short_segments: remove any segment less than a minimum duration - - take_majority_vote: take a "majority vote" within each segment bounded by the "unlabeled" label, + - take_majority_vote: take a "majority vote" within each segment bounded by the background label, and apply the most "popular" label within each segment to all timebins in that segment Additionally some of the functions have more than one unit test, @@ -28,6 +28,7 @@ import numpy as np import pytest +import vak.common # for constants import vak.common.files.spect import vak.common.labels import vak.transforms.frame_labels @@ -87,7 +88,7 @@ def test_from_segments(annot, spect_path, labelset): annot.seq.onsets_s, annot.seq.offsets_s, timebins, - unlabeled_label=labelmap['unlabeled'], + background_label=labelmap[vak.common.constants.DEFAULT_BACKGROUND_LABEL], ) assert lbl_tb.shape == timebins.shape assert all( @@ -98,10 +99,10 @@ def test_from_segments(annot, spect_path, labelset): @pytest.mark.parametrize( "lbl_tb, labelmap, labels_expected_int", [ - (np.array([0, 0, 1, 1, 0, 0, 2, 2, 0, 0]), {'unlabeled': 0, 'a': 1, 'b': 2}, [1, 2]), - (np.array([0, 0, 1, 1, 0, 0, 2, 2, 0, 0]), {'unlabeled': 0, '1': 1, '2': 2}, [1, 2]), - (np.array([0, 0, 21, 21, 0, 0, 22, 22, 0, 0]), {'unlabeled': 0, '21': 21, '22': 22}, [21, 22]), - (np.array([0, 0, 11, 11, 0, 0, 12, 12, 0, 0]), {'unlabeled': 0, '11': 11, '12': 12}, [11, 12]), + (np.array([0, 0, 1, 1, 0, 0, 2, 2, 0, 0]), {vak.common.constants.DEFAULT_BACKGROUND_LABEL: 0, 'a': 1, 'b': 2}, [1, 2]), + (np.array([0, 0, 1, 1, 0, 0, 2, 2, 0, 0]), {vak.common.constants.DEFAULT_BACKGROUND_LABEL: 0, '1': 1, '2': 2}, [1, 2]), + (np.array([0, 0, 21, 21, 0, 0, 22, 22, 0, 0]), {vak.common.constants.DEFAULT_BACKGROUND_LABEL: 0, '21': 21, '22': 22}, [21, 22]), + (np.array([0, 0, 11, 11, 0, 0, 12, 12, 0, 0]), {vak.common.constants.DEFAULT_BACKGROUND_LABEL: 0, '11': 11, '12': 12}, [11, 12]), ] ) def test_to_labels(lbl_tb, labelmap, labels_expected_int): @@ -109,7 +110,7 @@ def test_to_labels(lbl_tb, labelmap, labels_expected_int): # we can easily compare strings we get back with expected; # this is what core.eval does labelmap = vak.common.labels.multi_char_labels_to_single_char( - labelmap, skip=('unlabeled',) + labelmap, skip=(vak.common.constants.DEFAULT_BACKGROUND_LABEL,) ) labelmap_inv = {v: k for k, v in labelmap.items()} labels_expected = ''.join([labelmap_inv[lbl_int] for lbl_int in labels_expected_int]) @@ -150,7 +151,7 @@ def test_to_labels_real_data( # we can easily compare strings we get back with expected; # this is what core.eval does labelmap = vak.common.labels.multi_char_labels_to_single_char( - labelmap, skip=('unlabeled',) + labelmap, skip=(vak.common.constants.DEFAULT_BACKGROUND_LABEL,) ) TIMEBINS_KEY = "t" @@ -177,7 +178,7 @@ def test_to_labels_real_data( annot.seq.onsets_s, annot.seq.offsets_s, timebins, - unlabeled_label=labelmap["unlabeled"], + background_label=labelmap[vak.common.constants.DEFAULT_BACKGROUND_LABEL], ) labels = vak.transforms.frame_labels.to_labels( @@ -229,7 +230,7 @@ def test_to_segments_real_data( annot.seq.onsets_s, annot.seq.offsets_s, timebins, - unlabeled_label=labelmap["unlabeled"], + background_label=labelmap[vak.common.constants.DEFAULT_BACKGROUND_LABEL], ) labels, onsets_s, offsets_s = vak.transforms.frame_labels.to_segments( @@ -263,12 +264,12 @@ def test_to_segments_real_data( ), ], ) -def test_to_inds(frame_labels, seg_inds_list_expected): +def test_segment_inds_list_from_class_labels(frame_labels, seg_inds_list_expected): """Test ``to_inds`` works as expected""" UNLABELED = 0 - seg_inds_list = vak.transforms.frame_labels.to_inds_list( - frame_labels=frame_labels, unlabeled_label=UNLABELED + seg_inds_list = vak.transforms.frame_labels.segment_inds_list_from_class_labels( + frame_labels=frame_labels, background_label=UNLABELED ) assert np.array_equal(seg_inds_list, seg_inds_list_expected) @@ -296,15 +297,15 @@ def test_to_inds(frame_labels, seg_inds_list_expected): ) def test_remove_short_segments(lbl_tb, unlabeled, timebin_dur, min_segment_dur, lbl_tb_expected): """Test ``remove_short_segments`` works as expected""" - segment_inds_list = vak.transforms.frame_labels.to_inds_list( - lbl_tb, unlabeled_label=unlabeled + segment_inds_list = vak.transforms.frame_labels.segment_inds_list_from_class_labels( + lbl_tb, background_label=unlabeled ) lbl_tb_tfm, segment_inds_list_out = vak.transforms.frame_labels.remove_short_segments( lbl_tb, segment_inds_list, timebin_dur=timebin_dur, min_segment_dur=min_segment_dur, - unlabeled_label=unlabeled, + background_label=unlabeled, ) assert np.array_equal(lbl_tb_tfm, lbl_tb_expected) @@ -323,13 +324,13 @@ def test_remove_short_segments(lbl_tb, unlabeled, timebin_dur, min_segment_dur, 0, np.array([0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0]), ), - # test MajorityVote works when there is no 'unlabeled' segment at start of vector + # test MajorityVote works when there is no vak.common.constants.DEFAULT_BACKGROUND_LABEL segment at start of vector ( np.asarray([1, 1, 2, 1, 0, 0, 0, 0]), 0, np.asarray([1, 1, 1, 1, 0, 0, 0, 0]) ), - # test MajorityVote works when there is no 'unlabeled' segment at end of vector + # test MajorityVote works when there is no vak.common.constants.DEFAULT_BACKGROUND_LABEL segment at end of vector ( np.array([0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 2, 1]), 0, @@ -345,8 +346,8 @@ def test_remove_short_segments(lbl_tb, unlabeled, timebin_dur, min_segment_dur, ) def test_majority_vote(lbl_tb_in, unlabeled, lbl_tb_expected): """Test ``majority_vote`` works as expected""" - segment_inds_list = vak.transforms.frame_labels.to_inds_list( - lbl_tb_in, unlabeled_label=unlabeled + segment_inds_list = vak.transforms.frame_labels.segment_inds_list_from_class_labels( + lbl_tb_in, background_label=unlabeled ) lbl_tb_maj_vote = vak.transforms.frame_labels.take_majority_vote( lbl_tb_in, segment_inds_list @@ -389,14 +390,14 @@ def test_majority_vote(lbl_tb_in, unlabeled, lbl_tb_expected): # majority vote converts second segment to label "a" np.array([0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0]), ), - # test MajorityVote works when there is no 'unlabeled' segment at start of vector + # test MajorityVote works when there is no vak.common.constants.DEFAULT_BACKGROUND_LABEL segment at start of vector ( np.array([1, 1, 2, 1, 0, 0, 0, 0]), None, True, np.array([1, 1, 1, 1, 0, 0, 0, 0]), ), - # test MajorityVote works when there is no 'unlabeled' segment at end of vector + # test MajorityVote works when there is no vak.common.constants.DEFAULT_BACKGROUND_LABEL segment at end of vector ( np.array([0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 2, 1]), None, @@ -451,17 +452,17 @@ def test_majority_vote(lbl_tb_in, unlabeled, lbl_tb_expected): @pytest.mark.parametrize( - 'lbl_tb, timebin_dur, unlabeled_label, min_segment_dur, majority_vote, lbl_tb_expected', + 'lbl_tb, timebin_dur, background_label, min_segment_dur, majority_vote, lbl_tb_expected', POSTPROCESS_PARAMS_ARGVALS ) -def test_postprocess(lbl_tb, timebin_dur, unlabeled_label, min_segment_dur, majority_vote, lbl_tb_expected): +def test_postprocess(lbl_tb, timebin_dur, background_label, min_segment_dur, majority_vote, lbl_tb_expected): """Test that ``trasnforms.frame_labels.postprocess`` works as expected. Specifically test that we recover an expected string of labels, as would be used to compute edit distance.""" lbl_tb = vak.transforms.frame_labels.postprocess( lbl_tb, timebin_dur=timebin_dur, - unlabeled_label=UNLABELED_LABEL, + background_label=UNLABELED_LABEL, majority_vote=majority_vote, min_segment_dur=min_segment_dur, ) diff --git a/tests/test_transforms/test_frame_labels/test_transforms.py b/tests/test_transforms/test_frame_labels/test_transforms.py index c044fd8b0..399eee87e 100644 --- a/tests/test_transforms/test_frame_labels/test_transforms.py +++ b/tests/test_transforms/test_frame_labels/test_transforms.py @@ -2,6 +2,7 @@ import pytest import vak +import vak.common # for constants from .test_functional import ( @@ -36,7 +37,7 @@ def test_call(self, annot, spect_path, labelset): 'Annotation with label not in labelset, would not include in dataset' ) - from_segments_tfm = vak.transforms.frame_labels.FromSegments(unlabeled_label=labelmap['unlabeled']) + from_segments_tfm = vak.transforms.frame_labels.FromSegments(background_label=labelmap[vak.common.constants.DEFAULT_BACKGROUND_LABEL]) lbl_tb = from_segments_tfm( lbls_int, annot.seq.onsets_s, @@ -55,10 +56,10 @@ class TestToLabels: [tup[2] for tup in FROM_SEGMENTS_PARAMETRIZE_ARGVALS], ) def test_init(self, labelset): - # Note that we add an 'unlabeled' class because post-processing transforms *require* it + # Note that we add an vak.common.constants.DEFAULT_BACKGROUND_LABEL class because post-processing transforms *require* it # This is default, just making it explicit labelset = vak.common.converters.labelset_to_set(labelset) - labelmap = vak.common.labels.to_map(labelset, map_unlabeled=True) + labelmap = vak.common.labels.to_map(labelset, map_background=True) to_labels_tfm = vak.transforms.frame_labels.ToLabels( labelmap=labelmap, @@ -68,17 +69,17 @@ def test_init(self, labelset): @pytest.mark.parametrize( "lbl_tb, labelmap, labels_expected_int", [ - (np.array([0, 0, 1, 1, 0, 0, 2, 2, 0, 0]), {'unlabeled': 0, 'a': 1, 'b': 2}, [1, 2]), - (np.array([0, 0, 1, 1, 0, 0, 2, 2, 0, 0]), {'unlabeled': 0, '1': 1, '2': 2}, [1, 2]), - (np.array([0, 0, 21, 21, 0, 0, 22, 22, 0, 0]), {'unlabeled': 0, '21': 21, '22': 22}, [21, 22]), - (np.array([0, 0, 11, 11, 0, 0, 12, 12, 0, 0]), {'unlabeled': 0, '11': 11, '12': 12}, [11, 12]), + (np.array([0, 0, 1, 1, 0, 0, 2, 2, 0, 0]), {vak.common.constants.DEFAULT_BACKGROUND_LABEL: 0, 'a': 1, 'b': 2}, [1, 2]), + (np.array([0, 0, 1, 1, 0, 0, 2, 2, 0, 0]), {vak.common.constants.DEFAULT_BACKGROUND_LABEL: 0, '1': 1, '2': 2}, [1, 2]), + (np.array([0, 0, 21, 21, 0, 0, 22, 22, 0, 0]), {vak.common.constants.DEFAULT_BACKGROUND_LABEL: 0, '21': 21, '22': 22}, [21, 22]), + (np.array([0, 0, 11, 11, 0, 0, 12, 12, 0, 0]), {vak.common.constants.DEFAULT_BACKGROUND_LABEL: 0, '11': 11, '12': 12}, [11, 12]), ] ) def test_call(self, lbl_tb, labelmap, labels_expected_int): - # Note that we add an 'unlabeled' class because post-processing transforms *require* it + # Note that we add an vak.common.constants.DEFAULT_BACKGROUND_LABEL class because post-processing transforms *require* it # This is default, just making it explicit labelmap = vak.common.labels.multi_char_labels_to_single_char( - labelmap, skip=('unlabeled',) + labelmap, skip=(vak.common.constants.DEFAULT_BACKGROUND_LABEL,) ) labelmap_inv = {v: k for k, v in labelmap.items()} labels_expected = ''.join([labelmap_inv[lbl_int] for lbl_int in labels_expected_int]) @@ -103,7 +104,7 @@ def test_call_real_data( # we can easily compare strings we get back with expected; # this is what core.eval does labelmap = vak.common.labels.multi_char_labels_to_single_char( - labelmap, skip=('unlabeled',) + labelmap, skip=(vak.common.constants.DEFAULT_BACKGROUND_LABEL,) ) TIMEBINS_KEY = "t" @@ -130,7 +131,7 @@ def test_call_real_data( annot.seq.onsets_s, annot.seq.offsets_s, timebins, - unlabeled_label=labelmap["unlabeled"], + background_label=labelmap[vak.common.constants.DEFAULT_BACKGROUND_LABEL], ) to_labels_tfm = vak.transforms.frame_labels.ToLabels( @@ -152,10 +153,10 @@ class TestToSegments: [tup[2] for tup in FROM_SEGMENTS_PARAMETRIZE_ARGVALS], ) def test_init(self, labelset): - # Note that we add an 'unlabeled' class because post-processing transforms *require* it + # Note that we add an vak.common.constants.DEFAULT_BACKGROUND_LABEL class because post-processing transforms *require* it # This is default, just making it explicit labelset = vak.common.converters.labelset_to_set(labelset) - labelmap = vak.common.labels.to_map(labelset, map_unlabeled=True) + labelmap = vak.common.labels.to_map(labelset, map_background=True) to_segments_tfm = vak.transforms.frame_labels.ToSegments( labelmap=labelmap, @@ -195,7 +196,7 @@ def test_call_real_data(self, annot, spect_path, labelset): annot.seq.onsets_s, annot.seq.offsets_s, timebins, - unlabeled_label=labelmap["unlabeled"], + background_label=labelmap[vak.common.constants.DEFAULT_BACKGROUND_LABEL], ) to_segments_tfm = vak.transforms.frame_labels.ToSegments( @@ -218,7 +219,7 @@ class TestPostprocess: [argvals[3:5] + (TIMEBIN_DUR_FOR_PARAMETRIZE,) for argvals in POSTPROCESS_PARAMS_ARGVALS] ) def test_init(self, min_segment_dur, majority_vote, timebin_dur): - # Note that we add an 'unlabeled' class + # Note that we add an vak.common.constants.DEFAULT_BACKGROUND_LABEL class # because post-processing transforms *require* it # This is default, just making it explicit to_labels_tfm = vak.transforms.frame_labels.PostProcess( @@ -229,11 +230,11 @@ def test_init(self, min_segment_dur, majority_vote, timebin_dur): assert isinstance(to_labels_tfm, vak.transforms.frame_labels.PostProcess) @pytest.mark.parametrize( - 'lbl_tb, timebin_dur, unlabeled_label, min_segment_dur, majority_vote, lbl_tb_expected', + 'lbl_tb, timebin_dur, background_label, min_segment_dur, majority_vote, lbl_tb_expected', POSTPROCESS_PARAMS_ARGVALS ) - def test_call(self, lbl_tb, timebin_dur, unlabeled_label, min_segment_dur, majority_vote, lbl_tb_expected): - # Note that we add an 'unlabeled' class because post-processing transforms *require* it + def test_call(self, lbl_tb, timebin_dur, background_label, min_segment_dur, majority_vote, lbl_tb_expected): + # Note that we add an vak.common.constants.DEFAULT_BACKGROUND_LABEL class because post-processing transforms *require* it # This is default, just making it explicit postprocess_tfm = vak.transforms.frame_labels.PostProcess( min_segment_dur=min_segment_dur, diff --git a/tests/test_transforms/test_transforms.py b/tests/test_transforms/test_transforms.py index b71a42749..34ddb5ad6 100644 --- a/tests/test_transforms/test_transforms.py +++ b/tests/test_transforms/test_transforms.py @@ -4,9 +4,10 @@ import vak.transforms import vak.transforms.functional import vak.common.validators +from vak.datapipes.frame_classification import Metadata -class TestStandardizeSpect: +class TestFramesStandardizer: @pytest.mark.parametrize( 'mean_freqs, std_freqs, non_zero_std', @@ -19,10 +20,10 @@ class TestStandardizeSpect: ] ) def test_instance(self, mean_freqs, std_freqs, non_zero_std): - standardizer = vak.transforms.StandardizeSpect( + standardizer = vak.transforms.FramesStandardizer( mean_freqs=mean_freqs, std_freqs=std_freqs, non_zero_std=non_zero_std ) - assert isinstance(standardizer, vak.transforms.StandardizeSpect) + assert isinstance(standardizer, vak.transforms.FramesStandardizer) for attr_name, expected in zip( ('mean_freqs', 'std_freqs', 'non_zero_std',), (mean_freqs, std_freqs, non_zero_std), @@ -42,6 +43,27 @@ def test_instance(self, mean_freqs, std_freqs, non_zero_std): np.equal(spect_out, expected) ) + def test_fit(self): + spect = np.random.rand(513, 1000) + standardizer = vak.transforms.FramesStandardizer.fit(spect) + + expected_mean_freqs = np.mean(spect, axis=1) + expected_std_freqs = np.std(spect, axis=1) + expected_non_zero_std = np.argwhere(expected_std_freqs != 0) + # we convert to 1d vector in __init__ + expected_non_zero_std = vak.common.validators.column_or_1d(expected_non_zero_std) + + for attr_name, expected in zip( + ('mean_freqs', 'std_freqs', 'non_zero_std',), + (expected_mean_freqs, expected_std_freqs, expected_non_zero_std), + ): + assert hasattr(standardizer, attr_name) + attr = getattr(standardizer, attr_name) + assert isinstance(attr, np.ndarray) + assert np.all( + np.equal(attr, expected) + ) + @pytest.mark.parametrize( 'split', [ @@ -50,8 +72,7 @@ def test_instance(self, mean_freqs, std_freqs, non_zero_std): None ] ) - def test_fit_dataset_path(self, split, train_cbin_notmat_df, - specific_dataset_path, specific_dataset_csv_path): + def test_fit_inputs_targets_csv_path(self, split, train_cbin_notmat_df, specific_dataset_path): # we need dataset_path since paths in df are relative to it dataset_path = specific_dataset_path( config_type="train", @@ -59,6 +80,8 @@ def test_fit_dataset_path(self, split, train_cbin_notmat_df, audio_format="cbin", annot_format="notmat" ) + metadata = Metadata.from_dataset_path(dataset_path) + dataset_csv_path = dataset_path / metadata.dataset_csv_filename if split is None: split_to_test = 'train' @@ -83,10 +106,10 @@ def test_fit_dataset_path(self, split, train_cbin_notmat_df, # ---- actually do fit if split: - standardizer = vak.transforms.StandardizeSpect.fit_dataset_path(dataset_path, split=split) + standardizer = vak.transforms.FramesStandardizer.fit_inputs_targets_csv_path(dataset_csv_path, dataset_path, split=split) else: # this tests that default value for split 'train' works as expected - standardizer = vak.transforms.StandardizeSpect.fit_dataset_path(dataset_path) + standardizer = vak.transforms.FramesStandardizer.fit_inputs_targets_csv_path(dataset_csv_path, dataset_path) # ---- test for attr_name, expected in zip( @@ -100,16 +123,52 @@ def test_fit_dataset_path(self, split, train_cbin_notmat_df, np.equal(attr, expected) ) - def test_fit(self): - spect = np.random.rand(513, 1000) - standardizer = vak.transforms.StandardizeSpect.fit(spect) + @pytest.mark.parametrize( + 'split', + [ + 'train', + 'val', + None + ] + ) + def test_fit_dataset_path(self, split, train_cbin_notmat_df, specific_dataset_path): + # we need dataset_path since paths in df are relative to it + dataset_path = specific_dataset_path( + config_type="train", + model="TweetyNet", + audio_format="cbin", + annot_format="notmat" + ) - expected_mean_freqs = np.mean(spect, axis=1) - expected_std_freqs = np.std(spect, axis=1) + if split is None: + split_to_test = 'train' + else: + split_to_test = split + # ---- set up + df_split = train_cbin_notmat_df[train_cbin_notmat_df.split == split_to_test].copy() + spect_paths = df_split['frames_path'].values + spect = vak.common.files.spect.load(dataset_path / spect_paths[0])[vak.common.constants.SPECT_KEY] + mean_freqs = np.mean(spect, axis=1) + std_freqs = np.std(spect, axis=1) + + for spect_path in spect_paths[1:]: + spect = vak.common.files.spect.load(dataset_path / spect_path)[vak.common.constants.SPECT_KEY] + mean_freqs += np.mean(spect, axis=1) + std_freqs += np.std(spect, axis=1) + expected_mean_freqs = mean_freqs / len(spect_paths) + expected_std_freqs = std_freqs / len(spect_paths) expected_non_zero_std = np.argwhere(expected_std_freqs != 0) # we convert to 1d vector in __init__ expected_non_zero_std = vak.common.validators.column_or_1d(expected_non_zero_std) + # ---- actually do fit + if split: + standardizer = vak.transforms.FramesStandardizer.fit_dataset_path(dataset_path, split=split) + else: + # this tests that default value for split 'train' works as expected + standardizer = vak.transforms.FramesStandardizer.fit_dataset_path(dataset_path) + + # ---- test for attr_name, expected in zip( ('mean_freqs', 'std_freqs', 'non_zero_std',), (expected_mean_freqs, expected_std_freqs, expected_non_zero_std),