Test presets with Keras saving fixes

.cloudbuild/cloudbuild.yaml

@@ -50,6 +50,7 @@ steps:
     '--ext-str', 'backend=$_BACKEND',
     '-o', 'output.yaml',
   ]
+  secretEnv: ['KAGGLE_USERNAME', 'KAGGLE_KEY']
 - name: 'gcr.io/cloud-builders/gcloud'
   id: create-job
   waitFor:
@@ -75,3 +76,9 @@ options:
   volumes:
   - name: go-modules
     path: /go
+availableSecrets:
+  secretManager:
+  - versionName: projects/$PROJECT_ID/secrets/KAGGLE_USERNAME/versions/latest
+    env: 'KAGGLE_USERNAME'
+  - versionName: projects/$PROJECT_ID/secrets/KAGGLE_KEY/versions/latest
+    env: 'KAGGLE_KEY'

.cloudbuild/unit_test_jobs.jsonnet

@@ -34,7 +34,7 @@ local unittest = base.BaseTest {
     )
   ],
   command: [
-    'pytest --run_large --durations 0',
+    'pytest --run_extra_large --durations 0',
     'keras_cv/bounding_box',
     'keras_cv/callbacks',
     'keras_cv/losses',

.github/workflows/actions.yml

@@ -41,6 +41,8 @@ jobs:
         pip install -e ".[tests]" --progress-bar off --upgrade
     - name: Test with pytest
       env:
+        KAGGLE_USERNAME:  ${{ secrets.KAGGLE_USERNAME }}
+        KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
         TEST_CUSTOM_OPS: false
       run: |
         pytest keras_cv/ --ignore keras_cv/models/legacy/ --durations 0
@@ -77,6 +79,8 @@ jobs:
       env:
         TEST_CUSTOM_OPS: false # TODO(ianstenbit): test custom ops, or figure out what our story is here
         KERAS_BACKEND: ${{ matrix.backend }}
+        KAGGLE_USERNAME:  ${{ secrets.KAGGLE_USERNAME}}
+        KAGGLE_KEY: ${{ secrets.KAGGLE_KEY}}
         JAX_ENABLE_X64: true
       run: |
         pytest keras_cv/bounding_box \

keras_cv/__init__.py

@@ -42,4 +42,4 @@
 from keras_cv.core import NormalFactorSampler
 from keras_cv.core import UniformFactorSampler
 
-__version__ = "0.8.0"
+__version__ = "0.8.0.dev0"

keras_cv/models/backbones/backbone.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 """Base class for Backbone models."""
 
-import os
-
 from keras_cv.api_export import keras_cv_export
 from keras_cv.backend import keras
+from keras_cv.utils.preset_utils import check_preset_class
+from keras_cv.utils.preset_utils import load_from_preset
 from keras_cv.utils.python_utils import classproperty
 from keras_cv.utils.python_utils import format_docstring
 
@@ -95,43 +95,18 @@ def from_preset(
             load_weights=False,
         ```
         """
-
-        if not cls.presets:
-            raise NotImplementedError(
-                "No presets have been created for this class."
-            )
-
-        if preset not in cls.presets:
-            raise ValueError(
-                "`preset` must be one of "
-                f"""{", ".join(cls.presets)}. Received: {preset}."""
-            )
-
-        if load_weights and preset not in cls.presets_with_weights:
-            raise ValueError(
-                f"""Pretrained weights not available for preset "{preset}". """
-                "Set `load_weights=False` to use this preset or choose one of "
-                "the following presets with weights:"
-                f""" "{'", "'.join(cls.presets_with_weights)}"."""
-            )
-
-        metadata = cls.presets[preset]
-        config = metadata["config"]
-        model = cls.from_config({**config, **kwargs})
-
-        if preset not in cls.presets_with_weights or load_weights is False:
-            return model
-
-        weights = keras.utils.get_file(
-            "model.h5",
-            metadata["weights_url"],
-            cache_subdir=os.path.join("models", preset),
-            file_hash=metadata["weights_hash"],
+        # We support short IDs for official presets, e.g. `"bert_base_en"`.
+        # Map these to a Kaggle Models handle.
+        if preset in cls.presets:
+            preset = cls.presets[preset]["kaggle_handle"]
+
+        check_preset_class(preset, cls)
+        return load_from_preset(
+            preset,
+            load_weights=load_weights,
+            config_overrides=kwargs,
         )
 
-        model.load_weights(weights)
-        return model
-
     def __init_subclass__(cls, **kwargs):
         # Use __init_subclass__ to set up a correct docstring for from_preset.
         super().__init_subclass__(**kwargs)

keras_cv/models/backbones/csp_darknet/csp_darknet_backbone_presets.py

@@ -26,15 +26,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "class_name": "keras_cv>CSPDarkNetBackbone",
-        "config": {
-            "stackwise_channels": [48, 96, 192, 384],
-            "stackwise_depth": [1, 3, 3, 1],
-            "include_rescaling": True,
-            "use_depthwise": False,
-            "input_shape": (None, None, 3),
-            "input_tensor": None,
-        },
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_tiny/1",
     },
     "csp_darknet_s": {
         "metadata": {
@@ -47,15 +39,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "class_name": "keras_cv>CSPDarkNetBackbone",
-        "config": {
-            "stackwise_channels": [64, 128, 256, 512],
-            "stackwise_depth": [1, 3, 3, 1],
-            "include_rescaling": True,
-            "use_depthwise": False,
-            "input_shape": (None, None, 3),
-            "input_tensor": None,
-        },
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_s/1",
     },
     "csp_darknet_m": {
         "metadata": {
@@ -68,15 +52,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "class_name": "keras_cv>CSPDarkNetBackbone",
-        "config": {
-            "stackwise_channels": [96, 192, 384, 768],
-            "stackwise_depth": [2, 6, 6, 2],
-            "include_rescaling": True,
-            "use_depthwise": False,
-            "input_shape": (None, None, 3),
-            "input_tensor": None,
-        },
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_m/1",
     },
     "csp_darknet_l": {
         "metadata": {
@@ -89,15 +65,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "class_name": "keras_cv>CSPDarkNetBackbone",
-        "config": {
-            "stackwise_channels": [128, 256, 512, 1024],
-            "stackwise_depth": [3, 9, 9, 3],
-            "include_rescaling": True,
-            "use_depthwise": False,
-            "input_shape": (None, None, 3),
-            "input_tensor": None,
-        },
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_l/1",
     },
     "csp_darknet_xl": {
         "metadata": {
@@ -110,15 +78,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "class_name": "keras_cv>CSPDarkNetBackbone",
-        "config": {
-            "stackwise_channels": [170, 340, 680, 1360],
-            "stackwise_depth": [4, 12, 12, 4],
-            "include_rescaling": True,
-            "use_depthwise": False,
-            "input_shape": (None, None, 3),
-            "input_tensor": None,
-        },
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_xl/1",
     },
 }
 
@@ -135,10 +95,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "class_name": "keras_cv>CSPDarkNetBackbone",
-        "config": backbone_presets_no_weights["csp_darknet_tiny"]["config"],
-        "weights_url": "https://storage.googleapis.com/keras-cv/models/cspdarknettiny/imagenet/classification-v0-notop.h5",  # noqa: E501
-        "weights_hash": "0007ae82c95be4d4aef06368a7c38e006381324d77e5df029b04890e18a8ad19",  # noqa: E501
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_tiny_imagenet/1",  # noqa: E501
     },
     "csp_darknet_l_imagenet": {
         "metadata": {
@@ -152,10 +109,7 @@
             "official_name": "CSPDarkNet",
             "path": "csp_darknet",
         },
-        "class_name": "keras_cv>CSPDarkNetBackbone",
-        "config": backbone_presets_no_weights["csp_darknet_l"]["config"],
-        "weights_url": "https://storage.googleapis.com/keras-cv/models/cspdarknetl/imagenet/classification-v0-notop.h5",  # noqa: E501
-        "weights_hash": "9303aabfadffbff8447171fce1e941f96d230d8f3cef30d3f05a9c85097f8f1e",  # noqa: E501
+        "kaggle_handle": "kaggle://keras/cspdarknet/csp_darknet_l_imagenet/1",
     },
 }
 

keras_cv/models/backbones/densenet/densenet_backbone_presets.py

@@ -18,43 +18,19 @@
         "metadata": {
             "description": "DenseNet model with 121 layers.",
         },
-        "class_name": "keras_cv>DenseNetBackbone",
-        "config": {
-            "stackwise_num_repeats": [6, 12, 24, 16],
-            "include_rescaling": True,
-            "input_shape": (None, None, 3),
-            "input_tensor": None,
-            "compression_ratio": 0.5,
-            "growth_rate": 32,
-        },
+        "kaggle_handle": "kaggle://keras/densenet/densenet121/1",
     },
     "densenet169": {
         "metadata": {
             "description": "DenseNet model with 169 layers.",
         },
-        "class_name": "keras_cv>DenseNetBackbone",
-        "config": {
-            "stackwise_num_repeats": [6, 12, 32, 32],
-            "include_rescaling": True,
-            "input_shape": (None, None, 3),
-            "input_tensor": None,
-            "compression_ratio": 0.5,
-            "growth_rate": 32,
-        },
+        "kaggle_handle": "kaggle://keras/densenet/densenet169/1",
     },
     "densenet201": {
         "metadata": {
             "description": "DenseNet model with 201 layers.",
         },
-        "class_name": "keras_cv>DenseNetBackbone",
-        "config": {
-            "stackwise_num_repeats": [6, 12, 48, 32],
-            "include_rescaling": True,
-            "input_shape": (None, None, 3),
-            "input_tensor": None,
-            "compression_ratio": 0.5,
-            "growth_rate": 32,
-        },
+        "kaggle_handle": "kaggle://keras/densenet/densenet201/1",
     },
 }
 
@@ -66,10 +42,7 @@
                 "classification task."
             ),
         },
-        "class_name": "keras_cv>DenseNetBackbone",
-        "config": backbone_presets_no_weights["densenet121"]["config"],
-        "weights_url": "https://storage.googleapis.com/keras-cv/models/densenet121/imagenet/classification-v0-notop.h5",  # noqa: E501
-        "weights_hash": "709afe0321d9f2b2562e562ff9d0dc44cca10ed09e0e2cfba08d783ff4dab6bf",  # noqa: E501
+        "kaggle_handle": "kaggle://keras/densenet/densenet121_imagenet/1",
     },
     "densenet169_imagenet": {
         "metadata": {
@@ -78,10 +51,7 @@
                 "classification task."
             ),
         },
-        "class_name": "keras_cv>DenseNetBackbone",
-        "config": backbone_presets_no_weights["densenet169"]["config"],
-        "weights_url": "https://storage.googleapis.com/keras-cv/models/densenet169/imagenet/classification-v0-notop.h5",  # noqa: E501
-        "weights_hash": "a99d1bb2cbe1a59a1cdd1f435fb265453a97c2a7b723d26f4ebee96e5fb49d62",  # noqa: E501
+        "kaggle_handle": "kaggle://keras/densenet/densenet169_imagenet/1",
     },
     "densenet201_imagenet": {
         "metadata": {
@@ -90,10 +60,7 @@
                 "classification task."
             ),
         },
-        "class_name": "keras_cv>DenseNetBackbone",
-        "config": backbone_presets_no_weights["densenet201"]["config"],
-        "weights_url": "https://storage.googleapis.com/keras-cv/models/densenet201/imagenet/classification-v0-notop.h5",  # noqa: E501
-        "weights_hash": "c1189a934f12c1a676a9cf52238e5994401af925e2adfc0365bad8133c052060",  # noqa: E501
+        "kaggle_handle": "kaggle://keras/densenet/densenet201_imagenet/1",
     },
 }