update pre-commit

.pre-commit-config.yaml

@@ -1,69 +1,70 @@
 repos:
-  # Empty notebookds
-  - repo: local
-    hooks:
-      - id: clear-notebooks-output
-        name: clear-notebooks-output
-        files: tools/.*\.ipynb$
-        stages: [commit]
-        language: python
-        entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
-        additional_dependencies: [jupyter]
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
-    hooks:
-      - id: check-yaml # Check YAML files for syntax errors only
-        args: [--unsafe, --allow-multiple-documents]
-      - id: debug-statements # Check for debugger imports and py37+ breakpoint()
-      - id: end-of-file-fixer # Ensure files end in a newline
-      - id: trailing-whitespace # Trailing whitespace checker
-      - id: no-commit-to-branch # Prevent committing to main / master
-      - id: check-added-large-files # Check for large files added to git
-      - id: check-merge-conflict # Check for files that contain merge conflict
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.4.2
-    hooks:
-      - id: black
-        args: [--line-length=120]
-  - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        args:
-          - -l 120
-          - --force-single-line-imports
-          - --profile black
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.6
-    hooks:
-      - id: ruff
-        exclude: '(dev/.*|.*_)\.py$'
-        args:
-          - --line-length=120
-          - --fix
-          - --exit-non-zero-on-fix
-          - --preview
-  - repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.9.1
-    hooks:
-      - id: sphinx-lint
-  # For now, we use it. But it does not support a lot of sphinx features
-  - repo: https://github.com/dzhu/rstfmt
-    rev: v0.0.14
-    hooks:
-      - id: rstfmt
-        exclude: 'cli/.*' # Because we use argparse
-  - repo: https://github.com/b8raoult/pre-commit-docconvert
-    rev: "0.1.4"
-    hooks:
-      - id: docconvert
-        args: ["numpy"]
-  - repo: https://github.com/b8raoult/optional-dependencies-all
-    rev: "0.0.6"
-    hooks:
-    - id: optional-dependencies-all
-      args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=docs,tests"]
-  - repo: https://github.com/tox-dev/pyproject-fmt
-    rev: "2.1.3"
-    hooks:
-    - id: pyproject-fmt
+# Empty notebookds
+- repo: local
+  hooks:
+  - id: clear-notebooks-output
+    name: clear-notebooks-output
+    files: tools/.*\.ipynb$
+    stages: [commit]
+    language: python
+    entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
+    additional_dependencies: [jupyter]
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.6.0
+  hooks:
+  - id: check-yaml # Check YAML files for syntax errors only
+    args: [--unsafe, --allow-multiple-documents]
+  - id: debug-statements # Check for debugger imports and py37+ breakpoint()
+  - id: end-of-file-fixer # Ensure files end in a newline
+  - id: trailing-whitespace # Trailing whitespace checker
+  - id: no-commit-to-branch # Prevent committing to main / master
+  - id: check-added-large-files # Check for large files added to git
+  - id: check-merge-conflict # Check for files that contain merge conflict
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 24.4.2
+  hooks:
+  - id: black
+    args: [--line-length=120]
+- repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort
+    args:
+    - -l 120
+    - --force-single-line-imports
+    - --profile black
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.4.6
+  hooks:
+  - id: ruff
+    # Next line if for documenation cod snippets
+    exclude: '^(dev/.*|[A-Za-z].*_)\.py$'
+    args:
+    - --line-length=120
+    - --fix
+    - --exit-non-zero-on-fix
+    - --preview
+- repo: https://github.com/sphinx-contrib/sphinx-lint
+  rev: v0.9.1
+  hooks:
+  - id: sphinx-lint
+# For now, we use it. But it does not support a lot of sphinx features
+- repo: https://github.com/dzhu/rstfmt
+  rev: v0.0.14
+  hooks:
+  - id: rstfmt
+    exclude: 'cli/.*' # Because we use argparse
+- repo: https://github.com/b8raoult/pre-commit-docconvert
+  rev: "0.1.5"
+  hooks:
+  - id: docconvert
+    args: ["numpy"]
+- repo: https://github.com/b8raoult/optional-dependencies-all
+  rev: "0.0.6"
+  hooks:
+  - id: optional-dependencies-all
+    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
+- repo: https://github.com/tox-dev/pyproject-fmt
+  rev: "2.1.3"
+  hooks:
+  - id: pyproject-fmt

pyproject.toml

@@ -64,8 +64,10 @@ optional-dependencies.all = [
 ]
 
 optional-dependencies.dev = [
+  "ai-models>=0.6.1",
   "anemoi-datasets",
   "pytest",
+  "tqdm",
 ]
 
 optional-dependencies.plugin = [

src/anemoi/inference/checkpoint/metadata/patch.py

@@ -78,9 +78,6 @@ def patch(a, b):
 
 def list_to_dict(datasets, config):
 
-    # print(json.dumps(datasets, indent=4))
-    # os._exit(1)
-
     arguments = config["dataloader"]["training"]
     assert "dataset" in arguments
     assert isinstance(arguments["dataset"], list)

src/anemoi/inference/checkpoint/metadata/version_0_2_0.py

@@ -6,7 +6,6 @@
 # nor does it submit to any jurisdiction.
 
 
-import json
 import logging
 from functools import cached_property
 
@@ -59,21 +58,12 @@ def _as_string(r):
             r = dict(with_nans=nans)
             yield _as_string(r)
 
-    def dump_content(self, full, indent):
-
-        if full:
-            print("{", f'"{self.__class__.__name__}"', ":")
-            print(json.dumps(self.metadata, indent=2, default=str))
-            print("}")
-            return
+    def dump_content(self, indent):
 
         print()
         print(" " * indent, "-", self)
-        if full:
-            print(" " * indent, " ", json.dumps(self.metadata, indent=2, default=str))
-        else:
-            for n in self.mars_request():
-                print(" " * indent, " ", n)
+        for n in self.mars_request():
+            print(" " * indent, " ", n)
 
     @property
     def param_sfc(self):
@@ -128,8 +118,8 @@ def param_step_sfc_pairs(self):
     def variables_with_nans(self):
         return sorted(self.attributes.get("variables_with_nans", []))
 
-    def dump(self, full, indent=0):
-        self.dump_content(full, indent)
+    def dump(self, indent=0):
+        self.dump_content(indent)
 
 
 class Forward(DataRequest):
@@ -140,9 +130,9 @@ def forward(self):
     def __getattr__(self, name):
         return getattr(self.forward, name)
 
-    def dump(self, full, indent=0):
-        self.dump_content(full, indent)
-        self.forward.dump(indent + 2, full)
+    def dump(self, indent=0):
+        self.dump_content(indent)
+        self.forward.dump(indent + 2)
 
 
 class SubsetRequest(Forward):
@@ -174,8 +164,8 @@ def __init__(self, metadata):
     def forward(self):
         return self.datasets[0]
 
-    def dump(self, full, indent=0):
-        self.dump_content(full, indent)
+    def dump(self, indent=0):
+        self.dump_content(indent)
         for dataset in self.datasets:
             dataset.dump(indent + 2)
 

src/anemoi/inference/commands/inspect.py

@@ -26,11 +26,9 @@ def run(self, args):
         c = Checkpoint(args.path)
 
         if args.dump:
-            c.dump(full=True)
+            c.dump()
             return
 
-        c.dump(full=False)
-
         def _(f):
             try:
                 return f()

tests/test_datasets.py

@@ -61,7 +61,7 @@ def standard_test(dataset):
 
     ds = open_dataset(**dataset)
     md = Metadata.from_metadata({"version": "1.0.0", "dataset": ds.metadata()})
-    md.dump(full=True)
+    md.dump()
 
     return ds, md