Compatibility with Python 3.12 (#21)

Also, small fixes to address:
  - an issue with flexible channel priorities in mixed mode
  - an issue with specifying the python dependency in the requirements.txt or environment.yml file

metaflow_extensions/netflix_ext/cmd/environment/environment_cmd.py

@@ -47,13 +47,9 @@
     merge_dep_dicts,
 )
 
-from metaflow_extensions.netflix_ext.vendor.packaging.requirements import (
-    InvalidRequirement,
-    Requirement,
-)
-from metaflow_extensions.netflix_ext.vendor.packaging.utils import (
-    canonicalize_version,
-)
+from metaflow._vendor.packaging.requirements import InvalidRequirement, Requirement
+from metaflow._vendor.packaging.specifiers import SpecifierSet
+from metaflow._vendor.packaging.utils import canonicalize_version
 
 from .utils import download_mf_version
 
@@ -568,15 +564,33 @@ def resolve(
             )
 
     # Parse yaml first to put conda sources first to be consistent with step decorator
+    parsed_python_version = None
     if yml_file:
-        _parse_yml_file(
+        parsed_python_version = _parse_yml_file(
             yml_file, new_extras, new_sources, new_conda_deps, new_pypi_deps
         )
     if req_file:
-        _parse_req_file(
+        parsed_python_version = _parse_req_file(
             req_file, new_extras, new_sources, new_pypi_deps, new_np_conda_deps
         )
 
+    if base_env_python:
+        if parsed_python_version:
+            if using_pathspec is not None or using is not None:
+                # Check if the python version matches properly
+                if not SpecifierSet(parsed_python_version).contains(base_env_python):
+                    raise InvalidEnvironmentException(
+                        "The base environment's Python version (%s) does not match the "
+                        "one specified in the requirements file (%s)"
+                        % (base_env_python, parsed_python_version)
+                    )
+            else:
+                raise InvalidEnvironmentException(
+                    "Cannot specify --python if the python dependency is already set "
+                    "in the requirements file"
+                )
+    else:
+        base_env_python = parsed_python_version
     if base_env_python is None:
         base_env_python = platform.python_version()
 
@@ -905,7 +919,8 @@ def _parse_req_file(
     sources: Dict[str, List[str]],
     deps: Dict[str, str],
     np_deps: Dict[str, str],
-):
+) -> Optional[str]:
+    python_version = None
     with open(file_name, mode="r", encoding="utf-8") as f:
         for line in f:
             line = line.strip()
@@ -963,7 +978,12 @@ def _parse_req_file(
                 if parsed_req.url:
                     dep_name += "@%s" % parsed_req.url
                 specifier = str(parsed_req.specifier).lstrip(" =")
-                deps[dep_name] = str(specifier)
+                if dep_name == "python":
+                    if specifier:
+                        python_version = specifier
+                else:
+                    deps[dep_name] = specifier
+    return python_version
 
 
 def _parse_yml_file(
@@ -972,7 +992,8 @@ def _parse_yml_file(
     sources: Dict[str, List[str]],
     conda_deps: Dict[str, str],
     pypi_deps: Dict[str, str],
-):
+) -> Optional[str]:
+    python_version = None  # type: Optional[str]
     with open(file_name, mode="r", encoding="utf-8") as f:
         # Very poor man's yaml parsing
         mode = None
@@ -998,9 +1019,23 @@ def _parse_yml_file(
                     to_update = conda_deps if mode == "deps" else pypi_deps
                     splits = line.split("=", 1)
                     if len(splits) == 1:
-                        to_update[line] = ""
+                        if line != "python":
+                            to_update[line] = ""
                     else:
-                        to_update[splits[0].strip()] = splits[1].lstrip(" =")
+                        dep_name = splits[0].strip()
+                        dep_version = splits[1].lstrip(" =").rstrip()
+                        if dep_name == "python":
+                            if dep_version:
+                                if python_version:
+                                    raise InvalidEnvironmentException(
+                                        "Python versions specified multiple times in "
+                                        "the YAML file."
+                                    )
+                                python_version = dep_version
+                        else:
+                            to_update[dep_name] = dep_version
+
+    return python_version
 
 
 # @environment.command(help="List resolved environments for a set of dependencies")

metaflow_extensions/netflix_ext/plugins/conda/conda.py

@@ -17,8 +17,6 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from contextlib import closing
 from datetime import datetime
-from distutils.version import LooseVersion
-from itertools import chain
 from typing import (
     Any,
     Callable,
@@ -60,6 +58,7 @@
 from metaflow.metaflow_environment import InvalidEnvironmentException
 from metaflow.util import get_username
 
+from metaflow._vendor.packaging.version import parse as parse_version
 
 from .utils import (
     CONDA_FORMATS,
@@ -1900,25 +1899,26 @@ def _validate_conda_installation(self) -> Optional[Exception]:
             # We consider that locally installed environments are OK
             return None
 
-        # Check if the dependency solver exists.
-        if self._bins is None:
-            return InvalidEnvironmentException("No binaries configured for Conda")
-        # Check for other binaries
+        # Remove anything that has an invalid path
         to_remove = [
             k for k, v in self._bins.items() if v is None or not os.path.isfile(v)
         ]  # type: List[str]
         if to_remove:
             for k in to_remove:
                 del self._bins[k]
 
+        if "conda" not in self._bins:
+            return InvalidEnvironmentException(
+                "No %s binary found" % self._conda_executable_type
+            )
         # Check version requirements
         if "cph" in self._bins:
             cph_version = (
                 self.call_binary(["--version"], binary="cph")
                 .decode("utf-8")
                 .split()[-1]
             )
-            if LooseVersion(cph_version) < LooseVersion("1.9.0"):
+            if parse_version(cph_version) < parse_version("1.9.0"):
                 self.echo(
                     "cph is installed but not recent enough (1.9.0 or later is required) "
                     "-- ignoring"
@@ -1931,7 +1931,7 @@ def _validate_conda_installation(self) -> Optional[Exception]:
                 .decode("utf-8")
                 .split()[-1]
             )
-            if LooseVersion(conda_lock_version) < LooseVersion("2.0.0"):
+            if parse_version(conda_lock_version) < parse_version("2.0.0"):
                 self.echo(
                     "conda-lock is installed but not recent enough (2.0.0 or later "
                     "is required) --ignoring"
@@ -1942,22 +1942,22 @@ def _validate_conda_installation(self) -> Optional[Exception]:
                 1
             ]
             # 22.3 has PEP 658 support which can be a big performance boost
-            if LooseVersion(pip_version.decode("utf-8")) < LooseVersion("22.3"):
+            if parse_version(pip_version.decode("utf-8")) < parse_version("22.3"):
                 self.echo(
                     "pip is installed but not recent enough (22.3 or later is required) "
                     "-- ignoring"
                 )
                 del self._bins["pip"]
 
         if "micromamba version" in self._info_no_lock:
-            if LooseVersion(self._info_no_lock["micromamba version"]) < LooseVersion(
+            if parse_version(self._info_no_lock["micromamba version"]) < parse_version(
                 "1.4.0"
             ):
                 return InvalidEnvironmentException(
                     self._install_message_for_resolver("micromamba")
                 )
         else:
-            if LooseVersion(self._info_no_lock["conda_version"]) < LooseVersion(
+            if parse_version(self._info_no_lock["conda_version"]) < parse_version(
                 "4.14.0"
             ):
                 return InvalidEnvironmentException(

metaflow_extensions/netflix_ext/plugins/conda/conda_common_decorator.py

@@ -7,7 +7,7 @@
 from metaflow.metaflow_environment import InvalidEnvironmentException
 from metaflow.unbounded_foreach import UBF_CONTROL, UBF_TASK
 
-from metaflow_extensions.netflix_ext.vendor.packaging.utils import canonicalize_version
+from metaflow._vendor.packaging.utils import canonicalize_version
 
 from .utils import merge_dep_dicts
 

metaflow_extensions/netflix_ext/plugins/conda/conda_environment.py

@@ -40,7 +40,7 @@
 
 from metaflow.unbounded_foreach import UBF_TASK
 
-from metaflow_extensions.netflix_ext.vendor.packaging.utils import canonicalize_version
+from metaflow._vendor.packaging.utils import canonicalize_version
 
 from .envsresolver import EnvsResolver
 from .utils import (

metaflow_extensions/netflix_ext/plugins/conda/env_descr.py

@@ -8,7 +8,7 @@
 from hashlib import md5, sha1, sha256
 from itertools import chain
 
-from metaflow_extensions.netflix_ext.vendor.packaging.utils import (
+from metaflow._vendor.packaging.utils import (
     parse_sdist_filename,
     parse_wheel_filename,
 )

metaflow_extensions/netflix_ext/plugins/conda/envsresolver.py

@@ -28,7 +28,7 @@
 )
 from metaflow.metaflow_environment import InvalidEnvironmentException
 
-from metaflow_extensions.netflix_ext.vendor.packaging.version import parse as parse_version
+from metaflow._vendor.packaging.version import parse as parse_version
 from .env_descr import (
     EnvID,
     EnvType,

metaflow_extensions/netflix_ext/plugins/conda/pypi_package_builder.py

@@ -13,8 +13,8 @@
 
 from metaflow.debug import debug
 
-from metaflow_extensions.netflix_ext.vendor.packaging.tags import Tag
-from metaflow_extensions.netflix_ext.vendor.packaging.utils import parse_wheel_filename
+from metaflow._vendor.packaging.tags import Tag
+from metaflow._vendor.packaging.utils import parse_wheel_filename
 
 from .env_descr import (
     EnvType,

metaflow_extensions/netflix_ext/plugins/conda/resolvers/conda_lock_resolver.py

@@ -12,7 +12,7 @@
 from metaflow.debug import debug
 from metaflow.metaflow_config import CONDA_LOCAL_PATH
 
-from metaflow_extensions.netflix_ext.vendor.packaging.requirements import Requirement
+from metaflow._vendor.packaging.requirements import Requirement
 
 from ..env_descr import (
     CondaPackageSpecification,
@@ -155,7 +155,7 @@ def _poetry_exec(cmd: str, *args: str):
             toml_lines.append("\n")
             # TODO: Maybe we can make this better and only relax if :: is for channels
             # that don't exist in the list
-            if any(["::" in conda_deps]) or have_extra_channels:
+            if any(["::" in d for d in conda_deps]) or have_extra_channels:
                 addl_env = {"CONDA_CHANNEL_PRIORITY": "flexible"}
             else:
                 addl_env = {}

metaflow_extensions/netflix_ext/plugins/conda/utils.py

@@ -32,19 +32,17 @@
 from requests import PreparedRequest
 from requests.auth import AuthBase, HTTPBasicAuth
 
-from metaflow_extensions.netflix_ext.vendor.packaging.tags import (
+from metaflow._vendor.packaging.tags import (
     compatible_tags,
     _cpython_abis,
     cpython_tags,
     mac_platforms,
     Tag,
 )
 
-from metaflow_extensions.netflix_ext.vendor.packaging.utils import (
-    BuildTag,
-    parse_wheel_filename,
-)
-from metaflow_extensions.netflix_ext.vendor.packaging.version import Version
+from metaflow._vendor.packaging.utils import BuildTag, parse_wheel_filename
+
+from metaflow._vendor.packaging.version import Version
 
 from metaflow.debug import debug
 from metaflow.exception import MetaflowException

metaflow_extensions/netflix_ext/toplevel/mfextinit_netflixext.py

@@ -1 +1 @@
-toplevel = "netflixext_toplevel"
+toplevel = "netflixext_toplevel"

metaflow_extensions/netflix_ext/toplevel/netflixext_toplevel.py

@@ -1,10 +1,4 @@
-__mf_extensions__ = "netflix-ext"
-
-import pkg_resources
+from .netflixext_version import netflixext_version
 
-try:
-    __version__ = pkg_resources.get_distribution("metaflow-netflixext").version
-except:
-    # this happens on remote environments since the job package
-    # does not have a version
-    __version__ = None
+__mf_extensions__ = "netflix-ext"
+__version__ = netflixext_version

metaflow_extensions/netflix_ext/toplevel/netflixext_version.py

@@ -0,0 +1 @@
+netflixext_version = "1.0.5"