Merge pull request #1257 from mhsmith/python3.13

Add support for runtime Python version 3.13, and related changes

demo/app/build.gradle.kts

@@ -23,12 +23,12 @@ afterEvaluate {
 
 android {
     namespace = "com.chaquo.python.demo"
-    compileSdk = 34
+    compileSdk = 35
 
     defaultConfig {
         applicationId = "com.chaquo.python.demo3"
         minSdk = 24
-        targetSdk = 34
+        targetSdk = 35
 
         val plugins = buildscript.configurations.getByName("classpath")
              .resolvedConfiguration.resolvedArtifacts.map {
@@ -97,7 +97,7 @@ chaquopy {
     defaultConfig {
         // Android UI demo
         pip {
-            install("Pygments==2.2.0")  // Also used in Java API demo
+            install("Pygments==2.13.0")  // Also used in Java API demo
         }
         staticProxy("chaquopy.demo.ui_demo")
 

demo/app/src/utils/python/chaquopy/utils/console.py

@@ -77,7 +77,7 @@ def __repr__(self):
     def __getattribute__(self, name):
         # Forward all attributes that have useful implementations.
         if name in [
-            "close", "closed", "flush", "writable",  # IOBase
+            "close", "closed", "fileno", "flush", "writable",  # IOBase
             "encoding", "errors", "newlines", "buffer", "detach",  # TextIOBase
             "line_buffering", "write_through", "reconfigure",  # TextIOWrapper
         ]:
@@ -90,5 +90,9 @@ def write(self, s):
         # exception, the app crashes in the same way whether it's using
         # ConsoleOutputStream or not.
         result = self.stream.write(s)
+
+        # In case `s` is a str subclass that writes itself to stdout or stderr
+        # when we call its methods, convert it to an actual str.
+        s = str.__str__(s)
         self.method(s)
         return result

demo/app/src/utils/res/values-v35/utils.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+
+    <style name="AppTheme.V35" parent="AppTheme.Base">
+        <item name="android:windowOptOutEdgeToEdgeEnforcement">true</item>
+    </style>
+    <style name="AppTheme" parent="AppTheme.V35"/>
+
+</resources>

demo/app/src/utils/res/values/utils.xml

@@ -2,13 +2,14 @@
 <resources>
 
     <!-- Base application theme. -->
-    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
+    <style name="AppTheme.Base" parent="Theme.AppCompat.Light.DarkActionBar">
         <item name="preferenceTheme">@style/PreferenceThemeOverlay.v14.Material</item>
 
         <item name="colorPrimary">@color/colorPrimary</item>
         <item name="colorPrimaryDark">@color/colorPrimaryDark</item>
         <item name="colorAccent">@color/colorAccent</item>
     </style>
+    <style name="AppTheme" parent="AppTheme.Base"/>
 
     <color name="colorPrimary">#3F51B5</color>
     <color name="colorPrimaryDark">#303F9F</color>

product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java

@@ -10,19 +10,20 @@ public class Common {
     // Minimum Android Gradle plugin version
     public static final String MIN_AGP_VERSION = "7.0.0";
 
-    // This should match api_level in target/build-common.sh.
+    // This should match api_level in target/android-env.sh.
     public static final int MIN_SDK_VERSION = 24;
 
     public static final int COMPILE_SDK_VERSION = 34;
 
     public static final Map<String, String> PYTHON_VERSIONS = new LinkedHashMap<>();
     static {
         // Version, build number
-        PYTHON_VERSIONS.put("3.8.18", "0");
-        PYTHON_VERSIONS.put("3.9.18", "0");
-        PYTHON_VERSIONS.put("3.10.13", "0");
-        PYTHON_VERSIONS.put("3.11.6", "0");
-        PYTHON_VERSIONS.put("3.12.1", "0");
+        PYTHON_VERSIONS.put("3.8.20", "0");
+        PYTHON_VERSIONS.put("3.9.20", "0");
+        PYTHON_VERSIONS.put("3.10.15", "0");
+        PYTHON_VERSIONS.put("3.11.10", "0");
+        PYTHON_VERSIONS.put("3.12.7", "0");
+        PYTHON_VERSIONS.put("3.13.0", "0");
     }
 
     public static List<String> PYTHON_VERSIONS_SHORT = new ArrayList<>();

product/gradle-plugin/README.md

@@ -122,8 +122,10 @@ After stable release:
 
 * Increment Chaquopy major version if not already done.
 * Update `MIN_SDK_VERSION` in Common.java.
-* Update `api_level` in target/build-common.sh.
-* Update default API level in server/pypi/build-wheel.py.
+* Update `api_level` in target/android-env.sh.
+* In server/pypi/build-wheel.py:
+  * Update default API level.
+  * Update `STANDARD_LIBS` with any libraries added in the new level.
 * Search repository for other things that should be updated, including workarounds which
   are now unnecessary:
   * Useful regex: `api.?level|android.?ver|android \d|min.?sdk|SDK_INT`

product/gradle-plugin/src/main/kotlin/PythonTasks.kt

@@ -224,6 +224,9 @@ internal class TaskBuilder(
                     val customIndexUrl = listOf("--index-url", "-i").any {
                         it in python.pip.options
                     }
+                    val versionFull = pythonVersionInfo(python).key
+                    val versionFullNoPre =
+                        """\d+\.\d+\.\d+""".toRegex().find(versionFull)!!.value
 
                     execBuildPython {
                         args("-m", "chaquopy.pip_install")
@@ -237,7 +240,7 @@ internal class TaskBuilder(
                             args("--extra-index-url", "https://chaquo.com/pypi-13.1")
                         }
                         args("--implementation", Common.PYTHON_IMPLEMENTATION)
-                        args("--python-version", pythonVersionInfo(python).key)
+                        args("--python-version", versionFullNoPre)
                         args("--abi", (Common.PYTHON_IMPLEMENTATION +
                                        python.version!!.replace(".", "")))
                         args("--no-compile")
@@ -397,21 +400,36 @@ internal class TaskBuilder(
                 //
                 // If this list changes, search for references to this variable name to
                 // find the tests that need to be updated.
-                val BOOTSTRAP_NATIVE_STDLIB = listOf(
+                val BOOTSTRAP_NATIVE_STDLIB = mutableListOf(
                     "_bz2.so",  // zipfile < importer
                     "_ctypes.so",  // java.primitive and importer
                     "_datetime.so",  // calendar < importer (see test_datetime)
                     "_lzma.so",  // zipfile < importer
                     "_random.so",  // random < tempfile < zipimport
-                    "_sha2.so",  // random < tempfile < zipimport (Python >= 3.12)
-                    "_sha512.so",  // random < tempfile < zipimport (Python <= 3.11)
+                    "_sha512.so",  // random < tempfile < zipimport
                     "_struct.so",  // zipfile < importer
                     "binascii.so",  // zipfile < importer
                     "math.so",  // datetime < calendar < importer
                     "mmap.so",  // elftools < importer
                     "zlib.so"  // zipimport
                 )
 
+                val versionParts = python.version!!.split(".")
+                val versionInt =
+                    (versionParts[0].toInt() * 100) + versionParts[1].toInt()
+                if (versionInt >= 312) {
+                    BOOTSTRAP_NATIVE_STDLIB.removeAll(listOf("_sha512.so"))
+                    BOOTSTRAP_NATIVE_STDLIB.addAll(listOf(
+                        "_sha2.so"  // random < tempfile < zipimport
+                    ))
+                }
+                if (versionInt >= 313) {
+                    BOOTSTRAP_NATIVE_STDLIB.removeAll(listOf("_sha2.so"))
+                    BOOTSTRAP_NATIVE_STDLIB.addAll(listOf(
+                        "_opcode.so"  // opcode < dis < inspect < importer
+                    ))
+                }
+
                 for (abi in abis) {
                     project.copy {
                         from(project.zipTree(resolveArtifact(targetNative, abi).file))

product/gradle-plugin/src/main/python/chaquopy/pyc.py

@@ -15,14 +15,15 @@
 import warnings
 
 
-# See the list in importlib/_bootstrap_external.py.
+# See the CPython source code in Include/internal/pycore_magic_number.h or
+# Lib/importlib/_bootstrap_external.py.
 MAGIC = {
-    "3.7": 3394,
     "3.8": 3413,
     "3.9": 3425,
     "3.10": 3439,
     "3.11": 3495,
     "3.12": 3531,
+    "3.13": 3571,
 }
 
 

product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle

@@ -15,6 +15,8 @@ android {
         versionName "0.0.1"
         python {
             version "3.10"
+            pip { install "six" }
+            pyc { pip true }
         }
         ndk {
             abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"

product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle

@@ -15,6 +15,8 @@ android {
         versionName "0.0.1"
         python {
             version "3.11"
+            pip { install "six" }
+            pyc { pip true }
         }
         ndk {
             abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"

product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle

@@ -15,6 +15,8 @@ android {
         versionName "0.0.1"
         python {
             version "3.12"
+            pip { install "six" }
+            pyc { pip true }
         }
         ndk {
             abiFilters "arm64-v8a", "x86_64"

product/gradle-plugin/src/test/integration/data/PythonVersion/3.13/app/build.gradle

@@ -0,0 +1,25 @@
+plugins {
+    id 'com.android.application'
+    id 'com.chaquo.python'
+}
+
+android {
+    namespace "com.chaquo.python.test"
+    compileSdk 31
+
+    defaultConfig {
+        applicationId "com.chaquo.python.test"
+        minSdk 24
+        targetSdk 31
+        versionCode 1
+        versionName "0.0.1"
+        python {
+            version "3.13"
+            pip { install "six" }
+            pyc { pip true }
+        }
+        ndk {
+            abiFilters "arm64-v8a", "x86_64"
+        }
+    }
+}

product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle

@@ -15,6 +15,8 @@ android {
         versionName "0.0.1"
         python {
             version "3.8"
+            pip { install "six" }
+            pyc { pip true }
         }
         ndk {
             abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"

product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle

@@ -15,6 +15,8 @@ android {
         versionName "0.0.1"
         python {
             version "3.9"
+            pip { install "six" }
+            pyc { pip true }
         }
         ndk {
             abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"

product/gradle-plugin/src/test/integration/test_gradle_plugin.py

@@ -55,7 +55,7 @@ def list_versions(mode):
 for full_version in list_versions("micro").splitlines():
     version = full_version.rpartition(".")[0]
     PYTHON_VERSIONS[version] = full_version
-assert list(PYTHON_VERSIONS) == ["3.8", "3.9", "3.10", "3.11", "3.12"]
+assert list(PYTHON_VERSIONS) == ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 DEFAULT_PYTHON_VERSION_FULL = PYTHON_VERSIONS[DEFAULT_PYTHON_VERSION]
 
 NON_DEFAULT_PYTHON_VERSION = "3.10"
@@ -480,7 +480,9 @@ def check_version(self, run, version):
             abis = ["arm64-v8a", "x86_64"]
             if version in ["3.8", "3.9", "3.10", "3.11"]:
                 abis += ["armeabi-v7a", "x86"]
-            run.rerun(f"PythonVersion/{version}", python_version=version, abis=abis)
+            run.rerun(
+                f"PythonVersion/{version}", python_version=version, abis=abis,
+                requirements=["six.py"])
 
             if version == DEFAULT_PYTHON_VERSION:
                 self.assertNotInLong(self.WARNING.format(".*"), run.stdout, re=True)
@@ -1835,14 +1837,18 @@ def check_assets(self, apk_dir, kwargs):
 
         python_version_info = tuple(int(x) for x in python_version.split("."))
         stdlib_bootstrap_expected = {
-            # This is the list from our minimum Python version. For why each of these
-            # modules is needed, see BOOTSTRAP_NATIVE_STDLIB in PythonTasks.kt.
-            "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so",
-            "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so",
+            # For why each of these modules is needed, see BOOTSTRAP_NATIVE_STDLIB in
+            # PythonTasks.kt.
+            "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so",
+            "_random.so", "_sha512.so", "_struct.so", "binascii.so", "math.so",
+            "mmap.so", "zlib.so",
         }
         if python_version_info >= (3, 12):
             stdlib_bootstrap_expected -= {"_sha512.so"}
             stdlib_bootstrap_expected |= {"_sha2.so"}
+        if python_version_info >= (3, 13):
+            stdlib_bootstrap_expected -= {"_sha2.so"}
+            stdlib_bootstrap_expected |= {"_opcode.so"}
 
         bootstrap_native_dir = join(asset_dir, "bootstrap-native")
         self.test.assertCountEqual(abis, os.listdir(bootstrap_native_dir))
@@ -1863,11 +1869,13 @@ def check_assets(self, apk_dir, kwargs):
             if "stdlib" in pyc:
                 self.check_pyc(stdlib_zip, "argparse.pyc", kwargs)
 
-        # Data files packaged with stdlib: see target/package_target.sh.
-        for grammar_stem in ["Grammar", "PatternGrammar"]:
-            self.test.assertIn("lib2to3/{}{}.final.0.pickle".format(
-                                   grammar_stem, PYTHON_VERSIONS[python_version]),
-                               stdlib_files)
+        # Data files packaged with lib2to3: see target/package_target.sh.
+        # This module was removed in Python 3.13.
+        if python_version_info < (3, 13):
+            for grammar_stem in ["Grammar", "PatternGrammar"]:
+                self.test.assertIn("lib2to3/{}{}.final.0.pickle".format(
+                                       grammar_stem, PYTHON_VERSIONS[python_version]),
+                                   stdlib_files)
 
         stdlib_native_expected = {
             # This is the list from the minimum supported Python version.
@@ -1892,6 +1900,13 @@ def check_assets(self, apk_dir, kwargs):
         if python_version_info >= (3, 12):
             stdlib_native_expected -= {"_sha256.so", "_typing.so"}
             stdlib_native_expected |= {"_xxinterpchannels.so", "xxsubtype.so"}
+        if python_version_info >= (3, 13):
+            stdlib_native_expected -= {
+                "audioop.so", "_xxinterpchannels.so", "_multiprocessing.so",
+                "_opcode.so", "_xxsubinterpreters.so", "ossaudiodev.so"}
+            stdlib_native_expected |= {
+                "_interpreters.so", "_interpchannels.so", "_interpqueues.so",
+                "_sha2.so"}
 
         for abi in abis:
             stdlib_native_zip = ZipFile(join(asset_dir, f"stdlib-{abi}.imy"))
@@ -1919,14 +1934,15 @@ def check_assets(self, apk_dir, kwargs):
             build_json["assets"])
 
     def check_pyc(self, zip_file, pyc_filename, kwargs):
-        # See the list in importlib/_bootstrap_external.py.
+        # See the CPython source code at Include/internal/pycore_magic_number.h or
+        # Lib/importlib/_bootstrap_external.py.
         MAGIC = {
-            "3.7": 3394,
             "3.8": 3413,
             "3.9": 3425,
             "3.10": 3439,
             "3.11": 3495,
             "3.12": 3531,
+            "3.13": 3571,
         }
         with zip_file.open(pyc_filename) as pyc_file:
             self.test.assertEqual(
@@ -1940,9 +1956,16 @@ def check_lib(self, lib_dir, kwargs):
         for abi in abis:
             abi_dir = join(lib_dir, abi)
             self.test.assertCountEqual(
-                ["libchaquopy_java.so", "libcrypto_chaquopy.so",
-                 f"libpython{kwargs['python_version']}.so", "libssl_chaquopy.so",
-                 "libsqlite3_chaquopy.so"],
+                [
+                    "libchaquopy_java.so",
+                    "libcrypto_chaquopy.so",
+                    "libcrypto_python.so",
+                    f"libpython{python_version}.so",
+                    "libssl_chaquopy.so",
+                    "libssl_python.so",
+                    "libsqlite3_chaquopy.so",
+                    "libsqlite3_python.so",
+                ],
                 os.listdir(abi_dir))
             self.check_python_so(join(abi_dir, "libchaquopy_java.so"), python_version, abi)